Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / beffc4e84fd5710d4000664425addfda8760ebc2 / . / llvm / lib / Target / NVPTX / NVPTXISelDAGToDAG.cpp
blob: 22992b47f40f09b4a0e6fde96967dc44a3859164 [file] [log] [blame]
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the NVPTX target.
11//
12//===----------------------------------------------------------------------===//
13
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]14#include "NVPTXISelDAGToDAG.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +0000[diff] [blame]15#include "llvm/IR/GlobalValue.h"
16#include "llvm/IR/Instructions.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]17#include "llvm/Support/CommandLine.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]18#include "llvm/Support/Debug.h"
19#include "llvm/Support/ErrorHandling.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]20#include "llvm/Support/raw_ostream.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]21#include "llvm/Target/TargetIntrinsicInfo.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]22
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]23using namespace llvm;
24
Chandler Carruth84e68b22014-04-22 02:41:26 +0000[diff] [blame]25#define DEBUG_TYPE "nvptx-isel"
26
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]27static cl::opt<int> UsePrecDivF32(
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]28 "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]29 cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
Sylvestre Ledru469de192014-08-11 18:04:46 +0000[diff] [blame]30 " IEEE Compliant F32 div.rnd if available."),
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]31 cl::init(2));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]32
Justin Holewinski48f4ad32013-05-21 16:51:30 +0000[diff] [blame]33static cl::opt<bool>
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]34UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
Justin Holewinski48f4ad32013-05-21 16:51:30 +0000[diff] [blame]35 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
36 cl::init(true));
37
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]38static cl::opt<bool>
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]39FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]40 cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
41 cl::init(false));
42
43
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]44/// createNVPTXISelDag - This pass converts a legalized DAG into a
45/// NVPTX-specific DAG, ready for instruction scheduling.
46FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
47 llvm::CodeGenOpt::Level OptLevel) {
48 return new NVPTXDAGToDAGISel(TM, OptLevel);
49}
50
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]51NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
52 CodeGenOpt::Level OptLevel)
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]53 : SelectionDAGISel(tm, OptLevel) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]54 doMulWide = (OptLevel > 0);
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]55}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]56
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]57bool NVPTXDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
58 Subtarget = &static_cast<const NVPTXSubtarget &>(MF.getSubtarget());
59 return SelectionDAGISel::runOnMachineFunction(MF);
60}
61
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]62int NVPTXDAGToDAGISel::getDivF32Level() const {
63 if (UsePrecDivF32.getNumOccurrences() > 0) {
64 // If nvptx-prec-div32=N is used on the command-line, always honor it
65 return UsePrecDivF32;
66 } else {
67 // Otherwise, use div.approx if fast math is enabled
68 if (TM.Options.UnsafeFPMath)
69 return 0;
70 else
71 return 2;
72 }
73}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]74
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]75bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
76 if (UsePrecSqrtF32.getNumOccurrences() > 0) {
77 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
78 return UsePrecSqrtF32;
79 } else {
80 // Otherwise, use sqrt.approx if fast math is enabled
81 if (TM.Options.UnsafeFPMath)
82 return false;
83 else
84 return true;
85 }
86}
87
88bool NVPTXDAGToDAGISel::useF32FTZ() const {
89 if (FtzEnabled.getNumOccurrences() > 0) {
90 // If nvptx-f32ftz is used on the command-line, always honor it
91 return FtzEnabled;
92 } else {
93 const Function *F = MF->getFunction();
94 // Otherwise, check for an nvptx-f32ftz attribute on the function
95 if (F->hasFnAttribute("nvptx-f32ftz"))
Duncan P. N. Exon Smithb5054332015-02-14 15:35:43 +0000[diff] [blame]96 return F->getFnAttribute("nvptx-f32ftz").getValueAsString() == "true";
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]97 else
98 return false;
99 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]100}
101
Justin Holewinski428cf0e2014-07-17 18:10:09 +0000[diff] [blame]102bool NVPTXDAGToDAGISel::allowFMA() const {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]103 const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
Justin Holewinski428cf0e2014-07-17 18:10:09 +0000[diff] [blame]104 return TL->allowFMA(*MF, OptLevel);
105}
106
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]107/// Select - Select instructions not customized! Used for
108/// expanded, promoted and normal instructions.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]109SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]110
Tim Northover31d093c2013-09-22 08:21:56 +0000[diff] [blame]111 if (N->isMachineOpcode()) {
112 N->setNodeId(-1);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]113 return nullptr; // Already selected.
Tim Northover31d093c2013-09-22 08:21:56 +0000[diff] [blame]114 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]115
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]116 SDNode *ResNode = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]117 switch (N->getOpcode()) {
118 case ISD::LOAD:
119 ResNode = SelectLoad(N);
120 break;
121 case ISD::STORE:
122 ResNode = SelectStore(N);
123 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]124 case NVPTXISD::LoadV2:
125 case NVPTXISD::LoadV4:
126 ResNode = SelectLoadVector(N);
127 break;
128 case NVPTXISD::LDGV2:
129 case NVPTXISD::LDGV4:
130 case NVPTXISD::LDUV2:
131 case NVPTXISD::LDUV4:
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]132 ResNode = SelectLDGLDU(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]133 break;
134 case NVPTXISD::StoreV2:
135 case NVPTXISD::StoreV4:
136 ResNode = SelectStoreVector(N);
137 break;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]138 case NVPTXISD::LoadParam:
139 case NVPTXISD::LoadParamV2:
140 case NVPTXISD::LoadParamV4:
141 ResNode = SelectLoadParam(N);
142 break;
143 case NVPTXISD::StoreRetval:
144 case NVPTXISD::StoreRetvalV2:
145 case NVPTXISD::StoreRetvalV4:
146 ResNode = SelectStoreRetval(N);
147 break;
148 case NVPTXISD::StoreParam:
149 case NVPTXISD::StoreParamV2:
150 case NVPTXISD::StoreParamV4:
151 case NVPTXISD::StoreParamS32:
152 case NVPTXISD::StoreParamU32:
153 ResNode = SelectStoreParam(N);
154 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]155 case ISD::INTRINSIC_WO_CHAIN:
156 ResNode = SelectIntrinsicNoChain(N);
157 break;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]158 case ISD::INTRINSIC_W_CHAIN:
159 ResNode = SelectIntrinsicChain(N);
160 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]161 case NVPTXISD::Tex1DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]162 case NVPTXISD::Tex1DFloatFloat:
163 case NVPTXISD::Tex1DFloatFloatLevel:
164 case NVPTXISD::Tex1DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]165 case NVPTXISD::Tex1DS32S32:
166 case NVPTXISD::Tex1DS32Float:
167 case NVPTXISD::Tex1DS32FloatLevel:
168 case NVPTXISD::Tex1DS32FloatGrad:
169 case NVPTXISD::Tex1DU32S32:
170 case NVPTXISD::Tex1DU32Float:
171 case NVPTXISD::Tex1DU32FloatLevel:
172 case NVPTXISD::Tex1DU32FloatGrad:
173 case NVPTXISD::Tex1DArrayFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]174 case NVPTXISD::Tex1DArrayFloatFloat:
175 case NVPTXISD::Tex1DArrayFloatFloatLevel:
176 case NVPTXISD::Tex1DArrayFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]177 case NVPTXISD::Tex1DArrayS32S32:
178 case NVPTXISD::Tex1DArrayS32Float:
179 case NVPTXISD::Tex1DArrayS32FloatLevel:
180 case NVPTXISD::Tex1DArrayS32FloatGrad:
181 case NVPTXISD::Tex1DArrayU32S32:
182 case NVPTXISD::Tex1DArrayU32Float:
183 case NVPTXISD::Tex1DArrayU32FloatLevel:
184 case NVPTXISD::Tex1DArrayU32FloatGrad:
185 case NVPTXISD::Tex2DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]186 case NVPTXISD::Tex2DFloatFloat:
187 case NVPTXISD::Tex2DFloatFloatLevel:
188 case NVPTXISD::Tex2DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]189 case NVPTXISD::Tex2DS32S32:
190 case NVPTXISD::Tex2DS32Float:
191 case NVPTXISD::Tex2DS32FloatLevel:
192 case NVPTXISD::Tex2DS32FloatGrad:
193 case NVPTXISD::Tex2DU32S32:
194 case NVPTXISD::Tex2DU32Float:
195 case NVPTXISD::Tex2DU32FloatLevel:
196 case NVPTXISD::Tex2DU32FloatGrad:
197 case NVPTXISD::Tex2DArrayFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]198 case NVPTXISD::Tex2DArrayFloatFloat:
199 case NVPTXISD::Tex2DArrayFloatFloatLevel:
200 case NVPTXISD::Tex2DArrayFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]201 case NVPTXISD::Tex2DArrayS32S32:
202 case NVPTXISD::Tex2DArrayS32Float:
203 case NVPTXISD::Tex2DArrayS32FloatLevel:
204 case NVPTXISD::Tex2DArrayS32FloatGrad:
205 case NVPTXISD::Tex2DArrayU32S32:
206 case NVPTXISD::Tex2DArrayU32Float:
207 case NVPTXISD::Tex2DArrayU32FloatLevel:
208 case NVPTXISD::Tex2DArrayU32FloatGrad:
209 case NVPTXISD::Tex3DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]210 case NVPTXISD::Tex3DFloatFloat:
211 case NVPTXISD::Tex3DFloatFloatLevel:
212 case NVPTXISD::Tex3DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]213 case NVPTXISD::Tex3DS32S32:
214 case NVPTXISD::Tex3DS32Float:
215 case NVPTXISD::Tex3DS32FloatLevel:
216 case NVPTXISD::Tex3DS32FloatGrad:
217 case NVPTXISD::Tex3DU32S32:
218 case NVPTXISD::Tex3DU32Float:
219 case NVPTXISD::Tex3DU32FloatLevel:
220 case NVPTXISD::Tex3DU32FloatGrad:
221 case NVPTXISD::TexCubeFloatFloat:
222 case NVPTXISD::TexCubeFloatFloatLevel:
223 case NVPTXISD::TexCubeS32Float:
224 case NVPTXISD::TexCubeS32FloatLevel:
225 case NVPTXISD::TexCubeU32Float:
226 case NVPTXISD::TexCubeU32FloatLevel:
227 case NVPTXISD::TexCubeArrayFloatFloat:
228 case NVPTXISD::TexCubeArrayFloatFloatLevel:
229 case NVPTXISD::TexCubeArrayS32Float:
230 case NVPTXISD::TexCubeArrayS32FloatLevel:
231 case NVPTXISD::TexCubeArrayU32Float:
232 case NVPTXISD::TexCubeArrayU32FloatLevel:
233 case NVPTXISD::Tld4R2DFloatFloat:
234 case NVPTXISD::Tld4G2DFloatFloat:
235 case NVPTXISD::Tld4B2DFloatFloat:
236 case NVPTXISD::Tld4A2DFloatFloat:
237 case NVPTXISD::Tld4R2DS64Float:
238 case NVPTXISD::Tld4G2DS64Float:
239 case NVPTXISD::Tld4B2DS64Float:
240 case NVPTXISD::Tld4A2DS64Float:
241 case NVPTXISD::Tld4R2DU64Float:
242 case NVPTXISD::Tld4G2DU64Float:
243 case NVPTXISD::Tld4B2DU64Float:
244 case NVPTXISD::Tld4A2DU64Float:
245 case NVPTXISD::TexUnified1DFloatS32:
246 case NVPTXISD::TexUnified1DFloatFloat:
247 case NVPTXISD::TexUnified1DFloatFloatLevel:
248 case NVPTXISD::TexUnified1DFloatFloatGrad:
249 case NVPTXISD::TexUnified1DS32S32:
250 case NVPTXISD::TexUnified1DS32Float:
251 case NVPTXISD::TexUnified1DS32FloatLevel:
252 case NVPTXISD::TexUnified1DS32FloatGrad:
253 case NVPTXISD::TexUnified1DU32S32:
254 case NVPTXISD::TexUnified1DU32Float:
255 case NVPTXISD::TexUnified1DU32FloatLevel:
256 case NVPTXISD::TexUnified1DU32FloatGrad:
257 case NVPTXISD::TexUnified1DArrayFloatS32:
258 case NVPTXISD::TexUnified1DArrayFloatFloat:
259 case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
260 case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
261 case NVPTXISD::TexUnified1DArrayS32S32:
262 case NVPTXISD::TexUnified1DArrayS32Float:
263 case NVPTXISD::TexUnified1DArrayS32FloatLevel:
264 case NVPTXISD::TexUnified1DArrayS32FloatGrad:
265 case NVPTXISD::TexUnified1DArrayU32S32:
266 case NVPTXISD::TexUnified1DArrayU32Float:
267 case NVPTXISD::TexUnified1DArrayU32FloatLevel:
268 case NVPTXISD::TexUnified1DArrayU32FloatGrad:
269 case NVPTXISD::TexUnified2DFloatS32:
270 case NVPTXISD::TexUnified2DFloatFloat:
271 case NVPTXISD::TexUnified2DFloatFloatLevel:
272 case NVPTXISD::TexUnified2DFloatFloatGrad:
273 case NVPTXISD::TexUnified2DS32S32:
274 case NVPTXISD::TexUnified2DS32Float:
275 case NVPTXISD::TexUnified2DS32FloatLevel:
276 case NVPTXISD::TexUnified2DS32FloatGrad:
277 case NVPTXISD::TexUnified2DU32S32:
278 case NVPTXISD::TexUnified2DU32Float:
279 case NVPTXISD::TexUnified2DU32FloatLevel:
280 case NVPTXISD::TexUnified2DU32FloatGrad:
281 case NVPTXISD::TexUnified2DArrayFloatS32:
282 case NVPTXISD::TexUnified2DArrayFloatFloat:
283 case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
284 case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
285 case NVPTXISD::TexUnified2DArrayS32S32:
286 case NVPTXISD::TexUnified2DArrayS32Float:
287 case NVPTXISD::TexUnified2DArrayS32FloatLevel:
288 case NVPTXISD::TexUnified2DArrayS32FloatGrad:
289 case NVPTXISD::TexUnified2DArrayU32S32:
290 case NVPTXISD::TexUnified2DArrayU32Float:
291 case NVPTXISD::TexUnified2DArrayU32FloatLevel:
292 case NVPTXISD::TexUnified2DArrayU32FloatGrad:
293 case NVPTXISD::TexUnified3DFloatS32:
294 case NVPTXISD::TexUnified3DFloatFloat:
295 case NVPTXISD::TexUnified3DFloatFloatLevel:
296 case NVPTXISD::TexUnified3DFloatFloatGrad:
297 case NVPTXISD::TexUnified3DS32S32:
298 case NVPTXISD::TexUnified3DS32Float:
299 case NVPTXISD::TexUnified3DS32FloatLevel:
300 case NVPTXISD::TexUnified3DS32FloatGrad:
301 case NVPTXISD::TexUnified3DU32S32:
302 case NVPTXISD::TexUnified3DU32Float:
303 case NVPTXISD::TexUnified3DU32FloatLevel:
304 case NVPTXISD::TexUnified3DU32FloatGrad:
305 case NVPTXISD::TexUnifiedCubeFloatFloat:
306 case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
307 case NVPTXISD::TexUnifiedCubeS32Float:
308 case NVPTXISD::TexUnifiedCubeS32FloatLevel:
309 case NVPTXISD::TexUnifiedCubeU32Float:
310 case NVPTXISD::TexUnifiedCubeU32FloatLevel:
311 case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
312 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
313 case NVPTXISD::TexUnifiedCubeArrayS32Float:
314 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
315 case NVPTXISD::TexUnifiedCubeArrayU32Float:
316 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
317 case NVPTXISD::Tld4UnifiedR2DFloatFloat:
318 case NVPTXISD::Tld4UnifiedG2DFloatFloat:
319 case NVPTXISD::Tld4UnifiedB2DFloatFloat:
320 case NVPTXISD::Tld4UnifiedA2DFloatFloat:
321 case NVPTXISD::Tld4UnifiedR2DS64Float:
322 case NVPTXISD::Tld4UnifiedG2DS64Float:
323 case NVPTXISD::Tld4UnifiedB2DS64Float:
324 case NVPTXISD::Tld4UnifiedA2DS64Float:
325 case NVPTXISD::Tld4UnifiedR2DU64Float:
326 case NVPTXISD::Tld4UnifiedG2DU64Float:
327 case NVPTXISD::Tld4UnifiedB2DU64Float:
328 case NVPTXISD::Tld4UnifiedA2DU64Float:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]329 ResNode = SelectTextureIntrinsic(N);
330 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]331 case NVPTXISD::Suld1DI8Clamp:
332 case NVPTXISD::Suld1DI16Clamp:
333 case NVPTXISD::Suld1DI32Clamp:
334 case NVPTXISD::Suld1DI64Clamp:
335 case NVPTXISD::Suld1DV2I8Clamp:
336 case NVPTXISD::Suld1DV2I16Clamp:
337 case NVPTXISD::Suld1DV2I32Clamp:
338 case NVPTXISD::Suld1DV2I64Clamp:
339 case NVPTXISD::Suld1DV4I8Clamp:
340 case NVPTXISD::Suld1DV4I16Clamp:
341 case NVPTXISD::Suld1DV4I32Clamp:
342 case NVPTXISD::Suld1DArrayI8Clamp:
343 case NVPTXISD::Suld1DArrayI16Clamp:
344 case NVPTXISD::Suld1DArrayI32Clamp:
345 case NVPTXISD::Suld1DArrayI64Clamp:
346 case NVPTXISD::Suld1DArrayV2I8Clamp:
347 case NVPTXISD::Suld1DArrayV2I16Clamp:
348 case NVPTXISD::Suld1DArrayV2I32Clamp:
349 case NVPTXISD::Suld1DArrayV2I64Clamp:
350 case NVPTXISD::Suld1DArrayV4I8Clamp:
351 case NVPTXISD::Suld1DArrayV4I16Clamp:
352 case NVPTXISD::Suld1DArrayV4I32Clamp:
353 case NVPTXISD::Suld2DI8Clamp:
354 case NVPTXISD::Suld2DI16Clamp:
355 case NVPTXISD::Suld2DI32Clamp:
356 case NVPTXISD::Suld2DI64Clamp:
357 case NVPTXISD::Suld2DV2I8Clamp:
358 case NVPTXISD::Suld2DV2I16Clamp:
359 case NVPTXISD::Suld2DV2I32Clamp:
360 case NVPTXISD::Suld2DV2I64Clamp:
361 case NVPTXISD::Suld2DV4I8Clamp:
362 case NVPTXISD::Suld2DV4I16Clamp:
363 case NVPTXISD::Suld2DV4I32Clamp:
364 case NVPTXISD::Suld2DArrayI8Clamp:
365 case NVPTXISD::Suld2DArrayI16Clamp:
366 case NVPTXISD::Suld2DArrayI32Clamp:
367 case NVPTXISD::Suld2DArrayI64Clamp:
368 case NVPTXISD::Suld2DArrayV2I8Clamp:
369 case NVPTXISD::Suld2DArrayV2I16Clamp:
370 case NVPTXISD::Suld2DArrayV2I32Clamp:
371 case NVPTXISD::Suld2DArrayV2I64Clamp:
372 case NVPTXISD::Suld2DArrayV4I8Clamp:
373 case NVPTXISD::Suld2DArrayV4I16Clamp:
374 case NVPTXISD::Suld2DArrayV4I32Clamp:
375 case NVPTXISD::Suld3DI8Clamp:
376 case NVPTXISD::Suld3DI16Clamp:
377 case NVPTXISD::Suld3DI32Clamp:
378 case NVPTXISD::Suld3DI64Clamp:
379 case NVPTXISD::Suld3DV2I8Clamp:
380 case NVPTXISD::Suld3DV2I16Clamp:
381 case NVPTXISD::Suld3DV2I32Clamp:
382 case NVPTXISD::Suld3DV2I64Clamp:
383 case NVPTXISD::Suld3DV4I8Clamp:
384 case NVPTXISD::Suld3DV4I16Clamp:
385 case NVPTXISD::Suld3DV4I32Clamp:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]386 case NVPTXISD::Suld1DI8Trap:
387 case NVPTXISD::Suld1DI16Trap:
388 case NVPTXISD::Suld1DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]389 case NVPTXISD::Suld1DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]390 case NVPTXISD::Suld1DV2I8Trap:
391 case NVPTXISD::Suld1DV2I16Trap:
392 case NVPTXISD::Suld1DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]393 case NVPTXISD::Suld1DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]394 case NVPTXISD::Suld1DV4I8Trap:
395 case NVPTXISD::Suld1DV4I16Trap:
396 case NVPTXISD::Suld1DV4I32Trap:
397 case NVPTXISD::Suld1DArrayI8Trap:
398 case NVPTXISD::Suld1DArrayI16Trap:
399 case NVPTXISD::Suld1DArrayI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]400 case NVPTXISD::Suld1DArrayI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]401 case NVPTXISD::Suld1DArrayV2I8Trap:
402 case NVPTXISD::Suld1DArrayV2I16Trap:
403 case NVPTXISD::Suld1DArrayV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]404 case NVPTXISD::Suld1DArrayV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]405 case NVPTXISD::Suld1DArrayV4I8Trap:
406 case NVPTXISD::Suld1DArrayV4I16Trap:
407 case NVPTXISD::Suld1DArrayV4I32Trap:
408 case NVPTXISD::Suld2DI8Trap:
409 case NVPTXISD::Suld2DI16Trap:
410 case NVPTXISD::Suld2DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]411 case NVPTXISD::Suld2DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]412 case NVPTXISD::Suld2DV2I8Trap:
413 case NVPTXISD::Suld2DV2I16Trap:
414 case NVPTXISD::Suld2DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]415 case NVPTXISD::Suld2DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]416 case NVPTXISD::Suld2DV4I8Trap:
417 case NVPTXISD::Suld2DV4I16Trap:
418 case NVPTXISD::Suld2DV4I32Trap:
419 case NVPTXISD::Suld2DArrayI8Trap:
420 case NVPTXISD::Suld2DArrayI16Trap:
421 case NVPTXISD::Suld2DArrayI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]422 case NVPTXISD::Suld2DArrayI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]423 case NVPTXISD::Suld2DArrayV2I8Trap:
424 case NVPTXISD::Suld2DArrayV2I16Trap:
425 case NVPTXISD::Suld2DArrayV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]426 case NVPTXISD::Suld2DArrayV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]427 case NVPTXISD::Suld2DArrayV4I8Trap:
428 case NVPTXISD::Suld2DArrayV4I16Trap:
429 case NVPTXISD::Suld2DArrayV4I32Trap:
430 case NVPTXISD::Suld3DI8Trap:
431 case NVPTXISD::Suld3DI16Trap:
432 case NVPTXISD::Suld3DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]433 case NVPTXISD::Suld3DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]434 case NVPTXISD::Suld3DV2I8Trap:
435 case NVPTXISD::Suld3DV2I16Trap:
436 case NVPTXISD::Suld3DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]437 case NVPTXISD::Suld3DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]438 case NVPTXISD::Suld3DV4I8Trap:
439 case NVPTXISD::Suld3DV4I16Trap:
440 case NVPTXISD::Suld3DV4I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]441 case NVPTXISD::Suld1DI8Zero:
442 case NVPTXISD::Suld1DI16Zero:
443 case NVPTXISD::Suld1DI32Zero:
444 case NVPTXISD::Suld1DI64Zero:
445 case NVPTXISD::Suld1DV2I8Zero:
446 case NVPTXISD::Suld1DV2I16Zero:
447 case NVPTXISD::Suld1DV2I32Zero:
448 case NVPTXISD::Suld1DV2I64Zero:
449 case NVPTXISD::Suld1DV4I8Zero:
450 case NVPTXISD::Suld1DV4I16Zero:
451 case NVPTXISD::Suld1DV4I32Zero:
452 case NVPTXISD::Suld1DArrayI8Zero:
453 case NVPTXISD::Suld1DArrayI16Zero:
454 case NVPTXISD::Suld1DArrayI32Zero:
455 case NVPTXISD::Suld1DArrayI64Zero:
456 case NVPTXISD::Suld1DArrayV2I8Zero:
457 case NVPTXISD::Suld1DArrayV2I16Zero:
458 case NVPTXISD::Suld1DArrayV2I32Zero:
459 case NVPTXISD::Suld1DArrayV2I64Zero:
460 case NVPTXISD::Suld1DArrayV4I8Zero:
461 case NVPTXISD::Suld1DArrayV4I16Zero:
462 case NVPTXISD::Suld1DArrayV4I32Zero:
463 case NVPTXISD::Suld2DI8Zero:
464 case NVPTXISD::Suld2DI16Zero:
465 case NVPTXISD::Suld2DI32Zero:
466 case NVPTXISD::Suld2DI64Zero:
467 case NVPTXISD::Suld2DV2I8Zero:
468 case NVPTXISD::Suld2DV2I16Zero:
469 case NVPTXISD::Suld2DV2I32Zero:
470 case NVPTXISD::Suld2DV2I64Zero:
471 case NVPTXISD::Suld2DV4I8Zero:
472 case NVPTXISD::Suld2DV4I16Zero:
473 case NVPTXISD::Suld2DV4I32Zero:
474 case NVPTXISD::Suld2DArrayI8Zero:
475 case NVPTXISD::Suld2DArrayI16Zero:
476 case NVPTXISD::Suld2DArrayI32Zero:
477 case NVPTXISD::Suld2DArrayI64Zero:
478 case NVPTXISD::Suld2DArrayV2I8Zero:
479 case NVPTXISD::Suld2DArrayV2I16Zero:
480 case NVPTXISD::Suld2DArrayV2I32Zero:
481 case NVPTXISD::Suld2DArrayV2I64Zero:
482 case NVPTXISD::Suld2DArrayV4I8Zero:
483 case NVPTXISD::Suld2DArrayV4I16Zero:
484 case NVPTXISD::Suld2DArrayV4I32Zero:
485 case NVPTXISD::Suld3DI8Zero:
486 case NVPTXISD::Suld3DI16Zero:
487 case NVPTXISD::Suld3DI32Zero:
488 case NVPTXISD::Suld3DI64Zero:
489 case NVPTXISD::Suld3DV2I8Zero:
490 case NVPTXISD::Suld3DV2I16Zero:
491 case NVPTXISD::Suld3DV2I32Zero:
492 case NVPTXISD::Suld3DV2I64Zero:
493 case NVPTXISD::Suld3DV4I8Zero:
494 case NVPTXISD::Suld3DV4I16Zero:
495 case NVPTXISD::Suld3DV4I32Zero:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]496 ResNode = SelectSurfaceIntrinsic(N);
497 break;
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]498 case ISD::AND:
499 case ISD::SRA:
500 case ISD::SRL:
501 // Try to select BFE
502 ResNode = SelectBFE(N);
503 break;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]504 case ISD::ADDRSPACECAST:
505 ResNode = SelectAddrSpaceCast(N);
506 break;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]507 default:
508 break;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]509 }
510 if (ResNode)
511 return ResNode;
512 return SelectCode(N);
513}
514
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]515SDNode *NVPTXDAGToDAGISel::SelectIntrinsicChain(SDNode *N) {
516 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
517 switch (IID) {
518 default:
519 return NULL;
520 case Intrinsic::nvvm_ldg_global_f:
521 case Intrinsic::nvvm_ldg_global_i:
522 case Intrinsic::nvvm_ldg_global_p:
523 case Intrinsic::nvvm_ldu_global_f:
524 case Intrinsic::nvvm_ldu_global_i:
525 case Intrinsic::nvvm_ldu_global_p:
526 return SelectLDGLDU(N);
527 }
528}
529
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]530static unsigned int getCodeAddrSpace(MemSDNode *N) {
Nick Lewyckyaad475b2014-04-15 07:22:52 +0000[diff] [blame]531 const Value *Src = N->getMemOperand()->getValue();
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]532
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]533 if (!Src)
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]534 return NVPTX::PTXLdStInstCode::GENERIC;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]535
536 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
537 switch (PT->getAddressSpace()) {
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]538 case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
539 case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
540 case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
541 case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
542 case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
543 case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
544 default: break;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]545 }
546 }
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]547 return NVPTX::PTXLdStInstCode::GENERIC;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]548}
549
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]550SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) {
551 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
552 switch (IID) {
553 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]554 return nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]555 case Intrinsic::nvvm_texsurf_handle_internal:
556 return SelectTexSurfHandle(N);
557 }
558}
559
560SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
561 // Op 0 is the intrinsic ID
562 SDValue Wrapper = N->getOperand(1);
563 SDValue GlobalVal = Wrapper.getOperand(0);
564 return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64,
565 GlobalVal);
566}
567
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]568SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
569 SDValue Src = N->getOperand(0);
570 AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
571 unsigned SrcAddrSpace = CastN->getSrcAddressSpace();
572 unsigned DstAddrSpace = CastN->getDestAddressSpace();
573
574 assert(SrcAddrSpace != DstAddrSpace &&
575 "addrspacecast must be between different address spaces");
576
577 if (DstAddrSpace == ADDRESS_SPACE_GENERIC) {
578 // Specific to generic
579 unsigned Opc;
580 switch (SrcAddrSpace) {
581 default: report_fatal_error("Bad address space in addrspacecast");
582 case ADDRESS_SPACE_GLOBAL:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]583 Opc = Subtarget->is64Bit() ? NVPTX::cvta_global_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]584 : NVPTX::cvta_global_yes;
585 break;
586 case ADDRESS_SPACE_SHARED:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]587 Opc = Subtarget->is64Bit() ? NVPTX::cvta_shared_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]588 : NVPTX::cvta_shared_yes;
589 break;
590 case ADDRESS_SPACE_CONST:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]591 Opc = Subtarget->is64Bit() ? NVPTX::cvta_const_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]592 : NVPTX::cvta_const_yes;
593 break;
594 case ADDRESS_SPACE_LOCAL:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]595 Opc = Subtarget->is64Bit() ? NVPTX::cvta_local_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]596 : NVPTX::cvta_local_yes;
597 break;
598 }
599 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
600 } else {
601 // Generic to specific
602 if (SrcAddrSpace != 0)
603 report_fatal_error("Cannot cast between two non-generic address spaces");
604 unsigned Opc;
605 switch (DstAddrSpace) {
606 default: report_fatal_error("Bad address space in addrspacecast");
607 case ADDRESS_SPACE_GLOBAL:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]608 Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_global_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]609 : NVPTX::cvta_to_global_yes;
610 break;
611 case ADDRESS_SPACE_SHARED:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]612 Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_shared_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]613 : NVPTX::cvta_to_shared_yes;
614 break;
615 case ADDRESS_SPACE_CONST:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]616 Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_const_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]617 : NVPTX::cvta_to_const_yes;
618 break;
619 case ADDRESS_SPACE_LOCAL:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]620 Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_local_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]621 : NVPTX::cvta_to_local_yes;
622 break;
623 }
624 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
625 }
626}
627
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]628SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]629 SDLoc dl(N);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]630 LoadSDNode *LD = cast<LoadSDNode>(N);
631 EVT LoadedVT = LD->getMemoryVT();
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]632 SDNode *NVPTXLD = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]633
634 // do not support pre/post inc/dec
635 if (LD->isIndexed())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]636 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]637
638 if (!LoadedVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]639 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]640
641 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]642 unsigned int codeAddrSpace = getCodeAddrSpace(LD);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]643
644 // Volatile Setting
645 // - .volatile is only availalble for .global and .shared
646 bool isVolatile = LD->isVolatile();
647 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
648 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
649 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
650 isVolatile = false;
651
652 // Vector Setting
653 MVT SimpleVT = LoadedVT.getSimpleVT();
654 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
655 if (SimpleVT.isVector()) {
656 unsigned num = SimpleVT.getVectorNumElements();
657 if (num == 2)
658 vecType = NVPTX::PTXLdStInstCode::V2;
659 else if (num == 4)
660 vecType = NVPTX::PTXLdStInstCode::V4;
661 else
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]662 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]663 }
664
665 // Type Setting: fromType + fromTypeWidth
666 //
667 // Sign : ISD::SEXTLOAD
668 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
669 // type is integer
670 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
671 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski994d66a2013-05-30 12:22:39 +0000[diff] [blame]672 // Read at least 8 bits (predicates are stored as 8-bit values)
673 unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]674 unsigned int fromType;
675 if ((LD->getExtensionType() == ISD::SEXTLOAD))
676 fromType = NVPTX::PTXLdStInstCode::Signed;
677 else if (ScalarVT.isFloatingPoint())
678 fromType = NVPTX::PTXLdStInstCode::Float;
679 else
680 fromType = NVPTX::PTXLdStInstCode::Unsigned;
681
682 // Create the machine instruction DAG
683 SDValue Chain = N->getOperand(0);
684 SDValue N1 = N->getOperand(1);
685 SDValue Addr;
686 SDValue Offset, Base;
687 unsigned Opcode;
Craig Topperd9c27832013-08-15 02:44:19 +0000[diff] [blame]688 MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]689
690 if (SelectDirectAddr(N1, Addr)) {
691 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]692 case MVT::i8:
693 Opcode = NVPTX::LD_i8_avar;
694 break;
695 case MVT::i16:
696 Opcode = NVPTX::LD_i16_avar;
697 break;
698 case MVT::i32:
699 Opcode = NVPTX::LD_i32_avar;
700 break;
701 case MVT::i64:
702 Opcode = NVPTX::LD_i64_avar;
703 break;
704 case MVT::f32:
705 Opcode = NVPTX::LD_f32_avar;
706 break;
707 case MVT::f64:
708 Opcode = NVPTX::LD_f64_avar;
709 break;
710 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]711 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]712 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]713 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
714 getI32Imm(vecType), getI32Imm(fromType),
715 getI32Imm(fromTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]716 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]717 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]718 ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
719 : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]720 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]721 case MVT::i8:
722 Opcode = NVPTX::LD_i8_asi;
723 break;
724 case MVT::i16:
725 Opcode = NVPTX::LD_i16_asi;
726 break;
727 case MVT::i32:
728 Opcode = NVPTX::LD_i32_asi;
729 break;
730 case MVT::i64:
731 Opcode = NVPTX::LD_i64_asi;
732 break;
733 case MVT::f32:
734 Opcode = NVPTX::LD_f32_asi;
735 break;
736 case MVT::f64:
737 Opcode = NVPTX::LD_f64_asi;
738 break;
739 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]740 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]741 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]742 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
743 getI32Imm(vecType), getI32Imm(fromType),
744 getI32Imm(fromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]745 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]746 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]747 ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
748 : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]749 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]750 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]751 case MVT::i8:
752 Opcode = NVPTX::LD_i8_ari_64;
753 break;
754 case MVT::i16:
755 Opcode = NVPTX::LD_i16_ari_64;
756 break;
757 case MVT::i32:
758 Opcode = NVPTX::LD_i32_ari_64;
759 break;
760 case MVT::i64:
761 Opcode = NVPTX::LD_i64_ari_64;
762 break;
763 case MVT::f32:
764 Opcode = NVPTX::LD_f32_ari_64;
765 break;
766 case MVT::f64:
767 Opcode = NVPTX::LD_f64_ari_64;
768 break;
769 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]770 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]771 }
772 } else {
773 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]774 case MVT::i8:
775 Opcode = NVPTX::LD_i8_ari;
776 break;
777 case MVT::i16:
778 Opcode = NVPTX::LD_i16_ari;
779 break;
780 case MVT::i32:
781 Opcode = NVPTX::LD_i32_ari;
782 break;
783 case MVT::i64:
784 Opcode = NVPTX::LD_i64_ari;
785 break;
786 case MVT::f32:
787 Opcode = NVPTX::LD_f32_ari;
788 break;
789 case MVT::f64:
790 Opcode = NVPTX::LD_f64_ari;
791 break;
792 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]793 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]794 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]795 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]796 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
797 getI32Imm(vecType), getI32Imm(fromType),
798 getI32Imm(fromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]799 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]800 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]801 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]802 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]803 case MVT::i8:
804 Opcode = NVPTX::LD_i8_areg_64;
805 break;
806 case MVT::i16:
807 Opcode = NVPTX::LD_i16_areg_64;
808 break;
809 case MVT::i32:
810 Opcode = NVPTX::LD_i32_areg_64;
811 break;
812 case MVT::i64:
813 Opcode = NVPTX::LD_i64_areg_64;
814 break;
815 case MVT::f32:
816 Opcode = NVPTX::LD_f32_areg_64;
817 break;
818 case MVT::f64:
819 Opcode = NVPTX::LD_f64_areg_64;
820 break;
821 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]822 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]823 }
824 } else {
825 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]826 case MVT::i8:
827 Opcode = NVPTX::LD_i8_areg;
828 break;
829 case MVT::i16:
830 Opcode = NVPTX::LD_i16_areg;
831 break;
832 case MVT::i32:
833 Opcode = NVPTX::LD_i32_areg;
834 break;
835 case MVT::i64:
836 Opcode = NVPTX::LD_i64_areg;
837 break;
838 case MVT::f32:
839 Opcode = NVPTX::LD_f32_areg;
840 break;
841 case MVT::f64:
842 Opcode = NVPTX::LD_f64_areg;
843 break;
844 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]845 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]846 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]847 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]848 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
849 getI32Imm(vecType), getI32Imm(fromType),
850 getI32Imm(fromTypeWidth), N1, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]851 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]852 }
853
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]854 if (NVPTXLD) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]855 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
856 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
857 cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
858 }
859
860 return NVPTXLD;
861}
862
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]863SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
864
865 SDValue Chain = N->getOperand(0);
866 SDValue Op1 = N->getOperand(1);
867 SDValue Addr, Offset, Base;
868 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]869 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]870 SDNode *LD;
871 MemSDNode *MemSD = cast<MemSDNode>(N);
872 EVT LoadedVT = MemSD->getMemoryVT();
873
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]874 if (!LoadedVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]875 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]876
877 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]878 unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]879
880 // Volatile Setting
881 // - .volatile is only availalble for .global and .shared
882 bool IsVolatile = MemSD->isVolatile();
883 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
884 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
885 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
886 IsVolatile = false;
887
888 // Vector Setting
889 MVT SimpleVT = LoadedVT.getSimpleVT();
890
891 // Type Setting: fromType + fromTypeWidth
892 //
893 // Sign : ISD::SEXTLOAD
894 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
895 // type is integer
896 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
897 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski994d66a2013-05-30 12:22:39 +0000[diff] [blame]898 // Read at least 8 bits (predicates are stored as 8-bit values)
899 unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]900 unsigned int FromType;
901 // The last operand holds the original LoadSDNode::getExtensionType() value
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]902 unsigned ExtensionType = cast<ConstantSDNode>(
903 N->getOperand(N->getNumOperands() - 1))->getZExtValue();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]904 if (ExtensionType == ISD::SEXTLOAD)
905 FromType = NVPTX::PTXLdStInstCode::Signed;
906 else if (ScalarVT.isFloatingPoint())
907 FromType = NVPTX::PTXLdStInstCode::Float;
908 else
909 FromType = NVPTX::PTXLdStInstCode::Unsigned;
910
911 unsigned VecType;
912
913 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]914 case NVPTXISD::LoadV2:
915 VecType = NVPTX::PTXLdStInstCode::V2;
916 break;
917 case NVPTXISD::LoadV4:
918 VecType = NVPTX::PTXLdStInstCode::V4;
919 break;
920 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]921 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]922 }
923
924 EVT EltVT = N->getValueType(0);
925
926 if (SelectDirectAddr(Op1, Addr)) {
927 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]928 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]929 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]930 case NVPTXISD::LoadV2:
931 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]932 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]933 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]934 case MVT::i8:
935 Opcode = NVPTX::LDV_i8_v2_avar;
936 break;
937 case MVT::i16:
938 Opcode = NVPTX::LDV_i16_v2_avar;
939 break;
940 case MVT::i32:
941 Opcode = NVPTX::LDV_i32_v2_avar;
942 break;
943 case MVT::i64:
944 Opcode = NVPTX::LDV_i64_v2_avar;
945 break;
946 case MVT::f32:
947 Opcode = NVPTX::LDV_f32_v2_avar;
948 break;
949 case MVT::f64:
950 Opcode = NVPTX::LDV_f64_v2_avar;
951 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]952 }
953 break;
954 case NVPTXISD::LoadV4:
955 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]956 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]957 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]958 case MVT::i8:
959 Opcode = NVPTX::LDV_i8_v4_avar;
960 break;
961 case MVT::i16:
962 Opcode = NVPTX::LDV_i16_v4_avar;
963 break;
964 case MVT::i32:
965 Opcode = NVPTX::LDV_i32_v4_avar;
966 break;
967 case MVT::f32:
968 Opcode = NVPTX::LDV_f32_v4_avar;
969 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]970 }
971 break;
972 }
973
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]974 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
975 getI32Imm(VecType), getI32Imm(FromType),
976 getI32Imm(FromTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]977 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]978 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]979 ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
980 : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]981 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]982 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]983 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]984 case NVPTXISD::LoadV2:
985 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]986 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]987 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]988 case MVT::i8:
989 Opcode = NVPTX::LDV_i8_v2_asi;
990 break;
991 case MVT::i16:
992 Opcode = NVPTX::LDV_i16_v2_asi;
993 break;
994 case MVT::i32:
995 Opcode = NVPTX::LDV_i32_v2_asi;
996 break;
997 case MVT::i64:
998 Opcode = NVPTX::LDV_i64_v2_asi;
999 break;
1000 case MVT::f32:
1001 Opcode = NVPTX::LDV_f32_v2_asi;
1002 break;
1003 case MVT::f64:
1004 Opcode = NVPTX::LDV_f64_v2_asi;
1005 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1006 }
1007 break;
1008 case NVPTXISD::LoadV4:
1009 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1010 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1011 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1012 case MVT::i8:
1013 Opcode = NVPTX::LDV_i8_v4_asi;
1014 break;
1015 case MVT::i16:
1016 Opcode = NVPTX::LDV_i16_v4_asi;
1017 break;
1018 case MVT::i32:
1019 Opcode = NVPTX::LDV_i32_v4_asi;
1020 break;
1021 case MVT::f32:
1022 Opcode = NVPTX::LDV_f32_v4_asi;
1023 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1024 }
1025 break;
1026 }
1027
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1028 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1029 getI32Imm(VecType), getI32Imm(FromType),
1030 getI32Imm(FromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1031 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]1032 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1033 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
1034 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]1035 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1036 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1037 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1038 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1039 case NVPTXISD::LoadV2:
1040 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1041 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1042 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1043 case MVT::i8:
1044 Opcode = NVPTX::LDV_i8_v2_ari_64;
1045 break;
1046 case MVT::i16:
1047 Opcode = NVPTX::LDV_i16_v2_ari_64;
1048 break;
1049 case MVT::i32:
1050 Opcode = NVPTX::LDV_i32_v2_ari_64;
1051 break;
1052 case MVT::i64:
1053 Opcode = NVPTX::LDV_i64_v2_ari_64;
1054 break;
1055 case MVT::f32:
1056 Opcode = NVPTX::LDV_f32_v2_ari_64;
1057 break;
1058 case MVT::f64:
1059 Opcode = NVPTX::LDV_f64_v2_ari_64;
1060 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1061 }
1062 break;
1063 case NVPTXISD::LoadV4:
1064 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1065 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1066 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1067 case MVT::i8:
1068 Opcode = NVPTX::LDV_i8_v4_ari_64;
1069 break;
1070 case MVT::i16:
1071 Opcode = NVPTX::LDV_i16_v4_ari_64;
1072 break;
1073 case MVT::i32:
1074 Opcode = NVPTX::LDV_i32_v4_ari_64;
1075 break;
1076 case MVT::f32:
1077 Opcode = NVPTX::LDV_f32_v4_ari_64;
1078 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1079 }
1080 break;
1081 }
1082 } else {
1083 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1084 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1085 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1086 case NVPTXISD::LoadV2:
1087 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1088 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1089 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1090 case MVT::i8:
1091 Opcode = NVPTX::LDV_i8_v2_ari;
1092 break;
1093 case MVT::i16:
1094 Opcode = NVPTX::LDV_i16_v2_ari;
1095 break;
1096 case MVT::i32:
1097 Opcode = NVPTX::LDV_i32_v2_ari;
1098 break;
1099 case MVT::i64:
1100 Opcode = NVPTX::LDV_i64_v2_ari;
1101 break;
1102 case MVT::f32:
1103 Opcode = NVPTX::LDV_f32_v2_ari;
1104 break;
1105 case MVT::f64:
1106 Opcode = NVPTX::LDV_f64_v2_ari;
1107 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1108 }
1109 break;
1110 case NVPTXISD::LoadV4:
1111 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1112 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1113 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1114 case MVT::i8:
1115 Opcode = NVPTX::LDV_i8_v4_ari;
1116 break;
1117 case MVT::i16:
1118 Opcode = NVPTX::LDV_i16_v4_ari;
1119 break;
1120 case MVT::i32:
1121 Opcode = NVPTX::LDV_i32_v4_ari;
1122 break;
1123 case MVT::f32:
1124 Opcode = NVPTX::LDV_f32_v4_ari;
1125 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1126 }
1127 break;
1128 }
1129 }
1130
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1131 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1132 getI32Imm(VecType), getI32Imm(FromType),
1133 getI32Imm(FromTypeWidth), Base, Offset, Chain };
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1134
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1135 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1136 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]1137 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1138 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1139 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1140 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1141 case NVPTXISD::LoadV2:
1142 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1143 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1144 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1145 case MVT::i8:
1146 Opcode = NVPTX::LDV_i8_v2_areg_64;
1147 break;
1148 case MVT::i16:
1149 Opcode = NVPTX::LDV_i16_v2_areg_64;
1150 break;
1151 case MVT::i32:
1152 Opcode = NVPTX::LDV_i32_v2_areg_64;
1153 break;
1154 case MVT::i64:
1155 Opcode = NVPTX::LDV_i64_v2_areg_64;
1156 break;
1157 case MVT::f32:
1158 Opcode = NVPTX::LDV_f32_v2_areg_64;
1159 break;
1160 case MVT::f64:
1161 Opcode = NVPTX::LDV_f64_v2_areg_64;
1162 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1163 }
1164 break;
1165 case NVPTXISD::LoadV4:
1166 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1167 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1168 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1169 case MVT::i8:
1170 Opcode = NVPTX::LDV_i8_v4_areg_64;
1171 break;
1172 case MVT::i16:
1173 Opcode = NVPTX::LDV_i16_v4_areg_64;
1174 break;
1175 case MVT::i32:
1176 Opcode = NVPTX::LDV_i32_v4_areg_64;
1177 break;
1178 case MVT::f32:
1179 Opcode = NVPTX::LDV_f32_v4_areg_64;
1180 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1181 }
1182 break;
1183 }
1184 } else {
1185 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1186 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1187 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1188 case NVPTXISD::LoadV2:
1189 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1190 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1191 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1192 case MVT::i8:
1193 Opcode = NVPTX::LDV_i8_v2_areg;
1194 break;
1195 case MVT::i16:
1196 Opcode = NVPTX::LDV_i16_v2_areg;
1197 break;
1198 case MVT::i32:
1199 Opcode = NVPTX::LDV_i32_v2_areg;
1200 break;
1201 case MVT::i64:
1202 Opcode = NVPTX::LDV_i64_v2_areg;
1203 break;
1204 case MVT::f32:
1205 Opcode = NVPTX::LDV_f32_v2_areg;
1206 break;
1207 case MVT::f64:
1208 Opcode = NVPTX::LDV_f64_v2_areg;
1209 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1210 }
1211 break;
1212 case NVPTXISD::LoadV4:
1213 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1214 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1215 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1216 case MVT::i8:
1217 Opcode = NVPTX::LDV_i8_v4_areg;
1218 break;
1219 case MVT::i16:
1220 Opcode = NVPTX::LDV_i16_v4_areg;
1221 break;
1222 case MVT::i32:
1223 Opcode = NVPTX::LDV_i32_v4_areg;
1224 break;
1225 case MVT::f32:
1226 Opcode = NVPTX::LDV_f32_v4_areg;
1227 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1228 }
1229 break;
1230 }
1231 }
1232
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1233 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1234 getI32Imm(VecType), getI32Imm(FromType),
1235 getI32Imm(FromTypeWidth), Op1, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1236 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1237 }
1238
1239 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1240 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1241 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
1242
1243 return LD;
1244}
1245
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1246SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1247
1248 SDValue Chain = N->getOperand(0);
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1249 SDValue Op1;
1250 MemSDNode *Mem;
1251 bool IsLDG = true;
1252
1253 // If this is an LDG intrinsic, the address is the third operand. Its its an
1254 // LDG/LDU SD node (from custom vector handling), then its the second operand
1255 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
1256 Op1 = N->getOperand(2);
1257 Mem = cast<MemIntrinsicSDNode>(N);
1258 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1259 switch (IID) {
1260 default:
1261 return NULL;
1262 case Intrinsic::nvvm_ldg_global_f:
1263 case Intrinsic::nvvm_ldg_global_i:
1264 case Intrinsic::nvvm_ldg_global_p:
1265 IsLDG = true;
1266 break;
1267 case Intrinsic::nvvm_ldu_global_f:
1268 case Intrinsic::nvvm_ldu_global_i:
1269 case Intrinsic::nvvm_ldu_global_p:
1270 IsLDG = false;
1271 break;
1272 }
1273 } else {
1274 Op1 = N->getOperand(1);
1275 Mem = cast<MemSDNode>(N);
1276 }
1277
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1278 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1279 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1280 SDNode *LD;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1281 SDValue Base, Offset, Addr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1282
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1283 EVT EltVT = Mem->getMemoryVT();
1284 if (EltVT.isVector()) {
1285 EltVT = EltVT.getVectorElementType();
1286 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1287
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1288 if (SelectDirectAddr(Op1, Addr)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1289 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1290 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1291 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1292 case ISD::INTRINSIC_W_CHAIN:
1293 if (IsLDG) {
1294 switch (EltVT.getSimpleVT().SimpleTy) {
1295 default:
1296 return nullptr;
1297 case MVT::i8:
1298 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar;
1299 break;
1300 case MVT::i16:
1301 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar;
1302 break;
1303 case MVT::i32:
1304 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar;
1305 break;
1306 case MVT::i64:
1307 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar;
1308 break;
1309 case MVT::f32:
1310 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar;
1311 break;
1312 case MVT::f64:
1313 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar;
1314 break;
1315 }
1316 } else {
1317 switch (EltVT.getSimpleVT().SimpleTy) {
1318 default:
1319 return nullptr;
1320 case MVT::i8:
1321 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar;
1322 break;
1323 case MVT::i16:
1324 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar;
1325 break;
1326 case MVT::i32:
1327 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar;
1328 break;
1329 case MVT::i64:
1330 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar;
1331 break;
1332 case MVT::f32:
1333 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar;
1334 break;
1335 case MVT::f64:
1336 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar;
1337 break;
1338 }
1339 }
1340 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1341 case NVPTXISD::LDGV2:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1342 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1343 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1344 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1345 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1346 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1347 break;
1348 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1349 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1350 break;
1351 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1352 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1353 break;
1354 case MVT::i64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1355 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1356 break;
1357 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1358 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1359 break;
1360 case MVT::f64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1361 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1362 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1363 }
1364 break;
1365 case NVPTXISD::LDUV2:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1366 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1367 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1368 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1369 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1370 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1371 break;
1372 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1373 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1374 break;
1375 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1376 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1377 break;
1378 case MVT::i64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1379 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1380 break;
1381 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1382 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1383 break;
1384 case MVT::f64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1385 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
1386 break;
1387 }
1388 break;
1389 case NVPTXISD::LDGV4:
1390 switch (EltVT.getSimpleVT().SimpleTy) {
1391 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1392 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1393 case MVT::i8:
1394 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
1395 break;
1396 case MVT::i16:
1397 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
1398 break;
1399 case MVT::i32:
1400 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
1401 break;
1402 case MVT::f32:
1403 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1404 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1405 }
1406 break;
1407 case NVPTXISD::LDUV4:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1408 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1409 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1410 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1411 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1412 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1413 break;
1414 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1415 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1416 break;
1417 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1418 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1419 break;
1420 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1421 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1422 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1423 }
1424 break;
1425 }
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1426
1427 SDValue Ops[] = { Addr, Chain };
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1428 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]1429 } else if (Subtarget->is64Bit()
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1430 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
1431 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]1432 if (Subtarget->is64Bit()) {
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1433 switch (N->getOpcode()) {
1434 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1435 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1436 case ISD::INTRINSIC_W_CHAIN:
1437 if (IsLDG) {
1438 switch (EltVT.getSimpleVT().SimpleTy) {
1439 default:
1440 return nullptr;
1441 case MVT::i8:
1442 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64;
1443 break;
1444 case MVT::i16:
1445 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64;
1446 break;
1447 case MVT::i32:
1448 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64;
1449 break;
1450 case MVT::i64:
1451 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64;
1452 break;
1453 case MVT::f32:
1454 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64;
1455 break;
1456 case MVT::f64:
1457 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64;
1458 break;
1459 }
1460 } else {
1461 switch (EltVT.getSimpleVT().SimpleTy) {
1462 default:
1463 return nullptr;
1464 case MVT::i8:
1465 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64;
1466 break;
1467 case MVT::i16:
1468 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64;
1469 break;
1470 case MVT::i32:
1471 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64;
1472 break;
1473 case MVT::i64:
1474 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64;
1475 break;
1476 case MVT::f32:
1477 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64;
1478 break;
1479 case MVT::f64:
1480 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64;
1481 break;
1482 }
1483 }
1484 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1485 case NVPTXISD::LDGV2:
1486 switch (EltVT.getSimpleVT().SimpleTy) {
1487 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1488 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1489 case MVT::i8:
1490 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
1491 break;
1492 case MVT::i16:
1493 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
1494 break;
1495 case MVT::i32:
1496 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
1497 break;
1498 case MVT::i64:
1499 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
1500 break;
1501 case MVT::f32:
1502 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
1503 break;
1504 case MVT::f64:
1505 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
1506 break;
1507 }
1508 break;
1509 case NVPTXISD::LDUV2:
1510 switch (EltVT.getSimpleVT().SimpleTy) {
1511 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1512 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1513 case MVT::i8:
1514 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
1515 break;
1516 case MVT::i16:
1517 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
1518 break;
1519 case MVT::i32:
1520 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
1521 break;
1522 case MVT::i64:
1523 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
1524 break;
1525 case MVT::f32:
1526 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
1527 break;
1528 case MVT::f64:
1529 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
1530 break;
1531 }
1532 break;
1533 case NVPTXISD::LDGV4:
1534 switch (EltVT.getSimpleVT().SimpleTy) {
1535 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1536 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1537 case MVT::i8:
1538 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
1539 break;
1540 case MVT::i16:
1541 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
1542 break;
1543 case MVT::i32:
1544 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
1545 break;
1546 case MVT::f32:
1547 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
1548 break;
1549 }
1550 break;
1551 case NVPTXISD::LDUV4:
1552 switch (EltVT.getSimpleVT().SimpleTy) {
1553 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1554 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1555 case MVT::i8:
1556 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
1557 break;
1558 case MVT::i16:
1559 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
1560 break;
1561 case MVT::i32:
1562 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
1563 break;
1564 case MVT::f32:
1565 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
1566 break;
1567 }
1568 break;
1569 }
1570 } else {
1571 switch (N->getOpcode()) {
1572 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1573 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1574 case ISD::INTRINSIC_W_CHAIN:
1575 if (IsLDG) {
1576 switch (EltVT.getSimpleVT().SimpleTy) {
1577 default:
1578 return nullptr;
1579 case MVT::i8:
1580 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari;
1581 break;
1582 case MVT::i16:
1583 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari;
1584 break;
1585 case MVT::i32:
1586 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari;
1587 break;
1588 case MVT::i64:
1589 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari;
1590 break;
1591 case MVT::f32:
1592 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari;
1593 break;
1594 case MVT::f64:
1595 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari;
1596 break;
1597 }
1598 } else {
1599 switch (EltVT.getSimpleVT().SimpleTy) {
1600 default:
1601 return nullptr;
1602 case MVT::i8:
1603 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari;
1604 break;
1605 case MVT::i16:
1606 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari;
1607 break;
1608 case MVT::i32:
1609 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari;
1610 break;
1611 case MVT::i64:
1612 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari;
1613 break;
1614 case MVT::f32:
1615 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari;
1616 break;
1617 case MVT::f64:
1618 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari;
1619 break;
1620 }
1621 }
1622 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1623 case NVPTXISD::LDGV2:
1624 switch (EltVT.getSimpleVT().SimpleTy) {
1625 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1626 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1627 case MVT::i8:
1628 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
1629 break;
1630 case MVT::i16:
1631 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
1632 break;
1633 case MVT::i32:
1634 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
1635 break;
1636 case MVT::i64:
1637 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
1638 break;
1639 case MVT::f32:
1640 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
1641 break;
1642 case MVT::f64:
1643 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
1644 break;
1645 }
1646 break;
1647 case NVPTXISD::LDUV2:
1648 switch (EltVT.getSimpleVT().SimpleTy) {
1649 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1650 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1651 case MVT::i8:
1652 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
1653 break;
1654 case MVT::i16:
1655 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
1656 break;
1657 case MVT::i32:
1658 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
1659 break;
1660 case MVT::i64:
1661 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
1662 break;
1663 case MVT::f32:
1664 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
1665 break;
1666 case MVT::f64:
1667 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
1668 break;
1669 }
1670 break;
1671 case NVPTXISD::LDGV4:
1672 switch (EltVT.getSimpleVT().SimpleTy) {
1673 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1674 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1675 case MVT::i8:
1676 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
1677 break;
1678 case MVT::i16:
1679 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
1680 break;
1681 case MVT::i32:
1682 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
1683 break;
1684 case MVT::f32:
1685 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
1686 break;
1687 }
1688 break;
1689 case NVPTXISD::LDUV4:
1690 switch (EltVT.getSimpleVT().SimpleTy) {
1691 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1692 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1693 case MVT::i8:
1694 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
1695 break;
1696 case MVT::i16:
1697 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
1698 break;
1699 case MVT::i32:
1700 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
1701 break;
1702 case MVT::f32:
1703 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
1704 break;
1705 }
1706 break;
1707 }
1708 }
1709
1710 SDValue Ops[] = { Base, Offset, Chain };
1711
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1712 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1713 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]1714 if (Subtarget->is64Bit()) {
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1715 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1716 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1717 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1718 case ISD::INTRINSIC_W_CHAIN:
1719 if (IsLDG) {
1720 switch (EltVT.getSimpleVT().SimpleTy) {
1721 default:
1722 return nullptr;
1723 case MVT::i8:
1724 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64;
1725 break;
1726 case MVT::i16:
1727 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64;
1728 break;
1729 case MVT::i32:
1730 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64;
1731 break;
1732 case MVT::i64:
1733 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64;
1734 break;
1735 case MVT::f32:
1736 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64;
1737 break;
1738 case MVT::f64:
1739 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64;
1740 break;
1741 }
1742 } else {
1743 switch (EltVT.getSimpleVT().SimpleTy) {
1744 default:
1745 return nullptr;
1746 case MVT::i8:
1747 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64;
1748 break;
1749 case MVT::i16:
1750 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64;
1751 break;
1752 case MVT::i32:
1753 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64;
1754 break;
1755 case MVT::i64:
1756 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64;
1757 break;
1758 case MVT::f32:
1759 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64;
1760 break;
1761 case MVT::f64:
1762 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64;
1763 break;
1764 }
1765 }
1766 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1767 case NVPTXISD::LDGV2:
1768 switch (EltVT.getSimpleVT().SimpleTy) {
1769 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1770 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1771 case MVT::i8:
1772 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
1773 break;
1774 case MVT::i16:
1775 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
1776 break;
1777 case MVT::i32:
1778 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
1779 break;
1780 case MVT::i64:
1781 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
1782 break;
1783 case MVT::f32:
1784 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
1785 break;
1786 case MVT::f64:
1787 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
1788 break;
1789 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1790 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1791 case NVPTXISD::LDUV2:
1792 switch (EltVT.getSimpleVT().SimpleTy) {
1793 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1794 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1795 case MVT::i8:
1796 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
1797 break;
1798 case MVT::i16:
1799 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
1800 break;
1801 case MVT::i32:
1802 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
1803 break;
1804 case MVT::i64:
1805 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
1806 break;
1807 case MVT::f32:
1808 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
1809 break;
1810 case MVT::f64:
1811 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
1812 break;
1813 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1814 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1815 case NVPTXISD::LDGV4:
1816 switch (EltVT.getSimpleVT().SimpleTy) {
1817 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1818 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1819 case MVT::i8:
1820 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
1821 break;
1822 case MVT::i16:
1823 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
1824 break;
1825 case MVT::i32:
1826 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
1827 break;
1828 case MVT::f32:
1829 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
1830 break;
1831 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1832 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1833 case NVPTXISD::LDUV4:
1834 switch (EltVT.getSimpleVT().SimpleTy) {
1835 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1836 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1837 case MVT::i8:
1838 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
1839 break;
1840 case MVT::i16:
1841 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
1842 break;
1843 case MVT::i32:
1844 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
1845 break;
1846 case MVT::f32:
1847 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
1848 break;
1849 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1850 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1851 }
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1852 } else {
1853 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1854 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1855 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1856 case ISD::INTRINSIC_W_CHAIN:
1857 if (IsLDG) {
1858 switch (EltVT.getSimpleVT().SimpleTy) {
1859 default:
1860 return nullptr;
1861 case MVT::i8:
1862 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg;
1863 break;
1864 case MVT::i16:
1865 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg;
1866 break;
1867 case MVT::i32:
1868 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg;
1869 break;
1870 case MVT::i64:
1871 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg;
1872 break;
1873 case MVT::f32:
1874 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg;
1875 break;
1876 case MVT::f64:
1877 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg;
1878 break;
1879 }
1880 } else {
1881 switch (EltVT.getSimpleVT().SimpleTy) {
1882 default:
1883 return nullptr;
1884 case MVT::i8:
1885 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg;
1886 break;
1887 case MVT::i16:
1888 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg;
1889 break;
1890 case MVT::i32:
1891 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg;
1892 break;
1893 case MVT::i64:
1894 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg;
1895 break;
1896 case MVT::f32:
1897 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg;
1898 break;
1899 case MVT::f64:
1900 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg;
1901 break;
1902 }
1903 }
1904 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1905 case NVPTXISD::LDGV2:
1906 switch (EltVT.getSimpleVT().SimpleTy) {
1907 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1908 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1909 case MVT::i8:
1910 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
1911 break;
1912 case MVT::i16:
1913 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
1914 break;
1915 case MVT::i32:
1916 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
1917 break;
1918 case MVT::i64:
1919 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
1920 break;
1921 case MVT::f32:
1922 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
1923 break;
1924 case MVT::f64:
1925 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
1926 break;
1927 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1928 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1929 case NVPTXISD::LDUV2:
1930 switch (EltVT.getSimpleVT().SimpleTy) {
1931 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1932 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1933 case MVT::i8:
1934 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
1935 break;
1936 case MVT::i16:
1937 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
1938 break;
1939 case MVT::i32:
1940 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
1941 break;
1942 case MVT::i64:
1943 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
1944 break;
1945 case MVT::f32:
1946 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
1947 break;
1948 case MVT::f64:
1949 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
1950 break;
1951 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1952 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1953 case NVPTXISD::LDGV4:
1954 switch (EltVT.getSimpleVT().SimpleTy) {
1955 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1956 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1957 case MVT::i8:
1958 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
1959 break;
1960 case MVT::i16:
1961 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
1962 break;
1963 case MVT::i32:
1964 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
1965 break;
1966 case MVT::f32:
1967 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
1968 break;
1969 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1970 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1971 case NVPTXISD::LDUV4:
1972 switch (EltVT.getSimpleVT().SimpleTy) {
1973 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1974 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1975 case MVT::i8:
1976 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
1977 break;
1978 case MVT::i16:
1979 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
1980 break;
1981 case MVT::i32:
1982 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
1983 break;
1984 case MVT::f32:
1985 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
1986 break;
1987 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1988 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1989 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1990 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1991
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1992 SDValue Ops[] = { Op1, Chain };
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1993 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1994 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1995
1996 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1997 MemRefs0[0] = Mem->getMemOperand();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1998 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
1999
2000 return LD;
2001}
2002
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2003SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]2004 SDLoc dl(N);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2005 StoreSDNode *ST = cast<StoreSDNode>(N);
2006 EVT StoreVT = ST->getMemoryVT();
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2007 SDNode *NVPTXST = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2008
2009 // do not support pre/post inc/dec
2010 if (ST->isIndexed())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2011 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2012
2013 if (!StoreVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2014 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2015
2016 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]2017 unsigned int codeAddrSpace = getCodeAddrSpace(ST);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2018
2019 // Volatile Setting
2020 // - .volatile is only availalble for .global and .shared
2021 bool isVolatile = ST->isVolatile();
2022 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
2023 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
2024 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
2025 isVolatile = false;
2026
2027 // Vector Setting
2028 MVT SimpleVT = StoreVT.getSimpleVT();
2029 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
2030 if (SimpleVT.isVector()) {
2031 unsigned num = SimpleVT.getVectorNumElements();
2032 if (num == 2)
2033 vecType = NVPTX::PTXLdStInstCode::V2;
2034 else if (num == 4)
2035 vecType = NVPTX::PTXLdStInstCode::V4;
2036 else
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2037 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2038 }
2039
2040 // Type Setting: toType + toTypeWidth
2041 // - for integer type, always use 'u'
2042 //
2043 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2044 unsigned toTypeWidth = ScalarVT.getSizeInBits();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2045 unsigned int toType;
2046 if (ScalarVT.isFloatingPoint())
2047 toType = NVPTX::PTXLdStInstCode::Float;
2048 else
2049 toType = NVPTX::PTXLdStInstCode::Unsigned;
2050
2051 // Create the machine instruction DAG
2052 SDValue Chain = N->getOperand(0);
2053 SDValue N1 = N->getOperand(1);
2054 SDValue N2 = N->getOperand(2);
2055 SDValue Addr;
2056 SDValue Offset, Base;
2057 unsigned Opcode;
Craig Topperd9c27832013-08-15 02:44:19 +0000[diff] [blame]2058 MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2059
2060 if (SelectDirectAddr(N2, Addr)) {
2061 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2062 case MVT::i8:
2063 Opcode = NVPTX::ST_i8_avar;
2064 break;
2065 case MVT::i16:
2066 Opcode = NVPTX::ST_i16_avar;
2067 break;
2068 case MVT::i32:
2069 Opcode = NVPTX::ST_i32_avar;
2070 break;
2071 case MVT::i64:
2072 Opcode = NVPTX::ST_i64_avar;
2073 break;
2074 case MVT::f32:
2075 Opcode = NVPTX::ST_f32_avar;
2076 break;
2077 case MVT::f64:
2078 Opcode = NVPTX::ST_f64_avar;
2079 break;
2080 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2081 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2082 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2083 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2084 getI32Imm(vecType), getI32Imm(toType),
2085 getI32Imm(toTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2086 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2087 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2088 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
2089 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2090 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2091 case MVT::i8:
2092 Opcode = NVPTX::ST_i8_asi;
2093 break;
2094 case MVT::i16:
2095 Opcode = NVPTX::ST_i16_asi;
2096 break;
2097 case MVT::i32:
2098 Opcode = NVPTX::ST_i32_asi;
2099 break;
2100 case MVT::i64:
2101 Opcode = NVPTX::ST_i64_asi;
2102 break;
2103 case MVT::f32:
2104 Opcode = NVPTX::ST_f32_asi;
2105 break;
2106 case MVT::f64:
2107 Opcode = NVPTX::ST_f64_asi;
2108 break;
2109 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2110 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2111 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2112 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2113 getI32Imm(vecType), getI32Imm(toType),
2114 getI32Imm(toTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2115 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2116 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2117 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
2118 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2119 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2120 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2121 case MVT::i8:
2122 Opcode = NVPTX::ST_i8_ari_64;
2123 break;
2124 case MVT::i16:
2125 Opcode = NVPTX::ST_i16_ari_64;
2126 break;
2127 case MVT::i32:
2128 Opcode = NVPTX::ST_i32_ari_64;
2129 break;
2130 case MVT::i64:
2131 Opcode = NVPTX::ST_i64_ari_64;
2132 break;
2133 case MVT::f32:
2134 Opcode = NVPTX::ST_f32_ari_64;
2135 break;
2136 case MVT::f64:
2137 Opcode = NVPTX::ST_f64_ari_64;
2138 break;
2139 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2140 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2141 }
2142 } else {
2143 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2144 case MVT::i8:
2145 Opcode = NVPTX::ST_i8_ari;
2146 break;
2147 case MVT::i16:
2148 Opcode = NVPTX::ST_i16_ari;
2149 break;
2150 case MVT::i32:
2151 Opcode = NVPTX::ST_i32_ari;
2152 break;
2153 case MVT::i64:
2154 Opcode = NVPTX::ST_i64_ari;
2155 break;
2156 case MVT::f32:
2157 Opcode = NVPTX::ST_f32_ari;
2158 break;
2159 case MVT::f64:
2160 Opcode = NVPTX::ST_f64_ari;
2161 break;
2162 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2163 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2164 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2165 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2166 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2167 getI32Imm(vecType), getI32Imm(toType),
2168 getI32Imm(toTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2169 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2170 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2171 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2172 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2173 case MVT::i8:
2174 Opcode = NVPTX::ST_i8_areg_64;
2175 break;
2176 case MVT::i16:
2177 Opcode = NVPTX::ST_i16_areg_64;
2178 break;
2179 case MVT::i32:
2180 Opcode = NVPTX::ST_i32_areg_64;
2181 break;
2182 case MVT::i64:
2183 Opcode = NVPTX::ST_i64_areg_64;
2184 break;
2185 case MVT::f32:
2186 Opcode = NVPTX::ST_f32_areg_64;
2187 break;
2188 case MVT::f64:
2189 Opcode = NVPTX::ST_f64_areg_64;
2190 break;
2191 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2192 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2193 }
2194 } else {
2195 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2196 case MVT::i8:
2197 Opcode = NVPTX::ST_i8_areg;
2198 break;
2199 case MVT::i16:
2200 Opcode = NVPTX::ST_i16_areg;
2201 break;
2202 case MVT::i32:
2203 Opcode = NVPTX::ST_i32_areg;
2204 break;
2205 case MVT::i64:
2206 Opcode = NVPTX::ST_i64_areg;
2207 break;
2208 case MVT::f32:
2209 Opcode = NVPTX::ST_f32_areg;
2210 break;
2211 case MVT::f64:
2212 Opcode = NVPTX::ST_f64_areg;
2213 break;
2214 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2215 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2216 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2217 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2218 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2219 getI32Imm(vecType), getI32Imm(toType),
2220 getI32Imm(toTypeWidth), N2, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2221 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2222 }
2223
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2224 if (NVPTXST) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2225 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2226 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2227 cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2228 }
2229
2230 return NVPTXST;
2231}
2232
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2233SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
2234 SDValue Chain = N->getOperand(0);
2235 SDValue Op1 = N->getOperand(1);
2236 SDValue Addr, Offset, Base;
2237 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]2238 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2239 SDNode *ST;
2240 EVT EltVT = Op1.getValueType();
2241 MemSDNode *MemSD = cast<MemSDNode>(N);
2242 EVT StoreVT = MemSD->getMemoryVT();
2243
2244 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]2245 unsigned CodeAddrSpace = getCodeAddrSpace(MemSD);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2246
2247 if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
2248 report_fatal_error("Cannot store to pointer that points to constant "
2249 "memory space");
2250 }
2251
2252 // Volatile Setting
2253 // - .volatile is only availalble for .global and .shared
2254 bool IsVolatile = MemSD->isVolatile();
2255 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
2256 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
2257 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
2258 IsVolatile = false;
2259
2260 // Type Setting: toType + toTypeWidth
2261 // - for integer type, always use 'u'
2262 assert(StoreVT.isSimple() && "Store value is not simple");
2263 MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2264 unsigned ToTypeWidth = ScalarVT.getSizeInBits();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2265 unsigned ToType;
2266 if (ScalarVT.isFloatingPoint())
2267 ToType = NVPTX::PTXLdStInstCode::Float;
2268 else
2269 ToType = NVPTX::PTXLdStInstCode::Unsigned;
2270
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2271 SmallVector<SDValue, 12> StOps;
2272 SDValue N2;
2273 unsigned VecType;
2274
2275 switch (N->getOpcode()) {
2276 case NVPTXISD::StoreV2:
2277 VecType = NVPTX::PTXLdStInstCode::V2;
2278 StOps.push_back(N->getOperand(1));
2279 StOps.push_back(N->getOperand(2));
2280 N2 = N->getOperand(3);
2281 break;
2282 case NVPTXISD::StoreV4:
2283 VecType = NVPTX::PTXLdStInstCode::V4;
2284 StOps.push_back(N->getOperand(1));
2285 StOps.push_back(N->getOperand(2));
2286 StOps.push_back(N->getOperand(3));
2287 StOps.push_back(N->getOperand(4));
2288 N2 = N->getOperand(5);
2289 break;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2290 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2291 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2292 }
2293
2294 StOps.push_back(getI32Imm(IsVolatile));
2295 StOps.push_back(getI32Imm(CodeAddrSpace));
2296 StOps.push_back(getI32Imm(VecType));
2297 StOps.push_back(getI32Imm(ToType));
2298 StOps.push_back(getI32Imm(ToTypeWidth));
2299
2300 if (SelectDirectAddr(N2, Addr)) {
2301 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2302 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2303 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2304 case NVPTXISD::StoreV2:
2305 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2306 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2307 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2308 case MVT::i8:
2309 Opcode = NVPTX::STV_i8_v2_avar;
2310 break;
2311 case MVT::i16:
2312 Opcode = NVPTX::STV_i16_v2_avar;
2313 break;
2314 case MVT::i32:
2315 Opcode = NVPTX::STV_i32_v2_avar;
2316 break;
2317 case MVT::i64:
2318 Opcode = NVPTX::STV_i64_v2_avar;
2319 break;
2320 case MVT::f32:
2321 Opcode = NVPTX::STV_f32_v2_avar;
2322 break;
2323 case MVT::f64:
2324 Opcode = NVPTX::STV_f64_v2_avar;
2325 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2326 }
2327 break;
2328 case NVPTXISD::StoreV4:
2329 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2330 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2331 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2332 case MVT::i8:
2333 Opcode = NVPTX::STV_i8_v4_avar;
2334 break;
2335 case MVT::i16:
2336 Opcode = NVPTX::STV_i16_v4_avar;
2337 break;
2338 case MVT::i32:
2339 Opcode = NVPTX::STV_i32_v4_avar;
2340 break;
2341 case MVT::f32:
2342 Opcode = NVPTX::STV_f32_v4_avar;
2343 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2344 }
2345 break;
2346 }
2347 StOps.push_back(Addr);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2348 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2349 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
2350 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2351 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2352 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2353 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2354 case NVPTXISD::StoreV2:
2355 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2356 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2357 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2358 case MVT::i8:
2359 Opcode = NVPTX::STV_i8_v2_asi;
2360 break;
2361 case MVT::i16:
2362 Opcode = NVPTX::STV_i16_v2_asi;
2363 break;
2364 case MVT::i32:
2365 Opcode = NVPTX::STV_i32_v2_asi;
2366 break;
2367 case MVT::i64:
2368 Opcode = NVPTX::STV_i64_v2_asi;
2369 break;
2370 case MVT::f32:
2371 Opcode = NVPTX::STV_f32_v2_asi;
2372 break;
2373 case MVT::f64:
2374 Opcode = NVPTX::STV_f64_v2_asi;
2375 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2376 }
2377 break;
2378 case NVPTXISD::StoreV4:
2379 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2380 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2381 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2382 case MVT::i8:
2383 Opcode = NVPTX::STV_i8_v4_asi;
2384 break;
2385 case MVT::i16:
2386 Opcode = NVPTX::STV_i16_v4_asi;
2387 break;
2388 case MVT::i32:
2389 Opcode = NVPTX::STV_i32_v4_asi;
2390 break;
2391 case MVT::f32:
2392 Opcode = NVPTX::STV_f32_v4_asi;
2393 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2394 }
2395 break;
2396 }
2397 StOps.push_back(Base);
2398 StOps.push_back(Offset);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2399 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2400 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
2401 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2402 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2403 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2404 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2405 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2406 case NVPTXISD::StoreV2:
2407 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2408 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2409 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2410 case MVT::i8:
2411 Opcode = NVPTX::STV_i8_v2_ari_64;
2412 break;
2413 case MVT::i16:
2414 Opcode = NVPTX::STV_i16_v2_ari_64;
2415 break;
2416 case MVT::i32:
2417 Opcode = NVPTX::STV_i32_v2_ari_64;
2418 break;
2419 case MVT::i64:
2420 Opcode = NVPTX::STV_i64_v2_ari_64;
2421 break;
2422 case MVT::f32:
2423 Opcode = NVPTX::STV_f32_v2_ari_64;
2424 break;
2425 case MVT::f64:
2426 Opcode = NVPTX::STV_f64_v2_ari_64;
2427 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2428 }
2429 break;
2430 case NVPTXISD::StoreV4:
2431 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2432 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2433 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2434 case MVT::i8:
2435 Opcode = NVPTX::STV_i8_v4_ari_64;
2436 break;
2437 case MVT::i16:
2438 Opcode = NVPTX::STV_i16_v4_ari_64;
2439 break;
2440 case MVT::i32:
2441 Opcode = NVPTX::STV_i32_v4_ari_64;
2442 break;
2443 case MVT::f32:
2444 Opcode = NVPTX::STV_f32_v4_ari_64;
2445 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2446 }
2447 break;
2448 }
2449 } else {
2450 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2451 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2452 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2453 case NVPTXISD::StoreV2:
2454 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2455 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2456 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2457 case MVT::i8:
2458 Opcode = NVPTX::STV_i8_v2_ari;
2459 break;
2460 case MVT::i16:
2461 Opcode = NVPTX::STV_i16_v2_ari;
2462 break;
2463 case MVT::i32:
2464 Opcode = NVPTX::STV_i32_v2_ari;
2465 break;
2466 case MVT::i64:
2467 Opcode = NVPTX::STV_i64_v2_ari;
2468 break;
2469 case MVT::f32:
2470 Opcode = NVPTX::STV_f32_v2_ari;
2471 break;
2472 case MVT::f64:
2473 Opcode = NVPTX::STV_f64_v2_ari;
2474 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2475 }
2476 break;
2477 case NVPTXISD::StoreV4:
2478 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2479 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2480 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2481 case MVT::i8:
2482 Opcode = NVPTX::STV_i8_v4_ari;
2483 break;
2484 case MVT::i16:
2485 Opcode = NVPTX::STV_i16_v4_ari;
2486 break;
2487 case MVT::i32:
2488 Opcode = NVPTX::STV_i32_v4_ari;
2489 break;
2490 case MVT::f32:
2491 Opcode = NVPTX::STV_f32_v4_ari;
2492 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2493 }
2494 break;
2495 }
2496 }
2497 StOps.push_back(Base);
2498 StOps.push_back(Offset);
2499 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2500 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2501 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2502 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2503 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2504 case NVPTXISD::StoreV2:
2505 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2506 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2507 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2508 case MVT::i8:
2509 Opcode = NVPTX::STV_i8_v2_areg_64;
2510 break;
2511 case MVT::i16:
2512 Opcode = NVPTX::STV_i16_v2_areg_64;
2513 break;
2514 case MVT::i32:
2515 Opcode = NVPTX::STV_i32_v2_areg_64;
2516 break;
2517 case MVT::i64:
2518 Opcode = NVPTX::STV_i64_v2_areg_64;
2519 break;
2520 case MVT::f32:
2521 Opcode = NVPTX::STV_f32_v2_areg_64;
2522 break;
2523 case MVT::f64:
2524 Opcode = NVPTX::STV_f64_v2_areg_64;
2525 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2526 }
2527 break;
2528 case NVPTXISD::StoreV4:
2529 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2530 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2531 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2532 case MVT::i8:
2533 Opcode = NVPTX::STV_i8_v4_areg_64;
2534 break;
2535 case MVT::i16:
2536 Opcode = NVPTX::STV_i16_v4_areg_64;
2537 break;
2538 case MVT::i32:
2539 Opcode = NVPTX::STV_i32_v4_areg_64;
2540 break;
2541 case MVT::f32:
2542 Opcode = NVPTX::STV_f32_v4_areg_64;
2543 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2544 }
2545 break;
2546 }
2547 } else {
2548 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2549 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2550 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2551 case NVPTXISD::StoreV2:
2552 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2553 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2554 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2555 case MVT::i8:
2556 Opcode = NVPTX::STV_i8_v2_areg;
2557 break;
2558 case MVT::i16:
2559 Opcode = NVPTX::STV_i16_v2_areg;
2560 break;
2561 case MVT::i32:
2562 Opcode = NVPTX::STV_i32_v2_areg;
2563 break;
2564 case MVT::i64:
2565 Opcode = NVPTX::STV_i64_v2_areg;
2566 break;
2567 case MVT::f32:
2568 Opcode = NVPTX::STV_f32_v2_areg;
2569 break;
2570 case MVT::f64:
2571 Opcode = NVPTX::STV_f64_v2_areg;
2572 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2573 }
2574 break;
2575 case NVPTXISD::StoreV4:
2576 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2577 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2578 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2579 case MVT::i8:
2580 Opcode = NVPTX::STV_i8_v4_areg;
2581 break;
2582 case MVT::i16:
2583 Opcode = NVPTX::STV_i16_v4_areg;
2584 break;
2585 case MVT::i32:
2586 Opcode = NVPTX::STV_i32_v4_areg;
2587 break;
2588 case MVT::f32:
2589 Opcode = NVPTX::STV_f32_v4_areg;
2590 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2591 }
2592 break;
2593 }
2594 }
2595 StOps.push_back(N2);
2596 }
2597
2598 StOps.push_back(Chain);
2599
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2600 ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2601
2602 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2603 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2604 cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2605
2606 return ST;
2607}
2608
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2609SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
2610 SDValue Chain = Node->getOperand(0);
2611 SDValue Offset = Node->getOperand(2);
2612 SDValue Flag = Node->getOperand(3);
2613 SDLoc DL(Node);
2614 MemSDNode *Mem = cast<MemSDNode>(Node);
2615
2616 unsigned VecSize;
2617 switch (Node->getOpcode()) {
2618 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2619 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2620 case NVPTXISD::LoadParam:
2621 VecSize = 1;
2622 break;
2623 case NVPTXISD::LoadParamV2:
2624 VecSize = 2;
2625 break;
2626 case NVPTXISD::LoadParamV4:
2627 VecSize = 4;
2628 break;
2629 }
2630
2631 EVT EltVT = Node->getValueType(0);
2632 EVT MemVT = Mem->getMemoryVT();
2633
2634 unsigned Opc = 0;
2635
2636 switch (VecSize) {
2637 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2638 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2639 case 1:
2640 switch (MemVT.getSimpleVT().SimpleTy) {
2641 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2642 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2643 case MVT::i1:
2644 Opc = NVPTX::LoadParamMemI8;
2645 break;
2646 case MVT::i8:
2647 Opc = NVPTX::LoadParamMemI8;
2648 break;
2649 case MVT::i16:
2650 Opc = NVPTX::LoadParamMemI16;
2651 break;
2652 case MVT::i32:
2653 Opc = NVPTX::LoadParamMemI32;
2654 break;
2655 case MVT::i64:
2656 Opc = NVPTX::LoadParamMemI64;
2657 break;
2658 case MVT::f32:
2659 Opc = NVPTX::LoadParamMemF32;
2660 break;
2661 case MVT::f64:
2662 Opc = NVPTX::LoadParamMemF64;
2663 break;
2664 }
2665 break;
2666 case 2:
2667 switch (MemVT.getSimpleVT().SimpleTy) {
2668 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2669 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2670 case MVT::i1:
2671 Opc = NVPTX::LoadParamMemV2I8;
2672 break;
2673 case MVT::i8:
2674 Opc = NVPTX::LoadParamMemV2I8;
2675 break;
2676 case MVT::i16:
2677 Opc = NVPTX::LoadParamMemV2I16;
2678 break;
2679 case MVT::i32:
2680 Opc = NVPTX::LoadParamMemV2I32;
2681 break;
2682 case MVT::i64:
2683 Opc = NVPTX::LoadParamMemV2I64;
2684 break;
2685 case MVT::f32:
2686 Opc = NVPTX::LoadParamMemV2F32;
2687 break;
2688 case MVT::f64:
2689 Opc = NVPTX::LoadParamMemV2F64;
2690 break;
2691 }
2692 break;
2693 case 4:
2694 switch (MemVT.getSimpleVT().SimpleTy) {
2695 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2696 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2697 case MVT::i1:
2698 Opc = NVPTX::LoadParamMemV4I8;
2699 break;
2700 case MVT::i8:
2701 Opc = NVPTX::LoadParamMemV4I8;
2702 break;
2703 case MVT::i16:
2704 Opc = NVPTX::LoadParamMemV4I16;
2705 break;
2706 case MVT::i32:
2707 Opc = NVPTX::LoadParamMemV4I32;
2708 break;
2709 case MVT::f32:
2710 Opc = NVPTX::LoadParamMemV4F32;
2711 break;
2712 }
2713 break;
2714 }
2715
2716 SDVTList VTs;
2717 if (VecSize == 1) {
2718 VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
2719 } else if (VecSize == 2) {
2720 VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
2721 } else {
2722 EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
Craig Topperabb4ac72014-04-16 06:10:51 +0000[diff] [blame]2723 VTs = CurDAG->getVTList(EVTs);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2724 }
2725
2726 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2727
2728 SmallVector<SDValue, 2> Ops;
2729 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2730 Ops.push_back(Chain);
2731 Ops.push_back(Flag);
2732
2733 SDNode *Ret =
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]2734 CurDAG->getMachineNode(Opc, DL, VTs, Ops);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2735 return Ret;
2736}
2737
2738SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
2739 SDLoc DL(N);
2740 SDValue Chain = N->getOperand(0);
2741 SDValue Offset = N->getOperand(1);
2742 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2743 MemSDNode *Mem = cast<MemSDNode>(N);
2744
2745 // How many elements do we have?
2746 unsigned NumElts = 1;
2747 switch (N->getOpcode()) {
2748 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2749 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2750 case NVPTXISD::StoreRetval:
2751 NumElts = 1;
2752 break;
2753 case NVPTXISD::StoreRetvalV2:
2754 NumElts = 2;
2755 break;
2756 case NVPTXISD::StoreRetvalV4:
2757 NumElts = 4;
2758 break;
2759 }
2760
2761 // Build vector of operands
2762 SmallVector<SDValue, 6> Ops;
2763 for (unsigned i = 0; i < NumElts; ++i)
2764 Ops.push_back(N->getOperand(i + 2));
2765 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2766 Ops.push_back(Chain);
2767
2768 // Determine target opcode
2769 // If we have an i1, use an 8-bit store. The lowering code in
2770 // NVPTXISelLowering will have already emitted an upcast.
2771 unsigned Opcode = 0;
2772 switch (NumElts) {
2773 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2774 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2775 case 1:
2776 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2777 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2778 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2779 case MVT::i1:
2780 Opcode = NVPTX::StoreRetvalI8;
2781 break;
2782 case MVT::i8:
2783 Opcode = NVPTX::StoreRetvalI8;
2784 break;
2785 case MVT::i16:
2786 Opcode = NVPTX::StoreRetvalI16;
2787 break;
2788 case MVT::i32:
2789 Opcode = NVPTX::StoreRetvalI32;
2790 break;
2791 case MVT::i64:
2792 Opcode = NVPTX::StoreRetvalI64;
2793 break;
2794 case MVT::f32:
2795 Opcode = NVPTX::StoreRetvalF32;
2796 break;
2797 case MVT::f64:
2798 Opcode = NVPTX::StoreRetvalF64;
2799 break;
2800 }
2801 break;
2802 case 2:
2803 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2804 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2805 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2806 case MVT::i1:
2807 Opcode = NVPTX::StoreRetvalV2I8;
2808 break;
2809 case MVT::i8:
2810 Opcode = NVPTX::StoreRetvalV2I8;
2811 break;
2812 case MVT::i16:
2813 Opcode = NVPTX::StoreRetvalV2I16;
2814 break;
2815 case MVT::i32:
2816 Opcode = NVPTX::StoreRetvalV2I32;
2817 break;
2818 case MVT::i64:
2819 Opcode = NVPTX::StoreRetvalV2I64;
2820 break;
2821 case MVT::f32:
2822 Opcode = NVPTX::StoreRetvalV2F32;
2823 break;
2824 case MVT::f64:
2825 Opcode = NVPTX::StoreRetvalV2F64;
2826 break;
2827 }
2828 break;
2829 case 4:
2830 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2831 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2832 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2833 case MVT::i1:
2834 Opcode = NVPTX::StoreRetvalV4I8;
2835 break;
2836 case MVT::i8:
2837 Opcode = NVPTX::StoreRetvalV4I8;
2838 break;
2839 case MVT::i16:
2840 Opcode = NVPTX::StoreRetvalV4I16;
2841 break;
2842 case MVT::i32:
2843 Opcode = NVPTX::StoreRetvalV4I32;
2844 break;
2845 case MVT::f32:
2846 Opcode = NVPTX::StoreRetvalV4F32;
2847 break;
2848 }
2849 break;
2850 }
2851
2852 SDNode *Ret =
2853 CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
2854 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2855 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2856 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
2857
2858 return Ret;
2859}
2860
2861SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
2862 SDLoc DL(N);
2863 SDValue Chain = N->getOperand(0);
2864 SDValue Param = N->getOperand(1);
2865 unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
2866 SDValue Offset = N->getOperand(2);
2867 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2868 MemSDNode *Mem = cast<MemSDNode>(N);
2869 SDValue Flag = N->getOperand(N->getNumOperands() - 1);
2870
2871 // How many elements do we have?
2872 unsigned NumElts = 1;
2873 switch (N->getOpcode()) {
2874 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2875 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2876 case NVPTXISD::StoreParamU32:
2877 case NVPTXISD::StoreParamS32:
2878 case NVPTXISD::StoreParam:
2879 NumElts = 1;
2880 break;
2881 case NVPTXISD::StoreParamV2:
2882 NumElts = 2;
2883 break;
2884 case NVPTXISD::StoreParamV4:
2885 NumElts = 4;
2886 break;
2887 }
2888
2889 // Build vector of operands
2890 SmallVector<SDValue, 8> Ops;
2891 for (unsigned i = 0; i < NumElts; ++i)
2892 Ops.push_back(N->getOperand(i + 3));
2893 Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32));
2894 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2895 Ops.push_back(Chain);
2896 Ops.push_back(Flag);
2897
2898 // Determine target opcode
2899 // If we have an i1, use an 8-bit store. The lowering code in
2900 // NVPTXISelLowering will have already emitted an upcast.
2901 unsigned Opcode = 0;
2902 switch (N->getOpcode()) {
2903 default:
2904 switch (NumElts) {
2905 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2906 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2907 case 1:
2908 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2909 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2910 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2911 case MVT::i1:
2912 Opcode = NVPTX::StoreParamI8;
2913 break;
2914 case MVT::i8:
2915 Opcode = NVPTX::StoreParamI8;
2916 break;
2917 case MVT::i16:
2918 Opcode = NVPTX::StoreParamI16;
2919 break;
2920 case MVT::i32:
2921 Opcode = NVPTX::StoreParamI32;
2922 break;
2923 case MVT::i64:
2924 Opcode = NVPTX::StoreParamI64;
2925 break;
2926 case MVT::f32:
2927 Opcode = NVPTX::StoreParamF32;
2928 break;
2929 case MVT::f64:
2930 Opcode = NVPTX::StoreParamF64;
2931 break;
2932 }
2933 break;
2934 case 2:
2935 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2936 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2937 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2938 case MVT::i1:
2939 Opcode = NVPTX::StoreParamV2I8;
2940 break;
2941 case MVT::i8:
2942 Opcode = NVPTX::StoreParamV2I8;
2943 break;
2944 case MVT::i16:
2945 Opcode = NVPTX::StoreParamV2I16;
2946 break;
2947 case MVT::i32:
2948 Opcode = NVPTX::StoreParamV2I32;
2949 break;
2950 case MVT::i64:
2951 Opcode = NVPTX::StoreParamV2I64;
2952 break;
2953 case MVT::f32:
2954 Opcode = NVPTX::StoreParamV2F32;
2955 break;
2956 case MVT::f64:
2957 Opcode = NVPTX::StoreParamV2F64;
2958 break;
2959 }
2960 break;
2961 case 4:
2962 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2963 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2964 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2965 case MVT::i1:
2966 Opcode = NVPTX::StoreParamV4I8;
2967 break;
2968 case MVT::i8:
2969 Opcode = NVPTX::StoreParamV4I8;
2970 break;
2971 case MVT::i16:
2972 Opcode = NVPTX::StoreParamV4I16;
2973 break;
2974 case MVT::i32:
2975 Opcode = NVPTX::StoreParamV4I32;
2976 break;
2977 case MVT::f32:
2978 Opcode = NVPTX::StoreParamV4F32;
2979 break;
2980 }
2981 break;
2982 }
2983 break;
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2984 // Special case: if we have a sign-extend/zero-extend node, insert the
2985 // conversion instruction first, and use that as the value operand to
2986 // the selected StoreParam node.
2987 case NVPTXISD::StoreParamU32: {
2988 Opcode = NVPTX::StoreParamI32;
2989 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
2990 MVT::i32);
2991 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
2992 MVT::i32, Ops[0], CvtNone);
2993 Ops[0] = SDValue(Cvt, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2994 break;
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2995 }
2996 case NVPTXISD::StoreParamS32: {
2997 Opcode = NVPTX::StoreParamI32;
2998 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
2999 MVT::i32);
3000 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
3001 MVT::i32, Ops[0], CvtNone);
3002 Ops[0] = SDValue(Cvt, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3003 break;
3004 }
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]3005 }
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3006
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]3007 SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3008 SDNode *Ret =
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]3009 CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3010 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
3011 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
3012 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
3013
3014 return Ret;
3015}
3016
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3017SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) {
3018 SDValue Chain = N->getOperand(0);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3019 SDNode *Ret = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3020 unsigned Opc = 0;
3021 SmallVector<SDValue, 8> Ops;
3022
3023 switch (N->getOpcode()) {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3024 default: return nullptr;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3025 case NVPTXISD::Tex1DFloatS32:
3026 Opc = NVPTX::TEX_1D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3027 break;
3028 case NVPTXISD::Tex1DFloatFloat:
3029 Opc = NVPTX::TEX_1D_F32_F32;
3030 break;
3031 case NVPTXISD::Tex1DFloatFloatLevel:
3032 Opc = NVPTX::TEX_1D_F32_F32_LEVEL;
3033 break;
3034 case NVPTXISD::Tex1DFloatFloatGrad:
3035 Opc = NVPTX::TEX_1D_F32_F32_GRAD;
3036 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3037 case NVPTXISD::Tex1DS32S32:
3038 Opc = NVPTX::TEX_1D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3039 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3040 case NVPTXISD::Tex1DS32Float:
3041 Opc = NVPTX::TEX_1D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3042 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3043 case NVPTXISD::Tex1DS32FloatLevel:
3044 Opc = NVPTX::TEX_1D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3045 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3046 case NVPTXISD::Tex1DS32FloatGrad:
3047 Opc = NVPTX::TEX_1D_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3048 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3049 case NVPTXISD::Tex1DU32S32:
3050 Opc = NVPTX::TEX_1D_U32_S32;
3051 break;
3052 case NVPTXISD::Tex1DU32Float:
3053 Opc = NVPTX::TEX_1D_U32_F32;
3054 break;
3055 case NVPTXISD::Tex1DU32FloatLevel:
3056 Opc = NVPTX::TEX_1D_U32_F32_LEVEL;
3057 break;
3058 case NVPTXISD::Tex1DU32FloatGrad:
3059 Opc = NVPTX::TEX_1D_U32_F32_GRAD;
3060 break;
3061 case NVPTXISD::Tex1DArrayFloatS32:
3062 Opc = NVPTX::TEX_1D_ARRAY_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3063 break;
3064 case NVPTXISD::Tex1DArrayFloatFloat:
3065 Opc = NVPTX::TEX_1D_ARRAY_F32_F32;
3066 break;
3067 case NVPTXISD::Tex1DArrayFloatFloatLevel:
3068 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL;
3069 break;
3070 case NVPTXISD::Tex1DArrayFloatFloatGrad:
3071 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD;
3072 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3073 case NVPTXISD::Tex1DArrayS32S32:
3074 Opc = NVPTX::TEX_1D_ARRAY_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3075 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3076 case NVPTXISD::Tex1DArrayS32Float:
3077 Opc = NVPTX::TEX_1D_ARRAY_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3078 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3079 case NVPTXISD::Tex1DArrayS32FloatLevel:
3080 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3081 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3082 case NVPTXISD::Tex1DArrayS32FloatGrad:
3083 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3084 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3085 case NVPTXISD::Tex1DArrayU32S32:
3086 Opc = NVPTX::TEX_1D_ARRAY_U32_S32;
3087 break;
3088 case NVPTXISD::Tex1DArrayU32Float:
3089 Opc = NVPTX::TEX_1D_ARRAY_U32_F32;
3090 break;
3091 case NVPTXISD::Tex1DArrayU32FloatLevel:
3092 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL;
3093 break;
3094 case NVPTXISD::Tex1DArrayU32FloatGrad:
3095 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD;
3096 break;
3097 case NVPTXISD::Tex2DFloatS32:
3098 Opc = NVPTX::TEX_2D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3099 break;
3100 case NVPTXISD::Tex2DFloatFloat:
3101 Opc = NVPTX::TEX_2D_F32_F32;
3102 break;
3103 case NVPTXISD::Tex2DFloatFloatLevel:
3104 Opc = NVPTX::TEX_2D_F32_F32_LEVEL;
3105 break;
3106 case NVPTXISD::Tex2DFloatFloatGrad:
3107 Opc = NVPTX::TEX_2D_F32_F32_GRAD;
3108 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3109 case NVPTXISD::Tex2DS32S32:
3110 Opc = NVPTX::TEX_2D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3111 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3112 case NVPTXISD::Tex2DS32Float:
3113 Opc = NVPTX::TEX_2D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3114 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3115 case NVPTXISD::Tex2DS32FloatLevel:
3116 Opc = NVPTX::TEX_2D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3117 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3118 case NVPTXISD::Tex2DS32FloatGrad:
3119 Opc = NVPTX::TEX_2D_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3120 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3121 case NVPTXISD::Tex2DU32S32:
3122 Opc = NVPTX::TEX_2D_U32_S32;
3123 break;
3124 case NVPTXISD::Tex2DU32Float:
3125 Opc = NVPTX::TEX_2D_U32_F32;
3126 break;
3127 case NVPTXISD::Tex2DU32FloatLevel:
3128 Opc = NVPTX::TEX_2D_U32_F32_LEVEL;
3129 break;
3130 case NVPTXISD::Tex2DU32FloatGrad:
3131 Opc = NVPTX::TEX_2D_U32_F32_GRAD;
3132 break;
3133 case NVPTXISD::Tex2DArrayFloatS32:
3134 Opc = NVPTX::TEX_2D_ARRAY_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3135 break;
3136 case NVPTXISD::Tex2DArrayFloatFloat:
3137 Opc = NVPTX::TEX_2D_ARRAY_F32_F32;
3138 break;
3139 case NVPTXISD::Tex2DArrayFloatFloatLevel:
3140 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL;
3141 break;
3142 case NVPTXISD::Tex2DArrayFloatFloatGrad:
3143 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD;
3144 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3145 case NVPTXISD::Tex2DArrayS32S32:
3146 Opc = NVPTX::TEX_2D_ARRAY_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3147 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3148 case NVPTXISD::Tex2DArrayS32Float:
3149 Opc = NVPTX::TEX_2D_ARRAY_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3150 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3151 case NVPTXISD::Tex2DArrayS32FloatLevel:
3152 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3153 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3154 case NVPTXISD::Tex2DArrayS32FloatGrad:
3155 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3156 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3157 case NVPTXISD::Tex2DArrayU32S32:
3158 Opc = NVPTX::TEX_2D_ARRAY_U32_S32;
3159 break;
3160 case NVPTXISD::Tex2DArrayU32Float:
3161 Opc = NVPTX::TEX_2D_ARRAY_U32_F32;
3162 break;
3163 case NVPTXISD::Tex2DArrayU32FloatLevel:
3164 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL;
3165 break;
3166 case NVPTXISD::Tex2DArrayU32FloatGrad:
3167 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD;
3168 break;
3169 case NVPTXISD::Tex3DFloatS32:
3170 Opc = NVPTX::TEX_3D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3171 break;
3172 case NVPTXISD::Tex3DFloatFloat:
3173 Opc = NVPTX::TEX_3D_F32_F32;
3174 break;
3175 case NVPTXISD::Tex3DFloatFloatLevel:
3176 Opc = NVPTX::TEX_3D_F32_F32_LEVEL;
3177 break;
3178 case NVPTXISD::Tex3DFloatFloatGrad:
3179 Opc = NVPTX::TEX_3D_F32_F32_GRAD;
3180 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3181 case NVPTXISD::Tex3DS32S32:
3182 Opc = NVPTX::TEX_3D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3183 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3184 case NVPTXISD::Tex3DS32Float:
3185 Opc = NVPTX::TEX_3D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3186 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3187 case NVPTXISD::Tex3DS32FloatLevel:
3188 Opc = NVPTX::TEX_3D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3189 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3190 case NVPTXISD::Tex3DS32FloatGrad:
3191 Opc = NVPTX::TEX_3D_S32_F32_GRAD;
3192 break;
3193 case NVPTXISD::Tex3DU32S32:
3194 Opc = NVPTX::TEX_3D_U32_S32;
3195 break;
3196 case NVPTXISD::Tex3DU32Float:
3197 Opc = NVPTX::TEX_3D_U32_F32;
3198 break;
3199 case NVPTXISD::Tex3DU32FloatLevel:
3200 Opc = NVPTX::TEX_3D_U32_F32_LEVEL;
3201 break;
3202 case NVPTXISD::Tex3DU32FloatGrad:
3203 Opc = NVPTX::TEX_3D_U32_F32_GRAD;
3204 break;
3205 case NVPTXISD::TexCubeFloatFloat:
3206 Opc = NVPTX::TEX_CUBE_F32_F32;
3207 break;
3208 case NVPTXISD::TexCubeFloatFloatLevel:
3209 Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL;
3210 break;
3211 case NVPTXISD::TexCubeS32Float:
3212 Opc = NVPTX::TEX_CUBE_S32_F32;
3213 break;
3214 case NVPTXISD::TexCubeS32FloatLevel:
3215 Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL;
3216 break;
3217 case NVPTXISD::TexCubeU32Float:
3218 Opc = NVPTX::TEX_CUBE_U32_F32;
3219 break;
3220 case NVPTXISD::TexCubeU32FloatLevel:
3221 Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL;
3222 break;
3223 case NVPTXISD::TexCubeArrayFloatFloat:
3224 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32;
3225 break;
3226 case NVPTXISD::TexCubeArrayFloatFloatLevel:
3227 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL;
3228 break;
3229 case NVPTXISD::TexCubeArrayS32Float:
3230 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32;
3231 break;
3232 case NVPTXISD::TexCubeArrayS32FloatLevel:
3233 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL;
3234 break;
3235 case NVPTXISD::TexCubeArrayU32Float:
3236 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32;
3237 break;
3238 case NVPTXISD::TexCubeArrayU32FloatLevel:
3239 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL;
3240 break;
3241 case NVPTXISD::Tld4R2DFloatFloat:
3242 Opc = NVPTX::TLD4_R_2D_F32_F32;
3243 break;
3244 case NVPTXISD::Tld4G2DFloatFloat:
3245 Opc = NVPTX::TLD4_G_2D_F32_F32;
3246 break;
3247 case NVPTXISD::Tld4B2DFloatFloat:
3248 Opc = NVPTX::TLD4_B_2D_F32_F32;
3249 break;
3250 case NVPTXISD::Tld4A2DFloatFloat:
3251 Opc = NVPTX::TLD4_A_2D_F32_F32;
3252 break;
3253 case NVPTXISD::Tld4R2DS64Float:
3254 Opc = NVPTX::TLD4_R_2D_S32_F32;
3255 break;
3256 case NVPTXISD::Tld4G2DS64Float:
3257 Opc = NVPTX::TLD4_G_2D_S32_F32;
3258 break;
3259 case NVPTXISD::Tld4B2DS64Float:
3260 Opc = NVPTX::TLD4_B_2D_S32_F32;
3261 break;
3262 case NVPTXISD::Tld4A2DS64Float:
3263 Opc = NVPTX::TLD4_A_2D_S32_F32;
3264 break;
3265 case NVPTXISD::Tld4R2DU64Float:
3266 Opc = NVPTX::TLD4_R_2D_U32_F32;
3267 break;
3268 case NVPTXISD::Tld4G2DU64Float:
3269 Opc = NVPTX::TLD4_G_2D_U32_F32;
3270 break;
3271 case NVPTXISD::Tld4B2DU64Float:
3272 Opc = NVPTX::TLD4_B_2D_U32_F32;
3273 break;
3274 case NVPTXISD::Tld4A2DU64Float:
3275 Opc = NVPTX::TLD4_A_2D_U32_F32;
3276 break;
3277 case NVPTXISD::TexUnified1DFloatS32:
3278 Opc = NVPTX::TEX_UNIFIED_1D_F32_S32;
3279 break;
3280 case NVPTXISD::TexUnified1DFloatFloat:
3281 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32;
3282 break;
3283 case NVPTXISD::TexUnified1DFloatFloatLevel:
3284 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL;
3285 break;
3286 case NVPTXISD::TexUnified1DFloatFloatGrad:
3287 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD;
3288 break;
3289 case NVPTXISD::TexUnified1DS32S32:
3290 Opc = NVPTX::TEX_UNIFIED_1D_S32_S32;
3291 break;
3292 case NVPTXISD::TexUnified1DS32Float:
3293 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32;
3294 break;
3295 case NVPTXISD::TexUnified1DS32FloatLevel:
3296 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL;
3297 break;
3298 case NVPTXISD::TexUnified1DS32FloatGrad:
3299 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD;
3300 break;
3301 case NVPTXISD::TexUnified1DU32S32:
3302 Opc = NVPTX::TEX_UNIFIED_1D_U32_S32;
3303 break;
3304 case NVPTXISD::TexUnified1DU32Float:
3305 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32;
3306 break;
3307 case NVPTXISD::TexUnified1DU32FloatLevel:
3308 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL;
3309 break;
3310 case NVPTXISD::TexUnified1DU32FloatGrad:
3311 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD;
3312 break;
3313 case NVPTXISD::TexUnified1DArrayFloatS32:
3314 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32;
3315 break;
3316 case NVPTXISD::TexUnified1DArrayFloatFloat:
3317 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32;
3318 break;
3319 case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
3320 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL;
3321 break;
3322 case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
3323 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD;
3324 break;
3325 case NVPTXISD::TexUnified1DArrayS32S32:
3326 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32;
3327 break;
3328 case NVPTXISD::TexUnified1DArrayS32Float:
3329 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32;
3330 break;
3331 case NVPTXISD::TexUnified1DArrayS32FloatLevel:
3332 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL;
3333 break;
3334 case NVPTXISD::TexUnified1DArrayS32FloatGrad:
3335 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD;
3336 break;
3337 case NVPTXISD::TexUnified1DArrayU32S32:
3338 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32;
3339 break;
3340 case NVPTXISD::TexUnified1DArrayU32Float:
3341 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32;
3342 break;
3343 case NVPTXISD::TexUnified1DArrayU32FloatLevel:
3344 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL;
3345 break;
3346 case NVPTXISD::TexUnified1DArrayU32FloatGrad:
3347 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD;
3348 break;
3349 case NVPTXISD::TexUnified2DFloatS32:
3350 Opc = NVPTX::TEX_UNIFIED_2D_F32_S32;
3351 break;
3352 case NVPTXISD::TexUnified2DFloatFloat:
3353 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32;
3354 break;
3355 case NVPTXISD::TexUnified2DFloatFloatLevel:
3356 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL;
3357 break;
3358 case NVPTXISD::TexUnified2DFloatFloatGrad:
3359 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD;
3360 break;
3361 case NVPTXISD::TexUnified2DS32S32:
3362 Opc = NVPTX::TEX_UNIFIED_2D_S32_S32;
3363 break;
3364 case NVPTXISD::TexUnified2DS32Float:
3365 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32;
3366 break;
3367 case NVPTXISD::TexUnified2DS32FloatLevel:
3368 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL;
3369 break;
3370 case NVPTXISD::TexUnified2DS32FloatGrad:
3371 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD;
3372 break;
3373 case NVPTXISD::TexUnified2DU32S32:
3374 Opc = NVPTX::TEX_UNIFIED_2D_U32_S32;
3375 break;
3376 case NVPTXISD::TexUnified2DU32Float:
3377 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32;
3378 break;
3379 case NVPTXISD::TexUnified2DU32FloatLevel:
3380 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL;
3381 break;
3382 case NVPTXISD::TexUnified2DU32FloatGrad:
3383 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD;
3384 break;
3385 case NVPTXISD::TexUnified2DArrayFloatS32:
3386 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32;
3387 break;
3388 case NVPTXISD::TexUnified2DArrayFloatFloat:
3389 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32;
3390 break;
3391 case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
3392 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL;
3393 break;
3394 case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
3395 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD;
3396 break;
3397 case NVPTXISD::TexUnified2DArrayS32S32:
3398 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32;
3399 break;
3400 case NVPTXISD::TexUnified2DArrayS32Float:
3401 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32;
3402 break;
3403 case NVPTXISD::TexUnified2DArrayS32FloatLevel:
3404 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL;
3405 break;
3406 case NVPTXISD::TexUnified2DArrayS32FloatGrad:
3407 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD;
3408 break;
3409 case NVPTXISD::TexUnified2DArrayU32S32:
3410 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32;
3411 break;
3412 case NVPTXISD::TexUnified2DArrayU32Float:
3413 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32;
3414 break;
3415 case NVPTXISD::TexUnified2DArrayU32FloatLevel:
3416 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL;
3417 break;
3418 case NVPTXISD::TexUnified2DArrayU32FloatGrad:
3419 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD;
3420 break;
3421 case NVPTXISD::TexUnified3DFloatS32:
3422 Opc = NVPTX::TEX_UNIFIED_3D_F32_S32;
3423 break;
3424 case NVPTXISD::TexUnified3DFloatFloat:
3425 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32;
3426 break;
3427 case NVPTXISD::TexUnified3DFloatFloatLevel:
3428 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL;
3429 break;
3430 case NVPTXISD::TexUnified3DFloatFloatGrad:
3431 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD;
3432 break;
3433 case NVPTXISD::TexUnified3DS32S32:
3434 Opc = NVPTX::TEX_UNIFIED_3D_S32_S32;
3435 break;
3436 case NVPTXISD::TexUnified3DS32Float:
3437 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32;
3438 break;
3439 case NVPTXISD::TexUnified3DS32FloatLevel:
3440 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL;
3441 break;
3442 case NVPTXISD::TexUnified3DS32FloatGrad:
3443 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD;
3444 break;
3445 case NVPTXISD::TexUnified3DU32S32:
3446 Opc = NVPTX::TEX_UNIFIED_3D_U32_S32;
3447 break;
3448 case NVPTXISD::TexUnified3DU32Float:
3449 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32;
3450 break;
3451 case NVPTXISD::TexUnified3DU32FloatLevel:
3452 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL;
3453 break;
3454 case NVPTXISD::TexUnified3DU32FloatGrad:
3455 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD;
3456 break;
3457 case NVPTXISD::TexUnifiedCubeFloatFloat:
3458 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32;
3459 break;
3460 case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
3461 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL;
3462 break;
3463 case NVPTXISD::TexUnifiedCubeS32Float:
3464 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32;
3465 break;
3466 case NVPTXISD::TexUnifiedCubeS32FloatLevel:
3467 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL;
3468 break;
3469 case NVPTXISD::TexUnifiedCubeU32Float:
3470 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32;
3471 break;
3472 case NVPTXISD::TexUnifiedCubeU32FloatLevel:
3473 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL;
3474 break;
3475 case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
3476 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32;
3477 break;
3478 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
3479 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL;
3480 break;
3481 case NVPTXISD::TexUnifiedCubeArrayS32Float:
3482 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32;
3483 break;
3484 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
3485 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL;
3486 break;
3487 case NVPTXISD::TexUnifiedCubeArrayU32Float:
3488 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32;
3489 break;
3490 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
3491 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL;
3492 break;
3493 case NVPTXISD::Tld4UnifiedR2DFloatFloat:
3494 Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32;
3495 break;
3496 case NVPTXISD::Tld4UnifiedG2DFloatFloat:
3497 Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32;
3498 break;
3499 case NVPTXISD::Tld4UnifiedB2DFloatFloat:
3500 Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32;
3501 break;
3502 case NVPTXISD::Tld4UnifiedA2DFloatFloat:
3503 Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32;
3504 break;
3505 case NVPTXISD::Tld4UnifiedR2DS64Float:
3506 Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32;
3507 break;
3508 case NVPTXISD::Tld4UnifiedG2DS64Float:
3509 Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32;
3510 break;
3511 case NVPTXISD::Tld4UnifiedB2DS64Float:
3512 Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32;
3513 break;
3514 case NVPTXISD::Tld4UnifiedA2DS64Float:
3515 Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32;
3516 break;
3517 case NVPTXISD::Tld4UnifiedR2DU64Float:
3518 Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32;
3519 break;
3520 case NVPTXISD::Tld4UnifiedG2DU64Float:
3521 Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32;
3522 break;
3523 case NVPTXISD::Tld4UnifiedB2DU64Float:
3524 Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32;
3525 break;
3526 case NVPTXISD::Tld4UnifiedA2DU64Float:
3527 Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3528 break;
3529 }
3530
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3531 // Copy over operands
3532 for (unsigned i = 1; i < N->getNumOperands(); ++i) {
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3533 Ops.push_back(N->getOperand(i));
3534 }
3535
3536 Ops.push_back(Chain);
3537 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
3538 return Ret;
3539}
3540
3541SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) {
3542 SDValue Chain = N->getOperand(0);
3543 SDValue TexHandle = N->getOperand(1);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3544 SDNode *Ret = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3545 unsigned Opc = 0;
3546 SmallVector<SDValue, 8> Ops;
3547 switch (N->getOpcode()) {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3548 default: return nullptr;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3549 case NVPTXISD::Suld1DI8Clamp:
3550 Opc = NVPTX::SULD_1D_I8_CLAMP;
3551 Ops.push_back(TexHandle);
3552 Ops.push_back(N->getOperand(2));
3553 Ops.push_back(Chain);
3554 break;
3555 case NVPTXISD::Suld1DI16Clamp:
3556 Opc = NVPTX::SULD_1D_I16_CLAMP;
3557 Ops.push_back(TexHandle);
3558 Ops.push_back(N->getOperand(2));
3559 Ops.push_back(Chain);
3560 break;
3561 case NVPTXISD::Suld1DI32Clamp:
3562 Opc = NVPTX::SULD_1D_I32_CLAMP;
3563 Ops.push_back(TexHandle);
3564 Ops.push_back(N->getOperand(2));
3565 Ops.push_back(Chain);
3566 break;
3567 case NVPTXISD::Suld1DI64Clamp:
3568 Opc = NVPTX::SULD_1D_I64_CLAMP;
3569 Ops.push_back(TexHandle);
3570 Ops.push_back(N->getOperand(2));
3571 Ops.push_back(Chain);
3572 break;
3573 case NVPTXISD::Suld1DV2I8Clamp:
3574 Opc = NVPTX::SULD_1D_V2I8_CLAMP;
3575 Ops.push_back(TexHandle);
3576 Ops.push_back(N->getOperand(2));
3577 Ops.push_back(Chain);
3578 break;
3579 case NVPTXISD::Suld1DV2I16Clamp:
3580 Opc = NVPTX::SULD_1D_V2I16_CLAMP;
3581 Ops.push_back(TexHandle);
3582 Ops.push_back(N->getOperand(2));
3583 Ops.push_back(Chain);
3584 break;
3585 case NVPTXISD::Suld1DV2I32Clamp:
3586 Opc = NVPTX::SULD_1D_V2I32_CLAMP;
3587 Ops.push_back(TexHandle);
3588 Ops.push_back(N->getOperand(2));
3589 Ops.push_back(Chain);
3590 break;
3591 case NVPTXISD::Suld1DV2I64Clamp:
3592 Opc = NVPTX::SULD_1D_V2I64_CLAMP;
3593 Ops.push_back(TexHandle);
3594 Ops.push_back(N->getOperand(2));
3595 Ops.push_back(Chain);
3596 break;
3597 case NVPTXISD::Suld1DV4I8Clamp:
3598 Opc = NVPTX::SULD_1D_V4I8_CLAMP;
3599 Ops.push_back(TexHandle);
3600 Ops.push_back(N->getOperand(2));
3601 Ops.push_back(Chain);
3602 break;
3603 case NVPTXISD::Suld1DV4I16Clamp:
3604 Opc = NVPTX::SULD_1D_V4I16_CLAMP;
3605 Ops.push_back(TexHandle);
3606 Ops.push_back(N->getOperand(2));
3607 Ops.push_back(Chain);
3608 break;
3609 case NVPTXISD::Suld1DV4I32Clamp:
3610 Opc = NVPTX::SULD_1D_V4I32_CLAMP;
3611 Ops.push_back(TexHandle);
3612 Ops.push_back(N->getOperand(2));
3613 Ops.push_back(Chain);
3614 break;
3615 case NVPTXISD::Suld1DArrayI8Clamp:
3616 Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP;
3617 Ops.push_back(TexHandle);
3618 Ops.push_back(N->getOperand(2));
3619 Ops.push_back(N->getOperand(3));
3620 Ops.push_back(Chain);
3621 break;
3622 case NVPTXISD::Suld1DArrayI16Clamp:
3623 Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP;
3624 Ops.push_back(TexHandle);
3625 Ops.push_back(N->getOperand(2));
3626 Ops.push_back(N->getOperand(3));
3627 Ops.push_back(Chain);
3628 break;
3629 case NVPTXISD::Suld1DArrayI32Clamp:
3630 Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP;
3631 Ops.push_back(TexHandle);
3632 Ops.push_back(N->getOperand(2));
3633 Ops.push_back(N->getOperand(3));
3634 Ops.push_back(Chain);
3635 break;
3636 case NVPTXISD::Suld1DArrayI64Clamp:
3637 Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP;
3638 Ops.push_back(TexHandle);
3639 Ops.push_back(N->getOperand(2));
3640 Ops.push_back(N->getOperand(3));
3641 Ops.push_back(Chain);
3642 break;
3643 case NVPTXISD::Suld1DArrayV2I8Clamp:
3644 Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP;
3645 Ops.push_back(TexHandle);
3646 Ops.push_back(N->getOperand(2));
3647 Ops.push_back(N->getOperand(3));
3648 Ops.push_back(Chain);
3649 break;
3650 case NVPTXISD::Suld1DArrayV2I16Clamp:
3651 Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP;
3652 Ops.push_back(TexHandle);
3653 Ops.push_back(N->getOperand(2));
3654 Ops.push_back(N->getOperand(3));
3655 Ops.push_back(Chain);
3656 break;
3657 case NVPTXISD::Suld1DArrayV2I32Clamp:
3658 Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP;
3659 Ops.push_back(TexHandle);
3660 Ops.push_back(N->getOperand(2));
3661 Ops.push_back(N->getOperand(3));
3662 Ops.push_back(Chain);
3663 break;
3664 case NVPTXISD::Suld1DArrayV2I64Clamp:
3665 Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP;
3666 Ops.push_back(TexHandle);
3667 Ops.push_back(N->getOperand(2));
3668 Ops.push_back(N->getOperand(3));
3669 Ops.push_back(Chain);
3670 break;
3671 case NVPTXISD::Suld1DArrayV4I8Clamp:
3672 Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP;
3673 Ops.push_back(TexHandle);
3674 Ops.push_back(N->getOperand(2));
3675 Ops.push_back(N->getOperand(3));
3676 Ops.push_back(Chain);
3677 break;
3678 case NVPTXISD::Suld1DArrayV4I16Clamp:
3679 Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP;
3680 Ops.push_back(TexHandle);
3681 Ops.push_back(N->getOperand(2));
3682 Ops.push_back(N->getOperand(3));
3683 Ops.push_back(Chain);
3684 break;
3685 case NVPTXISD::Suld1DArrayV4I32Clamp:
3686 Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP;
3687 Ops.push_back(TexHandle);
3688 Ops.push_back(N->getOperand(2));
3689 Ops.push_back(N->getOperand(3));
3690 Ops.push_back(Chain);
3691 break;
3692 case NVPTXISD::Suld2DI8Clamp:
3693 Opc = NVPTX::SULD_2D_I8_CLAMP;
3694 Ops.push_back(TexHandle);
3695 Ops.push_back(N->getOperand(2));
3696 Ops.push_back(N->getOperand(3));
3697 Ops.push_back(Chain);
3698 break;
3699 case NVPTXISD::Suld2DI16Clamp:
3700 Opc = NVPTX::SULD_2D_I16_CLAMP;
3701 Ops.push_back(TexHandle);
3702 Ops.push_back(N->getOperand(2));
3703 Ops.push_back(N->getOperand(3));
3704 Ops.push_back(Chain);
3705 break;
3706 case NVPTXISD::Suld2DI32Clamp:
3707 Opc = NVPTX::SULD_2D_I32_CLAMP;
3708 Ops.push_back(TexHandle);
3709 Ops.push_back(N->getOperand(2));
3710 Ops.push_back(N->getOperand(3));
3711 Ops.push_back(Chain);
3712 break;
3713 case NVPTXISD::Suld2DI64Clamp:
3714 Opc = NVPTX::SULD_2D_I64_CLAMP;
3715 Ops.push_back(TexHandle);
3716 Ops.push_back(N->getOperand(2));
3717 Ops.push_back(N->getOperand(3));
3718 Ops.push_back(Chain);
3719 break;
3720 case NVPTXISD::Suld2DV2I8Clamp:
3721 Opc = NVPTX::SULD_2D_V2I8_CLAMP;
3722 Ops.push_back(TexHandle);
3723 Ops.push_back(N->getOperand(2));
3724 Ops.push_back(N->getOperand(3));
3725 Ops.push_back(Chain);
3726 break;
3727 case NVPTXISD::Suld2DV2I16Clamp:
3728 Opc = NVPTX::SULD_2D_V2I16_CLAMP;
3729 Ops.push_back(TexHandle);
3730 Ops.push_back(N->getOperand(2));
3731 Ops.push_back(N->getOperand(3));
3732 Ops.push_back(Chain);
3733 break;
3734 case NVPTXISD::Suld2DV2I32Clamp:
3735 Opc = NVPTX::SULD_2D_V2I32_CLAMP;
3736 Ops.push_back(TexHandle);
3737 Ops.push_back(N->getOperand(2));
3738 Ops.push_back(N->getOperand(3));
3739 Ops.push_back(Chain);
3740 break;
3741 case NVPTXISD::Suld2DV2I64Clamp:
3742 Opc = NVPTX::SULD_2D_V2I64_CLAMP;
3743 Ops.push_back(TexHandle);
3744 Ops.push_back(N->getOperand(2));
3745 Ops.push_back(N->getOperand(3));
3746 Ops.push_back(Chain);
3747 break;
3748 case NVPTXISD::Suld2DV4I8Clamp:
3749 Opc = NVPTX::SULD_2D_V4I8_CLAMP;
3750 Ops.push_back(TexHandle);
3751 Ops.push_back(N->getOperand(2));
3752 Ops.push_back(N->getOperand(3));
3753 Ops.push_back(Chain);
3754 break;
3755 case NVPTXISD::Suld2DV4I16Clamp:
3756 Opc = NVPTX::SULD_2D_V4I16_CLAMP;
3757 Ops.push_back(TexHandle);
3758 Ops.push_back(N->getOperand(2));
3759 Ops.push_back(N->getOperand(3));
3760 Ops.push_back(Chain);
3761 break;
3762 case NVPTXISD::Suld2DV4I32Clamp:
3763 Opc = NVPTX::SULD_2D_V4I32_CLAMP;
3764 Ops.push_back(TexHandle);
3765 Ops.push_back(N->getOperand(2));
3766 Ops.push_back(N->getOperand(3));
3767 Ops.push_back(Chain);
3768 break;
3769 case NVPTXISD::Suld2DArrayI8Clamp:
3770 Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP;
3771 Ops.push_back(TexHandle);
3772 Ops.push_back(N->getOperand(2));
3773 Ops.push_back(N->getOperand(3));
3774 Ops.push_back(N->getOperand(4));
3775 Ops.push_back(Chain);
3776 break;
3777 case NVPTXISD::Suld2DArrayI16Clamp:
3778 Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP;
3779 Ops.push_back(TexHandle);
3780 Ops.push_back(N->getOperand(2));
3781 Ops.push_back(N->getOperand(3));
3782 Ops.push_back(N->getOperand(4));
3783 Ops.push_back(Chain);
3784 break;
3785 case NVPTXISD::Suld2DArrayI32Clamp:
3786 Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP;
3787 Ops.push_back(TexHandle);
3788 Ops.push_back(N->getOperand(2));
3789 Ops.push_back(N->getOperand(3));
3790 Ops.push_back(N->getOperand(4));
3791 Ops.push_back(Chain);
3792 break;
3793 case NVPTXISD::Suld2DArrayI64Clamp:
3794 Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP;
3795 Ops.push_back(TexHandle);
3796 Ops.push_back(N->getOperand(2));
3797 Ops.push_back(N->getOperand(3));
3798 Ops.push_back(N->getOperand(4));
3799 Ops.push_back(Chain);
3800 break;
3801 case NVPTXISD::Suld2DArrayV2I8Clamp:
3802 Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP;
3803 Ops.push_back(TexHandle);
3804 Ops.push_back(N->getOperand(2));
3805 Ops.push_back(N->getOperand(3));
3806 Ops.push_back(N->getOperand(4));
3807 Ops.push_back(Chain);
3808 break;
3809 case NVPTXISD::Suld2DArrayV2I16Clamp:
3810 Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP;
3811 Ops.push_back(TexHandle);
3812 Ops.push_back(N->getOperand(2));
3813 Ops.push_back(N->getOperand(3));
3814 Ops.push_back(N->getOperand(4));
3815 Ops.push_back(Chain);
3816 break;
3817 case NVPTXISD::Suld2DArrayV2I32Clamp:
3818 Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP;
3819 Ops.push_back(TexHandle);
3820 Ops.push_back(N->getOperand(2));
3821 Ops.push_back(N->getOperand(3));
3822 Ops.push_back(N->getOperand(4));
3823 Ops.push_back(Chain);
3824 break;
3825 case NVPTXISD::Suld2DArrayV2I64Clamp:
3826 Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP;
3827 Ops.push_back(TexHandle);
3828 Ops.push_back(N->getOperand(2));
3829 Ops.push_back(N->getOperand(3));
3830 Ops.push_back(N->getOperand(4));
3831 Ops.push_back(Chain);
3832 break;
3833 case NVPTXISD::Suld2DArrayV4I8Clamp:
3834 Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP;
3835 Ops.push_back(TexHandle);
3836 Ops.push_back(N->getOperand(2));
3837 Ops.push_back(N->getOperand(3));
3838 Ops.push_back(N->getOperand(4));
3839 Ops.push_back(Chain);
3840 break;
3841 case NVPTXISD::Suld2DArrayV4I16Clamp:
3842 Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP;
3843 Ops.push_back(TexHandle);
3844 Ops.push_back(N->getOperand(2));
3845 Ops.push_back(N->getOperand(3));
3846 Ops.push_back(N->getOperand(4));
3847 Ops.push_back(Chain);
3848 break;
3849 case NVPTXISD::Suld2DArrayV4I32Clamp:
3850 Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP;
3851 Ops.push_back(TexHandle);
3852 Ops.push_back(N->getOperand(2));
3853 Ops.push_back(N->getOperand(3));
3854 Ops.push_back(N->getOperand(4));
3855 Ops.push_back(Chain);
3856 break;
3857 case NVPTXISD::Suld3DI8Clamp:
3858 Opc = NVPTX::SULD_3D_I8_CLAMP;
3859 Ops.push_back(TexHandle);
3860 Ops.push_back(N->getOperand(2));
3861 Ops.push_back(N->getOperand(3));
3862 Ops.push_back(N->getOperand(4));
3863 Ops.push_back(Chain);
3864 break;
3865 case NVPTXISD::Suld3DI16Clamp:
3866 Opc = NVPTX::SULD_3D_I16_CLAMP;
3867 Ops.push_back(TexHandle);
3868 Ops.push_back(N->getOperand(2));
3869 Ops.push_back(N->getOperand(3));
3870 Ops.push_back(N->getOperand(4));
3871 Ops.push_back(Chain);
3872 break;
3873 case NVPTXISD::Suld3DI32Clamp:
3874 Opc = NVPTX::SULD_3D_I32_CLAMP;
3875 Ops.push_back(TexHandle);
3876 Ops.push_back(N->getOperand(2));
3877 Ops.push_back(N->getOperand(3));
3878 Ops.push_back(N->getOperand(4));
3879 Ops.push_back(Chain);
3880 break;
3881 case NVPTXISD::Suld3DI64Clamp:
3882 Opc = NVPTX::SULD_3D_I64_CLAMP;
3883 Ops.push_back(TexHandle);
3884 Ops.push_back(N->getOperand(2));
3885 Ops.push_back(N->getOperand(3));
3886 Ops.push_back(N->getOperand(4));
3887 Ops.push_back(Chain);
3888 break;
3889 case NVPTXISD::Suld3DV2I8Clamp:
3890 Opc = NVPTX::SULD_3D_V2I8_CLAMP;
3891 Ops.push_back(TexHandle);
3892 Ops.push_back(N->getOperand(2));
3893 Ops.push_back(N->getOperand(3));
3894 Ops.push_back(N->getOperand(4));
3895 Ops.push_back(Chain);
3896 break;
3897 case NVPTXISD::Suld3DV2I16Clamp:
3898 Opc = NVPTX::SULD_3D_V2I16_CLAMP;
3899 Ops.push_back(TexHandle);
3900 Ops.push_back(N->getOperand(2));
3901 Ops.push_back(N->getOperand(3));
3902 Ops.push_back(N->getOperand(4));
3903 Ops.push_back(Chain);
3904 break;
3905 case NVPTXISD::Suld3DV2I32Clamp:
3906 Opc = NVPTX::SULD_3D_V2I32_CLAMP;
3907 Ops.push_back(TexHandle);
3908 Ops.push_back(N->getOperand(2));
3909 Ops.push_back(N->getOperand(3));
3910 Ops.push_back(N->getOperand(4));
3911 Ops.push_back(Chain);
3912 break;
3913 case NVPTXISD::Suld3DV2I64Clamp:
3914 Opc = NVPTX::SULD_3D_V2I64_CLAMP;
3915 Ops.push_back(TexHandle);
3916 Ops.push_back(N->getOperand(2));
3917 Ops.push_back(N->getOperand(3));
3918 Ops.push_back(N->getOperand(4));
3919 Ops.push_back(Chain);
3920 break;
3921 case NVPTXISD::Suld3DV4I8Clamp:
3922 Opc = NVPTX::SULD_3D_V4I8_CLAMP;
3923 Ops.push_back(TexHandle);
3924 Ops.push_back(N->getOperand(2));
3925 Ops.push_back(N->getOperand(3));
3926 Ops.push_back(N->getOperand(4));
3927 Ops.push_back(Chain);
3928 break;
3929 case NVPTXISD::Suld3DV4I16Clamp:
3930 Opc = NVPTX::SULD_3D_V4I16_CLAMP;
3931 Ops.push_back(TexHandle);
3932 Ops.push_back(N->getOperand(2));
3933 Ops.push_back(N->getOperand(3));
3934 Ops.push_back(N->getOperand(4));
3935 Ops.push_back(Chain);
3936 break;
3937 case NVPTXISD::Suld3DV4I32Clamp:
3938 Opc = NVPTX::SULD_3D_V4I32_CLAMP;
3939 Ops.push_back(TexHandle);
3940 Ops.push_back(N->getOperand(2));
3941 Ops.push_back(N->getOperand(3));
3942 Ops.push_back(N->getOperand(4));
3943 Ops.push_back(Chain);
3944 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3945 case NVPTXISD::Suld1DI8Trap:
3946 Opc = NVPTX::SULD_1D_I8_TRAP;
3947 Ops.push_back(TexHandle);
3948 Ops.push_back(N->getOperand(2));
3949 Ops.push_back(Chain);
3950 break;
3951 case NVPTXISD::Suld1DI16Trap:
3952 Opc = NVPTX::SULD_1D_I16_TRAP;
3953 Ops.push_back(TexHandle);
3954 Ops.push_back(N->getOperand(2));
3955 Ops.push_back(Chain);
3956 break;
3957 case NVPTXISD::Suld1DI32Trap:
3958 Opc = NVPTX::SULD_1D_I32_TRAP;
3959 Ops.push_back(TexHandle);
3960 Ops.push_back(N->getOperand(2));
3961 Ops.push_back(Chain);
3962 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3963 case NVPTXISD::Suld1DI64Trap:
3964 Opc = NVPTX::SULD_1D_I64_TRAP;
3965 Ops.push_back(TexHandle);
3966 Ops.push_back(N->getOperand(2));
3967 Ops.push_back(Chain);
3968 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3969 case NVPTXISD::Suld1DV2I8Trap:
3970 Opc = NVPTX::SULD_1D_V2I8_TRAP;
3971 Ops.push_back(TexHandle);
3972 Ops.push_back(N->getOperand(2));
3973 Ops.push_back(Chain);
3974 break;
3975 case NVPTXISD::Suld1DV2I16Trap:
3976 Opc = NVPTX::SULD_1D_V2I16_TRAP;
3977 Ops.push_back(TexHandle);
3978 Ops.push_back(N->getOperand(2));
3979 Ops.push_back(Chain);
3980 break;
3981 case NVPTXISD::Suld1DV2I32Trap:
3982 Opc = NVPTX::SULD_1D_V2I32_TRAP;
3983 Ops.push_back(TexHandle);
3984 Ops.push_back(N->getOperand(2));
3985 Ops.push_back(Chain);
3986 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3987 case NVPTXISD::Suld1DV2I64Trap:
3988 Opc = NVPTX::SULD_1D_V2I64_TRAP;
3989 Ops.push_back(TexHandle);
3990 Ops.push_back(N->getOperand(2));
3991 Ops.push_back(Chain);
3992 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3993 case NVPTXISD::Suld1DV4I8Trap:
3994 Opc = NVPTX::SULD_1D_V4I8_TRAP;
3995 Ops.push_back(TexHandle);
3996 Ops.push_back(N->getOperand(2));
3997 Ops.push_back(Chain);
3998 break;
3999 case NVPTXISD::Suld1DV4I16Trap:
4000 Opc = NVPTX::SULD_1D_V4I16_TRAP;
4001 Ops.push_back(TexHandle);
4002 Ops.push_back(N->getOperand(2));
4003 Ops.push_back(Chain);
4004 break;
4005 case NVPTXISD::Suld1DV4I32Trap:
4006 Opc = NVPTX::SULD_1D_V4I32_TRAP;
4007 Ops.push_back(TexHandle);
4008 Ops.push_back(N->getOperand(2));
4009 Ops.push_back(Chain);
4010 break;
4011 case NVPTXISD::Suld1DArrayI8Trap:
4012 Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP;
4013 Ops.push_back(TexHandle);
4014 Ops.push_back(N->getOperand(2));
4015 Ops.push_back(N->getOperand(3));
4016 Ops.push_back(Chain);
4017 break;
4018 case NVPTXISD::Suld1DArrayI16Trap:
4019 Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP;
4020 Ops.push_back(TexHandle);
4021 Ops.push_back(N->getOperand(2));
4022 Ops.push_back(N->getOperand(3));
4023 Ops.push_back(Chain);
4024 break;
4025 case NVPTXISD::Suld1DArrayI32Trap:
4026 Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP;
4027 Ops.push_back(TexHandle);
4028 Ops.push_back(N->getOperand(2));
4029 Ops.push_back(N->getOperand(3));
4030 Ops.push_back(Chain);
4031 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4032 case NVPTXISD::Suld1DArrayI64Trap:
4033 Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP;
4034 Ops.push_back(TexHandle);
4035 Ops.push_back(N->getOperand(2));
4036 Ops.push_back(N->getOperand(3));
4037 Ops.push_back(Chain);
4038 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4039 case NVPTXISD::Suld1DArrayV2I8Trap:
4040 Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP;
4041 Ops.push_back(TexHandle);
4042 Ops.push_back(N->getOperand(2));
4043 Ops.push_back(N->getOperand(3));
4044 Ops.push_back(Chain);
4045 break;
4046 case NVPTXISD::Suld1DArrayV2I16Trap:
4047 Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP;
4048 Ops.push_back(TexHandle);
4049 Ops.push_back(N->getOperand(2));
4050 Ops.push_back(N->getOperand(3));
4051 Ops.push_back(Chain);
4052 break;
4053 case NVPTXISD::Suld1DArrayV2I32Trap:
4054 Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP;
4055 Ops.push_back(TexHandle);
4056 Ops.push_back(N->getOperand(2));
4057 Ops.push_back(N->getOperand(3));
4058 Ops.push_back(Chain);
4059 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4060 case NVPTXISD::Suld1DArrayV2I64Trap:
4061 Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP;
4062 Ops.push_back(TexHandle);
4063 Ops.push_back(N->getOperand(2));
4064 Ops.push_back(N->getOperand(3));
4065 Ops.push_back(Chain);
4066 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4067 case NVPTXISD::Suld1DArrayV4I8Trap:
4068 Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP;
4069 Ops.push_back(TexHandle);
4070 Ops.push_back(N->getOperand(2));
4071 Ops.push_back(N->getOperand(3));
4072 Ops.push_back(Chain);
4073 break;
4074 case NVPTXISD::Suld1DArrayV4I16Trap:
4075 Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP;
4076 Ops.push_back(TexHandle);
4077 Ops.push_back(N->getOperand(2));
4078 Ops.push_back(N->getOperand(3));
4079 Ops.push_back(Chain);
4080 break;
4081 case NVPTXISD::Suld1DArrayV4I32Trap:
4082 Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP;
4083 Ops.push_back(TexHandle);
4084 Ops.push_back(N->getOperand(2));
4085 Ops.push_back(N->getOperand(3));
4086 Ops.push_back(Chain);
4087 break;
4088 case NVPTXISD::Suld2DI8Trap:
4089 Opc = NVPTX::SULD_2D_I8_TRAP;
4090 Ops.push_back(TexHandle);
4091 Ops.push_back(N->getOperand(2));
4092 Ops.push_back(N->getOperand(3));
4093 Ops.push_back(Chain);
4094 break;
4095 case NVPTXISD::Suld2DI16Trap:
4096 Opc = NVPTX::SULD_2D_I16_TRAP;
4097 Ops.push_back(TexHandle);
4098 Ops.push_back(N->getOperand(2));
4099 Ops.push_back(N->getOperand(3));
4100 Ops.push_back(Chain);
4101 break;
4102 case NVPTXISD::Suld2DI32Trap:
4103 Opc = NVPTX::SULD_2D_I32_TRAP;
4104 Ops.push_back(TexHandle);
4105 Ops.push_back(N->getOperand(2));
4106 Ops.push_back(N->getOperand(3));
4107 Ops.push_back(Chain);
4108 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4109 case NVPTXISD::Suld2DI64Trap:
4110 Opc = NVPTX::SULD_2D_I64_TRAP;
4111 Ops.push_back(TexHandle);
4112 Ops.push_back(N->getOperand(2));
4113 Ops.push_back(N->getOperand(3));
4114 Ops.push_back(Chain);
4115 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4116 case NVPTXISD::Suld2DV2I8Trap:
4117 Opc = NVPTX::SULD_2D_V2I8_TRAP;
4118 Ops.push_back(TexHandle);
4119 Ops.push_back(N->getOperand(2));
4120 Ops.push_back(N->getOperand(3));
4121 Ops.push_back(Chain);
4122 break;
4123 case NVPTXISD::Suld2DV2I16Trap:
4124 Opc = NVPTX::SULD_2D_V2I16_TRAP;
4125 Ops.push_back(TexHandle);
4126 Ops.push_back(N->getOperand(2));
4127 Ops.push_back(N->getOperand(3));
4128 Ops.push_back(Chain);
4129 break;
4130 case NVPTXISD::Suld2DV2I32Trap:
4131 Opc = NVPTX::SULD_2D_V2I32_TRAP;
4132 Ops.push_back(TexHandle);
4133 Ops.push_back(N->getOperand(2));
4134 Ops.push_back(N->getOperand(3));
4135 Ops.push_back(Chain);
4136 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4137 case NVPTXISD::Suld2DV2I64Trap:
4138 Opc = NVPTX::SULD_2D_V2I64_TRAP;
4139 Ops.push_back(TexHandle);
4140 Ops.push_back(N->getOperand(2));
4141 Ops.push_back(N->getOperand(3));
4142 Ops.push_back(Chain);
4143 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4144 case NVPTXISD::Suld2DV4I8Trap:
4145 Opc = NVPTX::SULD_2D_V4I8_TRAP;
4146 Ops.push_back(TexHandle);
4147 Ops.push_back(N->getOperand(2));
4148 Ops.push_back(N->getOperand(3));
4149 Ops.push_back(Chain);
4150 break;
4151 case NVPTXISD::Suld2DV4I16Trap:
4152 Opc = NVPTX::SULD_2D_V4I16_TRAP;
4153 Ops.push_back(TexHandle);
4154 Ops.push_back(N->getOperand(2));
4155 Ops.push_back(N->getOperand(3));
4156 Ops.push_back(Chain);
4157 break;
4158 case NVPTXISD::Suld2DV4I32Trap:
4159 Opc = NVPTX::SULD_2D_V4I32_TRAP;
4160 Ops.push_back(TexHandle);
4161 Ops.push_back(N->getOperand(2));
4162 Ops.push_back(N->getOperand(3));
4163 Ops.push_back(Chain);
4164 break;
4165 case NVPTXISD::Suld2DArrayI8Trap:
4166 Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP;
4167 Ops.push_back(TexHandle);
4168 Ops.push_back(N->getOperand(2));
4169 Ops.push_back(N->getOperand(3));
4170 Ops.push_back(N->getOperand(4));
4171 Ops.push_back(Chain);
4172 break;
4173 case NVPTXISD::Suld2DArrayI16Trap:
4174 Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP;
4175 Ops.push_back(TexHandle);
4176 Ops.push_back(N->getOperand(2));
4177 Ops.push_back(N->getOperand(3));
4178 Ops.push_back(N->getOperand(4));
4179 Ops.push_back(Chain);
4180 break;
4181 case NVPTXISD::Suld2DArrayI32Trap:
4182 Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP;
4183 Ops.push_back(TexHandle);
4184 Ops.push_back(N->getOperand(2));
4185 Ops.push_back(N->getOperand(3));
4186 Ops.push_back(N->getOperand(4));
4187 Ops.push_back(Chain);
4188 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4189 case NVPTXISD::Suld2DArrayI64Trap:
4190 Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP;
4191 Ops.push_back(TexHandle);
4192 Ops.push_back(N->getOperand(2));
4193 Ops.push_back(N->getOperand(3));
4194 Ops.push_back(N->getOperand(4));
4195 Ops.push_back(Chain);
4196 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4197 case NVPTXISD::Suld2DArrayV2I8Trap:
4198 Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP;
4199 Ops.push_back(TexHandle);
4200 Ops.push_back(N->getOperand(2));
4201 Ops.push_back(N->getOperand(3));
4202 Ops.push_back(N->getOperand(4));
4203 Ops.push_back(Chain);
4204 break;
4205 case NVPTXISD::Suld2DArrayV2I16Trap:
4206 Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP;
4207 Ops.push_back(TexHandle);
4208 Ops.push_back(N->getOperand(2));
4209 Ops.push_back(N->getOperand(3));
4210 Ops.push_back(N->getOperand(4));
4211 Ops.push_back(Chain);
4212 break;
4213 case NVPTXISD::Suld2DArrayV2I32Trap:
4214 Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP;
4215 Ops.push_back(TexHandle);
4216 Ops.push_back(N->getOperand(2));
4217 Ops.push_back(N->getOperand(3));
4218 Ops.push_back(N->getOperand(4));
4219 Ops.push_back(Chain);
4220 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4221 case NVPTXISD::Suld2DArrayV2I64Trap:
4222 Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP;
4223 Ops.push_back(TexHandle);
4224 Ops.push_back(N->getOperand(2));
4225 Ops.push_back(N->getOperand(3));
4226 Ops.push_back(N->getOperand(4));
4227 Ops.push_back(Chain);
4228 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4229 case NVPTXISD::Suld2DArrayV4I8Trap:
4230 Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP;
4231 Ops.push_back(TexHandle);
4232 Ops.push_back(N->getOperand(2));
4233 Ops.push_back(N->getOperand(3));
4234 Ops.push_back(N->getOperand(4));
4235 Ops.push_back(Chain);
4236 break;
4237 case NVPTXISD::Suld2DArrayV4I16Trap:
4238 Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP;
4239 Ops.push_back(TexHandle);
4240 Ops.push_back(N->getOperand(2));
4241 Ops.push_back(N->getOperand(3));
4242 Ops.push_back(N->getOperand(4));
4243 Ops.push_back(Chain);
4244 break;
4245 case NVPTXISD::Suld2DArrayV4I32Trap:
4246 Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP;
4247 Ops.push_back(TexHandle);
4248 Ops.push_back(N->getOperand(2));
4249 Ops.push_back(N->getOperand(3));
4250 Ops.push_back(N->getOperand(4));
4251 Ops.push_back(Chain);
4252 break;
4253 case NVPTXISD::Suld3DI8Trap:
4254 Opc = NVPTX::SULD_3D_I8_TRAP;
4255 Ops.push_back(TexHandle);
4256 Ops.push_back(N->getOperand(2));
4257 Ops.push_back(N->getOperand(3));
4258 Ops.push_back(N->getOperand(4));
4259 Ops.push_back(Chain);
4260 break;
4261 case NVPTXISD::Suld3DI16Trap:
4262 Opc = NVPTX::SULD_3D_I16_TRAP;
4263 Ops.push_back(TexHandle);
4264 Ops.push_back(N->getOperand(2));
4265 Ops.push_back(N->getOperand(3));
4266 Ops.push_back(N->getOperand(4));
4267 Ops.push_back(Chain);
4268 break;
4269 case NVPTXISD::Suld3DI32Trap:
4270 Opc = NVPTX::SULD_3D_I32_TRAP;
4271 Ops.push_back(TexHandle);
4272 Ops.push_back(N->getOperand(2));
4273 Ops.push_back(N->getOperand(3));
4274 Ops.push_back(N->getOperand(4));
4275 Ops.push_back(Chain);
4276 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4277 case NVPTXISD::Suld3DI64Trap:
4278 Opc = NVPTX::SULD_3D_I64_TRAP;
4279 Ops.push_back(TexHandle);
4280 Ops.push_back(N->getOperand(2));
4281 Ops.push_back(N->getOperand(3));
4282 Ops.push_back(N->getOperand(4));
4283 Ops.push_back(Chain);
4284 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4285 case NVPTXISD::Suld3DV2I8Trap:
4286 Opc = NVPTX::SULD_3D_V2I8_TRAP;
4287 Ops.push_back(TexHandle);
4288 Ops.push_back(N->getOperand(2));
4289 Ops.push_back(N->getOperand(3));
4290 Ops.push_back(N->getOperand(4));
4291 Ops.push_back(Chain);
4292 break;
4293 case NVPTXISD::Suld3DV2I16Trap:
4294 Opc = NVPTX::SULD_3D_V2I16_TRAP;
4295 Ops.push_back(TexHandle);
4296 Ops.push_back(N->getOperand(2));
4297 Ops.push_back(N->getOperand(3));
4298 Ops.push_back(N->getOperand(4));
4299 Ops.push_back(Chain);
4300 break;
4301 case NVPTXISD::Suld3DV2I32Trap:
4302 Opc = NVPTX::SULD_3D_V2I32_TRAP;
4303 Ops.push_back(TexHandle);
4304 Ops.push_back(N->getOperand(2));
4305 Ops.push_back(N->getOperand(3));
4306 Ops.push_back(N->getOperand(4));
4307 Ops.push_back(Chain);
4308 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4309 case NVPTXISD::Suld3DV2I64Trap:
4310 Opc = NVPTX::SULD_3D_V2I64_TRAP;
4311 Ops.push_back(TexHandle);
4312 Ops.push_back(N->getOperand(2));
4313 Ops.push_back(N->getOperand(3));
4314 Ops.push_back(N->getOperand(4));
4315 Ops.push_back(Chain);
4316 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4317 case NVPTXISD::Suld3DV4I8Trap:
4318 Opc = NVPTX::SULD_3D_V4I8_TRAP;
4319 Ops.push_back(TexHandle);
4320 Ops.push_back(N->getOperand(2));
4321 Ops.push_back(N->getOperand(3));
4322 Ops.push_back(N->getOperand(4));
4323 Ops.push_back(Chain);
4324 break;
4325 case NVPTXISD::Suld3DV4I16Trap:
4326 Opc = NVPTX::SULD_3D_V4I16_TRAP;
4327 Ops.push_back(TexHandle);
4328 Ops.push_back(N->getOperand(2));
4329 Ops.push_back(N->getOperand(3));
4330 Ops.push_back(N->getOperand(4));
4331 Ops.push_back(Chain);
4332 break;
4333 case NVPTXISD::Suld3DV4I32Trap:
4334 Opc = NVPTX::SULD_3D_V4I32_TRAP;
4335 Ops.push_back(TexHandle);
4336 Ops.push_back(N->getOperand(2));
4337 Ops.push_back(N->getOperand(3));
4338 Ops.push_back(N->getOperand(4));
4339 Ops.push_back(Chain);
4340 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4341 case NVPTXISD::Suld1DI8Zero:
4342 Opc = NVPTX::SULD_1D_I8_ZERO;
4343 Ops.push_back(TexHandle);
4344 Ops.push_back(N->getOperand(2));
4345 Ops.push_back(Chain);
4346 break;
4347 case NVPTXISD::Suld1DI16Zero:
4348 Opc = NVPTX::SULD_1D_I16_ZERO;
4349 Ops.push_back(TexHandle);
4350 Ops.push_back(N->getOperand(2));
4351 Ops.push_back(Chain);
4352 break;
4353 case NVPTXISD::Suld1DI32Zero:
4354 Opc = NVPTX::SULD_1D_I32_ZERO;
4355 Ops.push_back(TexHandle);
4356 Ops.push_back(N->getOperand(2));
4357 Ops.push_back(Chain);
4358 break;
4359 case NVPTXISD::Suld1DI64Zero:
4360 Opc = NVPTX::SULD_1D_I64_ZERO;
4361 Ops.push_back(TexHandle);
4362 Ops.push_back(N->getOperand(2));
4363 Ops.push_back(Chain);
4364 break;
4365 case NVPTXISD::Suld1DV2I8Zero:
4366 Opc = NVPTX::SULD_1D_V2I8_ZERO;
4367 Ops.push_back(TexHandle);
4368 Ops.push_back(N->getOperand(2));
4369 Ops.push_back(Chain);
4370 break;
4371 case NVPTXISD::Suld1DV2I16Zero:
4372 Opc = NVPTX::SULD_1D_V2I16_ZERO;
4373 Ops.push_back(TexHandle);
4374 Ops.push_back(N->getOperand(2));
4375 Ops.push_back(Chain);
4376 break;
4377 case NVPTXISD::Suld1DV2I32Zero:
4378 Opc = NVPTX::SULD_1D_V2I32_ZERO;
4379 Ops.push_back(TexHandle);
4380 Ops.push_back(N->getOperand(2));
4381 Ops.push_back(Chain);
4382 break;
4383 case NVPTXISD::Suld1DV2I64Zero:
4384 Opc = NVPTX::SULD_1D_V2I64_ZERO;
4385 Ops.push_back(TexHandle);
4386 Ops.push_back(N->getOperand(2));
4387 Ops.push_back(Chain);
4388 break;
4389 case NVPTXISD::Suld1DV4I8Zero:
4390 Opc = NVPTX::SULD_1D_V4I8_ZERO;
4391 Ops.push_back(TexHandle);
4392 Ops.push_back(N->getOperand(2));
4393 Ops.push_back(Chain);
4394 break;
4395 case NVPTXISD::Suld1DV4I16Zero:
4396 Opc = NVPTX::SULD_1D_V4I16_ZERO;
4397 Ops.push_back(TexHandle);
4398 Ops.push_back(N->getOperand(2));
4399 Ops.push_back(Chain);
4400 break;
4401 case NVPTXISD::Suld1DV4I32Zero:
4402 Opc = NVPTX::SULD_1D_V4I32_ZERO;
4403 Ops.push_back(TexHandle);
4404 Ops.push_back(N->getOperand(2));
4405 Ops.push_back(Chain);
4406 break;
4407 case NVPTXISD::Suld1DArrayI8Zero:
4408 Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO;
4409 Ops.push_back(TexHandle);
4410 Ops.push_back(N->getOperand(2));
4411 Ops.push_back(N->getOperand(3));
4412 Ops.push_back(Chain);
4413 break;
4414 case NVPTXISD::Suld1DArrayI16Zero:
4415 Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO;
4416 Ops.push_back(TexHandle);
4417 Ops.push_back(N->getOperand(2));
4418 Ops.push_back(N->getOperand(3));
4419 Ops.push_back(Chain);
4420 break;
4421 case NVPTXISD::Suld1DArrayI32Zero:
4422 Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO;
4423 Ops.push_back(TexHandle);
4424 Ops.push_back(N->getOperand(2));
4425 Ops.push_back(N->getOperand(3));
4426 Ops.push_back(Chain);
4427 break;
4428 case NVPTXISD::Suld1DArrayI64Zero:
4429 Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO;
4430 Ops.push_back(TexHandle);
4431 Ops.push_back(N->getOperand(2));
4432 Ops.push_back(N->getOperand(3));
4433 Ops.push_back(Chain);
4434 break;
4435 case NVPTXISD::Suld1DArrayV2I8Zero:
4436 Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO;
4437 Ops.push_back(TexHandle);
4438 Ops.push_back(N->getOperand(2));
4439 Ops.push_back(N->getOperand(3));
4440 Ops.push_back(Chain);
4441 break;
4442 case NVPTXISD::Suld1DArrayV2I16Zero:
4443 Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO;
4444 Ops.push_back(TexHandle);
4445 Ops.push_back(N->getOperand(2));
4446 Ops.push_back(N->getOperand(3));
4447 Ops.push_back(Chain);
4448 break;
4449 case NVPTXISD::Suld1DArrayV2I32Zero:
4450 Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO;
4451 Ops.push_back(TexHandle);
4452 Ops.push_back(N->getOperand(2));
4453 Ops.push_back(N->getOperand(3));
4454 Ops.push_back(Chain);
4455 break;
4456 case NVPTXISD::Suld1DArrayV2I64Zero:
4457 Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO;
4458 Ops.push_back(TexHandle);
4459 Ops.push_back(N->getOperand(2));
4460 Ops.push_back(N->getOperand(3));
4461 Ops.push_back(Chain);
4462 break;
4463 case NVPTXISD::Suld1DArrayV4I8Zero:
4464 Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO;
4465 Ops.push_back(TexHandle);
4466 Ops.push_back(N->getOperand(2));
4467 Ops.push_back(N->getOperand(3));
4468 Ops.push_back(Chain);
4469 break;
4470 case NVPTXISD::Suld1DArrayV4I16Zero:
4471 Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO;
4472 Ops.push_back(TexHandle);
4473 Ops.push_back(N->getOperand(2));
4474 Ops.push_back(N->getOperand(3));
4475 Ops.push_back(Chain);
4476 break;
4477 case NVPTXISD::Suld1DArrayV4I32Zero:
4478 Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO;
4479 Ops.push_back(TexHandle);
4480 Ops.push_back(N->getOperand(2));
4481 Ops.push_back(N->getOperand(3));
4482 Ops.push_back(Chain);
4483 break;
4484 case NVPTXISD::Suld2DI8Zero:
4485 Opc = NVPTX::SULD_2D_I8_ZERO;
4486 Ops.push_back(TexHandle);
4487 Ops.push_back(N->getOperand(2));
4488 Ops.push_back(N->getOperand(3));
4489 Ops.push_back(Chain);
4490 break;
4491 case NVPTXISD::Suld2DI16Zero:
4492 Opc = NVPTX::SULD_2D_I16_ZERO;
4493 Ops.push_back(TexHandle);
4494 Ops.push_back(N->getOperand(2));
4495 Ops.push_back(N->getOperand(3));
4496 Ops.push_back(Chain);
4497 break;
4498 case NVPTXISD::Suld2DI32Zero:
4499 Opc = NVPTX::SULD_2D_I32_ZERO;
4500 Ops.push_back(TexHandle);
4501 Ops.push_back(N->getOperand(2));
4502 Ops.push_back(N->getOperand(3));
4503 Ops.push_back(Chain);
4504 break;
4505 case NVPTXISD::Suld2DI64Zero:
4506 Opc = NVPTX::SULD_2D_I64_ZERO;
4507 Ops.push_back(TexHandle);
4508 Ops.push_back(N->getOperand(2));
4509 Ops.push_back(N->getOperand(3));
4510 Ops.push_back(Chain);
4511 break;
4512 case NVPTXISD::Suld2DV2I8Zero:
4513 Opc = NVPTX::SULD_2D_V2I8_ZERO;
4514 Ops.push_back(TexHandle);
4515 Ops.push_back(N->getOperand(2));
4516 Ops.push_back(N->getOperand(3));
4517 Ops.push_back(Chain);
4518 break;
4519 case NVPTXISD::Suld2DV2I16Zero:
4520 Opc = NVPTX::SULD_2D_V2I16_ZERO;
4521 Ops.push_back(TexHandle);
4522 Ops.push_back(N->getOperand(2));
4523 Ops.push_back(N->getOperand(3));
4524 Ops.push_back(Chain);
4525 break;
4526 case NVPTXISD::Suld2DV2I32Zero:
4527 Opc = NVPTX::SULD_2D_V2I32_ZERO;
4528 Ops.push_back(TexHandle);
4529 Ops.push_back(N->getOperand(2));
4530 Ops.push_back(N->getOperand(3));
4531 Ops.push_back(Chain);
4532 break;
4533 case NVPTXISD::Suld2DV2I64Zero:
4534 Opc = NVPTX::SULD_2D_V2I64_ZERO;
4535 Ops.push_back(TexHandle);
4536 Ops.push_back(N->getOperand(2));
4537 Ops.push_back(N->getOperand(3));
4538 Ops.push_back(Chain);
4539 break;
4540 case NVPTXISD::Suld2DV4I8Zero:
4541 Opc = NVPTX::SULD_2D_V4I8_ZERO;
4542 Ops.push_back(TexHandle);
4543 Ops.push_back(N->getOperand(2));
4544 Ops.push_back(N->getOperand(3));
4545 Ops.push_back(Chain);
4546 break;
4547 case NVPTXISD::Suld2DV4I16Zero:
4548 Opc = NVPTX::SULD_2D_V4I16_ZERO;
4549 Ops.push_back(TexHandle);
4550 Ops.push_back(N->getOperand(2));
4551 Ops.push_back(N->getOperand(3));
4552 Ops.push_back(Chain);
4553 break;
4554 case NVPTXISD::Suld2DV4I32Zero:
4555 Opc = NVPTX::SULD_2D_V4I32_ZERO;
4556 Ops.push_back(TexHandle);
4557 Ops.push_back(N->getOperand(2));
4558 Ops.push_back(N->getOperand(3));
4559 Ops.push_back(Chain);
4560 break;
4561 case NVPTXISD::Suld2DArrayI8Zero:
4562 Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO;
4563 Ops.push_back(TexHandle);
4564 Ops.push_back(N->getOperand(2));
4565 Ops.push_back(N->getOperand(3));
4566 Ops.push_back(N->getOperand(4));
4567 Ops.push_back(Chain);
4568 break;
4569 case NVPTXISD::Suld2DArrayI16Zero:
4570 Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO;
4571 Ops.push_back(TexHandle);
4572 Ops.push_back(N->getOperand(2));
4573 Ops.push_back(N->getOperand(3));
4574 Ops.push_back(N->getOperand(4));
4575 Ops.push_back(Chain);
4576 break;
4577 case NVPTXISD::Suld2DArrayI32Zero:
4578 Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO;
4579 Ops.push_back(TexHandle);
4580 Ops.push_back(N->getOperand(2));
4581 Ops.push_back(N->getOperand(3));
4582 Ops.push_back(N->getOperand(4));
4583 Ops.push_back(Chain);
4584 break;
4585 case NVPTXISD::Suld2DArrayI64Zero:
4586 Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO;
4587 Ops.push_back(TexHandle);
4588 Ops.push_back(N->getOperand(2));
4589 Ops.push_back(N->getOperand(3));
4590 Ops.push_back(N->getOperand(4));
4591 Ops.push_back(Chain);
4592 break;
4593 case NVPTXISD::Suld2DArrayV2I8Zero:
4594 Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO;
4595 Ops.push_back(TexHandle);
4596 Ops.push_back(N->getOperand(2));
4597 Ops.push_back(N->getOperand(3));
4598 Ops.push_back(N->getOperand(4));
4599 Ops.push_back(Chain);
4600 break;
4601 case NVPTXISD::Suld2DArrayV2I16Zero:
4602 Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO;
4603 Ops.push_back(TexHandle);
4604 Ops.push_back(N->getOperand(2));
4605 Ops.push_back(N->getOperand(3));
4606 Ops.push_back(N->getOperand(4));
4607 Ops.push_back(Chain);
4608 break;
4609 case NVPTXISD::Suld2DArrayV2I32Zero:
4610 Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO;
4611 Ops.push_back(TexHandle);
4612 Ops.push_back(N->getOperand(2));
4613 Ops.push_back(N->getOperand(3));
4614 Ops.push_back(N->getOperand(4));
4615 Ops.push_back(Chain);
4616 break;
4617 case NVPTXISD::Suld2DArrayV2I64Zero:
4618 Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO;
4619 Ops.push_back(TexHandle);
4620 Ops.push_back(N->getOperand(2));
4621 Ops.push_back(N->getOperand(3));
4622 Ops.push_back(N->getOperand(4));
4623 Ops.push_back(Chain);
4624 break;
4625 case NVPTXISD::Suld2DArrayV4I8Zero:
4626 Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO;
4627 Ops.push_back(TexHandle);
4628 Ops.push_back(N->getOperand(2));
4629 Ops.push_back(N->getOperand(3));
4630 Ops.push_back(N->getOperand(4));
4631 Ops.push_back(Chain);
4632 break;
4633 case NVPTXISD::Suld2DArrayV4I16Zero:
4634 Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO;
4635 Ops.push_back(TexHandle);
4636 Ops.push_back(N->getOperand(2));
4637 Ops.push_back(N->getOperand(3));
4638 Ops.push_back(N->getOperand(4));
4639 Ops.push_back(Chain);
4640 break;
4641 case NVPTXISD::Suld2DArrayV4I32Zero:
4642 Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO;
4643 Ops.push_back(TexHandle);
4644 Ops.push_back(N->getOperand(2));
4645 Ops.push_back(N->getOperand(3));
4646 Ops.push_back(N->getOperand(4));
4647 Ops.push_back(Chain);
4648 break;
4649 case NVPTXISD::Suld3DI8Zero:
4650 Opc = NVPTX::SULD_3D_I8_ZERO;
4651 Ops.push_back(TexHandle);
4652 Ops.push_back(N->getOperand(2));
4653 Ops.push_back(N->getOperand(3));
4654 Ops.push_back(N->getOperand(4));
4655 Ops.push_back(Chain);
4656 break;
4657 case NVPTXISD::Suld3DI16Zero:
4658 Opc = NVPTX::SULD_3D_I16_ZERO;
4659 Ops.push_back(TexHandle);
4660 Ops.push_back(N->getOperand(2));
4661 Ops.push_back(N->getOperand(3));
4662 Ops.push_back(N->getOperand(4));
4663 Ops.push_back(Chain);
4664 break;
4665 case NVPTXISD::Suld3DI32Zero:
4666 Opc = NVPTX::SULD_3D_I32_ZERO;
4667 Ops.push_back(TexHandle);
4668 Ops.push_back(N->getOperand(2));
4669 Ops.push_back(N->getOperand(3));
4670 Ops.push_back(N->getOperand(4));
4671 Ops.push_back(Chain);
4672 break;
4673 case NVPTXISD::Suld3DI64Zero:
4674 Opc = NVPTX::SULD_3D_I64_ZERO;
4675 Ops.push_back(TexHandle);
4676 Ops.push_back(N->getOperand(2));
4677 Ops.push_back(N->getOperand(3));
4678 Ops.push_back(N->getOperand(4));
4679 Ops.push_back(Chain);
4680 break;
4681 case NVPTXISD::Suld3DV2I8Zero:
4682 Opc = NVPTX::SULD_3D_V2I8_ZERO;
4683 Ops.push_back(TexHandle);
4684 Ops.push_back(N->getOperand(2));
4685 Ops.push_back(N->getOperand(3));
4686 Ops.push_back(N->getOperand(4));
4687 Ops.push_back(Chain);
4688 break;
4689 case NVPTXISD::Suld3DV2I16Zero:
4690 Opc = NVPTX::SULD_3D_V2I16_ZERO;
4691 Ops.push_back(TexHandle);
4692 Ops.push_back(N->getOperand(2));
4693 Ops.push_back(N->getOperand(3));
4694 Ops.push_back(N->getOperand(4));
4695 Ops.push_back(Chain);
4696 break;
4697 case NVPTXISD::Suld3DV2I32Zero:
4698 Opc = NVPTX::SULD_3D_V2I32_ZERO;
4699 Ops.push_back(TexHandle);
4700 Ops.push_back(N->getOperand(2));
4701 Ops.push_back(N->getOperand(3));
4702 Ops.push_back(N->getOperand(4));
4703 Ops.push_back(Chain);
4704 break;
4705 case NVPTXISD::Suld3DV2I64Zero:
4706 Opc = NVPTX::SULD_3D_V2I64_ZERO;
4707 Ops.push_back(TexHandle);
4708 Ops.push_back(N->getOperand(2));
4709 Ops.push_back(N->getOperand(3));
4710 Ops.push_back(N->getOperand(4));
4711 Ops.push_back(Chain);
4712 break;
4713 case NVPTXISD::Suld3DV4I8Zero:
4714 Opc = NVPTX::SULD_3D_V4I8_ZERO;
4715 Ops.push_back(TexHandle);
4716 Ops.push_back(N->getOperand(2));
4717 Ops.push_back(N->getOperand(3));
4718 Ops.push_back(N->getOperand(4));
4719 Ops.push_back(Chain);
4720 break;
4721 case NVPTXISD::Suld3DV4I16Zero:
4722 Opc = NVPTX::SULD_3D_V4I16_ZERO;
4723 Ops.push_back(TexHandle);
4724 Ops.push_back(N->getOperand(2));
4725 Ops.push_back(N->getOperand(3));
4726 Ops.push_back(N->getOperand(4));
4727 Ops.push_back(Chain);
4728 break;
4729 case NVPTXISD::Suld3DV4I32Zero:
4730 Opc = NVPTX::SULD_3D_V4I32_ZERO;
4731 Ops.push_back(TexHandle);
4732 Ops.push_back(N->getOperand(2));
4733 Ops.push_back(N->getOperand(3));
4734 Ops.push_back(N->getOperand(4));
4735 Ops.push_back(Chain);
4736 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4737 }
4738 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
4739 return Ret;
4740}
4741
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4742
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4743/// SelectBFE - Look for instruction sequences that can be made more efficient
4744/// by using the 'bfe' (bit-field extract) PTX instruction
4745SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
4746 SDValue LHS = N->getOperand(0);
4747 SDValue RHS = N->getOperand(1);
4748 SDValue Len;
4749 SDValue Start;
4750 SDValue Val;
4751 bool IsSigned = false;
4752
4753 if (N->getOpcode() == ISD::AND) {
4754 // Canonicalize the operands
4755 // We want 'and %val, %mask'
4756 if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) {
4757 std::swap(LHS, RHS);
4758 }
4759
4760 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS);
4761 if (!Mask) {
4762 // We need a constant mask on the RHS of the AND
4763 return NULL;
4764 }
4765
4766 // Extract the mask bits
4767 uint64_t MaskVal = Mask->getZExtValue();
4768 if (!isMask_64(MaskVal)) {
4769 // We *could* handle shifted masks here, but doing so would require an
4770 // 'and' operation to fix up the low-order bits so we would trade
4771 // shr+and for bfe+and, which has the same throughput
4772 return NULL;
4773 }
4774
4775 // How many bits are in our mask?
Benjamin Kramer5f6a9072015-02-12 15:35:40 +0000[diff] [blame]4776 uint64_t NumBits = countTrailingOnes(MaskVal);
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4777 Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
4778
4779 if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) {
4780 // We have a 'srl/and' pair, extract the effective start bit and length
4781 Val = LHS.getNode()->getOperand(0);
4782 Start = LHS.getNode()->getOperand(1);
4783 ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start);
4784 if (StartConst) {
4785 uint64_t StartVal = StartConst->getZExtValue();
4786 // How many "good" bits do we have left? "good" is defined here as bits
4787 // that exist in the original value, not shifted in.
4788 uint64_t GoodBits = Start.getValueType().getSizeInBits() - StartVal;
4789 if (NumBits > GoodBits) {
4790 // Do not handle the case where bits have been shifted in. In theory
4791 // we could handle this, but the cost is likely higher than just
4792 // emitting the srl/and pair.
4793 return NULL;
4794 }
4795 Start = CurDAG->getTargetConstant(StartVal, MVT::i32);
4796 } else {
4797 // Do not handle the case where the shift amount (can be zero if no srl
4798 // was found) is not constant. We could handle this case, but it would
4799 // require run-time logic that would be more expensive than just
4800 // emitting the srl/and pair.
4801 return NULL;
4802 }
4803 } else {
4804 // Do not handle the case where the LHS of the and is not a shift. While
4805 // it would be trivial to handle this case, it would just transform
4806 // 'and' -> 'bfe', but 'and' has higher-throughput.
4807 return NULL;
4808 }
4809 } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) {
4810 if (LHS->getOpcode() == ISD::AND) {
4811 ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS);
4812 if (!ShiftCnst) {
4813 // Shift amount must be constant
4814 return NULL;
4815 }
4816
4817 uint64_t ShiftAmt = ShiftCnst->getZExtValue();
4818
4819 SDValue AndLHS = LHS->getOperand(0);
4820 SDValue AndRHS = LHS->getOperand(1);
4821
4822 // Canonicalize the AND to have the mask on the RHS
4823 if (isa<ConstantSDNode>(AndLHS)) {
4824 std::swap(AndLHS, AndRHS);
4825 }
4826
4827 ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS);
4828 if (!MaskCnst) {
4829 // Mask must be constant
4830 return NULL;
4831 }
4832
4833 uint64_t MaskVal = MaskCnst->getZExtValue();
4834 uint64_t NumZeros;
4835 uint64_t NumBits;
4836 if (isMask_64(MaskVal)) {
4837 NumZeros = 0;
4838 // The number of bits in the result bitfield will be the number of
4839 // trailing ones (the AND) minus the number of bits we shift off
Benjamin Kramer5f6a9072015-02-12 15:35:40 +0000[diff] [blame]4840 NumBits = countTrailingOnes(MaskVal) - ShiftAmt;
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4841 } else if (isShiftedMask_64(MaskVal)) {
4842 NumZeros = countTrailingZeros(MaskVal);
Benjamin Kramer5f6a9072015-02-12 15:35:40 +0000[diff] [blame]4843 unsigned NumOnes = countTrailingOnes(MaskVal >> NumZeros);
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4844 // The number of bits in the result bitfield will be the number of
4845 // trailing zeros plus the number of set bits in the mask minus the
4846 // number of bits we shift off
4847 NumBits = NumZeros + NumOnes - ShiftAmt;
4848 } else {
4849 // This is not a mask we can handle
4850 return NULL;
4851 }
4852
4853 if (ShiftAmt < NumZeros) {
4854 // Handling this case would require extra logic that would make this
4855 // transformation non-profitable
4856 return NULL;
4857 }
4858
4859 Val = AndLHS;
4860 Start = CurDAG->getTargetConstant(ShiftAmt, MVT::i32);
4861 Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
4862 } else if (LHS->getOpcode() == ISD::SHL) {
4863 // Here, we have a pattern like:
4864 //
4865 // (sra (shl val, NN), MM)
4866 // or
4867 // (srl (shl val, NN), MM)
4868 //
4869 // If MM >= NN, we can efficiently optimize this with bfe
4870 Val = LHS->getOperand(0);
4871
4872 SDValue ShlRHS = LHS->getOperand(1);
4873 ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS);
4874 if (!ShlCnst) {
4875 // Shift amount must be constant
4876 return NULL;
4877 }
4878 uint64_t InnerShiftAmt = ShlCnst->getZExtValue();
4879
4880 SDValue ShrRHS = RHS;
4881 ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS);
4882 if (!ShrCnst) {
4883 // Shift amount must be constant
4884 return NULL;
4885 }
4886 uint64_t OuterShiftAmt = ShrCnst->getZExtValue();
4887
4888 // To avoid extra codegen and be profitable, we need Outer >= Inner
4889 if (OuterShiftAmt < InnerShiftAmt) {
4890 return NULL;
4891 }
4892
4893 // If the outer shift is more than the type size, we have no bitfield to
4894 // extract (since we also check that the inner shift is <= the outer shift
4895 // then this also implies that the inner shift is < the type size)
4896 if (OuterShiftAmt >= Val.getValueType().getSizeInBits()) {
4897 return NULL;
4898 }
4899
4900 Start =
4901 CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, MVT::i32);
4902 Len =
4903 CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() -
4904 OuterShiftAmt, MVT::i32);
4905
4906 if (N->getOpcode() == ISD::SRA) {
4907 // If we have a arithmetic right shift, we need to use the signed bfe
4908 // variant
4909 IsSigned = true;
4910 }
4911 } else {
4912 // No can do...
4913 return NULL;
4914 }
4915 } else {
4916 // No can do...
4917 return NULL;
4918 }
4919
4920
4921 unsigned Opc;
4922 // For the BFE operations we form here from "and" and "srl", always use the
4923 // unsigned variants.
4924 if (Val.getValueType() == MVT::i32) {
4925 if (IsSigned) {
4926 Opc = NVPTX::BFE_S32rii;
4927 } else {
4928 Opc = NVPTX::BFE_U32rii;
4929 }
4930 } else if (Val.getValueType() == MVT::i64) {
4931 if (IsSigned) {
4932 Opc = NVPTX::BFE_S64rii;
4933 } else {
4934 Opc = NVPTX::BFE_U64rii;
4935 }
4936 } else {
4937 // We cannot handle this type
4938 return NULL;
4939 }
4940
4941 SDValue Ops[] = {
4942 Val, Start, Len
4943 };
4944
4945 SDNode *Ret =
4946 CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
4947
4948 return Ret;
4949}
4950
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4951// SelectDirectAddr - Match a direct address for DAG.
4952// A direct address could be a globaladdress or externalsymbol.
4953bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
4954 // Return true if TGA or ES.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4955 if (N.getOpcode() == ISD::TargetGlobalAddress ||
4956 N.getOpcode() == ISD::TargetExternalSymbol) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4957 Address = N;
4958 return true;
4959 }
4960 if (N.getOpcode() == NVPTXISD::Wrapper) {
4961 Address = N.getOperand(0);
4962 return true;
4963 }
4964 if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
4965 unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue();
4966 if (IID == Intrinsic::nvvm_ptr_gen_to_param)
4967 if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam)
4968 return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address));
4969 }
4970 return false;
4971}
4972
4973// symbol+offset
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4974bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
4975 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4976 if (Addr.getOpcode() == ISD::ADD) {
4977 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4978 SDValue base = Addr.getOperand(0);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4979 if (SelectDirectAddr(base, Base)) {
4980 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
4981 return true;
4982 }
4983 }
4984 }
4985 return false;
4986}
4987
4988// symbol+offset
4989bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
4990 SDValue &Base, SDValue &Offset) {
4991 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
4992}
4993
4994// symbol+offset
4995bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
4996 SDValue &Base, SDValue &Offset) {
4997 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
4998}
4999
5000// register+offset
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5001bool NVPTXDAGToDAGISel::SelectADDRri_imp(
5002 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5003 if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
5004 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
5005 Offset = CurDAG->getTargetConstant(0, mvt);
5006 return true;
5007 }
5008 if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
5009 Addr.getOpcode() == ISD::TargetGlobalAddress)
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5010 return false; // direct calls.
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5011
5012 if (Addr.getOpcode() == ISD::ADD) {
5013 if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
5014 return false;
5015 }
5016 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
5017 if (FrameIndexSDNode *FIN =
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5018 dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5019 // Constant offset from frame ref.
5020 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
5021 else
5022 Base = Addr.getOperand(0);
5023 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
5024 return true;
5025 }
5026 }
5027 return false;
5028}
5029
5030// register+offset
5031bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
5032 SDValue &Base, SDValue &Offset) {
5033 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
5034}
5035
5036// register+offset
5037bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
5038 SDValue &Base, SDValue &Offset) {
5039 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
5040}
5041
5042bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
5043 unsigned int spN) const {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]5044 const Value *Src = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5045 if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
Nick Lewyckyaad475b2014-04-15 07:22:52 +0000[diff] [blame]5046 if (spN == 0 && mN->getMemOperand()->getPseudoValue())
5047 return true;
5048 Src = mN->getMemOperand()->getValue();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5049 }
5050 if (!Src)
5051 return false;
5052 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
5053 return (PT->getAddressSpace() == spN);
5054 return false;
5055}
5056
5057/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
5058/// inline asm expressions.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5059bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
5060 const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5061 SDValue Op0, Op1;
5062 switch (ConstraintCode) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5063 default:
5064 return true;
5065 case 'm': // memory
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5066 if (SelectDirectAddr(Op, Op0)) {
5067 OutOps.push_back(Op0);
5068 OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
5069 return false;
5070 }
5071 if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
5072 OutOps.push_back(Op0);
5073 OutOps.push_back(Op1);
5074 return false;
5075 }
5076 break;
5077 }
5078 return true;
5079}