Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / 9a2350e45950e13d3d3a0ff318c10912ffef9d32 / . / llvm / lib / Target / NVPTX / NVPTXISelDAGToDAG.cpp
blob: 35d553227e88308b2a66b47047b36b3c78038ac2 [file] [log] [blame]
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the NVPTX target.
11//
12//===----------------------------------------------------------------------===//
13
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]14#include "NVPTXISelDAGToDAG.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +0000[diff] [blame]15#include "llvm/IR/GlobalValue.h"
16#include "llvm/IR/Instructions.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]17#include "llvm/Support/CommandLine.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]18#include "llvm/Support/Debug.h"
19#include "llvm/Support/ErrorHandling.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]20#include "llvm/Support/raw_ostream.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]21#include "llvm/Target/TargetIntrinsicInfo.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]22
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]23using namespace llvm;
24
Chandler Carruth84e68b22014-04-22 02:41:26 +0000[diff] [blame]25#define DEBUG_TYPE "nvptx-isel"
26
Justin Holewinskieafe26d2014-06-27 18:35:37 +0000[diff] [blame]27unsigned FMAContractLevel = 0;
28
29static cl::opt<unsigned, true>
30FMAContractLevelOpt("nvptx-fma-level", cl::ZeroOrMore, cl::Hidden,
31 cl::desc("NVPTX Specific: FMA contraction (0: don't do it"
32 " 1: do it 2: do it aggressively"),
33 cl::location(FMAContractLevel),
34 cl::init(2));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]35
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]36static cl::opt<int> UsePrecDivF32(
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]37 "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]38 cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
39 " IEEE Compliant F32 div.rnd if avaiable."),
40 cl::init(2));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]41
Justin Holewinski48f4ad32013-05-21 16:51:30 +0000[diff] [blame]42static cl::opt<bool>
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]43UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
Justin Holewinski48f4ad32013-05-21 16:51:30 +0000[diff] [blame]44 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
45 cl::init(true));
46
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]47static cl::opt<bool>
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]48FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]49 cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
50 cl::init(false));
51
52
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]53/// createNVPTXISelDag - This pass converts a legalized DAG into a
54/// NVPTX-specific DAG, ready for instruction scheduling.
55FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
56 llvm::CodeGenOpt::Level OptLevel) {
57 return new NVPTXDAGToDAGISel(TM, OptLevel);
58}
59
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]60NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
61 CodeGenOpt::Level OptLevel)
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]62 : SelectionDAGISel(tm, OptLevel),
63 Subtarget(tm.getSubtarget<NVPTXSubtarget>()) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]64
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]65 doFMAF32 = (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel >= 1);
66 doFMAF64 = (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel >= 1);
67 doFMAF32AGG =
68 (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel == 2);
69 doFMAF64AGG =
70 (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel == 2);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]71
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]72 allowFMA = (FMAContractLevel >= 1);
Benjamin Kramera25a61b2012-05-05 11:22:02 +0000[diff] [blame]73
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]74 doMulWide = (OptLevel > 0);
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]75}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]76
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]77int NVPTXDAGToDAGISel::getDivF32Level() const {
78 if (UsePrecDivF32.getNumOccurrences() > 0) {
79 // If nvptx-prec-div32=N is used on the command-line, always honor it
80 return UsePrecDivF32;
81 } else {
82 // Otherwise, use div.approx if fast math is enabled
83 if (TM.Options.UnsafeFPMath)
84 return 0;
85 else
86 return 2;
87 }
88}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]89
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]90bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
91 if (UsePrecSqrtF32.getNumOccurrences() > 0) {
92 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
93 return UsePrecSqrtF32;
94 } else {
95 // Otherwise, use sqrt.approx if fast math is enabled
96 if (TM.Options.UnsafeFPMath)
97 return false;
98 else
99 return true;
100 }
101}
102
103bool NVPTXDAGToDAGISel::useF32FTZ() const {
104 if (FtzEnabled.getNumOccurrences() > 0) {
105 // If nvptx-f32ftz is used on the command-line, always honor it
106 return FtzEnabled;
107 } else {
108 const Function *F = MF->getFunction();
109 // Otherwise, check for an nvptx-f32ftz attribute on the function
110 if (F->hasFnAttribute("nvptx-f32ftz"))
111 return (F->getAttributes().getAttribute(AttributeSet::FunctionIndex,
112 "nvptx-f32ftz")
113 .getValueAsString() == "true");
114 else
115 return false;
116 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]117}
118
119/// Select - Select instructions not customized! Used for
120/// expanded, promoted and normal instructions.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]121SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]122
Tim Northover31d093c2013-09-22 08:21:56 +0000[diff] [blame]123 if (N->isMachineOpcode()) {
124 N->setNodeId(-1);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]125 return nullptr; // Already selected.
Tim Northover31d093c2013-09-22 08:21:56 +0000[diff] [blame]126 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]127
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]128 SDNode *ResNode = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]129 switch (N->getOpcode()) {
130 case ISD::LOAD:
131 ResNode = SelectLoad(N);
132 break;
133 case ISD::STORE:
134 ResNode = SelectStore(N);
135 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]136 case NVPTXISD::LoadV2:
137 case NVPTXISD::LoadV4:
138 ResNode = SelectLoadVector(N);
139 break;
140 case NVPTXISD::LDGV2:
141 case NVPTXISD::LDGV4:
142 case NVPTXISD::LDUV2:
143 case NVPTXISD::LDUV4:
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]144 ResNode = SelectLDGLDU(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]145 break;
146 case NVPTXISD::StoreV2:
147 case NVPTXISD::StoreV4:
148 ResNode = SelectStoreVector(N);
149 break;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]150 case NVPTXISD::LoadParam:
151 case NVPTXISD::LoadParamV2:
152 case NVPTXISD::LoadParamV4:
153 ResNode = SelectLoadParam(N);
154 break;
155 case NVPTXISD::StoreRetval:
156 case NVPTXISD::StoreRetvalV2:
157 case NVPTXISD::StoreRetvalV4:
158 ResNode = SelectStoreRetval(N);
159 break;
160 case NVPTXISD::StoreParam:
161 case NVPTXISD::StoreParamV2:
162 case NVPTXISD::StoreParamV4:
163 case NVPTXISD::StoreParamS32:
164 case NVPTXISD::StoreParamU32:
165 ResNode = SelectStoreParam(N);
166 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]167 case ISD::INTRINSIC_WO_CHAIN:
168 ResNode = SelectIntrinsicNoChain(N);
169 break;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]170 case ISD::INTRINSIC_W_CHAIN:
171 ResNode = SelectIntrinsicChain(N);
172 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]173 case NVPTXISD::Tex1DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]174 case NVPTXISD::Tex1DFloatFloat:
175 case NVPTXISD::Tex1DFloatFloatLevel:
176 case NVPTXISD::Tex1DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]177 case NVPTXISD::Tex1DS32S32:
178 case NVPTXISD::Tex1DS32Float:
179 case NVPTXISD::Tex1DS32FloatLevel:
180 case NVPTXISD::Tex1DS32FloatGrad:
181 case NVPTXISD::Tex1DU32S32:
182 case NVPTXISD::Tex1DU32Float:
183 case NVPTXISD::Tex1DU32FloatLevel:
184 case NVPTXISD::Tex1DU32FloatGrad:
185 case NVPTXISD::Tex1DArrayFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]186 case NVPTXISD::Tex1DArrayFloatFloat:
187 case NVPTXISD::Tex1DArrayFloatFloatLevel:
188 case NVPTXISD::Tex1DArrayFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]189 case NVPTXISD::Tex1DArrayS32S32:
190 case NVPTXISD::Tex1DArrayS32Float:
191 case NVPTXISD::Tex1DArrayS32FloatLevel:
192 case NVPTXISD::Tex1DArrayS32FloatGrad:
193 case NVPTXISD::Tex1DArrayU32S32:
194 case NVPTXISD::Tex1DArrayU32Float:
195 case NVPTXISD::Tex1DArrayU32FloatLevel:
196 case NVPTXISD::Tex1DArrayU32FloatGrad:
197 case NVPTXISD::Tex2DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]198 case NVPTXISD::Tex2DFloatFloat:
199 case NVPTXISD::Tex2DFloatFloatLevel:
200 case NVPTXISD::Tex2DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]201 case NVPTXISD::Tex2DS32S32:
202 case NVPTXISD::Tex2DS32Float:
203 case NVPTXISD::Tex2DS32FloatLevel:
204 case NVPTXISD::Tex2DS32FloatGrad:
205 case NVPTXISD::Tex2DU32S32:
206 case NVPTXISD::Tex2DU32Float:
207 case NVPTXISD::Tex2DU32FloatLevel:
208 case NVPTXISD::Tex2DU32FloatGrad:
209 case NVPTXISD::Tex2DArrayFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]210 case NVPTXISD::Tex2DArrayFloatFloat:
211 case NVPTXISD::Tex2DArrayFloatFloatLevel:
212 case NVPTXISD::Tex2DArrayFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]213 case NVPTXISD::Tex2DArrayS32S32:
214 case NVPTXISD::Tex2DArrayS32Float:
215 case NVPTXISD::Tex2DArrayS32FloatLevel:
216 case NVPTXISD::Tex2DArrayS32FloatGrad:
217 case NVPTXISD::Tex2DArrayU32S32:
218 case NVPTXISD::Tex2DArrayU32Float:
219 case NVPTXISD::Tex2DArrayU32FloatLevel:
220 case NVPTXISD::Tex2DArrayU32FloatGrad:
221 case NVPTXISD::Tex3DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]222 case NVPTXISD::Tex3DFloatFloat:
223 case NVPTXISD::Tex3DFloatFloatLevel:
224 case NVPTXISD::Tex3DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]225 case NVPTXISD::Tex3DS32S32:
226 case NVPTXISD::Tex3DS32Float:
227 case NVPTXISD::Tex3DS32FloatLevel:
228 case NVPTXISD::Tex3DS32FloatGrad:
229 case NVPTXISD::Tex3DU32S32:
230 case NVPTXISD::Tex3DU32Float:
231 case NVPTXISD::Tex3DU32FloatLevel:
232 case NVPTXISD::Tex3DU32FloatGrad:
233 case NVPTXISD::TexCubeFloatFloat:
234 case NVPTXISD::TexCubeFloatFloatLevel:
235 case NVPTXISD::TexCubeS32Float:
236 case NVPTXISD::TexCubeS32FloatLevel:
237 case NVPTXISD::TexCubeU32Float:
238 case NVPTXISD::TexCubeU32FloatLevel:
239 case NVPTXISD::TexCubeArrayFloatFloat:
240 case NVPTXISD::TexCubeArrayFloatFloatLevel:
241 case NVPTXISD::TexCubeArrayS32Float:
242 case NVPTXISD::TexCubeArrayS32FloatLevel:
243 case NVPTXISD::TexCubeArrayU32Float:
244 case NVPTXISD::TexCubeArrayU32FloatLevel:
245 case NVPTXISD::Tld4R2DFloatFloat:
246 case NVPTXISD::Tld4G2DFloatFloat:
247 case NVPTXISD::Tld4B2DFloatFloat:
248 case NVPTXISD::Tld4A2DFloatFloat:
249 case NVPTXISD::Tld4R2DS64Float:
250 case NVPTXISD::Tld4G2DS64Float:
251 case NVPTXISD::Tld4B2DS64Float:
252 case NVPTXISD::Tld4A2DS64Float:
253 case NVPTXISD::Tld4R2DU64Float:
254 case NVPTXISD::Tld4G2DU64Float:
255 case NVPTXISD::Tld4B2DU64Float:
256 case NVPTXISD::Tld4A2DU64Float:
257 case NVPTXISD::TexUnified1DFloatS32:
258 case NVPTXISD::TexUnified1DFloatFloat:
259 case NVPTXISD::TexUnified1DFloatFloatLevel:
260 case NVPTXISD::TexUnified1DFloatFloatGrad:
261 case NVPTXISD::TexUnified1DS32S32:
262 case NVPTXISD::TexUnified1DS32Float:
263 case NVPTXISD::TexUnified1DS32FloatLevel:
264 case NVPTXISD::TexUnified1DS32FloatGrad:
265 case NVPTXISD::TexUnified1DU32S32:
266 case NVPTXISD::TexUnified1DU32Float:
267 case NVPTXISD::TexUnified1DU32FloatLevel:
268 case NVPTXISD::TexUnified1DU32FloatGrad:
269 case NVPTXISD::TexUnified1DArrayFloatS32:
270 case NVPTXISD::TexUnified1DArrayFloatFloat:
271 case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
272 case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
273 case NVPTXISD::TexUnified1DArrayS32S32:
274 case NVPTXISD::TexUnified1DArrayS32Float:
275 case NVPTXISD::TexUnified1DArrayS32FloatLevel:
276 case NVPTXISD::TexUnified1DArrayS32FloatGrad:
277 case NVPTXISD::TexUnified1DArrayU32S32:
278 case NVPTXISD::TexUnified1DArrayU32Float:
279 case NVPTXISD::TexUnified1DArrayU32FloatLevel:
280 case NVPTXISD::TexUnified1DArrayU32FloatGrad:
281 case NVPTXISD::TexUnified2DFloatS32:
282 case NVPTXISD::TexUnified2DFloatFloat:
283 case NVPTXISD::TexUnified2DFloatFloatLevel:
284 case NVPTXISD::TexUnified2DFloatFloatGrad:
285 case NVPTXISD::TexUnified2DS32S32:
286 case NVPTXISD::TexUnified2DS32Float:
287 case NVPTXISD::TexUnified2DS32FloatLevel:
288 case NVPTXISD::TexUnified2DS32FloatGrad:
289 case NVPTXISD::TexUnified2DU32S32:
290 case NVPTXISD::TexUnified2DU32Float:
291 case NVPTXISD::TexUnified2DU32FloatLevel:
292 case NVPTXISD::TexUnified2DU32FloatGrad:
293 case NVPTXISD::TexUnified2DArrayFloatS32:
294 case NVPTXISD::TexUnified2DArrayFloatFloat:
295 case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
296 case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
297 case NVPTXISD::TexUnified2DArrayS32S32:
298 case NVPTXISD::TexUnified2DArrayS32Float:
299 case NVPTXISD::TexUnified2DArrayS32FloatLevel:
300 case NVPTXISD::TexUnified2DArrayS32FloatGrad:
301 case NVPTXISD::TexUnified2DArrayU32S32:
302 case NVPTXISD::TexUnified2DArrayU32Float:
303 case NVPTXISD::TexUnified2DArrayU32FloatLevel:
304 case NVPTXISD::TexUnified2DArrayU32FloatGrad:
305 case NVPTXISD::TexUnified3DFloatS32:
306 case NVPTXISD::TexUnified3DFloatFloat:
307 case NVPTXISD::TexUnified3DFloatFloatLevel:
308 case NVPTXISD::TexUnified3DFloatFloatGrad:
309 case NVPTXISD::TexUnified3DS32S32:
310 case NVPTXISD::TexUnified3DS32Float:
311 case NVPTXISD::TexUnified3DS32FloatLevel:
312 case NVPTXISD::TexUnified3DS32FloatGrad:
313 case NVPTXISD::TexUnified3DU32S32:
314 case NVPTXISD::TexUnified3DU32Float:
315 case NVPTXISD::TexUnified3DU32FloatLevel:
316 case NVPTXISD::TexUnified3DU32FloatGrad:
317 case NVPTXISD::TexUnifiedCubeFloatFloat:
318 case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
319 case NVPTXISD::TexUnifiedCubeS32Float:
320 case NVPTXISD::TexUnifiedCubeS32FloatLevel:
321 case NVPTXISD::TexUnifiedCubeU32Float:
322 case NVPTXISD::TexUnifiedCubeU32FloatLevel:
323 case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
324 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
325 case NVPTXISD::TexUnifiedCubeArrayS32Float:
326 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
327 case NVPTXISD::TexUnifiedCubeArrayU32Float:
328 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
329 case NVPTXISD::Tld4UnifiedR2DFloatFloat:
330 case NVPTXISD::Tld4UnifiedG2DFloatFloat:
331 case NVPTXISD::Tld4UnifiedB2DFloatFloat:
332 case NVPTXISD::Tld4UnifiedA2DFloatFloat:
333 case NVPTXISD::Tld4UnifiedR2DS64Float:
334 case NVPTXISD::Tld4UnifiedG2DS64Float:
335 case NVPTXISD::Tld4UnifiedB2DS64Float:
336 case NVPTXISD::Tld4UnifiedA2DS64Float:
337 case NVPTXISD::Tld4UnifiedR2DU64Float:
338 case NVPTXISD::Tld4UnifiedG2DU64Float:
339 case NVPTXISD::Tld4UnifiedB2DU64Float:
340 case NVPTXISD::Tld4UnifiedA2DU64Float:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]341 ResNode = SelectTextureIntrinsic(N);
342 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]343 case NVPTXISD::Suld1DI8Clamp:
344 case NVPTXISD::Suld1DI16Clamp:
345 case NVPTXISD::Suld1DI32Clamp:
346 case NVPTXISD::Suld1DI64Clamp:
347 case NVPTXISD::Suld1DV2I8Clamp:
348 case NVPTXISD::Suld1DV2I16Clamp:
349 case NVPTXISD::Suld1DV2I32Clamp:
350 case NVPTXISD::Suld1DV2I64Clamp:
351 case NVPTXISD::Suld1DV4I8Clamp:
352 case NVPTXISD::Suld1DV4I16Clamp:
353 case NVPTXISD::Suld1DV4I32Clamp:
354 case NVPTXISD::Suld1DArrayI8Clamp:
355 case NVPTXISD::Suld1DArrayI16Clamp:
356 case NVPTXISD::Suld1DArrayI32Clamp:
357 case NVPTXISD::Suld1DArrayI64Clamp:
358 case NVPTXISD::Suld1DArrayV2I8Clamp:
359 case NVPTXISD::Suld1DArrayV2I16Clamp:
360 case NVPTXISD::Suld1DArrayV2I32Clamp:
361 case NVPTXISD::Suld1DArrayV2I64Clamp:
362 case NVPTXISD::Suld1DArrayV4I8Clamp:
363 case NVPTXISD::Suld1DArrayV4I16Clamp:
364 case NVPTXISD::Suld1DArrayV4I32Clamp:
365 case NVPTXISD::Suld2DI8Clamp:
366 case NVPTXISD::Suld2DI16Clamp:
367 case NVPTXISD::Suld2DI32Clamp:
368 case NVPTXISD::Suld2DI64Clamp:
369 case NVPTXISD::Suld2DV2I8Clamp:
370 case NVPTXISD::Suld2DV2I16Clamp:
371 case NVPTXISD::Suld2DV2I32Clamp:
372 case NVPTXISD::Suld2DV2I64Clamp:
373 case NVPTXISD::Suld2DV4I8Clamp:
374 case NVPTXISD::Suld2DV4I16Clamp:
375 case NVPTXISD::Suld2DV4I32Clamp:
376 case NVPTXISD::Suld2DArrayI8Clamp:
377 case NVPTXISD::Suld2DArrayI16Clamp:
378 case NVPTXISD::Suld2DArrayI32Clamp:
379 case NVPTXISD::Suld2DArrayI64Clamp:
380 case NVPTXISD::Suld2DArrayV2I8Clamp:
381 case NVPTXISD::Suld2DArrayV2I16Clamp:
382 case NVPTXISD::Suld2DArrayV2I32Clamp:
383 case NVPTXISD::Suld2DArrayV2I64Clamp:
384 case NVPTXISD::Suld2DArrayV4I8Clamp:
385 case NVPTXISD::Suld2DArrayV4I16Clamp:
386 case NVPTXISD::Suld2DArrayV4I32Clamp:
387 case NVPTXISD::Suld3DI8Clamp:
388 case NVPTXISD::Suld3DI16Clamp:
389 case NVPTXISD::Suld3DI32Clamp:
390 case NVPTXISD::Suld3DI64Clamp:
391 case NVPTXISD::Suld3DV2I8Clamp:
392 case NVPTXISD::Suld3DV2I16Clamp:
393 case NVPTXISD::Suld3DV2I32Clamp:
394 case NVPTXISD::Suld3DV2I64Clamp:
395 case NVPTXISD::Suld3DV4I8Clamp:
396 case NVPTXISD::Suld3DV4I16Clamp:
397 case NVPTXISD::Suld3DV4I32Clamp:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]398 case NVPTXISD::Suld1DI8Trap:
399 case NVPTXISD::Suld1DI16Trap:
400 case NVPTXISD::Suld1DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]401 case NVPTXISD::Suld1DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]402 case NVPTXISD::Suld1DV2I8Trap:
403 case NVPTXISD::Suld1DV2I16Trap:
404 case NVPTXISD::Suld1DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]405 case NVPTXISD::Suld1DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]406 case NVPTXISD::Suld1DV4I8Trap:
407 case NVPTXISD::Suld1DV4I16Trap:
408 case NVPTXISD::Suld1DV4I32Trap:
409 case NVPTXISD::Suld1DArrayI8Trap:
410 case NVPTXISD::Suld1DArrayI16Trap:
411 case NVPTXISD::Suld1DArrayI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]412 case NVPTXISD::Suld1DArrayI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]413 case NVPTXISD::Suld1DArrayV2I8Trap:
414 case NVPTXISD::Suld1DArrayV2I16Trap:
415 case NVPTXISD::Suld1DArrayV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]416 case NVPTXISD::Suld1DArrayV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]417 case NVPTXISD::Suld1DArrayV4I8Trap:
418 case NVPTXISD::Suld1DArrayV4I16Trap:
419 case NVPTXISD::Suld1DArrayV4I32Trap:
420 case NVPTXISD::Suld2DI8Trap:
421 case NVPTXISD::Suld2DI16Trap:
422 case NVPTXISD::Suld2DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]423 case NVPTXISD::Suld2DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]424 case NVPTXISD::Suld2DV2I8Trap:
425 case NVPTXISD::Suld2DV2I16Trap:
426 case NVPTXISD::Suld2DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]427 case NVPTXISD::Suld2DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]428 case NVPTXISD::Suld2DV4I8Trap:
429 case NVPTXISD::Suld2DV4I16Trap:
430 case NVPTXISD::Suld2DV4I32Trap:
431 case NVPTXISD::Suld2DArrayI8Trap:
432 case NVPTXISD::Suld2DArrayI16Trap:
433 case NVPTXISD::Suld2DArrayI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]434 case NVPTXISD::Suld2DArrayI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]435 case NVPTXISD::Suld2DArrayV2I8Trap:
436 case NVPTXISD::Suld2DArrayV2I16Trap:
437 case NVPTXISD::Suld2DArrayV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]438 case NVPTXISD::Suld2DArrayV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]439 case NVPTXISD::Suld2DArrayV4I8Trap:
440 case NVPTXISD::Suld2DArrayV4I16Trap:
441 case NVPTXISD::Suld2DArrayV4I32Trap:
442 case NVPTXISD::Suld3DI8Trap:
443 case NVPTXISD::Suld3DI16Trap:
444 case NVPTXISD::Suld3DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]445 case NVPTXISD::Suld3DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]446 case NVPTXISD::Suld3DV2I8Trap:
447 case NVPTXISD::Suld3DV2I16Trap:
448 case NVPTXISD::Suld3DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]449 case NVPTXISD::Suld3DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]450 case NVPTXISD::Suld3DV4I8Trap:
451 case NVPTXISD::Suld3DV4I16Trap:
452 case NVPTXISD::Suld3DV4I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]453 case NVPTXISD::Suld1DI8Zero:
454 case NVPTXISD::Suld1DI16Zero:
455 case NVPTXISD::Suld1DI32Zero:
456 case NVPTXISD::Suld1DI64Zero:
457 case NVPTXISD::Suld1DV2I8Zero:
458 case NVPTXISD::Suld1DV2I16Zero:
459 case NVPTXISD::Suld1DV2I32Zero:
460 case NVPTXISD::Suld1DV2I64Zero:
461 case NVPTXISD::Suld1DV4I8Zero:
462 case NVPTXISD::Suld1DV4I16Zero:
463 case NVPTXISD::Suld1DV4I32Zero:
464 case NVPTXISD::Suld1DArrayI8Zero:
465 case NVPTXISD::Suld1DArrayI16Zero:
466 case NVPTXISD::Suld1DArrayI32Zero:
467 case NVPTXISD::Suld1DArrayI64Zero:
468 case NVPTXISD::Suld1DArrayV2I8Zero:
469 case NVPTXISD::Suld1DArrayV2I16Zero:
470 case NVPTXISD::Suld1DArrayV2I32Zero:
471 case NVPTXISD::Suld1DArrayV2I64Zero:
472 case NVPTXISD::Suld1DArrayV4I8Zero:
473 case NVPTXISD::Suld1DArrayV4I16Zero:
474 case NVPTXISD::Suld1DArrayV4I32Zero:
475 case NVPTXISD::Suld2DI8Zero:
476 case NVPTXISD::Suld2DI16Zero:
477 case NVPTXISD::Suld2DI32Zero:
478 case NVPTXISD::Suld2DI64Zero:
479 case NVPTXISD::Suld2DV2I8Zero:
480 case NVPTXISD::Suld2DV2I16Zero:
481 case NVPTXISD::Suld2DV2I32Zero:
482 case NVPTXISD::Suld2DV2I64Zero:
483 case NVPTXISD::Suld2DV4I8Zero:
484 case NVPTXISD::Suld2DV4I16Zero:
485 case NVPTXISD::Suld2DV4I32Zero:
486 case NVPTXISD::Suld2DArrayI8Zero:
487 case NVPTXISD::Suld2DArrayI16Zero:
488 case NVPTXISD::Suld2DArrayI32Zero:
489 case NVPTXISD::Suld2DArrayI64Zero:
490 case NVPTXISD::Suld2DArrayV2I8Zero:
491 case NVPTXISD::Suld2DArrayV2I16Zero:
492 case NVPTXISD::Suld2DArrayV2I32Zero:
493 case NVPTXISD::Suld2DArrayV2I64Zero:
494 case NVPTXISD::Suld2DArrayV4I8Zero:
495 case NVPTXISD::Suld2DArrayV4I16Zero:
496 case NVPTXISD::Suld2DArrayV4I32Zero:
497 case NVPTXISD::Suld3DI8Zero:
498 case NVPTXISD::Suld3DI16Zero:
499 case NVPTXISD::Suld3DI32Zero:
500 case NVPTXISD::Suld3DI64Zero:
501 case NVPTXISD::Suld3DV2I8Zero:
502 case NVPTXISD::Suld3DV2I16Zero:
503 case NVPTXISD::Suld3DV2I32Zero:
504 case NVPTXISD::Suld3DV2I64Zero:
505 case NVPTXISD::Suld3DV4I8Zero:
506 case NVPTXISD::Suld3DV4I16Zero:
507 case NVPTXISD::Suld3DV4I32Zero:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]508 ResNode = SelectSurfaceIntrinsic(N);
509 break;
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]510 case ISD::AND:
511 case ISD::SRA:
512 case ISD::SRL:
513 // Try to select BFE
514 ResNode = SelectBFE(N);
515 break;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]516 case ISD::ADDRSPACECAST:
517 ResNode = SelectAddrSpaceCast(N);
518 break;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]519 default:
520 break;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]521 }
522 if (ResNode)
523 return ResNode;
524 return SelectCode(N);
525}
526
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]527SDNode *NVPTXDAGToDAGISel::SelectIntrinsicChain(SDNode *N) {
528 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
529 switch (IID) {
530 default:
531 return NULL;
532 case Intrinsic::nvvm_ldg_global_f:
533 case Intrinsic::nvvm_ldg_global_i:
534 case Intrinsic::nvvm_ldg_global_p:
535 case Intrinsic::nvvm_ldu_global_f:
536 case Intrinsic::nvvm_ldu_global_i:
537 case Intrinsic::nvvm_ldu_global_p:
538 return SelectLDGLDU(N);
539 }
540}
541
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]542static unsigned int getCodeAddrSpace(MemSDNode *N,
543 const NVPTXSubtarget &Subtarget) {
Nick Lewyckyaad475b2014-04-15 07:22:52 +0000[diff] [blame]544 const Value *Src = N->getMemOperand()->getValue();
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]545
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]546 if (!Src)
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]547 return NVPTX::PTXLdStInstCode::GENERIC;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]548
549 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
550 switch (PT->getAddressSpace()) {
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]551 case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
552 case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
553 case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
554 case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
555 case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
556 case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
557 default: break;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]558 }
559 }
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]560 return NVPTX::PTXLdStInstCode::GENERIC;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]561}
562
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]563SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) {
564 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
565 switch (IID) {
566 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]567 return nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]568 case Intrinsic::nvvm_texsurf_handle_internal:
569 return SelectTexSurfHandle(N);
570 }
571}
572
573SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
574 // Op 0 is the intrinsic ID
575 SDValue Wrapper = N->getOperand(1);
576 SDValue GlobalVal = Wrapper.getOperand(0);
577 return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64,
578 GlobalVal);
579}
580
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]581SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
582 SDValue Src = N->getOperand(0);
583 AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
584 unsigned SrcAddrSpace = CastN->getSrcAddressSpace();
585 unsigned DstAddrSpace = CastN->getDestAddressSpace();
586
587 assert(SrcAddrSpace != DstAddrSpace &&
588 "addrspacecast must be between different address spaces");
589
590 if (DstAddrSpace == ADDRESS_SPACE_GENERIC) {
591 // Specific to generic
592 unsigned Opc;
593 switch (SrcAddrSpace) {
594 default: report_fatal_error("Bad address space in addrspacecast");
595 case ADDRESS_SPACE_GLOBAL:
596 Opc = Subtarget.is64Bit() ? NVPTX::cvta_global_yes_64
597 : NVPTX::cvta_global_yes;
598 break;
599 case ADDRESS_SPACE_SHARED:
600 Opc = Subtarget.is64Bit() ? NVPTX::cvta_shared_yes_64
601 : NVPTX::cvta_shared_yes;
602 break;
603 case ADDRESS_SPACE_CONST:
604 Opc = Subtarget.is64Bit() ? NVPTX::cvta_const_yes_64
605 : NVPTX::cvta_const_yes;
606 break;
607 case ADDRESS_SPACE_LOCAL:
608 Opc = Subtarget.is64Bit() ? NVPTX::cvta_local_yes_64
609 : NVPTX::cvta_local_yes;
610 break;
611 }
612 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
613 } else {
614 // Generic to specific
615 if (SrcAddrSpace != 0)
616 report_fatal_error("Cannot cast between two non-generic address spaces");
617 unsigned Opc;
618 switch (DstAddrSpace) {
619 default: report_fatal_error("Bad address space in addrspacecast");
620 case ADDRESS_SPACE_GLOBAL:
621 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_global_yes_64
622 : NVPTX::cvta_to_global_yes;
623 break;
624 case ADDRESS_SPACE_SHARED:
625 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_shared_yes_64
626 : NVPTX::cvta_to_shared_yes;
627 break;
628 case ADDRESS_SPACE_CONST:
629 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_const_yes_64
630 : NVPTX::cvta_to_const_yes;
631 break;
632 case ADDRESS_SPACE_LOCAL:
633 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_local_yes_64
634 : NVPTX::cvta_to_local_yes;
635 break;
636 }
637 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
638 }
639}
640
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]641SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]642 SDLoc dl(N);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]643 LoadSDNode *LD = cast<LoadSDNode>(N);
644 EVT LoadedVT = LD->getMemoryVT();
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]645 SDNode *NVPTXLD = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]646
647 // do not support pre/post inc/dec
648 if (LD->isIndexed())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]649 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]650
651 if (!LoadedVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]652 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]653
654 // Address Space Setting
655 unsigned int codeAddrSpace = getCodeAddrSpace(LD, Subtarget);
656
657 // Volatile Setting
658 // - .volatile is only availalble for .global and .shared
659 bool isVolatile = LD->isVolatile();
660 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
661 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
662 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
663 isVolatile = false;
664
665 // Vector Setting
666 MVT SimpleVT = LoadedVT.getSimpleVT();
667 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
668 if (SimpleVT.isVector()) {
669 unsigned num = SimpleVT.getVectorNumElements();
670 if (num == 2)
671 vecType = NVPTX::PTXLdStInstCode::V2;
672 else if (num == 4)
673 vecType = NVPTX::PTXLdStInstCode::V4;
674 else
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]675 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]676 }
677
678 // Type Setting: fromType + fromTypeWidth
679 //
680 // Sign : ISD::SEXTLOAD
681 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
682 // type is integer
683 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
684 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski994d66a2013-05-30 12:22:39 +0000[diff] [blame]685 // Read at least 8 bits (predicates are stored as 8-bit values)
686 unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]687 unsigned int fromType;
688 if ((LD->getExtensionType() == ISD::SEXTLOAD))
689 fromType = NVPTX::PTXLdStInstCode::Signed;
690 else if (ScalarVT.isFloatingPoint())
691 fromType = NVPTX::PTXLdStInstCode::Float;
692 else
693 fromType = NVPTX::PTXLdStInstCode::Unsigned;
694
695 // Create the machine instruction DAG
696 SDValue Chain = N->getOperand(0);
697 SDValue N1 = N->getOperand(1);
698 SDValue Addr;
699 SDValue Offset, Base;
700 unsigned Opcode;
Craig Topperd9c27832013-08-15 02:44:19 +0000[diff] [blame]701 MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]702
703 if (SelectDirectAddr(N1, Addr)) {
704 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]705 case MVT::i8:
706 Opcode = NVPTX::LD_i8_avar;
707 break;
708 case MVT::i16:
709 Opcode = NVPTX::LD_i16_avar;
710 break;
711 case MVT::i32:
712 Opcode = NVPTX::LD_i32_avar;
713 break;
714 case MVT::i64:
715 Opcode = NVPTX::LD_i64_avar;
716 break;
717 case MVT::f32:
718 Opcode = NVPTX::LD_f32_avar;
719 break;
720 case MVT::f64:
721 Opcode = NVPTX::LD_f64_avar;
722 break;
723 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]724 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]725 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]726 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
727 getI32Imm(vecType), getI32Imm(fromType),
728 getI32Imm(fromTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]729 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]730 } else if (Subtarget.is64Bit()
731 ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
732 : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]733 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]734 case MVT::i8:
735 Opcode = NVPTX::LD_i8_asi;
736 break;
737 case MVT::i16:
738 Opcode = NVPTX::LD_i16_asi;
739 break;
740 case MVT::i32:
741 Opcode = NVPTX::LD_i32_asi;
742 break;
743 case MVT::i64:
744 Opcode = NVPTX::LD_i64_asi;
745 break;
746 case MVT::f32:
747 Opcode = NVPTX::LD_f32_asi;
748 break;
749 case MVT::f64:
750 Opcode = NVPTX::LD_f64_asi;
751 break;
752 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]753 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]754 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]755 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
756 getI32Imm(vecType), getI32Imm(fromType),
757 getI32Imm(fromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]758 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]759 } else if (Subtarget.is64Bit()
760 ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
761 : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]762 if (Subtarget.is64Bit()) {
763 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]764 case MVT::i8:
765 Opcode = NVPTX::LD_i8_ari_64;
766 break;
767 case MVT::i16:
768 Opcode = NVPTX::LD_i16_ari_64;
769 break;
770 case MVT::i32:
771 Opcode = NVPTX::LD_i32_ari_64;
772 break;
773 case MVT::i64:
774 Opcode = NVPTX::LD_i64_ari_64;
775 break;
776 case MVT::f32:
777 Opcode = NVPTX::LD_f32_ari_64;
778 break;
779 case MVT::f64:
780 Opcode = NVPTX::LD_f64_ari_64;
781 break;
782 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]783 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]784 }
785 } else {
786 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]787 case MVT::i8:
788 Opcode = NVPTX::LD_i8_ari;
789 break;
790 case MVT::i16:
791 Opcode = NVPTX::LD_i16_ari;
792 break;
793 case MVT::i32:
794 Opcode = NVPTX::LD_i32_ari;
795 break;
796 case MVT::i64:
797 Opcode = NVPTX::LD_i64_ari;
798 break;
799 case MVT::f32:
800 Opcode = NVPTX::LD_f32_ari;
801 break;
802 case MVT::f64:
803 Opcode = NVPTX::LD_f64_ari;
804 break;
805 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]806 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]807 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]808 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]809 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
810 getI32Imm(vecType), getI32Imm(fromType),
811 getI32Imm(fromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]812 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]813 } else {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]814 if (Subtarget.is64Bit()) {
815 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]816 case MVT::i8:
817 Opcode = NVPTX::LD_i8_areg_64;
818 break;
819 case MVT::i16:
820 Opcode = NVPTX::LD_i16_areg_64;
821 break;
822 case MVT::i32:
823 Opcode = NVPTX::LD_i32_areg_64;
824 break;
825 case MVT::i64:
826 Opcode = NVPTX::LD_i64_areg_64;
827 break;
828 case MVT::f32:
829 Opcode = NVPTX::LD_f32_areg_64;
830 break;
831 case MVT::f64:
832 Opcode = NVPTX::LD_f64_areg_64;
833 break;
834 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]835 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]836 }
837 } else {
838 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]839 case MVT::i8:
840 Opcode = NVPTX::LD_i8_areg;
841 break;
842 case MVT::i16:
843 Opcode = NVPTX::LD_i16_areg;
844 break;
845 case MVT::i32:
846 Opcode = NVPTX::LD_i32_areg;
847 break;
848 case MVT::i64:
849 Opcode = NVPTX::LD_i64_areg;
850 break;
851 case MVT::f32:
852 Opcode = NVPTX::LD_f32_areg;
853 break;
854 case MVT::f64:
855 Opcode = NVPTX::LD_f64_areg;
856 break;
857 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]858 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]859 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]860 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]861 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
862 getI32Imm(vecType), getI32Imm(fromType),
863 getI32Imm(fromTypeWidth), N1, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]864 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]865 }
866
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]867 if (NVPTXLD) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]868 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
869 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
870 cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
871 }
872
873 return NVPTXLD;
874}
875
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]876SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
877
878 SDValue Chain = N->getOperand(0);
879 SDValue Op1 = N->getOperand(1);
880 SDValue Addr, Offset, Base;
881 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]882 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]883 SDNode *LD;
884 MemSDNode *MemSD = cast<MemSDNode>(N);
885 EVT LoadedVT = MemSD->getMemoryVT();
886
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]887 if (!LoadedVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]888 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]889
890 // Address Space Setting
891 unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget);
892
893 // Volatile Setting
894 // - .volatile is only availalble for .global and .shared
895 bool IsVolatile = MemSD->isVolatile();
896 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
897 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
898 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
899 IsVolatile = false;
900
901 // Vector Setting
902 MVT SimpleVT = LoadedVT.getSimpleVT();
903
904 // Type Setting: fromType + fromTypeWidth
905 //
906 // Sign : ISD::SEXTLOAD
907 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
908 // type is integer
909 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
910 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski994d66a2013-05-30 12:22:39 +0000[diff] [blame]911 // Read at least 8 bits (predicates are stored as 8-bit values)
912 unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]913 unsigned int FromType;
914 // The last operand holds the original LoadSDNode::getExtensionType() value
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]915 unsigned ExtensionType = cast<ConstantSDNode>(
916 N->getOperand(N->getNumOperands() - 1))->getZExtValue();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]917 if (ExtensionType == ISD::SEXTLOAD)
918 FromType = NVPTX::PTXLdStInstCode::Signed;
919 else if (ScalarVT.isFloatingPoint())
920 FromType = NVPTX::PTXLdStInstCode::Float;
921 else
922 FromType = NVPTX::PTXLdStInstCode::Unsigned;
923
924 unsigned VecType;
925
926 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]927 case NVPTXISD::LoadV2:
928 VecType = NVPTX::PTXLdStInstCode::V2;
929 break;
930 case NVPTXISD::LoadV4:
931 VecType = NVPTX::PTXLdStInstCode::V4;
932 break;
933 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]934 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]935 }
936
937 EVT EltVT = N->getValueType(0);
938
939 if (SelectDirectAddr(Op1, Addr)) {
940 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]941 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]942 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]943 case NVPTXISD::LoadV2:
944 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]945 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]946 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]947 case MVT::i8:
948 Opcode = NVPTX::LDV_i8_v2_avar;
949 break;
950 case MVT::i16:
951 Opcode = NVPTX::LDV_i16_v2_avar;
952 break;
953 case MVT::i32:
954 Opcode = NVPTX::LDV_i32_v2_avar;
955 break;
956 case MVT::i64:
957 Opcode = NVPTX::LDV_i64_v2_avar;
958 break;
959 case MVT::f32:
960 Opcode = NVPTX::LDV_f32_v2_avar;
961 break;
962 case MVT::f64:
963 Opcode = NVPTX::LDV_f64_v2_avar;
964 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]965 }
966 break;
967 case NVPTXISD::LoadV4:
968 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]969 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]970 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]971 case MVT::i8:
972 Opcode = NVPTX::LDV_i8_v4_avar;
973 break;
974 case MVT::i16:
975 Opcode = NVPTX::LDV_i16_v4_avar;
976 break;
977 case MVT::i32:
978 Opcode = NVPTX::LDV_i32_v4_avar;
979 break;
980 case MVT::f32:
981 Opcode = NVPTX::LDV_f32_v4_avar;
982 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]983 }
984 break;
985 }
986
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]987 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
988 getI32Imm(VecType), getI32Imm(FromType),
989 getI32Imm(FromTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]990 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]991 } else if (Subtarget.is64Bit()
992 ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
993 : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]994 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]995 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]996 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]997 case NVPTXISD::LoadV2:
998 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]999 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1000 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1001 case MVT::i8:
1002 Opcode = NVPTX::LDV_i8_v2_asi;
1003 break;
1004 case MVT::i16:
1005 Opcode = NVPTX::LDV_i16_v2_asi;
1006 break;
1007 case MVT::i32:
1008 Opcode = NVPTX::LDV_i32_v2_asi;
1009 break;
1010 case MVT::i64:
1011 Opcode = NVPTX::LDV_i64_v2_asi;
1012 break;
1013 case MVT::f32:
1014 Opcode = NVPTX::LDV_f32_v2_asi;
1015 break;
1016 case MVT::f64:
1017 Opcode = NVPTX::LDV_f64_v2_asi;
1018 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1019 }
1020 break;
1021 case NVPTXISD::LoadV4:
1022 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1023 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1024 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1025 case MVT::i8:
1026 Opcode = NVPTX::LDV_i8_v4_asi;
1027 break;
1028 case MVT::i16:
1029 Opcode = NVPTX::LDV_i16_v4_asi;
1030 break;
1031 case MVT::i32:
1032 Opcode = NVPTX::LDV_i32_v4_asi;
1033 break;
1034 case MVT::f32:
1035 Opcode = NVPTX::LDV_f32_v4_asi;
1036 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1037 }
1038 break;
1039 }
1040
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1041 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1042 getI32Imm(VecType), getI32Imm(FromType),
1043 getI32Imm(FromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1044 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1045 } else if (Subtarget.is64Bit()
1046 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
1047 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1048 if (Subtarget.is64Bit()) {
1049 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1050 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1051 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1052 case NVPTXISD::LoadV2:
1053 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1054 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1055 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1056 case MVT::i8:
1057 Opcode = NVPTX::LDV_i8_v2_ari_64;
1058 break;
1059 case MVT::i16:
1060 Opcode = NVPTX::LDV_i16_v2_ari_64;
1061 break;
1062 case MVT::i32:
1063 Opcode = NVPTX::LDV_i32_v2_ari_64;
1064 break;
1065 case MVT::i64:
1066 Opcode = NVPTX::LDV_i64_v2_ari_64;
1067 break;
1068 case MVT::f32:
1069 Opcode = NVPTX::LDV_f32_v2_ari_64;
1070 break;
1071 case MVT::f64:
1072 Opcode = NVPTX::LDV_f64_v2_ari_64;
1073 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1074 }
1075 break;
1076 case NVPTXISD::LoadV4:
1077 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1078 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1079 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1080 case MVT::i8:
1081 Opcode = NVPTX::LDV_i8_v4_ari_64;
1082 break;
1083 case MVT::i16:
1084 Opcode = NVPTX::LDV_i16_v4_ari_64;
1085 break;
1086 case MVT::i32:
1087 Opcode = NVPTX::LDV_i32_v4_ari_64;
1088 break;
1089 case MVT::f32:
1090 Opcode = NVPTX::LDV_f32_v4_ari_64;
1091 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1092 }
1093 break;
1094 }
1095 } else {
1096 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1097 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1098 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1099 case NVPTXISD::LoadV2:
1100 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1101 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1102 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1103 case MVT::i8:
1104 Opcode = NVPTX::LDV_i8_v2_ari;
1105 break;
1106 case MVT::i16:
1107 Opcode = NVPTX::LDV_i16_v2_ari;
1108 break;
1109 case MVT::i32:
1110 Opcode = NVPTX::LDV_i32_v2_ari;
1111 break;
1112 case MVT::i64:
1113 Opcode = NVPTX::LDV_i64_v2_ari;
1114 break;
1115 case MVT::f32:
1116 Opcode = NVPTX::LDV_f32_v2_ari;
1117 break;
1118 case MVT::f64:
1119 Opcode = NVPTX::LDV_f64_v2_ari;
1120 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1121 }
1122 break;
1123 case NVPTXISD::LoadV4:
1124 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1125 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1126 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1127 case MVT::i8:
1128 Opcode = NVPTX::LDV_i8_v4_ari;
1129 break;
1130 case MVT::i16:
1131 Opcode = NVPTX::LDV_i16_v4_ari;
1132 break;
1133 case MVT::i32:
1134 Opcode = NVPTX::LDV_i32_v4_ari;
1135 break;
1136 case MVT::f32:
1137 Opcode = NVPTX::LDV_f32_v4_ari;
1138 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1139 }
1140 break;
1141 }
1142 }
1143
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1144 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1145 getI32Imm(VecType), getI32Imm(FromType),
1146 getI32Imm(FromTypeWidth), Base, Offset, Chain };
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1147
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1148 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1149 } else {
1150 if (Subtarget.is64Bit()) {
1151 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1152 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1153 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1154 case NVPTXISD::LoadV2:
1155 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1156 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1157 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1158 case MVT::i8:
1159 Opcode = NVPTX::LDV_i8_v2_areg_64;
1160 break;
1161 case MVT::i16:
1162 Opcode = NVPTX::LDV_i16_v2_areg_64;
1163 break;
1164 case MVT::i32:
1165 Opcode = NVPTX::LDV_i32_v2_areg_64;
1166 break;
1167 case MVT::i64:
1168 Opcode = NVPTX::LDV_i64_v2_areg_64;
1169 break;
1170 case MVT::f32:
1171 Opcode = NVPTX::LDV_f32_v2_areg_64;
1172 break;
1173 case MVT::f64:
1174 Opcode = NVPTX::LDV_f64_v2_areg_64;
1175 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1176 }
1177 break;
1178 case NVPTXISD::LoadV4:
1179 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1180 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1181 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1182 case MVT::i8:
1183 Opcode = NVPTX::LDV_i8_v4_areg_64;
1184 break;
1185 case MVT::i16:
1186 Opcode = NVPTX::LDV_i16_v4_areg_64;
1187 break;
1188 case MVT::i32:
1189 Opcode = NVPTX::LDV_i32_v4_areg_64;
1190 break;
1191 case MVT::f32:
1192 Opcode = NVPTX::LDV_f32_v4_areg_64;
1193 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1194 }
1195 break;
1196 }
1197 } else {
1198 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1199 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1200 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1201 case NVPTXISD::LoadV2:
1202 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1203 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1204 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1205 case MVT::i8:
1206 Opcode = NVPTX::LDV_i8_v2_areg;
1207 break;
1208 case MVT::i16:
1209 Opcode = NVPTX::LDV_i16_v2_areg;
1210 break;
1211 case MVT::i32:
1212 Opcode = NVPTX::LDV_i32_v2_areg;
1213 break;
1214 case MVT::i64:
1215 Opcode = NVPTX::LDV_i64_v2_areg;
1216 break;
1217 case MVT::f32:
1218 Opcode = NVPTX::LDV_f32_v2_areg;
1219 break;
1220 case MVT::f64:
1221 Opcode = NVPTX::LDV_f64_v2_areg;
1222 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1223 }
1224 break;
1225 case NVPTXISD::LoadV4:
1226 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1227 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1228 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1229 case MVT::i8:
1230 Opcode = NVPTX::LDV_i8_v4_areg;
1231 break;
1232 case MVT::i16:
1233 Opcode = NVPTX::LDV_i16_v4_areg;
1234 break;
1235 case MVT::i32:
1236 Opcode = NVPTX::LDV_i32_v4_areg;
1237 break;
1238 case MVT::f32:
1239 Opcode = NVPTX::LDV_f32_v4_areg;
1240 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1241 }
1242 break;
1243 }
1244 }
1245
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1246 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1247 getI32Imm(VecType), getI32Imm(FromType),
1248 getI32Imm(FromTypeWidth), Op1, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1249 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1250 }
1251
1252 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1253 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1254 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
1255
1256 return LD;
1257}
1258
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1259SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1260
1261 SDValue Chain = N->getOperand(0);
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1262 SDValue Op1;
1263 MemSDNode *Mem;
1264 bool IsLDG = true;
1265
1266 // If this is an LDG intrinsic, the address is the third operand. Its its an
1267 // LDG/LDU SD node (from custom vector handling), then its the second operand
1268 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
1269 Op1 = N->getOperand(2);
1270 Mem = cast<MemIntrinsicSDNode>(N);
1271 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1272 switch (IID) {
1273 default:
1274 return NULL;
1275 case Intrinsic::nvvm_ldg_global_f:
1276 case Intrinsic::nvvm_ldg_global_i:
1277 case Intrinsic::nvvm_ldg_global_p:
1278 IsLDG = true;
1279 break;
1280 case Intrinsic::nvvm_ldu_global_f:
1281 case Intrinsic::nvvm_ldu_global_i:
1282 case Intrinsic::nvvm_ldu_global_p:
1283 IsLDG = false;
1284 break;
1285 }
1286 } else {
1287 Op1 = N->getOperand(1);
1288 Mem = cast<MemSDNode>(N);
1289 }
1290
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1291 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1292 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1293 SDNode *LD;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1294 SDValue Base, Offset, Addr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1295
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1296 EVT EltVT = Mem->getMemoryVT();
1297 if (EltVT.isVector()) {
1298 EltVT = EltVT.getVectorElementType();
1299 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1300
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1301 if (SelectDirectAddr(Op1, Addr)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1302 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1303 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1304 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1305 case ISD::INTRINSIC_W_CHAIN:
1306 if (IsLDG) {
1307 switch (EltVT.getSimpleVT().SimpleTy) {
1308 default:
1309 return nullptr;
1310 case MVT::i8:
1311 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar;
1312 break;
1313 case MVT::i16:
1314 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar;
1315 break;
1316 case MVT::i32:
1317 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar;
1318 break;
1319 case MVT::i64:
1320 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar;
1321 break;
1322 case MVT::f32:
1323 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar;
1324 break;
1325 case MVT::f64:
1326 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar;
1327 break;
1328 }
1329 } else {
1330 switch (EltVT.getSimpleVT().SimpleTy) {
1331 default:
1332 return nullptr;
1333 case MVT::i8:
1334 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar;
1335 break;
1336 case MVT::i16:
1337 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar;
1338 break;
1339 case MVT::i32:
1340 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar;
1341 break;
1342 case MVT::i64:
1343 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar;
1344 break;
1345 case MVT::f32:
1346 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar;
1347 break;
1348 case MVT::f64:
1349 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar;
1350 break;
1351 }
1352 }
1353 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1354 case NVPTXISD::LDGV2:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1355 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1356 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1357 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1358 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1359 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1360 break;
1361 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1362 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1363 break;
1364 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1365 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1366 break;
1367 case MVT::i64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1368 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1369 break;
1370 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1371 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1372 break;
1373 case MVT::f64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1374 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1375 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1376 }
1377 break;
1378 case NVPTXISD::LDUV2:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1379 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1380 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1381 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1382 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1383 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1384 break;
1385 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1386 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1387 break;
1388 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1389 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1390 break;
1391 case MVT::i64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1392 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1393 break;
1394 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1395 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1396 break;
1397 case MVT::f64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1398 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
1399 break;
1400 }
1401 break;
1402 case NVPTXISD::LDGV4:
1403 switch (EltVT.getSimpleVT().SimpleTy) {
1404 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1405 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1406 case MVT::i8:
1407 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
1408 break;
1409 case MVT::i16:
1410 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
1411 break;
1412 case MVT::i32:
1413 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
1414 break;
1415 case MVT::f32:
1416 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1417 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1418 }
1419 break;
1420 case NVPTXISD::LDUV4:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1421 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1422 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1423 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1424 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1425 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1426 break;
1427 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1428 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1429 break;
1430 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1431 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1432 break;
1433 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1434 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1435 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1436 }
1437 break;
1438 }
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1439
1440 SDValue Ops[] = { Addr, Chain };
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1441 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1442 } else if (Subtarget.is64Bit()
1443 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
1444 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
1445 if (Subtarget.is64Bit()) {
1446 switch (N->getOpcode()) {
1447 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1448 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1449 case ISD::INTRINSIC_W_CHAIN:
1450 if (IsLDG) {
1451 switch (EltVT.getSimpleVT().SimpleTy) {
1452 default:
1453 return nullptr;
1454 case MVT::i8:
1455 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64;
1456 break;
1457 case MVT::i16:
1458 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64;
1459 break;
1460 case MVT::i32:
1461 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64;
1462 break;
1463 case MVT::i64:
1464 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64;
1465 break;
1466 case MVT::f32:
1467 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64;
1468 break;
1469 case MVT::f64:
1470 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64;
1471 break;
1472 }
1473 } else {
1474 switch (EltVT.getSimpleVT().SimpleTy) {
1475 default:
1476 return nullptr;
1477 case MVT::i8:
1478 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64;
1479 break;
1480 case MVT::i16:
1481 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64;
1482 break;
1483 case MVT::i32:
1484 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64;
1485 break;
1486 case MVT::i64:
1487 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64;
1488 break;
1489 case MVT::f32:
1490 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64;
1491 break;
1492 case MVT::f64:
1493 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64;
1494 break;
1495 }
1496 }
1497 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1498 case NVPTXISD::LDGV2:
1499 switch (EltVT.getSimpleVT().SimpleTy) {
1500 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1501 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1502 case MVT::i8:
1503 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
1504 break;
1505 case MVT::i16:
1506 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
1507 break;
1508 case MVT::i32:
1509 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
1510 break;
1511 case MVT::i64:
1512 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
1513 break;
1514 case MVT::f32:
1515 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
1516 break;
1517 case MVT::f64:
1518 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
1519 break;
1520 }
1521 break;
1522 case NVPTXISD::LDUV2:
1523 switch (EltVT.getSimpleVT().SimpleTy) {
1524 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1525 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1526 case MVT::i8:
1527 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
1528 break;
1529 case MVT::i16:
1530 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
1531 break;
1532 case MVT::i32:
1533 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
1534 break;
1535 case MVT::i64:
1536 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
1537 break;
1538 case MVT::f32:
1539 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
1540 break;
1541 case MVT::f64:
1542 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
1543 break;
1544 }
1545 break;
1546 case NVPTXISD::LDGV4:
1547 switch (EltVT.getSimpleVT().SimpleTy) {
1548 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1549 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1550 case MVT::i8:
1551 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
1552 break;
1553 case MVT::i16:
1554 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
1555 break;
1556 case MVT::i32:
1557 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
1558 break;
1559 case MVT::f32:
1560 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
1561 break;
1562 }
1563 break;
1564 case NVPTXISD::LDUV4:
1565 switch (EltVT.getSimpleVT().SimpleTy) {
1566 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1567 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1568 case MVT::i8:
1569 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
1570 break;
1571 case MVT::i16:
1572 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
1573 break;
1574 case MVT::i32:
1575 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
1576 break;
1577 case MVT::f32:
1578 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
1579 break;
1580 }
1581 break;
1582 }
1583 } else {
1584 switch (N->getOpcode()) {
1585 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1586 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1587 case ISD::INTRINSIC_W_CHAIN:
1588 if (IsLDG) {
1589 switch (EltVT.getSimpleVT().SimpleTy) {
1590 default:
1591 return nullptr;
1592 case MVT::i8:
1593 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari;
1594 break;
1595 case MVT::i16:
1596 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari;
1597 break;
1598 case MVT::i32:
1599 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari;
1600 break;
1601 case MVT::i64:
1602 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari;
1603 break;
1604 case MVT::f32:
1605 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari;
1606 break;
1607 case MVT::f64:
1608 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari;
1609 break;
1610 }
1611 } else {
1612 switch (EltVT.getSimpleVT().SimpleTy) {
1613 default:
1614 return nullptr;
1615 case MVT::i8:
1616 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari;
1617 break;
1618 case MVT::i16:
1619 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari;
1620 break;
1621 case MVT::i32:
1622 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari;
1623 break;
1624 case MVT::i64:
1625 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari;
1626 break;
1627 case MVT::f32:
1628 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari;
1629 break;
1630 case MVT::f64:
1631 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari;
1632 break;
1633 }
1634 }
1635 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1636 case NVPTXISD::LDGV2:
1637 switch (EltVT.getSimpleVT().SimpleTy) {
1638 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1639 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1640 case MVT::i8:
1641 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
1642 break;
1643 case MVT::i16:
1644 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
1645 break;
1646 case MVT::i32:
1647 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
1648 break;
1649 case MVT::i64:
1650 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
1651 break;
1652 case MVT::f32:
1653 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
1654 break;
1655 case MVT::f64:
1656 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
1657 break;
1658 }
1659 break;
1660 case NVPTXISD::LDUV2:
1661 switch (EltVT.getSimpleVT().SimpleTy) {
1662 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1663 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1664 case MVT::i8:
1665 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
1666 break;
1667 case MVT::i16:
1668 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
1669 break;
1670 case MVT::i32:
1671 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
1672 break;
1673 case MVT::i64:
1674 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
1675 break;
1676 case MVT::f32:
1677 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
1678 break;
1679 case MVT::f64:
1680 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
1681 break;
1682 }
1683 break;
1684 case NVPTXISD::LDGV4:
1685 switch (EltVT.getSimpleVT().SimpleTy) {
1686 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1687 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1688 case MVT::i8:
1689 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
1690 break;
1691 case MVT::i16:
1692 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
1693 break;
1694 case MVT::i32:
1695 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
1696 break;
1697 case MVT::f32:
1698 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
1699 break;
1700 }
1701 break;
1702 case NVPTXISD::LDUV4:
1703 switch (EltVT.getSimpleVT().SimpleTy) {
1704 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1705 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1706 case MVT::i8:
1707 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
1708 break;
1709 case MVT::i16:
1710 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
1711 break;
1712 case MVT::i32:
1713 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
1714 break;
1715 case MVT::f32:
1716 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
1717 break;
1718 }
1719 break;
1720 }
1721 }
1722
1723 SDValue Ops[] = { Base, Offset, Chain };
1724
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1725 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1726 } else {
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1727 if (Subtarget.is64Bit()) {
1728 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1729 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1730 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1731 case ISD::INTRINSIC_W_CHAIN:
1732 if (IsLDG) {
1733 switch (EltVT.getSimpleVT().SimpleTy) {
1734 default:
1735 return nullptr;
1736 case MVT::i8:
1737 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64;
1738 break;
1739 case MVT::i16:
1740 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64;
1741 break;
1742 case MVT::i32:
1743 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64;
1744 break;
1745 case MVT::i64:
1746 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64;
1747 break;
1748 case MVT::f32:
1749 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64;
1750 break;
1751 case MVT::f64:
1752 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64;
1753 break;
1754 }
1755 } else {
1756 switch (EltVT.getSimpleVT().SimpleTy) {
1757 default:
1758 return nullptr;
1759 case MVT::i8:
1760 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64;
1761 break;
1762 case MVT::i16:
1763 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64;
1764 break;
1765 case MVT::i32:
1766 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64;
1767 break;
1768 case MVT::i64:
1769 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64;
1770 break;
1771 case MVT::f32:
1772 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64;
1773 break;
1774 case MVT::f64:
1775 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64;
1776 break;
1777 }
1778 }
1779 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1780 case NVPTXISD::LDGV2:
1781 switch (EltVT.getSimpleVT().SimpleTy) {
1782 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1783 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1784 case MVT::i8:
1785 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
1786 break;
1787 case MVT::i16:
1788 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
1789 break;
1790 case MVT::i32:
1791 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
1792 break;
1793 case MVT::i64:
1794 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
1795 break;
1796 case MVT::f32:
1797 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
1798 break;
1799 case MVT::f64:
1800 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
1801 break;
1802 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1803 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1804 case NVPTXISD::LDUV2:
1805 switch (EltVT.getSimpleVT().SimpleTy) {
1806 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1807 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1808 case MVT::i8:
1809 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
1810 break;
1811 case MVT::i16:
1812 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
1813 break;
1814 case MVT::i32:
1815 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
1816 break;
1817 case MVT::i64:
1818 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
1819 break;
1820 case MVT::f32:
1821 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
1822 break;
1823 case MVT::f64:
1824 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
1825 break;
1826 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1827 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1828 case NVPTXISD::LDGV4:
1829 switch (EltVT.getSimpleVT().SimpleTy) {
1830 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1831 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1832 case MVT::i8:
1833 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
1834 break;
1835 case MVT::i16:
1836 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
1837 break;
1838 case MVT::i32:
1839 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
1840 break;
1841 case MVT::f32:
1842 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
1843 break;
1844 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1845 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1846 case NVPTXISD::LDUV4:
1847 switch (EltVT.getSimpleVT().SimpleTy) {
1848 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1849 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1850 case MVT::i8:
1851 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
1852 break;
1853 case MVT::i16:
1854 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
1855 break;
1856 case MVT::i32:
1857 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
1858 break;
1859 case MVT::f32:
1860 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
1861 break;
1862 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1863 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1864 }
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1865 } else {
1866 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1867 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1868 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1869 case ISD::INTRINSIC_W_CHAIN:
1870 if (IsLDG) {
1871 switch (EltVT.getSimpleVT().SimpleTy) {
1872 default:
1873 return nullptr;
1874 case MVT::i8:
1875 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg;
1876 break;
1877 case MVT::i16:
1878 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg;
1879 break;
1880 case MVT::i32:
1881 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg;
1882 break;
1883 case MVT::i64:
1884 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg;
1885 break;
1886 case MVT::f32:
1887 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg;
1888 break;
1889 case MVT::f64:
1890 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg;
1891 break;
1892 }
1893 } else {
1894 switch (EltVT.getSimpleVT().SimpleTy) {
1895 default:
1896 return nullptr;
1897 case MVT::i8:
1898 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg;
1899 break;
1900 case MVT::i16:
1901 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg;
1902 break;
1903 case MVT::i32:
1904 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg;
1905 break;
1906 case MVT::i64:
1907 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg;
1908 break;
1909 case MVT::f32:
1910 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg;
1911 break;
1912 case MVT::f64:
1913 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg;
1914 break;
1915 }
1916 }
1917 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1918 case NVPTXISD::LDGV2:
1919 switch (EltVT.getSimpleVT().SimpleTy) {
1920 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1921 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1922 case MVT::i8:
1923 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
1924 break;
1925 case MVT::i16:
1926 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
1927 break;
1928 case MVT::i32:
1929 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
1930 break;
1931 case MVT::i64:
1932 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
1933 break;
1934 case MVT::f32:
1935 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
1936 break;
1937 case MVT::f64:
1938 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
1939 break;
1940 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1941 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1942 case NVPTXISD::LDUV2:
1943 switch (EltVT.getSimpleVT().SimpleTy) {
1944 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1945 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1946 case MVT::i8:
1947 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
1948 break;
1949 case MVT::i16:
1950 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
1951 break;
1952 case MVT::i32:
1953 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
1954 break;
1955 case MVT::i64:
1956 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
1957 break;
1958 case MVT::f32:
1959 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
1960 break;
1961 case MVT::f64:
1962 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
1963 break;
1964 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1965 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1966 case NVPTXISD::LDGV4:
1967 switch (EltVT.getSimpleVT().SimpleTy) {
1968 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1969 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1970 case MVT::i8:
1971 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
1972 break;
1973 case MVT::i16:
1974 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
1975 break;
1976 case MVT::i32:
1977 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
1978 break;
1979 case MVT::f32:
1980 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
1981 break;
1982 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1983 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1984 case NVPTXISD::LDUV4:
1985 switch (EltVT.getSimpleVT().SimpleTy) {
1986 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1987 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1988 case MVT::i8:
1989 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
1990 break;
1991 case MVT::i16:
1992 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
1993 break;
1994 case MVT::i32:
1995 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
1996 break;
1997 case MVT::f32:
1998 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
1999 break;
2000 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2001 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2002 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2003 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2004
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]2005 SDValue Ops[] = { Op1, Chain };
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]2006 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]2007 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2008
2009 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]2010 MemRefs0[0] = Mem->getMemOperand();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2011 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
2012
2013 return LD;
2014}
2015
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2016SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]2017 SDLoc dl(N);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2018 StoreSDNode *ST = cast<StoreSDNode>(N);
2019 EVT StoreVT = ST->getMemoryVT();
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2020 SDNode *NVPTXST = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2021
2022 // do not support pre/post inc/dec
2023 if (ST->isIndexed())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2024 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2025
2026 if (!StoreVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2027 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2028
2029 // Address Space Setting
2030 unsigned int codeAddrSpace = getCodeAddrSpace(ST, Subtarget);
2031
2032 // Volatile Setting
2033 // - .volatile is only availalble for .global and .shared
2034 bool isVolatile = ST->isVolatile();
2035 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
2036 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
2037 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
2038 isVolatile = false;
2039
2040 // Vector Setting
2041 MVT SimpleVT = StoreVT.getSimpleVT();
2042 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
2043 if (SimpleVT.isVector()) {
2044 unsigned num = SimpleVT.getVectorNumElements();
2045 if (num == 2)
2046 vecType = NVPTX::PTXLdStInstCode::V2;
2047 else if (num == 4)
2048 vecType = NVPTX::PTXLdStInstCode::V4;
2049 else
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2050 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2051 }
2052
2053 // Type Setting: toType + toTypeWidth
2054 // - for integer type, always use 'u'
2055 //
2056 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2057 unsigned toTypeWidth = ScalarVT.getSizeInBits();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2058 unsigned int toType;
2059 if (ScalarVT.isFloatingPoint())
2060 toType = NVPTX::PTXLdStInstCode::Float;
2061 else
2062 toType = NVPTX::PTXLdStInstCode::Unsigned;
2063
2064 // Create the machine instruction DAG
2065 SDValue Chain = N->getOperand(0);
2066 SDValue N1 = N->getOperand(1);
2067 SDValue N2 = N->getOperand(2);
2068 SDValue Addr;
2069 SDValue Offset, Base;
2070 unsigned Opcode;
Craig Topperd9c27832013-08-15 02:44:19 +0000[diff] [blame]2071 MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2072
2073 if (SelectDirectAddr(N2, Addr)) {
2074 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2075 case MVT::i8:
2076 Opcode = NVPTX::ST_i8_avar;
2077 break;
2078 case MVT::i16:
2079 Opcode = NVPTX::ST_i16_avar;
2080 break;
2081 case MVT::i32:
2082 Opcode = NVPTX::ST_i32_avar;
2083 break;
2084 case MVT::i64:
2085 Opcode = NVPTX::ST_i64_avar;
2086 break;
2087 case MVT::f32:
2088 Opcode = NVPTX::ST_f32_avar;
2089 break;
2090 case MVT::f64:
2091 Opcode = NVPTX::ST_f64_avar;
2092 break;
2093 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2094 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2095 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2096 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2097 getI32Imm(vecType), getI32Imm(toType),
2098 getI32Imm(toTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2099 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2100 } else if (Subtarget.is64Bit()
2101 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
2102 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2103 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2104 case MVT::i8:
2105 Opcode = NVPTX::ST_i8_asi;
2106 break;
2107 case MVT::i16:
2108 Opcode = NVPTX::ST_i16_asi;
2109 break;
2110 case MVT::i32:
2111 Opcode = NVPTX::ST_i32_asi;
2112 break;
2113 case MVT::i64:
2114 Opcode = NVPTX::ST_i64_asi;
2115 break;
2116 case MVT::f32:
2117 Opcode = NVPTX::ST_f32_asi;
2118 break;
2119 case MVT::f64:
2120 Opcode = NVPTX::ST_f64_asi;
2121 break;
2122 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2123 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2124 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2125 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2126 getI32Imm(vecType), getI32Imm(toType),
2127 getI32Imm(toTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2128 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2129 } else if (Subtarget.is64Bit()
2130 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
2131 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2132 if (Subtarget.is64Bit()) {
2133 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2134 case MVT::i8:
2135 Opcode = NVPTX::ST_i8_ari_64;
2136 break;
2137 case MVT::i16:
2138 Opcode = NVPTX::ST_i16_ari_64;
2139 break;
2140 case MVT::i32:
2141 Opcode = NVPTX::ST_i32_ari_64;
2142 break;
2143 case MVT::i64:
2144 Opcode = NVPTX::ST_i64_ari_64;
2145 break;
2146 case MVT::f32:
2147 Opcode = NVPTX::ST_f32_ari_64;
2148 break;
2149 case MVT::f64:
2150 Opcode = NVPTX::ST_f64_ari_64;
2151 break;
2152 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2153 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2154 }
2155 } else {
2156 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2157 case MVT::i8:
2158 Opcode = NVPTX::ST_i8_ari;
2159 break;
2160 case MVT::i16:
2161 Opcode = NVPTX::ST_i16_ari;
2162 break;
2163 case MVT::i32:
2164 Opcode = NVPTX::ST_i32_ari;
2165 break;
2166 case MVT::i64:
2167 Opcode = NVPTX::ST_i64_ari;
2168 break;
2169 case MVT::f32:
2170 Opcode = NVPTX::ST_f32_ari;
2171 break;
2172 case MVT::f64:
2173 Opcode = NVPTX::ST_f64_ari;
2174 break;
2175 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2176 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2177 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2178 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2179 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2180 getI32Imm(vecType), getI32Imm(toType),
2181 getI32Imm(toTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2182 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2183 } else {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2184 if (Subtarget.is64Bit()) {
2185 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2186 case MVT::i8:
2187 Opcode = NVPTX::ST_i8_areg_64;
2188 break;
2189 case MVT::i16:
2190 Opcode = NVPTX::ST_i16_areg_64;
2191 break;
2192 case MVT::i32:
2193 Opcode = NVPTX::ST_i32_areg_64;
2194 break;
2195 case MVT::i64:
2196 Opcode = NVPTX::ST_i64_areg_64;
2197 break;
2198 case MVT::f32:
2199 Opcode = NVPTX::ST_f32_areg_64;
2200 break;
2201 case MVT::f64:
2202 Opcode = NVPTX::ST_f64_areg_64;
2203 break;
2204 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2205 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2206 }
2207 } else {
2208 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2209 case MVT::i8:
2210 Opcode = NVPTX::ST_i8_areg;
2211 break;
2212 case MVT::i16:
2213 Opcode = NVPTX::ST_i16_areg;
2214 break;
2215 case MVT::i32:
2216 Opcode = NVPTX::ST_i32_areg;
2217 break;
2218 case MVT::i64:
2219 Opcode = NVPTX::ST_i64_areg;
2220 break;
2221 case MVT::f32:
2222 Opcode = NVPTX::ST_f32_areg;
2223 break;
2224 case MVT::f64:
2225 Opcode = NVPTX::ST_f64_areg;
2226 break;
2227 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2228 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2229 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2230 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2231 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2232 getI32Imm(vecType), getI32Imm(toType),
2233 getI32Imm(toTypeWidth), N2, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2234 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2235 }
2236
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2237 if (NVPTXST) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2238 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2239 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2240 cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2241 }
2242
2243 return NVPTXST;
2244}
2245
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2246SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
2247 SDValue Chain = N->getOperand(0);
2248 SDValue Op1 = N->getOperand(1);
2249 SDValue Addr, Offset, Base;
2250 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]2251 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2252 SDNode *ST;
2253 EVT EltVT = Op1.getValueType();
2254 MemSDNode *MemSD = cast<MemSDNode>(N);
2255 EVT StoreVT = MemSD->getMemoryVT();
2256
2257 // Address Space Setting
2258 unsigned CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget);
2259
2260 if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
2261 report_fatal_error("Cannot store to pointer that points to constant "
2262 "memory space");
2263 }
2264
2265 // Volatile Setting
2266 // - .volatile is only availalble for .global and .shared
2267 bool IsVolatile = MemSD->isVolatile();
2268 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
2269 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
2270 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
2271 IsVolatile = false;
2272
2273 // Type Setting: toType + toTypeWidth
2274 // - for integer type, always use 'u'
2275 assert(StoreVT.isSimple() && "Store value is not simple");
2276 MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2277 unsigned ToTypeWidth = ScalarVT.getSizeInBits();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2278 unsigned ToType;
2279 if (ScalarVT.isFloatingPoint())
2280 ToType = NVPTX::PTXLdStInstCode::Float;
2281 else
2282 ToType = NVPTX::PTXLdStInstCode::Unsigned;
2283
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2284 SmallVector<SDValue, 12> StOps;
2285 SDValue N2;
2286 unsigned VecType;
2287
2288 switch (N->getOpcode()) {
2289 case NVPTXISD::StoreV2:
2290 VecType = NVPTX::PTXLdStInstCode::V2;
2291 StOps.push_back(N->getOperand(1));
2292 StOps.push_back(N->getOperand(2));
2293 N2 = N->getOperand(3);
2294 break;
2295 case NVPTXISD::StoreV4:
2296 VecType = NVPTX::PTXLdStInstCode::V4;
2297 StOps.push_back(N->getOperand(1));
2298 StOps.push_back(N->getOperand(2));
2299 StOps.push_back(N->getOperand(3));
2300 StOps.push_back(N->getOperand(4));
2301 N2 = N->getOperand(5);
2302 break;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2303 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2304 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2305 }
2306
2307 StOps.push_back(getI32Imm(IsVolatile));
2308 StOps.push_back(getI32Imm(CodeAddrSpace));
2309 StOps.push_back(getI32Imm(VecType));
2310 StOps.push_back(getI32Imm(ToType));
2311 StOps.push_back(getI32Imm(ToTypeWidth));
2312
2313 if (SelectDirectAddr(N2, Addr)) {
2314 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2315 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2316 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2317 case NVPTXISD::StoreV2:
2318 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2319 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2320 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2321 case MVT::i8:
2322 Opcode = NVPTX::STV_i8_v2_avar;
2323 break;
2324 case MVT::i16:
2325 Opcode = NVPTX::STV_i16_v2_avar;
2326 break;
2327 case MVT::i32:
2328 Opcode = NVPTX::STV_i32_v2_avar;
2329 break;
2330 case MVT::i64:
2331 Opcode = NVPTX::STV_i64_v2_avar;
2332 break;
2333 case MVT::f32:
2334 Opcode = NVPTX::STV_f32_v2_avar;
2335 break;
2336 case MVT::f64:
2337 Opcode = NVPTX::STV_f64_v2_avar;
2338 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2339 }
2340 break;
2341 case NVPTXISD::StoreV4:
2342 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2343 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2344 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2345 case MVT::i8:
2346 Opcode = NVPTX::STV_i8_v4_avar;
2347 break;
2348 case MVT::i16:
2349 Opcode = NVPTX::STV_i16_v4_avar;
2350 break;
2351 case MVT::i32:
2352 Opcode = NVPTX::STV_i32_v4_avar;
2353 break;
2354 case MVT::f32:
2355 Opcode = NVPTX::STV_f32_v4_avar;
2356 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2357 }
2358 break;
2359 }
2360 StOps.push_back(Addr);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2361 } else if (Subtarget.is64Bit()
2362 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
2363 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2364 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2365 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2366 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2367 case NVPTXISD::StoreV2:
2368 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2369 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2370 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2371 case MVT::i8:
2372 Opcode = NVPTX::STV_i8_v2_asi;
2373 break;
2374 case MVT::i16:
2375 Opcode = NVPTX::STV_i16_v2_asi;
2376 break;
2377 case MVT::i32:
2378 Opcode = NVPTX::STV_i32_v2_asi;
2379 break;
2380 case MVT::i64:
2381 Opcode = NVPTX::STV_i64_v2_asi;
2382 break;
2383 case MVT::f32:
2384 Opcode = NVPTX::STV_f32_v2_asi;
2385 break;
2386 case MVT::f64:
2387 Opcode = NVPTX::STV_f64_v2_asi;
2388 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2389 }
2390 break;
2391 case NVPTXISD::StoreV4:
2392 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2393 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2394 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2395 case MVT::i8:
2396 Opcode = NVPTX::STV_i8_v4_asi;
2397 break;
2398 case MVT::i16:
2399 Opcode = NVPTX::STV_i16_v4_asi;
2400 break;
2401 case MVT::i32:
2402 Opcode = NVPTX::STV_i32_v4_asi;
2403 break;
2404 case MVT::f32:
2405 Opcode = NVPTX::STV_f32_v4_asi;
2406 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2407 }
2408 break;
2409 }
2410 StOps.push_back(Base);
2411 StOps.push_back(Offset);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2412 } else if (Subtarget.is64Bit()
2413 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
2414 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2415 if (Subtarget.is64Bit()) {
2416 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2417 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2418 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2419 case NVPTXISD::StoreV2:
2420 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2421 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2422 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2423 case MVT::i8:
2424 Opcode = NVPTX::STV_i8_v2_ari_64;
2425 break;
2426 case MVT::i16:
2427 Opcode = NVPTX::STV_i16_v2_ari_64;
2428 break;
2429 case MVT::i32:
2430 Opcode = NVPTX::STV_i32_v2_ari_64;
2431 break;
2432 case MVT::i64:
2433 Opcode = NVPTX::STV_i64_v2_ari_64;
2434 break;
2435 case MVT::f32:
2436 Opcode = NVPTX::STV_f32_v2_ari_64;
2437 break;
2438 case MVT::f64:
2439 Opcode = NVPTX::STV_f64_v2_ari_64;
2440 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2441 }
2442 break;
2443 case NVPTXISD::StoreV4:
2444 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2445 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2446 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2447 case MVT::i8:
2448 Opcode = NVPTX::STV_i8_v4_ari_64;
2449 break;
2450 case MVT::i16:
2451 Opcode = NVPTX::STV_i16_v4_ari_64;
2452 break;
2453 case MVT::i32:
2454 Opcode = NVPTX::STV_i32_v4_ari_64;
2455 break;
2456 case MVT::f32:
2457 Opcode = NVPTX::STV_f32_v4_ari_64;
2458 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2459 }
2460 break;
2461 }
2462 } else {
2463 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2464 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2465 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2466 case NVPTXISD::StoreV2:
2467 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2468 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2469 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2470 case MVT::i8:
2471 Opcode = NVPTX::STV_i8_v2_ari;
2472 break;
2473 case MVT::i16:
2474 Opcode = NVPTX::STV_i16_v2_ari;
2475 break;
2476 case MVT::i32:
2477 Opcode = NVPTX::STV_i32_v2_ari;
2478 break;
2479 case MVT::i64:
2480 Opcode = NVPTX::STV_i64_v2_ari;
2481 break;
2482 case MVT::f32:
2483 Opcode = NVPTX::STV_f32_v2_ari;
2484 break;
2485 case MVT::f64:
2486 Opcode = NVPTX::STV_f64_v2_ari;
2487 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2488 }
2489 break;
2490 case NVPTXISD::StoreV4:
2491 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2492 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2493 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2494 case MVT::i8:
2495 Opcode = NVPTX::STV_i8_v4_ari;
2496 break;
2497 case MVT::i16:
2498 Opcode = NVPTX::STV_i16_v4_ari;
2499 break;
2500 case MVT::i32:
2501 Opcode = NVPTX::STV_i32_v4_ari;
2502 break;
2503 case MVT::f32:
2504 Opcode = NVPTX::STV_f32_v4_ari;
2505 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2506 }
2507 break;
2508 }
2509 }
2510 StOps.push_back(Base);
2511 StOps.push_back(Offset);
2512 } else {
2513 if (Subtarget.is64Bit()) {
2514 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2515 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2516 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2517 case NVPTXISD::StoreV2:
2518 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2519 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2520 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2521 case MVT::i8:
2522 Opcode = NVPTX::STV_i8_v2_areg_64;
2523 break;
2524 case MVT::i16:
2525 Opcode = NVPTX::STV_i16_v2_areg_64;
2526 break;
2527 case MVT::i32:
2528 Opcode = NVPTX::STV_i32_v2_areg_64;
2529 break;
2530 case MVT::i64:
2531 Opcode = NVPTX::STV_i64_v2_areg_64;
2532 break;
2533 case MVT::f32:
2534 Opcode = NVPTX::STV_f32_v2_areg_64;
2535 break;
2536 case MVT::f64:
2537 Opcode = NVPTX::STV_f64_v2_areg_64;
2538 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2539 }
2540 break;
2541 case NVPTXISD::StoreV4:
2542 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2543 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2544 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2545 case MVT::i8:
2546 Opcode = NVPTX::STV_i8_v4_areg_64;
2547 break;
2548 case MVT::i16:
2549 Opcode = NVPTX::STV_i16_v4_areg_64;
2550 break;
2551 case MVT::i32:
2552 Opcode = NVPTX::STV_i32_v4_areg_64;
2553 break;
2554 case MVT::f32:
2555 Opcode = NVPTX::STV_f32_v4_areg_64;
2556 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2557 }
2558 break;
2559 }
2560 } else {
2561 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2562 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2563 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2564 case NVPTXISD::StoreV2:
2565 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2566 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2567 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2568 case MVT::i8:
2569 Opcode = NVPTX::STV_i8_v2_areg;
2570 break;
2571 case MVT::i16:
2572 Opcode = NVPTX::STV_i16_v2_areg;
2573 break;
2574 case MVT::i32:
2575 Opcode = NVPTX::STV_i32_v2_areg;
2576 break;
2577 case MVT::i64:
2578 Opcode = NVPTX::STV_i64_v2_areg;
2579 break;
2580 case MVT::f32:
2581 Opcode = NVPTX::STV_f32_v2_areg;
2582 break;
2583 case MVT::f64:
2584 Opcode = NVPTX::STV_f64_v2_areg;
2585 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2586 }
2587 break;
2588 case NVPTXISD::StoreV4:
2589 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2590 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2591 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2592 case MVT::i8:
2593 Opcode = NVPTX::STV_i8_v4_areg;
2594 break;
2595 case MVT::i16:
2596 Opcode = NVPTX::STV_i16_v4_areg;
2597 break;
2598 case MVT::i32:
2599 Opcode = NVPTX::STV_i32_v4_areg;
2600 break;
2601 case MVT::f32:
2602 Opcode = NVPTX::STV_f32_v4_areg;
2603 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2604 }
2605 break;
2606 }
2607 }
2608 StOps.push_back(N2);
2609 }
2610
2611 StOps.push_back(Chain);
2612
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2613 ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2614
2615 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2616 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2617 cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2618
2619 return ST;
2620}
2621
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2622SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
2623 SDValue Chain = Node->getOperand(0);
2624 SDValue Offset = Node->getOperand(2);
2625 SDValue Flag = Node->getOperand(3);
2626 SDLoc DL(Node);
2627 MemSDNode *Mem = cast<MemSDNode>(Node);
2628
2629 unsigned VecSize;
2630 switch (Node->getOpcode()) {
2631 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2632 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2633 case NVPTXISD::LoadParam:
2634 VecSize = 1;
2635 break;
2636 case NVPTXISD::LoadParamV2:
2637 VecSize = 2;
2638 break;
2639 case NVPTXISD::LoadParamV4:
2640 VecSize = 4;
2641 break;
2642 }
2643
2644 EVT EltVT = Node->getValueType(0);
2645 EVT MemVT = Mem->getMemoryVT();
2646
2647 unsigned Opc = 0;
2648
2649 switch (VecSize) {
2650 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2651 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2652 case 1:
2653 switch (MemVT.getSimpleVT().SimpleTy) {
2654 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2655 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2656 case MVT::i1:
2657 Opc = NVPTX::LoadParamMemI8;
2658 break;
2659 case MVT::i8:
2660 Opc = NVPTX::LoadParamMemI8;
2661 break;
2662 case MVT::i16:
2663 Opc = NVPTX::LoadParamMemI16;
2664 break;
2665 case MVT::i32:
2666 Opc = NVPTX::LoadParamMemI32;
2667 break;
2668 case MVT::i64:
2669 Opc = NVPTX::LoadParamMemI64;
2670 break;
2671 case MVT::f32:
2672 Opc = NVPTX::LoadParamMemF32;
2673 break;
2674 case MVT::f64:
2675 Opc = NVPTX::LoadParamMemF64;
2676 break;
2677 }
2678 break;
2679 case 2:
2680 switch (MemVT.getSimpleVT().SimpleTy) {
2681 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2682 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2683 case MVT::i1:
2684 Opc = NVPTX::LoadParamMemV2I8;
2685 break;
2686 case MVT::i8:
2687 Opc = NVPTX::LoadParamMemV2I8;
2688 break;
2689 case MVT::i16:
2690 Opc = NVPTX::LoadParamMemV2I16;
2691 break;
2692 case MVT::i32:
2693 Opc = NVPTX::LoadParamMemV2I32;
2694 break;
2695 case MVT::i64:
2696 Opc = NVPTX::LoadParamMemV2I64;
2697 break;
2698 case MVT::f32:
2699 Opc = NVPTX::LoadParamMemV2F32;
2700 break;
2701 case MVT::f64:
2702 Opc = NVPTX::LoadParamMemV2F64;
2703 break;
2704 }
2705 break;
2706 case 4:
2707 switch (MemVT.getSimpleVT().SimpleTy) {
2708 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2709 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2710 case MVT::i1:
2711 Opc = NVPTX::LoadParamMemV4I8;
2712 break;
2713 case MVT::i8:
2714 Opc = NVPTX::LoadParamMemV4I8;
2715 break;
2716 case MVT::i16:
2717 Opc = NVPTX::LoadParamMemV4I16;
2718 break;
2719 case MVT::i32:
2720 Opc = NVPTX::LoadParamMemV4I32;
2721 break;
2722 case MVT::f32:
2723 Opc = NVPTX::LoadParamMemV4F32;
2724 break;
2725 }
2726 break;
2727 }
2728
2729 SDVTList VTs;
2730 if (VecSize == 1) {
2731 VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
2732 } else if (VecSize == 2) {
2733 VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
2734 } else {
2735 EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
Craig Topperabb4ac72014-04-16 06:10:51 +0000[diff] [blame]2736 VTs = CurDAG->getVTList(EVTs);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2737 }
2738
2739 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2740
2741 SmallVector<SDValue, 2> Ops;
2742 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2743 Ops.push_back(Chain);
2744 Ops.push_back(Flag);
2745
2746 SDNode *Ret =
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]2747 CurDAG->getMachineNode(Opc, DL, VTs, Ops);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2748 return Ret;
2749}
2750
2751SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
2752 SDLoc DL(N);
2753 SDValue Chain = N->getOperand(0);
2754 SDValue Offset = N->getOperand(1);
2755 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2756 MemSDNode *Mem = cast<MemSDNode>(N);
2757
2758 // How many elements do we have?
2759 unsigned NumElts = 1;
2760 switch (N->getOpcode()) {
2761 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2762 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2763 case NVPTXISD::StoreRetval:
2764 NumElts = 1;
2765 break;
2766 case NVPTXISD::StoreRetvalV2:
2767 NumElts = 2;
2768 break;
2769 case NVPTXISD::StoreRetvalV4:
2770 NumElts = 4;
2771 break;
2772 }
2773
2774 // Build vector of operands
2775 SmallVector<SDValue, 6> Ops;
2776 for (unsigned i = 0; i < NumElts; ++i)
2777 Ops.push_back(N->getOperand(i + 2));
2778 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2779 Ops.push_back(Chain);
2780
2781 // Determine target opcode
2782 // If we have an i1, use an 8-bit store. The lowering code in
2783 // NVPTXISelLowering will have already emitted an upcast.
2784 unsigned Opcode = 0;
2785 switch (NumElts) {
2786 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2787 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2788 case 1:
2789 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2790 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2791 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2792 case MVT::i1:
2793 Opcode = NVPTX::StoreRetvalI8;
2794 break;
2795 case MVT::i8:
2796 Opcode = NVPTX::StoreRetvalI8;
2797 break;
2798 case MVT::i16:
2799 Opcode = NVPTX::StoreRetvalI16;
2800 break;
2801 case MVT::i32:
2802 Opcode = NVPTX::StoreRetvalI32;
2803 break;
2804 case MVT::i64:
2805 Opcode = NVPTX::StoreRetvalI64;
2806 break;
2807 case MVT::f32:
2808 Opcode = NVPTX::StoreRetvalF32;
2809 break;
2810 case MVT::f64:
2811 Opcode = NVPTX::StoreRetvalF64;
2812 break;
2813 }
2814 break;
2815 case 2:
2816 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2817 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2818 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2819 case MVT::i1:
2820 Opcode = NVPTX::StoreRetvalV2I8;
2821 break;
2822 case MVT::i8:
2823 Opcode = NVPTX::StoreRetvalV2I8;
2824 break;
2825 case MVT::i16:
2826 Opcode = NVPTX::StoreRetvalV2I16;
2827 break;
2828 case MVT::i32:
2829 Opcode = NVPTX::StoreRetvalV2I32;
2830 break;
2831 case MVT::i64:
2832 Opcode = NVPTX::StoreRetvalV2I64;
2833 break;
2834 case MVT::f32:
2835 Opcode = NVPTX::StoreRetvalV2F32;
2836 break;
2837 case MVT::f64:
2838 Opcode = NVPTX::StoreRetvalV2F64;
2839 break;
2840 }
2841 break;
2842 case 4:
2843 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2844 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2845 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2846 case MVT::i1:
2847 Opcode = NVPTX::StoreRetvalV4I8;
2848 break;
2849 case MVT::i8:
2850 Opcode = NVPTX::StoreRetvalV4I8;
2851 break;
2852 case MVT::i16:
2853 Opcode = NVPTX::StoreRetvalV4I16;
2854 break;
2855 case MVT::i32:
2856 Opcode = NVPTX::StoreRetvalV4I32;
2857 break;
2858 case MVT::f32:
2859 Opcode = NVPTX::StoreRetvalV4F32;
2860 break;
2861 }
2862 break;
2863 }
2864
2865 SDNode *Ret =
2866 CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
2867 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2868 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2869 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
2870
2871 return Ret;
2872}
2873
2874SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
2875 SDLoc DL(N);
2876 SDValue Chain = N->getOperand(0);
2877 SDValue Param = N->getOperand(1);
2878 unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
2879 SDValue Offset = N->getOperand(2);
2880 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2881 MemSDNode *Mem = cast<MemSDNode>(N);
2882 SDValue Flag = N->getOperand(N->getNumOperands() - 1);
2883
2884 // How many elements do we have?
2885 unsigned NumElts = 1;
2886 switch (N->getOpcode()) {
2887 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2888 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2889 case NVPTXISD::StoreParamU32:
2890 case NVPTXISD::StoreParamS32:
2891 case NVPTXISD::StoreParam:
2892 NumElts = 1;
2893 break;
2894 case NVPTXISD::StoreParamV2:
2895 NumElts = 2;
2896 break;
2897 case NVPTXISD::StoreParamV4:
2898 NumElts = 4;
2899 break;
2900 }
2901
2902 // Build vector of operands
2903 SmallVector<SDValue, 8> Ops;
2904 for (unsigned i = 0; i < NumElts; ++i)
2905 Ops.push_back(N->getOperand(i + 3));
2906 Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32));
2907 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2908 Ops.push_back(Chain);
2909 Ops.push_back(Flag);
2910
2911 // Determine target opcode
2912 // If we have an i1, use an 8-bit store. The lowering code in
2913 // NVPTXISelLowering will have already emitted an upcast.
2914 unsigned Opcode = 0;
2915 switch (N->getOpcode()) {
2916 default:
2917 switch (NumElts) {
2918 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2919 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2920 case 1:
2921 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2922 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2923 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2924 case MVT::i1:
2925 Opcode = NVPTX::StoreParamI8;
2926 break;
2927 case MVT::i8:
2928 Opcode = NVPTX::StoreParamI8;
2929 break;
2930 case MVT::i16:
2931 Opcode = NVPTX::StoreParamI16;
2932 break;
2933 case MVT::i32:
2934 Opcode = NVPTX::StoreParamI32;
2935 break;
2936 case MVT::i64:
2937 Opcode = NVPTX::StoreParamI64;
2938 break;
2939 case MVT::f32:
2940 Opcode = NVPTX::StoreParamF32;
2941 break;
2942 case MVT::f64:
2943 Opcode = NVPTX::StoreParamF64;
2944 break;
2945 }
2946 break;
2947 case 2:
2948 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2949 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2950 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2951 case MVT::i1:
2952 Opcode = NVPTX::StoreParamV2I8;
2953 break;
2954 case MVT::i8:
2955 Opcode = NVPTX::StoreParamV2I8;
2956 break;
2957 case MVT::i16:
2958 Opcode = NVPTX::StoreParamV2I16;
2959 break;
2960 case MVT::i32:
2961 Opcode = NVPTX::StoreParamV2I32;
2962 break;
2963 case MVT::i64:
2964 Opcode = NVPTX::StoreParamV2I64;
2965 break;
2966 case MVT::f32:
2967 Opcode = NVPTX::StoreParamV2F32;
2968 break;
2969 case MVT::f64:
2970 Opcode = NVPTX::StoreParamV2F64;
2971 break;
2972 }
2973 break;
2974 case 4:
2975 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2976 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2977 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2978 case MVT::i1:
2979 Opcode = NVPTX::StoreParamV4I8;
2980 break;
2981 case MVT::i8:
2982 Opcode = NVPTX::StoreParamV4I8;
2983 break;
2984 case MVT::i16:
2985 Opcode = NVPTX::StoreParamV4I16;
2986 break;
2987 case MVT::i32:
2988 Opcode = NVPTX::StoreParamV4I32;
2989 break;
2990 case MVT::f32:
2991 Opcode = NVPTX::StoreParamV4F32;
2992 break;
2993 }
2994 break;
2995 }
2996 break;
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2997 // Special case: if we have a sign-extend/zero-extend node, insert the
2998 // conversion instruction first, and use that as the value operand to
2999 // the selected StoreParam node.
3000 case NVPTXISD::StoreParamU32: {
3001 Opcode = NVPTX::StoreParamI32;
3002 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
3003 MVT::i32);
3004 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
3005 MVT::i32, Ops[0], CvtNone);
3006 Ops[0] = SDValue(Cvt, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3007 break;
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]3008 }
3009 case NVPTXISD::StoreParamS32: {
3010 Opcode = NVPTX::StoreParamI32;
3011 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
3012 MVT::i32);
3013 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
3014 MVT::i32, Ops[0], CvtNone);
3015 Ops[0] = SDValue(Cvt, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3016 break;
3017 }
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]3018 }
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3019
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]3020 SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3021 SDNode *Ret =
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]3022 CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3023 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
3024 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
3025 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
3026
3027 return Ret;
3028}
3029
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3030SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) {
3031 SDValue Chain = N->getOperand(0);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3032 SDNode *Ret = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3033 unsigned Opc = 0;
3034 SmallVector<SDValue, 8> Ops;
3035
3036 switch (N->getOpcode()) {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3037 default: return nullptr;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3038 case NVPTXISD::Tex1DFloatS32:
3039 Opc = NVPTX::TEX_1D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3040 break;
3041 case NVPTXISD::Tex1DFloatFloat:
3042 Opc = NVPTX::TEX_1D_F32_F32;
3043 break;
3044 case NVPTXISD::Tex1DFloatFloatLevel:
3045 Opc = NVPTX::TEX_1D_F32_F32_LEVEL;
3046 break;
3047 case NVPTXISD::Tex1DFloatFloatGrad:
3048 Opc = NVPTX::TEX_1D_F32_F32_GRAD;
3049 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3050 case NVPTXISD::Tex1DS32S32:
3051 Opc = NVPTX::TEX_1D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3052 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3053 case NVPTXISD::Tex1DS32Float:
3054 Opc = NVPTX::TEX_1D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3055 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3056 case NVPTXISD::Tex1DS32FloatLevel:
3057 Opc = NVPTX::TEX_1D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3058 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3059 case NVPTXISD::Tex1DS32FloatGrad:
3060 Opc = NVPTX::TEX_1D_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3061 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3062 case NVPTXISD::Tex1DU32S32:
3063 Opc = NVPTX::TEX_1D_U32_S32;
3064 break;
3065 case NVPTXISD::Tex1DU32Float:
3066 Opc = NVPTX::TEX_1D_U32_F32;
3067 break;
3068 case NVPTXISD::Tex1DU32FloatLevel:
3069 Opc = NVPTX::TEX_1D_U32_F32_LEVEL;
3070 break;
3071 case NVPTXISD::Tex1DU32FloatGrad:
3072 Opc = NVPTX::TEX_1D_U32_F32_GRAD;
3073 break;
3074 case NVPTXISD::Tex1DArrayFloatS32:
3075 Opc = NVPTX::TEX_1D_ARRAY_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3076 break;
3077 case NVPTXISD::Tex1DArrayFloatFloat:
3078 Opc = NVPTX::TEX_1D_ARRAY_F32_F32;
3079 break;
3080 case NVPTXISD::Tex1DArrayFloatFloatLevel:
3081 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL;
3082 break;
3083 case NVPTXISD::Tex1DArrayFloatFloatGrad:
3084 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD;
3085 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3086 case NVPTXISD::Tex1DArrayS32S32:
3087 Opc = NVPTX::TEX_1D_ARRAY_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3088 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3089 case NVPTXISD::Tex1DArrayS32Float:
3090 Opc = NVPTX::TEX_1D_ARRAY_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3091 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3092 case NVPTXISD::Tex1DArrayS32FloatLevel:
3093 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3094 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3095 case NVPTXISD::Tex1DArrayS32FloatGrad:
3096 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3097 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3098 case NVPTXISD::Tex1DArrayU32S32:
3099 Opc = NVPTX::TEX_1D_ARRAY_U32_S32;
3100 break;
3101 case NVPTXISD::Tex1DArrayU32Float:
3102 Opc = NVPTX::TEX_1D_ARRAY_U32_F32;
3103 break;
3104 case NVPTXISD::Tex1DArrayU32FloatLevel:
3105 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL;
3106 break;
3107 case NVPTXISD::Tex1DArrayU32FloatGrad:
3108 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD;
3109 break;
3110 case NVPTXISD::Tex2DFloatS32:
3111 Opc = NVPTX::TEX_2D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3112 break;
3113 case NVPTXISD::Tex2DFloatFloat:
3114 Opc = NVPTX::TEX_2D_F32_F32;
3115 break;
3116 case NVPTXISD::Tex2DFloatFloatLevel:
3117 Opc = NVPTX::TEX_2D_F32_F32_LEVEL;
3118 break;
3119 case NVPTXISD::Tex2DFloatFloatGrad:
3120 Opc = NVPTX::TEX_2D_F32_F32_GRAD;
3121 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3122 case NVPTXISD::Tex2DS32S32:
3123 Opc = NVPTX::TEX_2D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3124 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3125 case NVPTXISD::Tex2DS32Float:
3126 Opc = NVPTX::TEX_2D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3127 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3128 case NVPTXISD::Tex2DS32FloatLevel:
3129 Opc = NVPTX::TEX_2D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3130 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3131 case NVPTXISD::Tex2DS32FloatGrad:
3132 Opc = NVPTX::TEX_2D_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3133 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3134 case NVPTXISD::Tex2DU32S32:
3135 Opc = NVPTX::TEX_2D_U32_S32;
3136 break;
3137 case NVPTXISD::Tex2DU32Float:
3138 Opc = NVPTX::TEX_2D_U32_F32;
3139 break;
3140 case NVPTXISD::Tex2DU32FloatLevel:
3141 Opc = NVPTX::TEX_2D_U32_F32_LEVEL;
3142 break;
3143 case NVPTXISD::Tex2DU32FloatGrad:
3144 Opc = NVPTX::TEX_2D_U32_F32_GRAD;
3145 break;
3146 case NVPTXISD::Tex2DArrayFloatS32:
3147 Opc = NVPTX::TEX_2D_ARRAY_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3148 break;
3149 case NVPTXISD::Tex2DArrayFloatFloat:
3150 Opc = NVPTX::TEX_2D_ARRAY_F32_F32;
3151 break;
3152 case NVPTXISD::Tex2DArrayFloatFloatLevel:
3153 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL;
3154 break;
3155 case NVPTXISD::Tex2DArrayFloatFloatGrad:
3156 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD;
3157 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3158 case NVPTXISD::Tex2DArrayS32S32:
3159 Opc = NVPTX::TEX_2D_ARRAY_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3160 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3161 case NVPTXISD::Tex2DArrayS32Float:
3162 Opc = NVPTX::TEX_2D_ARRAY_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3163 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3164 case NVPTXISD::Tex2DArrayS32FloatLevel:
3165 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3166 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3167 case NVPTXISD::Tex2DArrayS32FloatGrad:
3168 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3169 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3170 case NVPTXISD::Tex2DArrayU32S32:
3171 Opc = NVPTX::TEX_2D_ARRAY_U32_S32;
3172 break;
3173 case NVPTXISD::Tex2DArrayU32Float:
3174 Opc = NVPTX::TEX_2D_ARRAY_U32_F32;
3175 break;
3176 case NVPTXISD::Tex2DArrayU32FloatLevel:
3177 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL;
3178 break;
3179 case NVPTXISD::Tex2DArrayU32FloatGrad:
3180 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD;
3181 break;
3182 case NVPTXISD::Tex3DFloatS32:
3183 Opc = NVPTX::TEX_3D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3184 break;
3185 case NVPTXISD::Tex3DFloatFloat:
3186 Opc = NVPTX::TEX_3D_F32_F32;
3187 break;
3188 case NVPTXISD::Tex3DFloatFloatLevel:
3189 Opc = NVPTX::TEX_3D_F32_F32_LEVEL;
3190 break;
3191 case NVPTXISD::Tex3DFloatFloatGrad:
3192 Opc = NVPTX::TEX_3D_F32_F32_GRAD;
3193 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3194 case NVPTXISD::Tex3DS32S32:
3195 Opc = NVPTX::TEX_3D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3196 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3197 case NVPTXISD::Tex3DS32Float:
3198 Opc = NVPTX::TEX_3D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3199 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3200 case NVPTXISD::Tex3DS32FloatLevel:
3201 Opc = NVPTX::TEX_3D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3202 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3203 case NVPTXISD::Tex3DS32FloatGrad:
3204 Opc = NVPTX::TEX_3D_S32_F32_GRAD;
3205 break;
3206 case NVPTXISD::Tex3DU32S32:
3207 Opc = NVPTX::TEX_3D_U32_S32;
3208 break;
3209 case NVPTXISD::Tex3DU32Float:
3210 Opc = NVPTX::TEX_3D_U32_F32;
3211 break;
3212 case NVPTXISD::Tex3DU32FloatLevel:
3213 Opc = NVPTX::TEX_3D_U32_F32_LEVEL;
3214 break;
3215 case NVPTXISD::Tex3DU32FloatGrad:
3216 Opc = NVPTX::TEX_3D_U32_F32_GRAD;
3217 break;
3218 case NVPTXISD::TexCubeFloatFloat:
3219 Opc = NVPTX::TEX_CUBE_F32_F32;
3220 break;
3221 case NVPTXISD::TexCubeFloatFloatLevel:
3222 Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL;
3223 break;
3224 case NVPTXISD::TexCubeS32Float:
3225 Opc = NVPTX::TEX_CUBE_S32_F32;
3226 break;
3227 case NVPTXISD::TexCubeS32FloatLevel:
3228 Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL;
3229 break;
3230 case NVPTXISD::TexCubeU32Float:
3231 Opc = NVPTX::TEX_CUBE_U32_F32;
3232 break;
3233 case NVPTXISD::TexCubeU32FloatLevel:
3234 Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL;
3235 break;
3236 case NVPTXISD::TexCubeArrayFloatFloat:
3237 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32;
3238 break;
3239 case NVPTXISD::TexCubeArrayFloatFloatLevel:
3240 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL;
3241 break;
3242 case NVPTXISD::TexCubeArrayS32Float:
3243 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32;
3244 break;
3245 case NVPTXISD::TexCubeArrayS32FloatLevel:
3246 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL;
3247 break;
3248 case NVPTXISD::TexCubeArrayU32Float:
3249 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32;
3250 break;
3251 case NVPTXISD::TexCubeArrayU32FloatLevel:
3252 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL;
3253 break;
3254 case NVPTXISD::Tld4R2DFloatFloat:
3255 Opc = NVPTX::TLD4_R_2D_F32_F32;
3256 break;
3257 case NVPTXISD::Tld4G2DFloatFloat:
3258 Opc = NVPTX::TLD4_G_2D_F32_F32;
3259 break;
3260 case NVPTXISD::Tld4B2DFloatFloat:
3261 Opc = NVPTX::TLD4_B_2D_F32_F32;
3262 break;
3263 case NVPTXISD::Tld4A2DFloatFloat:
3264 Opc = NVPTX::TLD4_A_2D_F32_F32;
3265 break;
3266 case NVPTXISD::Tld4R2DS64Float:
3267 Opc = NVPTX::TLD4_R_2D_S32_F32;
3268 break;
3269 case NVPTXISD::Tld4G2DS64Float:
3270 Opc = NVPTX::TLD4_G_2D_S32_F32;
3271 break;
3272 case NVPTXISD::Tld4B2DS64Float:
3273 Opc = NVPTX::TLD4_B_2D_S32_F32;
3274 break;
3275 case NVPTXISD::Tld4A2DS64Float:
3276 Opc = NVPTX::TLD4_A_2D_S32_F32;
3277 break;
3278 case NVPTXISD::Tld4R2DU64Float:
3279 Opc = NVPTX::TLD4_R_2D_U32_F32;
3280 break;
3281 case NVPTXISD::Tld4G2DU64Float:
3282 Opc = NVPTX::TLD4_G_2D_U32_F32;
3283 break;
3284 case NVPTXISD::Tld4B2DU64Float:
3285 Opc = NVPTX::TLD4_B_2D_U32_F32;
3286 break;
3287 case NVPTXISD::Tld4A2DU64Float:
3288 Opc = NVPTX::TLD4_A_2D_U32_F32;
3289 break;
3290 case NVPTXISD::TexUnified1DFloatS32:
3291 Opc = NVPTX::TEX_UNIFIED_1D_F32_S32;
3292 break;
3293 case NVPTXISD::TexUnified1DFloatFloat:
3294 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32;
3295 break;
3296 case NVPTXISD::TexUnified1DFloatFloatLevel:
3297 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL;
3298 break;
3299 case NVPTXISD::TexUnified1DFloatFloatGrad:
3300 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD;
3301 break;
3302 case NVPTXISD::TexUnified1DS32S32:
3303 Opc = NVPTX::TEX_UNIFIED_1D_S32_S32;
3304 break;
3305 case NVPTXISD::TexUnified1DS32Float:
3306 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32;
3307 break;
3308 case NVPTXISD::TexUnified1DS32FloatLevel:
3309 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL;
3310 break;
3311 case NVPTXISD::TexUnified1DS32FloatGrad:
3312 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD;
3313 break;
3314 case NVPTXISD::TexUnified1DU32S32:
3315 Opc = NVPTX::TEX_UNIFIED_1D_U32_S32;
3316 break;
3317 case NVPTXISD::TexUnified1DU32Float:
3318 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32;
3319 break;
3320 case NVPTXISD::TexUnified1DU32FloatLevel:
3321 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL;
3322 break;
3323 case NVPTXISD::TexUnified1DU32FloatGrad:
3324 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD;
3325 break;
3326 case NVPTXISD::TexUnified1DArrayFloatS32:
3327 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32;
3328 break;
3329 case NVPTXISD::TexUnified1DArrayFloatFloat:
3330 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32;
3331 break;
3332 case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
3333 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL;
3334 break;
3335 case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
3336 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD;
3337 break;
3338 case NVPTXISD::TexUnified1DArrayS32S32:
3339 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32;
3340 break;
3341 case NVPTXISD::TexUnified1DArrayS32Float:
3342 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32;
3343 break;
3344 case NVPTXISD::TexUnified1DArrayS32FloatLevel:
3345 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL;
3346 break;
3347 case NVPTXISD::TexUnified1DArrayS32FloatGrad:
3348 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD;
3349 break;
3350 case NVPTXISD::TexUnified1DArrayU32S32:
3351 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32;
3352 break;
3353 case NVPTXISD::TexUnified1DArrayU32Float:
3354 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32;
3355 break;
3356 case NVPTXISD::TexUnified1DArrayU32FloatLevel:
3357 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL;
3358 break;
3359 case NVPTXISD::TexUnified1DArrayU32FloatGrad:
3360 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD;
3361 break;
3362 case NVPTXISD::TexUnified2DFloatS32:
3363 Opc = NVPTX::TEX_UNIFIED_2D_F32_S32;
3364 break;
3365 case NVPTXISD::TexUnified2DFloatFloat:
3366 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32;
3367 break;
3368 case NVPTXISD::TexUnified2DFloatFloatLevel:
3369 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL;
3370 break;
3371 case NVPTXISD::TexUnified2DFloatFloatGrad:
3372 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD;
3373 break;
3374 case NVPTXISD::TexUnified2DS32S32:
3375 Opc = NVPTX::TEX_UNIFIED_2D_S32_S32;
3376 break;
3377 case NVPTXISD::TexUnified2DS32Float:
3378 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32;
3379 break;
3380 case NVPTXISD::TexUnified2DS32FloatLevel:
3381 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL;
3382 break;
3383 case NVPTXISD::TexUnified2DS32FloatGrad:
3384 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD;
3385 break;
3386 case NVPTXISD::TexUnified2DU32S32:
3387 Opc = NVPTX::TEX_UNIFIED_2D_U32_S32;
3388 break;
3389 case NVPTXISD::TexUnified2DU32Float:
3390 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32;
3391 break;
3392 case NVPTXISD::TexUnified2DU32FloatLevel:
3393 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL;
3394 break;
3395 case NVPTXISD::TexUnified2DU32FloatGrad:
3396 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD;
3397 break;
3398 case NVPTXISD::TexUnified2DArrayFloatS32:
3399 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32;
3400 break;
3401 case NVPTXISD::TexUnified2DArrayFloatFloat:
3402 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32;
3403 break;
3404 case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
3405 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL;
3406 break;
3407 case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
3408 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD;
3409 break;
3410 case NVPTXISD::TexUnified2DArrayS32S32:
3411 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32;
3412 break;
3413 case NVPTXISD::TexUnified2DArrayS32Float:
3414 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32;
3415 break;
3416 case NVPTXISD::TexUnified2DArrayS32FloatLevel:
3417 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL;
3418 break;
3419 case NVPTXISD::TexUnified2DArrayS32FloatGrad:
3420 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD;
3421 break;
3422 case NVPTXISD::TexUnified2DArrayU32S32:
3423 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32;
3424 break;
3425 case NVPTXISD::TexUnified2DArrayU32Float:
3426 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32;
3427 break;
3428 case NVPTXISD::TexUnified2DArrayU32FloatLevel:
3429 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL;
3430 break;
3431 case NVPTXISD::TexUnified2DArrayU32FloatGrad:
3432 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD;
3433 break;
3434 case NVPTXISD::TexUnified3DFloatS32:
3435 Opc = NVPTX::TEX_UNIFIED_3D_F32_S32;
3436 break;
3437 case NVPTXISD::TexUnified3DFloatFloat:
3438 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32;
3439 break;
3440 case NVPTXISD::TexUnified3DFloatFloatLevel:
3441 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL;
3442 break;
3443 case NVPTXISD::TexUnified3DFloatFloatGrad:
3444 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD;
3445 break;
3446 case NVPTXISD::TexUnified3DS32S32:
3447 Opc = NVPTX::TEX_UNIFIED_3D_S32_S32;
3448 break;
3449 case NVPTXISD::TexUnified3DS32Float:
3450 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32;
3451 break;
3452 case NVPTXISD::TexUnified3DS32FloatLevel:
3453 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL;
3454 break;
3455 case NVPTXISD::TexUnified3DS32FloatGrad:
3456 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD;
3457 break;
3458 case NVPTXISD::TexUnified3DU32S32:
3459 Opc = NVPTX::TEX_UNIFIED_3D_U32_S32;
3460 break;
3461 case NVPTXISD::TexUnified3DU32Float:
3462 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32;
3463 break;
3464 case NVPTXISD::TexUnified3DU32FloatLevel:
3465 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL;
3466 break;
3467 case NVPTXISD::TexUnified3DU32FloatGrad:
3468 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD;
3469 break;
3470 case NVPTXISD::TexUnifiedCubeFloatFloat:
3471 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32;
3472 break;
3473 case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
3474 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL;
3475 break;
3476 case NVPTXISD::TexUnifiedCubeS32Float:
3477 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32;
3478 break;
3479 case NVPTXISD::TexUnifiedCubeS32FloatLevel:
3480 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL;
3481 break;
3482 case NVPTXISD::TexUnifiedCubeU32Float:
3483 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32;
3484 break;
3485 case NVPTXISD::TexUnifiedCubeU32FloatLevel:
3486 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL;
3487 break;
3488 case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
3489 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32;
3490 break;
3491 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
3492 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL;
3493 break;
3494 case NVPTXISD::TexUnifiedCubeArrayS32Float:
3495 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32;
3496 break;
3497 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
3498 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL;
3499 break;
3500 case NVPTXISD::TexUnifiedCubeArrayU32Float:
3501 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32;
3502 break;
3503 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
3504 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL;
3505 break;
3506 case NVPTXISD::Tld4UnifiedR2DFloatFloat:
3507 Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32;
3508 break;
3509 case NVPTXISD::Tld4UnifiedG2DFloatFloat:
3510 Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32;
3511 break;
3512 case NVPTXISD::Tld4UnifiedB2DFloatFloat:
3513 Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32;
3514 break;
3515 case NVPTXISD::Tld4UnifiedA2DFloatFloat:
3516 Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32;
3517 break;
3518 case NVPTXISD::Tld4UnifiedR2DS64Float:
3519 Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32;
3520 break;
3521 case NVPTXISD::Tld4UnifiedG2DS64Float:
3522 Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32;
3523 break;
3524 case NVPTXISD::Tld4UnifiedB2DS64Float:
3525 Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32;
3526 break;
3527 case NVPTXISD::Tld4UnifiedA2DS64Float:
3528 Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32;
3529 break;
3530 case NVPTXISD::Tld4UnifiedR2DU64Float:
3531 Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32;
3532 break;
3533 case NVPTXISD::Tld4UnifiedG2DU64Float:
3534 Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32;
3535 break;
3536 case NVPTXISD::Tld4UnifiedB2DU64Float:
3537 Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32;
3538 break;
3539 case NVPTXISD::Tld4UnifiedA2DU64Float:
3540 Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3541 break;
3542 }
3543
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3544 // Copy over operands
3545 for (unsigned i = 1; i < N->getNumOperands(); ++i) {
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3546 Ops.push_back(N->getOperand(i));
3547 }
3548
3549 Ops.push_back(Chain);
3550 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
3551 return Ret;
3552}
3553
3554SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) {
3555 SDValue Chain = N->getOperand(0);
3556 SDValue TexHandle = N->getOperand(1);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3557 SDNode *Ret = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3558 unsigned Opc = 0;
3559 SmallVector<SDValue, 8> Ops;
3560 switch (N->getOpcode()) {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3561 default: return nullptr;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3562 case NVPTXISD::Suld1DI8Clamp:
3563 Opc = NVPTX::SULD_1D_I8_CLAMP;
3564 Ops.push_back(TexHandle);
3565 Ops.push_back(N->getOperand(2));
3566 Ops.push_back(Chain);
3567 break;
3568 case NVPTXISD::Suld1DI16Clamp:
3569 Opc = NVPTX::SULD_1D_I16_CLAMP;
3570 Ops.push_back(TexHandle);
3571 Ops.push_back(N->getOperand(2));
3572 Ops.push_back(Chain);
3573 break;
3574 case NVPTXISD::Suld1DI32Clamp:
3575 Opc = NVPTX::SULD_1D_I32_CLAMP;
3576 Ops.push_back(TexHandle);
3577 Ops.push_back(N->getOperand(2));
3578 Ops.push_back(Chain);
3579 break;
3580 case NVPTXISD::Suld1DI64Clamp:
3581 Opc = NVPTX::SULD_1D_I64_CLAMP;
3582 Ops.push_back(TexHandle);
3583 Ops.push_back(N->getOperand(2));
3584 Ops.push_back(Chain);
3585 break;
3586 case NVPTXISD::Suld1DV2I8Clamp:
3587 Opc = NVPTX::SULD_1D_V2I8_CLAMP;
3588 Ops.push_back(TexHandle);
3589 Ops.push_back(N->getOperand(2));
3590 Ops.push_back(Chain);
3591 break;
3592 case NVPTXISD::Suld1DV2I16Clamp:
3593 Opc = NVPTX::SULD_1D_V2I16_CLAMP;
3594 Ops.push_back(TexHandle);
3595 Ops.push_back(N->getOperand(2));
3596 Ops.push_back(Chain);
3597 break;
3598 case NVPTXISD::Suld1DV2I32Clamp:
3599 Opc = NVPTX::SULD_1D_V2I32_CLAMP;
3600 Ops.push_back(TexHandle);
3601 Ops.push_back(N->getOperand(2));
3602 Ops.push_back(Chain);
3603 break;
3604 case NVPTXISD::Suld1DV2I64Clamp:
3605 Opc = NVPTX::SULD_1D_V2I64_CLAMP;
3606 Ops.push_back(TexHandle);
3607 Ops.push_back(N->getOperand(2));
3608 Ops.push_back(Chain);
3609 break;
3610 case NVPTXISD::Suld1DV4I8Clamp:
3611 Opc = NVPTX::SULD_1D_V4I8_CLAMP;
3612 Ops.push_back(TexHandle);
3613 Ops.push_back(N->getOperand(2));
3614 Ops.push_back(Chain);
3615 break;
3616 case NVPTXISD::Suld1DV4I16Clamp:
3617 Opc = NVPTX::SULD_1D_V4I16_CLAMP;
3618 Ops.push_back(TexHandle);
3619 Ops.push_back(N->getOperand(2));
3620 Ops.push_back(Chain);
3621 break;
3622 case NVPTXISD::Suld1DV4I32Clamp:
3623 Opc = NVPTX::SULD_1D_V4I32_CLAMP;
3624 Ops.push_back(TexHandle);
3625 Ops.push_back(N->getOperand(2));
3626 Ops.push_back(Chain);
3627 break;
3628 case NVPTXISD::Suld1DArrayI8Clamp:
3629 Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP;
3630 Ops.push_back(TexHandle);
3631 Ops.push_back(N->getOperand(2));
3632 Ops.push_back(N->getOperand(3));
3633 Ops.push_back(Chain);
3634 break;
3635 case NVPTXISD::Suld1DArrayI16Clamp:
3636 Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP;
3637 Ops.push_back(TexHandle);
3638 Ops.push_back(N->getOperand(2));
3639 Ops.push_back(N->getOperand(3));
3640 Ops.push_back(Chain);
3641 break;
3642 case NVPTXISD::Suld1DArrayI32Clamp:
3643 Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP;
3644 Ops.push_back(TexHandle);
3645 Ops.push_back(N->getOperand(2));
3646 Ops.push_back(N->getOperand(3));
3647 Ops.push_back(Chain);
3648 break;
3649 case NVPTXISD::Suld1DArrayI64Clamp:
3650 Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP;
3651 Ops.push_back(TexHandle);
3652 Ops.push_back(N->getOperand(2));
3653 Ops.push_back(N->getOperand(3));
3654 Ops.push_back(Chain);
3655 break;
3656 case NVPTXISD::Suld1DArrayV2I8Clamp:
3657 Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP;
3658 Ops.push_back(TexHandle);
3659 Ops.push_back(N->getOperand(2));
3660 Ops.push_back(N->getOperand(3));
3661 Ops.push_back(Chain);
3662 break;
3663 case NVPTXISD::Suld1DArrayV2I16Clamp:
3664 Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP;
3665 Ops.push_back(TexHandle);
3666 Ops.push_back(N->getOperand(2));
3667 Ops.push_back(N->getOperand(3));
3668 Ops.push_back(Chain);
3669 break;
3670 case NVPTXISD::Suld1DArrayV2I32Clamp:
3671 Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP;
3672 Ops.push_back(TexHandle);
3673 Ops.push_back(N->getOperand(2));
3674 Ops.push_back(N->getOperand(3));
3675 Ops.push_back(Chain);
3676 break;
3677 case NVPTXISD::Suld1DArrayV2I64Clamp:
3678 Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP;
3679 Ops.push_back(TexHandle);
3680 Ops.push_back(N->getOperand(2));
3681 Ops.push_back(N->getOperand(3));
3682 Ops.push_back(Chain);
3683 break;
3684 case NVPTXISD::Suld1DArrayV4I8Clamp:
3685 Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP;
3686 Ops.push_back(TexHandle);
3687 Ops.push_back(N->getOperand(2));
3688 Ops.push_back(N->getOperand(3));
3689 Ops.push_back(Chain);
3690 break;
3691 case NVPTXISD::Suld1DArrayV4I16Clamp:
3692 Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP;
3693 Ops.push_back(TexHandle);
3694 Ops.push_back(N->getOperand(2));
3695 Ops.push_back(N->getOperand(3));
3696 Ops.push_back(Chain);
3697 break;
3698 case NVPTXISD::Suld1DArrayV4I32Clamp:
3699 Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP;
3700 Ops.push_back(TexHandle);
3701 Ops.push_back(N->getOperand(2));
3702 Ops.push_back(N->getOperand(3));
3703 Ops.push_back(Chain);
3704 break;
3705 case NVPTXISD::Suld2DI8Clamp:
3706 Opc = NVPTX::SULD_2D_I8_CLAMP;
3707 Ops.push_back(TexHandle);
3708 Ops.push_back(N->getOperand(2));
3709 Ops.push_back(N->getOperand(3));
3710 Ops.push_back(Chain);
3711 break;
3712 case NVPTXISD::Suld2DI16Clamp:
3713 Opc = NVPTX::SULD_2D_I16_CLAMP;
3714 Ops.push_back(TexHandle);
3715 Ops.push_back(N->getOperand(2));
3716 Ops.push_back(N->getOperand(3));
3717 Ops.push_back(Chain);
3718 break;
3719 case NVPTXISD::Suld2DI32Clamp:
3720 Opc = NVPTX::SULD_2D_I32_CLAMP;
3721 Ops.push_back(TexHandle);
3722 Ops.push_back(N->getOperand(2));
3723 Ops.push_back(N->getOperand(3));
3724 Ops.push_back(Chain);
3725 break;
3726 case NVPTXISD::Suld2DI64Clamp:
3727 Opc = NVPTX::SULD_2D_I64_CLAMP;
3728 Ops.push_back(TexHandle);
3729 Ops.push_back(N->getOperand(2));
3730 Ops.push_back(N->getOperand(3));
3731 Ops.push_back(Chain);
3732 break;
3733 case NVPTXISD::Suld2DV2I8Clamp:
3734 Opc = NVPTX::SULD_2D_V2I8_CLAMP;
3735 Ops.push_back(TexHandle);
3736 Ops.push_back(N->getOperand(2));
3737 Ops.push_back(N->getOperand(3));
3738 Ops.push_back(Chain);
3739 break;
3740 case NVPTXISD::Suld2DV2I16Clamp:
3741 Opc = NVPTX::SULD_2D_V2I16_CLAMP;
3742 Ops.push_back(TexHandle);
3743 Ops.push_back(N->getOperand(2));
3744 Ops.push_back(N->getOperand(3));
3745 Ops.push_back(Chain);
3746 break;
3747 case NVPTXISD::Suld2DV2I32Clamp:
3748 Opc = NVPTX::SULD_2D_V2I32_CLAMP;
3749 Ops.push_back(TexHandle);
3750 Ops.push_back(N->getOperand(2));
3751 Ops.push_back(N->getOperand(3));
3752 Ops.push_back(Chain);
3753 break;
3754 case NVPTXISD::Suld2DV2I64Clamp:
3755 Opc = NVPTX::SULD_2D_V2I64_CLAMP;
3756 Ops.push_back(TexHandle);
3757 Ops.push_back(N->getOperand(2));
3758 Ops.push_back(N->getOperand(3));
3759 Ops.push_back(Chain);
3760 break;
3761 case NVPTXISD::Suld2DV4I8Clamp:
3762 Opc = NVPTX::SULD_2D_V4I8_CLAMP;
3763 Ops.push_back(TexHandle);
3764 Ops.push_back(N->getOperand(2));
3765 Ops.push_back(N->getOperand(3));
3766 Ops.push_back(Chain);
3767 break;
3768 case NVPTXISD::Suld2DV4I16Clamp:
3769 Opc = NVPTX::SULD_2D_V4I16_CLAMP;
3770 Ops.push_back(TexHandle);
3771 Ops.push_back(N->getOperand(2));
3772 Ops.push_back(N->getOperand(3));
3773 Ops.push_back(Chain);
3774 break;
3775 case NVPTXISD::Suld2DV4I32Clamp:
3776 Opc = NVPTX::SULD_2D_V4I32_CLAMP;
3777 Ops.push_back(TexHandle);
3778 Ops.push_back(N->getOperand(2));
3779 Ops.push_back(N->getOperand(3));
3780 Ops.push_back(Chain);
3781 break;
3782 case NVPTXISD::Suld2DArrayI8Clamp:
3783 Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP;
3784 Ops.push_back(TexHandle);
3785 Ops.push_back(N->getOperand(2));
3786 Ops.push_back(N->getOperand(3));
3787 Ops.push_back(N->getOperand(4));
3788 Ops.push_back(Chain);
3789 break;
3790 case NVPTXISD::Suld2DArrayI16Clamp:
3791 Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP;
3792 Ops.push_back(TexHandle);
3793 Ops.push_back(N->getOperand(2));
3794 Ops.push_back(N->getOperand(3));
3795 Ops.push_back(N->getOperand(4));
3796 Ops.push_back(Chain);
3797 break;
3798 case NVPTXISD::Suld2DArrayI32Clamp:
3799 Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP;
3800 Ops.push_back(TexHandle);
3801 Ops.push_back(N->getOperand(2));
3802 Ops.push_back(N->getOperand(3));
3803 Ops.push_back(N->getOperand(4));
3804 Ops.push_back(Chain);
3805 break;
3806 case NVPTXISD::Suld2DArrayI64Clamp:
3807 Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP;
3808 Ops.push_back(TexHandle);
3809 Ops.push_back(N->getOperand(2));
3810 Ops.push_back(N->getOperand(3));
3811 Ops.push_back(N->getOperand(4));
3812 Ops.push_back(Chain);
3813 break;
3814 case NVPTXISD::Suld2DArrayV2I8Clamp:
3815 Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP;
3816 Ops.push_back(TexHandle);
3817 Ops.push_back(N->getOperand(2));
3818 Ops.push_back(N->getOperand(3));
3819 Ops.push_back(N->getOperand(4));
3820 Ops.push_back(Chain);
3821 break;
3822 case NVPTXISD::Suld2DArrayV2I16Clamp:
3823 Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP;
3824 Ops.push_back(TexHandle);
3825 Ops.push_back(N->getOperand(2));
3826 Ops.push_back(N->getOperand(3));
3827 Ops.push_back(N->getOperand(4));
3828 Ops.push_back(Chain);
3829 break;
3830 case NVPTXISD::Suld2DArrayV2I32Clamp:
3831 Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP;
3832 Ops.push_back(TexHandle);
3833 Ops.push_back(N->getOperand(2));
3834 Ops.push_back(N->getOperand(3));
3835 Ops.push_back(N->getOperand(4));
3836 Ops.push_back(Chain);
3837 break;
3838 case NVPTXISD::Suld2DArrayV2I64Clamp:
3839 Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP;
3840 Ops.push_back(TexHandle);
3841 Ops.push_back(N->getOperand(2));
3842 Ops.push_back(N->getOperand(3));
3843 Ops.push_back(N->getOperand(4));
3844 Ops.push_back(Chain);
3845 break;
3846 case NVPTXISD::Suld2DArrayV4I8Clamp:
3847 Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP;
3848 Ops.push_back(TexHandle);
3849 Ops.push_back(N->getOperand(2));
3850 Ops.push_back(N->getOperand(3));
3851 Ops.push_back(N->getOperand(4));
3852 Ops.push_back(Chain);
3853 break;
3854 case NVPTXISD::Suld2DArrayV4I16Clamp:
3855 Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP;
3856 Ops.push_back(TexHandle);
3857 Ops.push_back(N->getOperand(2));
3858 Ops.push_back(N->getOperand(3));
3859 Ops.push_back(N->getOperand(4));
3860 Ops.push_back(Chain);
3861 break;
3862 case NVPTXISD::Suld2DArrayV4I32Clamp:
3863 Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP;
3864 Ops.push_back(TexHandle);
3865 Ops.push_back(N->getOperand(2));
3866 Ops.push_back(N->getOperand(3));
3867 Ops.push_back(N->getOperand(4));
3868 Ops.push_back(Chain);
3869 break;
3870 case NVPTXISD::Suld3DI8Clamp:
3871 Opc = NVPTX::SULD_3D_I8_CLAMP;
3872 Ops.push_back(TexHandle);
3873 Ops.push_back(N->getOperand(2));
3874 Ops.push_back(N->getOperand(3));
3875 Ops.push_back(N->getOperand(4));
3876 Ops.push_back(Chain);
3877 break;
3878 case NVPTXISD::Suld3DI16Clamp:
3879 Opc = NVPTX::SULD_3D_I16_CLAMP;
3880 Ops.push_back(TexHandle);
3881 Ops.push_back(N->getOperand(2));
3882 Ops.push_back(N->getOperand(3));
3883 Ops.push_back(N->getOperand(4));
3884 Ops.push_back(Chain);
3885 break;
3886 case NVPTXISD::Suld3DI32Clamp:
3887 Opc = NVPTX::SULD_3D_I32_CLAMP;
3888 Ops.push_back(TexHandle);
3889 Ops.push_back(N->getOperand(2));
3890 Ops.push_back(N->getOperand(3));
3891 Ops.push_back(N->getOperand(4));
3892 Ops.push_back(Chain);
3893 break;
3894 case NVPTXISD::Suld3DI64Clamp:
3895 Opc = NVPTX::SULD_3D_I64_CLAMP;
3896 Ops.push_back(TexHandle);
3897 Ops.push_back(N->getOperand(2));
3898 Ops.push_back(N->getOperand(3));
3899 Ops.push_back(N->getOperand(4));
3900 Ops.push_back(Chain);
3901 break;
3902 case NVPTXISD::Suld3DV2I8Clamp:
3903 Opc = NVPTX::SULD_3D_V2I8_CLAMP;
3904 Ops.push_back(TexHandle);
3905 Ops.push_back(N->getOperand(2));
3906 Ops.push_back(N->getOperand(3));
3907 Ops.push_back(N->getOperand(4));
3908 Ops.push_back(Chain);
3909 break;
3910 case NVPTXISD::Suld3DV2I16Clamp:
3911 Opc = NVPTX::SULD_3D_V2I16_CLAMP;
3912 Ops.push_back(TexHandle);
3913 Ops.push_back(N->getOperand(2));
3914 Ops.push_back(N->getOperand(3));
3915 Ops.push_back(N->getOperand(4));
3916 Ops.push_back(Chain);
3917 break;
3918 case NVPTXISD::Suld3DV2I32Clamp:
3919 Opc = NVPTX::SULD_3D_V2I32_CLAMP;
3920 Ops.push_back(TexHandle);
3921 Ops.push_back(N->getOperand(2));
3922 Ops.push_back(N->getOperand(3));
3923 Ops.push_back(N->getOperand(4));
3924 Ops.push_back(Chain);
3925 break;
3926 case NVPTXISD::Suld3DV2I64Clamp:
3927 Opc = NVPTX::SULD_3D_V2I64_CLAMP;
3928 Ops.push_back(TexHandle);
3929 Ops.push_back(N->getOperand(2));
3930 Ops.push_back(N->getOperand(3));
3931 Ops.push_back(N->getOperand(4));
3932 Ops.push_back(Chain);
3933 break;
3934 case NVPTXISD::Suld3DV4I8Clamp:
3935 Opc = NVPTX::SULD_3D_V4I8_CLAMP;
3936 Ops.push_back(TexHandle);
3937 Ops.push_back(N->getOperand(2));
3938 Ops.push_back(N->getOperand(3));
3939 Ops.push_back(N->getOperand(4));
3940 Ops.push_back(Chain);
3941 break;
3942 case NVPTXISD::Suld3DV4I16Clamp:
3943 Opc = NVPTX::SULD_3D_V4I16_CLAMP;
3944 Ops.push_back(TexHandle);
3945 Ops.push_back(N->getOperand(2));
3946 Ops.push_back(N->getOperand(3));
3947 Ops.push_back(N->getOperand(4));
3948 Ops.push_back(Chain);
3949 break;
3950 case NVPTXISD::Suld3DV4I32Clamp:
3951 Opc = NVPTX::SULD_3D_V4I32_CLAMP;
3952 Ops.push_back(TexHandle);
3953 Ops.push_back(N->getOperand(2));
3954 Ops.push_back(N->getOperand(3));
3955 Ops.push_back(N->getOperand(4));
3956 Ops.push_back(Chain);
3957 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3958 case NVPTXISD::Suld1DI8Trap:
3959 Opc = NVPTX::SULD_1D_I8_TRAP;
3960 Ops.push_back(TexHandle);
3961 Ops.push_back(N->getOperand(2));
3962 Ops.push_back(Chain);
3963 break;
3964 case NVPTXISD::Suld1DI16Trap:
3965 Opc = NVPTX::SULD_1D_I16_TRAP;
3966 Ops.push_back(TexHandle);
3967 Ops.push_back(N->getOperand(2));
3968 Ops.push_back(Chain);
3969 break;
3970 case NVPTXISD::Suld1DI32Trap:
3971 Opc = NVPTX::SULD_1D_I32_TRAP;
3972 Ops.push_back(TexHandle);
3973 Ops.push_back(N->getOperand(2));
3974 Ops.push_back(Chain);
3975 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]3976 case NVPTXISD::Suld1DI64Trap:
3977 Opc = NVPTX::SULD_1D_I64_TRAP;
3978 Ops.push_back(TexHandle);
3979 Ops.push_back(N->getOperand(2));
3980 Ops.push_back(Chain);
3981 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3982 case NVPTXISD::Suld1DV2I8Trap:
3983 Opc = NVPTX::SULD_1D_V2I8_TRAP;
3984 Ops.push_back(TexHandle);
3985 Ops.push_back(N->getOperand(2));
3986 Ops.push_back(Chain);
3987 break;
3988 case NVPTXISD::Suld1DV2I16Trap:
3989 Opc = NVPTX::SULD_1D_V2I16_TRAP;
3990 Ops.push_back(TexHandle);
3991 Ops.push_back(N->getOperand(2));
3992 Ops.push_back(Chain);
3993 break;
3994 case NVPTXISD::Suld1DV2I32Trap:
3995 Opc = NVPTX::SULD_1D_V2I32_TRAP;
3996 Ops.push_back(TexHandle);
3997 Ops.push_back(N->getOperand(2));
3998 Ops.push_back(Chain);
3999 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]4000 case NVPTXISD::Suld1DV2I64Trap:
4001 Opc = NVPTX::SULD_1D_V2I64_TRAP;
4002 Ops.push_back(TexHandle);
4003 Ops.push_back(N->getOperand(2));
4004 Ops.push_back(Chain);
4005 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4006 case NVPTXISD::Suld1DV4I8Trap:
4007 Opc = NVPTX::SULD_1D_V4I8_TRAP;
4008 Ops.push_back(TexHandle);
4009 Ops.push_back(N->getOperand(2));
4010 Ops.push_back(Chain);
4011 break;
4012 case NVPTXISD::Suld1DV4I16Trap:
4013 Opc = NVPTX::SULD_1D_V4I16_TRAP;
4014 Ops.push_back(TexHandle);
4015 Ops.push_back(N->getOperand(2));
4016 Ops.push_back(Chain);
4017 break;
4018 case NVPTXISD::Suld1DV4I32Trap:
4019 Opc = NVPTX::SULD_1D_V4I32_TRAP;
4020 Ops.push_back(TexHandle);
4021 Ops.push_back(N->getOperand(2));
4022 Ops.push_back(Chain);
4023 break;
4024 case NVPTXISD::Suld1DArrayI8Trap:
4025 Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP;
4026 Ops.push_back(TexHandle);
4027 Ops.push_back(N->getOperand(2));
4028 Ops.push_back(N->getOperand(3));
4029 Ops.push_back(Chain);
4030 break;
4031 case NVPTXISD::Suld1DArrayI16Trap:
4032 Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP;
4033 Ops.push_back(TexHandle);
4034 Ops.push_back(N->getOperand(2));
4035 Ops.push_back(N->getOperand(3));
4036 Ops.push_back(Chain);
4037 break;
4038 case NVPTXISD::Suld1DArrayI32Trap:
4039 Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP;
4040 Ops.push_back(TexHandle);
4041 Ops.push_back(N->getOperand(2));
4042 Ops.push_back(N->getOperand(3));
4043 Ops.push_back(Chain);
4044 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]4045 case NVPTXISD::Suld1DArrayI64Trap:
4046 Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP;
4047 Ops.push_back(TexHandle);
4048 Ops.push_back(N->getOperand(2));
4049 Ops.push_back(N->getOperand(3));
4050 Ops.push_back(Chain);
4051 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4052 case NVPTXISD::Suld1DArrayV2I8Trap:
4053 Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP;
4054 Ops.push_back(TexHandle);
4055 Ops.push_back(N->getOperand(2));
4056 Ops.push_back(N->getOperand(3));
4057 Ops.push_back(Chain);
4058 break;
4059 case NVPTXISD::Suld1DArrayV2I16Trap:
4060 Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP;
4061 Ops.push_back(TexHandle);
4062 Ops.push_back(N->getOperand(2));
4063 Ops.push_back(N->getOperand(3));
4064 Ops.push_back(Chain);
4065 break;
4066 case NVPTXISD::Suld1DArrayV2I32Trap:
4067 Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP;
4068 Ops.push_back(TexHandle);
4069 Ops.push_back(N->getOperand(2));
4070 Ops.push_back(N->getOperand(3));
4071 Ops.push_back(Chain);
4072 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]4073 case NVPTXISD::Suld1DArrayV2I64Trap:
4074 Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP;
4075 Ops.push_back(TexHandle);
4076 Ops.push_back(N->getOperand(2));
4077 Ops.push_back(N->getOperand(3));
4078 Ops.push_back(Chain);
4079 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4080 case NVPTXISD::Suld1DArrayV4I8Trap:
4081 Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP;
4082 Ops.push_back(TexHandle);
4083 Ops.push_back(N->getOperand(2));
4084 Ops.push_back(N->getOperand(3));
4085 Ops.push_back(Chain);
4086 break;
4087 case NVPTXISD::Suld1DArrayV4I16Trap:
4088 Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP;
4089 Ops.push_back(TexHandle);
4090 Ops.push_back(N->getOperand(2));
4091 Ops.push_back(N->getOperand(3));
4092 Ops.push_back(Chain);
4093 break;
4094 case NVPTXISD::Suld1DArrayV4I32Trap:
4095 Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP;
4096 Ops.push_back(TexHandle);
4097 Ops.push_back(N->getOperand(2));
4098 Ops.push_back(N->getOperand(3));
4099 Ops.push_back(Chain);
4100 break;
4101 case NVPTXISD::Suld2DI8Trap:
4102 Opc = NVPTX::SULD_2D_I8_TRAP;
4103 Ops.push_back(TexHandle);
4104 Ops.push_back(N->getOperand(2));
4105 Ops.push_back(N->getOperand(3));
4106 Ops.push_back(Chain);
4107 break;
4108 case NVPTXISD::Suld2DI16Trap:
4109 Opc = NVPTX::SULD_2D_I16_TRAP;
4110 Ops.push_back(TexHandle);
4111 Ops.push_back(N->getOperand(2));
4112 Ops.push_back(N->getOperand(3));
4113 Ops.push_back(Chain);
4114 break;
4115 case NVPTXISD::Suld2DI32Trap:
4116 Opc = NVPTX::SULD_2D_I32_TRAP;
4117 Ops.push_back(TexHandle);
4118 Ops.push_back(N->getOperand(2));
4119 Ops.push_back(N->getOperand(3));
4120 Ops.push_back(Chain);
4121 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]4122 case NVPTXISD::Suld2DI64Trap:
4123 Opc = NVPTX::SULD_2D_I64_TRAP;
4124 Ops.push_back(TexHandle);
4125 Ops.push_back(N->getOperand(2));
4126 Ops.push_back(N->getOperand(3));
4127 Ops.push_back(Chain);
4128 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4129 case NVPTXISD::Suld2DV2I8Trap:
4130 Opc = NVPTX::SULD_2D_V2I8_TRAP;
4131 Ops.push_back(TexHandle);
4132 Ops.push_back(N->getOperand(2));
4133 Ops.push_back(N->getOperand(3));
4134 Ops.push_back(Chain);
4135 break;
4136 case NVPTXISD::Suld2DV2I16Trap:
4137 Opc = NVPTX::SULD_2D_V2I16_TRAP;
4138 Ops.push_back(TexHandle);
4139 Ops.push_back(N->getOperand(2));
4140 Ops.push_back(N->getOperand(3));
4141 Ops.push_back(Chain);
4142 break;
4143 case NVPTXISD::Suld2DV2I32Trap:
4144 Opc = NVPTX::SULD_2D_V2I32_TRAP;
4145 Ops.push_back(TexHandle);
4146 Ops.push_back(N->getOperand(2));
4147 Ops.push_back(N->getOperand(3));
4148 Ops.push_back(Chain);
4149 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]4150 case NVPTXISD::Suld2DV2I64Trap:
4151 Opc = NVPTX::SULD_2D_V2I64_TRAP;
4152 Ops.push_back(TexHandle);
4153 Ops.push_back(N->getOperand(2));
4154 Ops.push_back(N->getOperand(3));
4155 Ops.push_back(Chain);
4156 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4157 case NVPTXISD::Suld2DV4I8Trap:
4158 Opc = NVPTX::SULD_2D_V4I8_TRAP;
4159 Ops.push_back(TexHandle);
4160 Ops.push_back(N->getOperand(2));
4161 Ops.push_back(N->getOperand(3));
4162 Ops.push_back(Chain);
4163 break;
4164 case NVPTXISD::Suld2DV4I16Trap:
4165 Opc = NVPTX::SULD_2D_V4I16_TRAP;
4166 Ops.push_back(TexHandle);
4167 Ops.push_back(N->getOperand(2));
4168 Ops.push_back(N->getOperand(3));
4169 Ops.push_back(Chain);
4170 break;
4171 case NVPTXISD::Suld2DV4I32Trap:
4172 Opc = NVPTX::SULD_2D_V4I32_TRAP;
4173 Ops.push_back(TexHandle);
4174 Ops.push_back(N->getOperand(2));
4175 Ops.push_back(N->getOperand(3));
4176 Ops.push_back(Chain);
4177 break;
4178 case NVPTXISD::Suld2DArrayI8Trap:
4179 Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP;
4180 Ops.push_back(TexHandle);
4181 Ops.push_back(N->getOperand(2));
4182 Ops.push_back(N->getOperand(3));
4183 Ops.push_back(N->getOperand(4));
4184 Ops.push_back(Chain);
4185 break;
4186 case NVPTXISD::Suld2DArrayI16Trap:
4187 Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP;
4188 Ops.push_back(TexHandle);
4189 Ops.push_back(N->getOperand(2));
4190 Ops.push_back(N->getOperand(3));
4191 Ops.push_back(N->getOperand(4));
4192 Ops.push_back(Chain);
4193 break;
4194 case NVPTXISD::Suld2DArrayI32Trap:
4195 Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP;
4196 Ops.push_back(TexHandle);
4197 Ops.push_back(N->getOperand(2));
4198 Ops.push_back(N->getOperand(3));
4199 Ops.push_back(N->getOperand(4));
4200 Ops.push_back(Chain);
4201 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]4202 case NVPTXISD::Suld2DArrayI64Trap:
4203 Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP;
4204 Ops.push_back(TexHandle);
4205 Ops.push_back(N->getOperand(2));
4206 Ops.push_back(N->getOperand(3));
4207 Ops.push_back(N->getOperand(4));
4208 Ops.push_back(Chain);
4209 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4210 case NVPTXISD::Suld2DArrayV2I8Trap:
4211 Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP;
4212 Ops.push_back(TexHandle);
4213 Ops.push_back(N->getOperand(2));
4214 Ops.push_back(N->getOperand(3));
4215 Ops.push_back(N->getOperand(4));
4216 Ops.push_back(Chain);
4217 break;
4218 case NVPTXISD::Suld2DArrayV2I16Trap:
4219 Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP;
4220 Ops.push_back(TexHandle);
4221 Ops.push_back(N->getOperand(2));
4222 Ops.push_back(N->getOperand(3));
4223 Ops.push_back(N->getOperand(4));
4224 Ops.push_back(Chain);
4225 break;
4226 case NVPTXISD::Suld2DArrayV2I32Trap:
4227 Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP;
4228 Ops.push_back(TexHandle);
4229 Ops.push_back(N->getOperand(2));
4230 Ops.push_back(N->getOperand(3));
4231 Ops.push_back(N->getOperand(4));
4232 Ops.push_back(Chain);
4233 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]4234 case NVPTXISD::Suld2DArrayV2I64Trap:
4235 Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP;
4236 Ops.push_back(TexHandle);
4237 Ops.push_back(N->getOperand(2));
4238 Ops.push_back(N->getOperand(3));
4239 Ops.push_back(N->getOperand(4));
4240 Ops.push_back(Chain);
4241 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4242 case NVPTXISD::Suld2DArrayV4I8Trap:
4243 Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP;
4244 Ops.push_back(TexHandle);
4245 Ops.push_back(N->getOperand(2));
4246 Ops.push_back(N->getOperand(3));
4247 Ops.push_back(N->getOperand(4));
4248 Ops.push_back(Chain);
4249 break;
4250 case NVPTXISD::Suld2DArrayV4I16Trap:
4251 Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP;
4252 Ops.push_back(TexHandle);
4253 Ops.push_back(N->getOperand(2));
4254 Ops.push_back(N->getOperand(3));
4255 Ops.push_back(N->getOperand(4));
4256 Ops.push_back(Chain);
4257 break;
4258 case NVPTXISD::Suld2DArrayV4I32Trap:
4259 Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP;
4260 Ops.push_back(TexHandle);
4261 Ops.push_back(N->getOperand(2));
4262 Ops.push_back(N->getOperand(3));
4263 Ops.push_back(N->getOperand(4));
4264 Ops.push_back(Chain);
4265 break;
4266 case NVPTXISD::Suld3DI8Trap:
4267 Opc = NVPTX::SULD_3D_I8_TRAP;
4268 Ops.push_back(TexHandle);
4269 Ops.push_back(N->getOperand(2));
4270 Ops.push_back(N->getOperand(3));
4271 Ops.push_back(N->getOperand(4));
4272 Ops.push_back(Chain);
4273 break;
4274 case NVPTXISD::Suld3DI16Trap:
4275 Opc = NVPTX::SULD_3D_I16_TRAP;
4276 Ops.push_back(TexHandle);
4277 Ops.push_back(N->getOperand(2));
4278 Ops.push_back(N->getOperand(3));
4279 Ops.push_back(N->getOperand(4));
4280 Ops.push_back(Chain);
4281 break;
4282 case NVPTXISD::Suld3DI32Trap:
4283 Opc = NVPTX::SULD_3D_I32_TRAP;
4284 Ops.push_back(TexHandle);
4285 Ops.push_back(N->getOperand(2));
4286 Ops.push_back(N->getOperand(3));
4287 Ops.push_back(N->getOperand(4));
4288 Ops.push_back(Chain);
4289 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]4290 case NVPTXISD::Suld3DI64Trap:
4291 Opc = NVPTX::SULD_3D_I64_TRAP;
4292 Ops.push_back(TexHandle);
4293 Ops.push_back(N->getOperand(2));
4294 Ops.push_back(N->getOperand(3));
4295 Ops.push_back(N->getOperand(4));
4296 Ops.push_back(Chain);
4297 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4298 case NVPTXISD::Suld3DV2I8Trap:
4299 Opc = NVPTX::SULD_3D_V2I8_TRAP;
4300 Ops.push_back(TexHandle);
4301 Ops.push_back(N->getOperand(2));
4302 Ops.push_back(N->getOperand(3));
4303 Ops.push_back(N->getOperand(4));
4304 Ops.push_back(Chain);
4305 break;
4306 case NVPTXISD::Suld3DV2I16Trap:
4307 Opc = NVPTX::SULD_3D_V2I16_TRAP;
4308 Ops.push_back(TexHandle);
4309 Ops.push_back(N->getOperand(2));
4310 Ops.push_back(N->getOperand(3));
4311 Ops.push_back(N->getOperand(4));
4312 Ops.push_back(Chain);
4313 break;
4314 case NVPTXISD::Suld3DV2I32Trap:
4315 Opc = NVPTX::SULD_3D_V2I32_TRAP;
4316 Ops.push_back(TexHandle);
4317 Ops.push_back(N->getOperand(2));
4318 Ops.push_back(N->getOperand(3));
4319 Ops.push_back(N->getOperand(4));
4320 Ops.push_back(Chain);
4321 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]4322 case NVPTXISD::Suld3DV2I64Trap:
4323 Opc = NVPTX::SULD_3D_V2I64_TRAP;
4324 Ops.push_back(TexHandle);
4325 Ops.push_back(N->getOperand(2));
4326 Ops.push_back(N->getOperand(3));
4327 Ops.push_back(N->getOperand(4));
4328 Ops.push_back(Chain);
4329 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4330 case NVPTXISD::Suld3DV4I8Trap:
4331 Opc = NVPTX::SULD_3D_V4I8_TRAP;
4332 Ops.push_back(TexHandle);
4333 Ops.push_back(N->getOperand(2));
4334 Ops.push_back(N->getOperand(3));
4335 Ops.push_back(N->getOperand(4));
4336 Ops.push_back(Chain);
4337 break;
4338 case NVPTXISD::Suld3DV4I16Trap:
4339 Opc = NVPTX::SULD_3D_V4I16_TRAP;
4340 Ops.push_back(TexHandle);
4341 Ops.push_back(N->getOperand(2));
4342 Ops.push_back(N->getOperand(3));
4343 Ops.push_back(N->getOperand(4));
4344 Ops.push_back(Chain);
4345 break;
4346 case NVPTXISD::Suld3DV4I32Trap:
4347 Opc = NVPTX::SULD_3D_V4I32_TRAP;
4348 Ops.push_back(TexHandle);
4349 Ops.push_back(N->getOperand(2));
4350 Ops.push_back(N->getOperand(3));
4351 Ops.push_back(N->getOperand(4));
4352 Ops.push_back(Chain);
4353 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]4354 case NVPTXISD::Suld1DI8Zero:
4355 Opc = NVPTX::SULD_1D_I8_ZERO;
4356 Ops.push_back(TexHandle);
4357 Ops.push_back(N->getOperand(2));
4358 Ops.push_back(Chain);
4359 break;
4360 case NVPTXISD::Suld1DI16Zero:
4361 Opc = NVPTX::SULD_1D_I16_ZERO;
4362 Ops.push_back(TexHandle);
4363 Ops.push_back(N->getOperand(2));
4364 Ops.push_back(Chain);
4365 break;
4366 case NVPTXISD::Suld1DI32Zero:
4367 Opc = NVPTX::SULD_1D_I32_ZERO;
4368 Ops.push_back(TexHandle);
4369 Ops.push_back(N->getOperand(2));
4370 Ops.push_back(Chain);
4371 break;
4372 case NVPTXISD::Suld1DI64Zero:
4373 Opc = NVPTX::SULD_1D_I64_ZERO;
4374 Ops.push_back(TexHandle);
4375 Ops.push_back(N->getOperand(2));
4376 Ops.push_back(Chain);
4377 break;
4378 case NVPTXISD::Suld1DV2I8Zero:
4379 Opc = NVPTX::SULD_1D_V2I8_ZERO;
4380 Ops.push_back(TexHandle);
4381 Ops.push_back(N->getOperand(2));
4382 Ops.push_back(Chain);
4383 break;
4384 case NVPTXISD::Suld1DV2I16Zero:
4385 Opc = NVPTX::SULD_1D_V2I16_ZERO;
4386 Ops.push_back(TexHandle);
4387 Ops.push_back(N->getOperand(2));
4388 Ops.push_back(Chain);
4389 break;
4390 case NVPTXISD::Suld1DV2I32Zero:
4391 Opc = NVPTX::SULD_1D_V2I32_ZERO;
4392 Ops.push_back(TexHandle);
4393 Ops.push_back(N->getOperand(2));
4394 Ops.push_back(Chain);
4395 break;
4396 case NVPTXISD::Suld1DV2I64Zero:
4397 Opc = NVPTX::SULD_1D_V2I64_ZERO;
4398 Ops.push_back(TexHandle);
4399 Ops.push_back(N->getOperand(2));
4400 Ops.push_back(Chain);
4401 break;
4402 case NVPTXISD::Suld1DV4I8Zero:
4403 Opc = NVPTX::SULD_1D_V4I8_ZERO;
4404 Ops.push_back(TexHandle);
4405 Ops.push_back(N->getOperand(2));
4406 Ops.push_back(Chain);
4407 break;
4408 case NVPTXISD::Suld1DV4I16Zero:
4409 Opc = NVPTX::SULD_1D_V4I16_ZERO;
4410 Ops.push_back(TexHandle);
4411 Ops.push_back(N->getOperand(2));
4412 Ops.push_back(Chain);
4413 break;
4414 case NVPTXISD::Suld1DV4I32Zero:
4415 Opc = NVPTX::SULD_1D_V4I32_ZERO;
4416 Ops.push_back(TexHandle);
4417 Ops.push_back(N->getOperand(2));
4418 Ops.push_back(Chain);
4419 break;
4420 case NVPTXISD::Suld1DArrayI8Zero:
4421 Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO;
4422 Ops.push_back(TexHandle);
4423 Ops.push_back(N->getOperand(2));
4424 Ops.push_back(N->getOperand(3));
4425 Ops.push_back(Chain);
4426 break;
4427 case NVPTXISD::Suld1DArrayI16Zero:
4428 Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO;
4429 Ops.push_back(TexHandle);
4430 Ops.push_back(N->getOperand(2));
4431 Ops.push_back(N->getOperand(3));
4432 Ops.push_back(Chain);
4433 break;
4434 case NVPTXISD::Suld1DArrayI32Zero:
4435 Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO;
4436 Ops.push_back(TexHandle);
4437 Ops.push_back(N->getOperand(2));
4438 Ops.push_back(N->getOperand(3));
4439 Ops.push_back(Chain);
4440 break;
4441 case NVPTXISD::Suld1DArrayI64Zero:
4442 Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO;
4443 Ops.push_back(TexHandle);
4444 Ops.push_back(N->getOperand(2));
4445 Ops.push_back(N->getOperand(3));
4446 Ops.push_back(Chain);
4447 break;
4448 case NVPTXISD::Suld1DArrayV2I8Zero:
4449 Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO;
4450 Ops.push_back(TexHandle);
4451 Ops.push_back(N->getOperand(2));
4452 Ops.push_back(N->getOperand(3));
4453 Ops.push_back(Chain);
4454 break;
4455 case NVPTXISD::Suld1DArrayV2I16Zero:
4456 Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO;
4457 Ops.push_back(TexHandle);
4458 Ops.push_back(N->getOperand(2));
4459 Ops.push_back(N->getOperand(3));
4460 Ops.push_back(Chain);
4461 break;
4462 case NVPTXISD::Suld1DArrayV2I32Zero:
4463 Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO;
4464 Ops.push_back(TexHandle);
4465 Ops.push_back(N->getOperand(2));
4466 Ops.push_back(N->getOperand(3));
4467 Ops.push_back(Chain);
4468 break;
4469 case NVPTXISD::Suld1DArrayV2I64Zero:
4470 Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO;
4471 Ops.push_back(TexHandle);
4472 Ops.push_back(N->getOperand(2));
4473 Ops.push_back(N->getOperand(3));
4474 Ops.push_back(Chain);
4475 break;
4476 case NVPTXISD::Suld1DArrayV4I8Zero:
4477 Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO;
4478 Ops.push_back(TexHandle);
4479 Ops.push_back(N->getOperand(2));
4480 Ops.push_back(N->getOperand(3));
4481 Ops.push_back(Chain);
4482 break;
4483 case NVPTXISD::Suld1DArrayV4I16Zero:
4484 Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO;
4485 Ops.push_back(TexHandle);
4486 Ops.push_back(N->getOperand(2));
4487 Ops.push_back(N->getOperand(3));
4488 Ops.push_back(Chain);
4489 break;
4490 case NVPTXISD::Suld1DArrayV4I32Zero:
4491 Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO;
4492 Ops.push_back(TexHandle);
4493 Ops.push_back(N->getOperand(2));
4494 Ops.push_back(N->getOperand(3));
4495 Ops.push_back(Chain);
4496 break;
4497 case NVPTXISD::Suld2DI8Zero:
4498 Opc = NVPTX::SULD_2D_I8_ZERO;
4499 Ops.push_back(TexHandle);
4500 Ops.push_back(N->getOperand(2));
4501 Ops.push_back(N->getOperand(3));
4502 Ops.push_back(Chain);
4503 break;
4504 case NVPTXISD::Suld2DI16Zero:
4505 Opc = NVPTX::SULD_2D_I16_ZERO;
4506 Ops.push_back(TexHandle);
4507 Ops.push_back(N->getOperand(2));
4508 Ops.push_back(N->getOperand(3));
4509 Ops.push_back(Chain);
4510 break;
4511 case NVPTXISD::Suld2DI32Zero:
4512 Opc = NVPTX::SULD_2D_I32_ZERO;
4513 Ops.push_back(TexHandle);
4514 Ops.push_back(N->getOperand(2));
4515 Ops.push_back(N->getOperand(3));
4516 Ops.push_back(Chain);
4517 break;
4518 case NVPTXISD::Suld2DI64Zero:
4519 Opc = NVPTX::SULD_2D_I64_ZERO;
4520 Ops.push_back(TexHandle);
4521 Ops.push_back(N->getOperand(2));
4522 Ops.push_back(N->getOperand(3));
4523 Ops.push_back(Chain);
4524 break;
4525 case NVPTXISD::Suld2DV2I8Zero:
4526 Opc = NVPTX::SULD_2D_V2I8_ZERO;
4527 Ops.push_back(TexHandle);
4528 Ops.push_back(N->getOperand(2));
4529 Ops.push_back(N->getOperand(3));
4530 Ops.push_back(Chain);
4531 break;
4532 case NVPTXISD::Suld2DV2I16Zero:
4533 Opc = NVPTX::SULD_2D_V2I16_ZERO;
4534 Ops.push_back(TexHandle);
4535 Ops.push_back(N->getOperand(2));
4536 Ops.push_back(N->getOperand(3));
4537 Ops.push_back(Chain);
4538 break;
4539 case NVPTXISD::Suld2DV2I32Zero:
4540 Opc = NVPTX::SULD_2D_V2I32_ZERO;
4541 Ops.push_back(TexHandle);
4542 Ops.push_back(N->getOperand(2));
4543 Ops.push_back(N->getOperand(3));
4544 Ops.push_back(Chain);
4545 break;
4546 case NVPTXISD::Suld2DV2I64Zero:
4547 Opc = NVPTX::SULD_2D_V2I64_ZERO;
4548 Ops.push_back(TexHandle);
4549 Ops.push_back(N->getOperand(2));
4550 Ops.push_back(N->getOperand(3));
4551 Ops.push_back(Chain);
4552 break;
4553 case NVPTXISD::Suld2DV4I8Zero:
4554 Opc = NVPTX::SULD_2D_V4I8_ZERO;
4555 Ops.push_back(TexHandle);
4556 Ops.push_back(N->getOperand(2));
4557 Ops.push_back(N->getOperand(3));
4558 Ops.push_back(Chain);
4559 break;
4560 case NVPTXISD::Suld2DV4I16Zero:
4561 Opc = NVPTX::SULD_2D_V4I16_ZERO;
4562 Ops.push_back(TexHandle);
4563 Ops.push_back(N->getOperand(2));
4564 Ops.push_back(N->getOperand(3));
4565 Ops.push_back(Chain);
4566 break;
4567 case NVPTXISD::Suld2DV4I32Zero:
4568 Opc = NVPTX::SULD_2D_V4I32_ZERO;
4569 Ops.push_back(TexHandle);
4570 Ops.push_back(N->getOperand(2));
4571 Ops.push_back(N->getOperand(3));
4572 Ops.push_back(Chain);
4573 break;
4574 case NVPTXISD::Suld2DArrayI8Zero:
4575 Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO;
4576 Ops.push_back(TexHandle);
4577 Ops.push_back(N->getOperand(2));
4578 Ops.push_back(N->getOperand(3));
4579 Ops.push_back(N->getOperand(4));
4580 Ops.push_back(Chain);
4581 break;
4582 case NVPTXISD::Suld2DArrayI16Zero:
4583 Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO;
4584 Ops.push_back(TexHandle);
4585 Ops.push_back(N->getOperand(2));
4586 Ops.push_back(N->getOperand(3));
4587 Ops.push_back(N->getOperand(4));
4588 Ops.push_back(Chain);
4589 break;
4590 case NVPTXISD::Suld2DArrayI32Zero:
4591 Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO;
4592 Ops.push_back(TexHandle);
4593 Ops.push_back(N->getOperand(2));
4594 Ops.push_back(N->getOperand(3));
4595 Ops.push_back(N->getOperand(4));
4596 Ops.push_back(Chain);
4597 break;
4598 case NVPTXISD::Suld2DArrayI64Zero:
4599 Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO;
4600 Ops.push_back(TexHandle);
4601 Ops.push_back(N->getOperand(2));
4602 Ops.push_back(N->getOperand(3));
4603 Ops.push_back(N->getOperand(4));
4604 Ops.push_back(Chain);
4605 break;
4606 case NVPTXISD::Suld2DArrayV2I8Zero:
4607 Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO;
4608 Ops.push_back(TexHandle);
4609 Ops.push_back(N->getOperand(2));
4610 Ops.push_back(N->getOperand(3));
4611 Ops.push_back(N->getOperand(4));
4612 Ops.push_back(Chain);
4613 break;
4614 case NVPTXISD::Suld2DArrayV2I16Zero:
4615 Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO;
4616 Ops.push_back(TexHandle);
4617 Ops.push_back(N->getOperand(2));
4618 Ops.push_back(N->getOperand(3));
4619 Ops.push_back(N->getOperand(4));
4620 Ops.push_back(Chain);
4621 break;
4622 case NVPTXISD::Suld2DArrayV2I32Zero:
4623 Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO;
4624 Ops.push_back(TexHandle);
4625 Ops.push_back(N->getOperand(2));
4626 Ops.push_back(N->getOperand(3));
4627 Ops.push_back(N->getOperand(4));
4628 Ops.push_back(Chain);
4629 break;
4630 case NVPTXISD::Suld2DArrayV2I64Zero:
4631 Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO;
4632 Ops.push_back(TexHandle);
4633 Ops.push_back(N->getOperand(2));
4634 Ops.push_back(N->getOperand(3));
4635 Ops.push_back(N->getOperand(4));
4636 Ops.push_back(Chain);
4637 break;
4638 case NVPTXISD::Suld2DArrayV4I8Zero:
4639 Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO;
4640 Ops.push_back(TexHandle);
4641 Ops.push_back(N->getOperand(2));
4642 Ops.push_back(N->getOperand(3));
4643 Ops.push_back(N->getOperand(4));
4644 Ops.push_back(Chain);
4645 break;
4646 case NVPTXISD::Suld2DArrayV4I16Zero:
4647 Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO;
4648 Ops.push_back(TexHandle);
4649 Ops.push_back(N->getOperand(2));
4650 Ops.push_back(N->getOperand(3));
4651 Ops.push_back(N->getOperand(4));
4652 Ops.push_back(Chain);
4653 break;
4654 case NVPTXISD::Suld2DArrayV4I32Zero:
4655 Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO;
4656 Ops.push_back(TexHandle);
4657 Ops.push_back(N->getOperand(2));
4658 Ops.push_back(N->getOperand(3));
4659 Ops.push_back(N->getOperand(4));
4660 Ops.push_back(Chain);
4661 break;
4662 case NVPTXISD::Suld3DI8Zero:
4663 Opc = NVPTX::SULD_3D_I8_ZERO;
4664 Ops.push_back(TexHandle);
4665 Ops.push_back(N->getOperand(2));
4666 Ops.push_back(N->getOperand(3));
4667 Ops.push_back(N->getOperand(4));
4668 Ops.push_back(Chain);
4669 break;
4670 case NVPTXISD::Suld3DI16Zero:
4671 Opc = NVPTX::SULD_3D_I16_ZERO;
4672 Ops.push_back(TexHandle);
4673 Ops.push_back(N->getOperand(2));
4674 Ops.push_back(N->getOperand(3));
4675 Ops.push_back(N->getOperand(4));
4676 Ops.push_back(Chain);
4677 break;
4678 case NVPTXISD::Suld3DI32Zero:
4679 Opc = NVPTX::SULD_3D_I32_ZERO;
4680 Ops.push_back(TexHandle);
4681 Ops.push_back(N->getOperand(2));
4682 Ops.push_back(N->getOperand(3));
4683 Ops.push_back(N->getOperand(4));
4684 Ops.push_back(Chain);
4685 break;
4686 case NVPTXISD::Suld3DI64Zero:
4687 Opc = NVPTX::SULD_3D_I64_ZERO;
4688 Ops.push_back(TexHandle);
4689 Ops.push_back(N->getOperand(2));
4690 Ops.push_back(N->getOperand(3));
4691 Ops.push_back(N->getOperand(4));
4692 Ops.push_back(Chain);
4693 break;
4694 case NVPTXISD::Suld3DV2I8Zero:
4695 Opc = NVPTX::SULD_3D_V2I8_ZERO;
4696 Ops.push_back(TexHandle);
4697 Ops.push_back(N->getOperand(2));
4698 Ops.push_back(N->getOperand(3));
4699 Ops.push_back(N->getOperand(4));
4700 Ops.push_back(Chain);
4701 break;
4702 case NVPTXISD::Suld3DV2I16Zero:
4703 Opc = NVPTX::SULD_3D_V2I16_ZERO;
4704 Ops.push_back(TexHandle);
4705 Ops.push_back(N->getOperand(2));
4706 Ops.push_back(N->getOperand(3));
4707 Ops.push_back(N->getOperand(4));
4708 Ops.push_back(Chain);
4709 break;
4710 case NVPTXISD::Suld3DV2I32Zero:
4711 Opc = NVPTX::SULD_3D_V2I32_ZERO;
4712 Ops.push_back(TexHandle);
4713 Ops.push_back(N->getOperand(2));
4714 Ops.push_back(N->getOperand(3));
4715 Ops.push_back(N->getOperand(4));
4716 Ops.push_back(Chain);
4717 break;
4718 case NVPTXISD::Suld3DV2I64Zero:
4719 Opc = NVPTX::SULD_3D_V2I64_ZERO;
4720 Ops.push_back(TexHandle);
4721 Ops.push_back(N->getOperand(2));
4722 Ops.push_back(N->getOperand(3));
4723 Ops.push_back(N->getOperand(4));
4724 Ops.push_back(Chain);
4725 break;
4726 case NVPTXISD::Suld3DV4I8Zero:
4727 Opc = NVPTX::SULD_3D_V4I8_ZERO;
4728 Ops.push_back(TexHandle);
4729 Ops.push_back(N->getOperand(2));
4730 Ops.push_back(N->getOperand(3));
4731 Ops.push_back(N->getOperand(4));
4732 Ops.push_back(Chain);
4733 break;
4734 case NVPTXISD::Suld3DV4I16Zero:
4735 Opc = NVPTX::SULD_3D_V4I16_ZERO;
4736 Ops.push_back(TexHandle);
4737 Ops.push_back(N->getOperand(2));
4738 Ops.push_back(N->getOperand(3));
4739 Ops.push_back(N->getOperand(4));
4740 Ops.push_back(Chain);
4741 break;
4742 case NVPTXISD::Suld3DV4I32Zero:
4743 Opc = NVPTX::SULD_3D_V4I32_ZERO;
4744 Ops.push_back(TexHandle);
4745 Ops.push_back(N->getOperand(2));
4746 Ops.push_back(N->getOperand(3));
4747 Ops.push_back(N->getOperand(4));
4748 Ops.push_back(Chain);
4749 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4750 }
4751 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
4752 return Ret;
4753}
4754
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame^]4755
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4756/// SelectBFE - Look for instruction sequences that can be made more efficient
4757/// by using the 'bfe' (bit-field extract) PTX instruction
4758SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
4759 SDValue LHS = N->getOperand(0);
4760 SDValue RHS = N->getOperand(1);
4761 SDValue Len;
4762 SDValue Start;
4763 SDValue Val;
4764 bool IsSigned = false;
4765
4766 if (N->getOpcode() == ISD::AND) {
4767 // Canonicalize the operands
4768 // We want 'and %val, %mask'
4769 if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) {
4770 std::swap(LHS, RHS);
4771 }
4772
4773 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS);
4774 if (!Mask) {
4775 // We need a constant mask on the RHS of the AND
4776 return NULL;
4777 }
4778
4779 // Extract the mask bits
4780 uint64_t MaskVal = Mask->getZExtValue();
4781 if (!isMask_64(MaskVal)) {
4782 // We *could* handle shifted masks here, but doing so would require an
4783 // 'and' operation to fix up the low-order bits so we would trade
4784 // shr+and for bfe+and, which has the same throughput
4785 return NULL;
4786 }
4787
4788 // How many bits are in our mask?
4789 uint64_t NumBits = CountTrailingOnes_64(MaskVal);
4790 Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
4791
4792 if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) {
4793 // We have a 'srl/and' pair, extract the effective start bit and length
4794 Val = LHS.getNode()->getOperand(0);
4795 Start = LHS.getNode()->getOperand(1);
4796 ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start);
4797 if (StartConst) {
4798 uint64_t StartVal = StartConst->getZExtValue();
4799 // How many "good" bits do we have left? "good" is defined here as bits
4800 // that exist in the original value, not shifted in.
4801 uint64_t GoodBits = Start.getValueType().getSizeInBits() - StartVal;
4802 if (NumBits > GoodBits) {
4803 // Do not handle the case where bits have been shifted in. In theory
4804 // we could handle this, but the cost is likely higher than just
4805 // emitting the srl/and pair.
4806 return NULL;
4807 }
4808 Start = CurDAG->getTargetConstant(StartVal, MVT::i32);
4809 } else {
4810 // Do not handle the case where the shift amount (can be zero if no srl
4811 // was found) is not constant. We could handle this case, but it would
4812 // require run-time logic that would be more expensive than just
4813 // emitting the srl/and pair.
4814 return NULL;
4815 }
4816 } else {
4817 // Do not handle the case where the LHS of the and is not a shift. While
4818 // it would be trivial to handle this case, it would just transform
4819 // 'and' -> 'bfe', but 'and' has higher-throughput.
4820 return NULL;
4821 }
4822 } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) {
4823 if (LHS->getOpcode() == ISD::AND) {
4824 ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS);
4825 if (!ShiftCnst) {
4826 // Shift amount must be constant
4827 return NULL;
4828 }
4829
4830 uint64_t ShiftAmt = ShiftCnst->getZExtValue();
4831
4832 SDValue AndLHS = LHS->getOperand(0);
4833 SDValue AndRHS = LHS->getOperand(1);
4834
4835 // Canonicalize the AND to have the mask on the RHS
4836 if (isa<ConstantSDNode>(AndLHS)) {
4837 std::swap(AndLHS, AndRHS);
4838 }
4839
4840 ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS);
4841 if (!MaskCnst) {
4842 // Mask must be constant
4843 return NULL;
4844 }
4845
4846 uint64_t MaskVal = MaskCnst->getZExtValue();
4847 uint64_t NumZeros;
4848 uint64_t NumBits;
4849 if (isMask_64(MaskVal)) {
4850 NumZeros = 0;
4851 // The number of bits in the result bitfield will be the number of
4852 // trailing ones (the AND) minus the number of bits we shift off
4853 NumBits = CountTrailingOnes_64(MaskVal) - ShiftAmt;
4854 } else if (isShiftedMask_64(MaskVal)) {
4855 NumZeros = countTrailingZeros(MaskVal);
4856 unsigned NumOnes = CountTrailingOnes_64(MaskVal >> NumZeros);
4857 // The number of bits in the result bitfield will be the number of
4858 // trailing zeros plus the number of set bits in the mask minus the
4859 // number of bits we shift off
4860 NumBits = NumZeros + NumOnes - ShiftAmt;
4861 } else {
4862 // This is not a mask we can handle
4863 return NULL;
4864 }
4865
4866 if (ShiftAmt < NumZeros) {
4867 // Handling this case would require extra logic that would make this
4868 // transformation non-profitable
4869 return NULL;
4870 }
4871
4872 Val = AndLHS;
4873 Start = CurDAG->getTargetConstant(ShiftAmt, MVT::i32);
4874 Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
4875 } else if (LHS->getOpcode() == ISD::SHL) {
4876 // Here, we have a pattern like:
4877 //
4878 // (sra (shl val, NN), MM)
4879 // or
4880 // (srl (shl val, NN), MM)
4881 //
4882 // If MM >= NN, we can efficiently optimize this with bfe
4883 Val = LHS->getOperand(0);
4884
4885 SDValue ShlRHS = LHS->getOperand(1);
4886 ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS);
4887 if (!ShlCnst) {
4888 // Shift amount must be constant
4889 return NULL;
4890 }
4891 uint64_t InnerShiftAmt = ShlCnst->getZExtValue();
4892
4893 SDValue ShrRHS = RHS;
4894 ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS);
4895 if (!ShrCnst) {
4896 // Shift amount must be constant
4897 return NULL;
4898 }
4899 uint64_t OuterShiftAmt = ShrCnst->getZExtValue();
4900
4901 // To avoid extra codegen and be profitable, we need Outer >= Inner
4902 if (OuterShiftAmt < InnerShiftAmt) {
4903 return NULL;
4904 }
4905
4906 // If the outer shift is more than the type size, we have no bitfield to
4907 // extract (since we also check that the inner shift is <= the outer shift
4908 // then this also implies that the inner shift is < the type size)
4909 if (OuterShiftAmt >= Val.getValueType().getSizeInBits()) {
4910 return NULL;
4911 }
4912
4913 Start =
4914 CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, MVT::i32);
4915 Len =
4916 CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() -
4917 OuterShiftAmt, MVT::i32);
4918
4919 if (N->getOpcode() == ISD::SRA) {
4920 // If we have a arithmetic right shift, we need to use the signed bfe
4921 // variant
4922 IsSigned = true;
4923 }
4924 } else {
4925 // No can do...
4926 return NULL;
4927 }
4928 } else {
4929 // No can do...
4930 return NULL;
4931 }
4932
4933
4934 unsigned Opc;
4935 // For the BFE operations we form here from "and" and "srl", always use the
4936 // unsigned variants.
4937 if (Val.getValueType() == MVT::i32) {
4938 if (IsSigned) {
4939 Opc = NVPTX::BFE_S32rii;
4940 } else {
4941 Opc = NVPTX::BFE_U32rii;
4942 }
4943 } else if (Val.getValueType() == MVT::i64) {
4944 if (IsSigned) {
4945 Opc = NVPTX::BFE_S64rii;
4946 } else {
4947 Opc = NVPTX::BFE_U64rii;
4948 }
4949 } else {
4950 // We cannot handle this type
4951 return NULL;
4952 }
4953
4954 SDValue Ops[] = {
4955 Val, Start, Len
4956 };
4957
4958 SDNode *Ret =
4959 CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
4960
4961 return Ret;
4962}
4963
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4964// SelectDirectAddr - Match a direct address for DAG.
4965// A direct address could be a globaladdress or externalsymbol.
4966bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
4967 // Return true if TGA or ES.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4968 if (N.getOpcode() == ISD::TargetGlobalAddress ||
4969 N.getOpcode() == ISD::TargetExternalSymbol) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4970 Address = N;
4971 return true;
4972 }
4973 if (N.getOpcode() == NVPTXISD::Wrapper) {
4974 Address = N.getOperand(0);
4975 return true;
4976 }
4977 if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
4978 unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue();
4979 if (IID == Intrinsic::nvvm_ptr_gen_to_param)
4980 if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam)
4981 return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address));
4982 }
4983 return false;
4984}
4985
4986// symbol+offset
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4987bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
4988 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4989 if (Addr.getOpcode() == ISD::ADD) {
4990 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4991 SDValue base = Addr.getOperand(0);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4992 if (SelectDirectAddr(base, Base)) {
4993 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
4994 return true;
4995 }
4996 }
4997 }
4998 return false;
4999}
5000
5001// symbol+offset
5002bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
5003 SDValue &Base, SDValue &Offset) {
5004 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
5005}
5006
5007// symbol+offset
5008bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
5009 SDValue &Base, SDValue &Offset) {
5010 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
5011}
5012
5013// register+offset
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5014bool NVPTXDAGToDAGISel::SelectADDRri_imp(
5015 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5016 if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
5017 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
5018 Offset = CurDAG->getTargetConstant(0, mvt);
5019 return true;
5020 }
5021 if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
5022 Addr.getOpcode() == ISD::TargetGlobalAddress)
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5023 return false; // direct calls.
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5024
5025 if (Addr.getOpcode() == ISD::ADD) {
5026 if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
5027 return false;
5028 }
5029 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
5030 if (FrameIndexSDNode *FIN =
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5031 dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5032 // Constant offset from frame ref.
5033 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
5034 else
5035 Base = Addr.getOperand(0);
5036 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
5037 return true;
5038 }
5039 }
5040 return false;
5041}
5042
5043// register+offset
5044bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
5045 SDValue &Base, SDValue &Offset) {
5046 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
5047}
5048
5049// register+offset
5050bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
5051 SDValue &Base, SDValue &Offset) {
5052 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
5053}
5054
5055bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
5056 unsigned int spN) const {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]5057 const Value *Src = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5058 // Even though MemIntrinsicSDNode is a subclas of MemSDNode,
5059 // the classof() for MemSDNode does not include MemIntrinsicSDNode
5060 // (See SelectionDAGNodes.h). So we need to check for both.
5061 if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
Nick Lewyckyaad475b2014-04-15 07:22:52 +0000[diff] [blame]5062 if (spN == 0 && mN->getMemOperand()->getPseudoValue())
5063 return true;
5064 Src = mN->getMemOperand()->getValue();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5065 } else if (MemSDNode *mN = dyn_cast<MemIntrinsicSDNode>(N)) {
Nick Lewyckyaad475b2014-04-15 07:22:52 +0000[diff] [blame]5066 if (spN == 0 && mN->getMemOperand()->getPseudoValue())
5067 return true;
5068 Src = mN->getMemOperand()->getValue();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5069 }
5070 if (!Src)
5071 return false;
5072 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
5073 return (PT->getAddressSpace() == spN);
5074 return false;
5075}
5076
5077/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
5078/// inline asm expressions.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5079bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
5080 const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5081 SDValue Op0, Op1;
5082 switch (ConstraintCode) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5083 default:
5084 return true;
5085 case 'm': // memory
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5086 if (SelectDirectAddr(Op, Op0)) {
5087 OutOps.push_back(Op0);
5088 OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
5089 return false;
5090 }
5091 if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
5092 OutOps.push_back(Op0);
5093 OutOps.push_back(Op1);
5094 return false;
5095 }
5096 break;
5097 }
5098 return true;
5099}