blob: 775df19be162e9d06f5e86c970b44611b7d1cb71 [file] [log] [blame]
Justin Holewinskiae556d32012-05-04 20:18:50 +00001//===- NVPTXVector.td - NVPTX Vector Specific Instruction defs -*- tblgen-*-==//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9
10//-----------------------------------
11// Vector Specific
12//-----------------------------------
13
14//
15// All vector instructions derive from NVPTXVecInst
16//
17
18class NVPTXVecInst<dag outs, dag ins, string asmstr, list<dag> pattern,
19 NVPTXInst sInst=NOP>
20 : NVPTXInst<outs, ins, asmstr, pattern> {
21 NVPTXInst scalarInst=sInst;
22}
23
24let isAsCheapAsAMove=1, VecInstType=isVecExtract.Value in {
25// Extract v2i16
26def V2i16Extract : NVPTXVecInst<(outs Int16Regs:$dst),
27 (ins V2I16Regs:$src, i8imm:$c),
28 "mov.u16 \t$dst, $src${c:vecelem};",
29 [(set Int16Regs:$dst, (vector_extract
30 (v2i16 V2I16Regs:$src), imm:$c))],
31 IMOV16rr>;
32
33// Extract v4i16
34def V4i16Extract : NVPTXVecInst<(outs Int16Regs:$dst),
35 (ins V4I16Regs:$src, i8imm:$c),
36 "mov.u16 \t$dst, $src${c:vecelem};",
37 [(set Int16Regs:$dst, (vector_extract
38 (v4i16 V4I16Regs:$src), imm:$c))],
39 IMOV16rr>;
40
41// Extract v2i8
42def V2i8Extract : NVPTXVecInst<(outs Int8Regs:$dst),
43 (ins V2I8Regs:$src, i8imm:$c),
44 "mov.u16 \t$dst, $src${c:vecelem};",
45 [(set Int8Regs:$dst, (vector_extract
46 (v2i8 V2I8Regs:$src), imm:$c))],
47 IMOV8rr>;
48
49// Extract v4i8
50def V4i8Extract : NVPTXVecInst<(outs Int8Regs:$dst),
51 (ins V4I8Regs:$src, i8imm:$c),
52 "mov.u16 \t$dst, $src${c:vecelem};",
53 [(set Int8Regs:$dst, (vector_extract
54 (v4i8 V4I8Regs:$src), imm:$c))],
55 IMOV8rr>;
56
57// Extract v2i32
58def V2i32Extract : NVPTXVecInst<(outs Int32Regs:$dst),
59 (ins V2I32Regs:$src, i8imm:$c),
60 "mov.u32 \t$dst, $src${c:vecelem};",
61 [(set Int32Regs:$dst, (vector_extract
62 (v2i32 V2I32Regs:$src), imm:$c))],
63 IMOV32rr>;
64
65// Extract v2f32
66def V2f32Extract : NVPTXVecInst<(outs Float32Regs:$dst),
67 (ins V2F32Regs:$src, i8imm:$c),
68 "mov.f32 \t$dst, $src${c:vecelem};",
69 [(set Float32Regs:$dst, (vector_extract
70 (v2f32 V2F32Regs:$src), imm:$c))],
71 FMOV32rr>;
72
73// Extract v2i64
74def V2i64Extract : NVPTXVecInst<(outs Int64Regs:$dst),
75 (ins V2I64Regs:$src, i8imm:$c),
76 "mov.u64 \t$dst, $src${c:vecelem};",
77 [(set Int64Regs:$dst, (vector_extract
78 (v2i64 V2I64Regs:$src), imm:$c))],
79 IMOV64rr>;
80
81// Extract v2f64
82def V2f64Extract : NVPTXVecInst<(outs Float64Regs:$dst),
83 (ins V2F64Regs:$src, i8imm:$c),
84 "mov.f64 \t$dst, $src${c:vecelem};",
85 [(set Float64Regs:$dst, (vector_extract
86 (v2f64 V2F64Regs:$src), imm:$c))],
87 FMOV64rr>;
88
89// Extract v4i32
90def V4i32Extract : NVPTXVecInst<(outs Int32Regs:$dst),
91 (ins V4I32Regs:$src, i8imm:$c),
92 "mov.u32 \t$dst, $src${c:vecelem};",
93 [(set Int32Regs:$dst, (vector_extract
94 (v4i32 V4I32Regs:$src), imm:$c))],
95 IMOV32rr>;
96
97// Extract v4f32
98def V4f32Extract : NVPTXVecInst<(outs Float32Regs:$dst),
99 (ins V4F32Regs:$src, i8imm:$c),
100 "mov.f32 \t$dst, $src${c:vecelem};",
101 [(set Float32Regs:$dst, (vector_extract
102 (v4f32 V4F32Regs:$src), imm:$c))],
103 FMOV32rr>;
104}
105
106let isAsCheapAsAMove=1, VecInstType=isVecInsert.Value in {
107// Insert v2i8
108def V2i8Insert : NVPTXVecInst<(outs V2I8Regs:$dst),
109 (ins V2I8Regs:$src, Int8Regs:$val, i8imm:$c),
110 "mov.v2.u16 \t${dst:vecfull}, ${src:vecfull};"
111 "\n\tmov.u16 \t$dst${c:vecelem}, $val;",
112 [(set V2I8Regs:$dst,
113 (vector_insert V2I8Regs:$src, Int8Regs:$val, imm:$c))],
114 IMOV8rr>;
115
116// Insert v4i8
117def V4i8Insert : NVPTXVecInst<(outs V4I8Regs:$dst),
118 (ins V4I8Regs:$src, Int8Regs:$val, i8imm:$c),
119 "mov.v4.u16 \t${dst:vecfull}, ${src:vecfull};"
120 "\n\tmov.u16 \t$dst${c:vecelem}, $val;",
121 [(set V4I8Regs:$dst,
122 (vector_insert V4I8Regs:$src, Int8Regs:$val, imm:$c))],
123 IMOV8rr>;
124
125// Insert v2i16
126def V2i16Insert : NVPTXVecInst<(outs V2I16Regs:$dst),
127 (ins V2I16Regs:$src, Int16Regs:$val, i8imm:$c),
128 "mov.v2.u16 \t${dst:vecfull}, ${src:vecfull};"
129 "\n\tmov.u16 \t$dst${c:vecelem}, $val;",
130 [(set V2I16Regs:$dst,
131 (vector_insert V2I16Regs:$src, Int16Regs:$val, imm:$c))],
132 IMOV16rr>;
133
134// Insert v4i16
135def V4i16Insert : NVPTXVecInst<(outs V4I16Regs:$dst),
136 (ins V4I16Regs:$src, Int16Regs:$val, i8imm:$c),
137 "mov.v4.u16 \t${dst:vecfull}, ${src:vecfull};"
138 "\n\tmov.u16 \t$dst${c:vecelem}, $val;",
139 [(set V4I16Regs:$dst,
140 (vector_insert V4I16Regs:$src, Int16Regs:$val, imm:$c))],
141 IMOV16rr>;
142
143// Insert v2i32
144def V2i32Insert : NVPTXVecInst<(outs V2I32Regs:$dst),
145 (ins V2I32Regs:$src, Int32Regs:$val, i8imm:$c),
146 "mov.v2.u32 \t${dst:vecfull}, ${src:vecfull};"
147 "\n\tmov.u32 \t$dst${c:vecelem}, $val;",
148 [(set V2I32Regs:$dst,
149 (vector_insert V2I32Regs:$src, Int32Regs:$val, imm:$c))],
150 IMOV32rr>;
151
152// Insert v2f32
153def V2f32Insert : NVPTXVecInst<(outs V2F32Regs:$dst),
154 (ins V2F32Regs:$src, Float32Regs:$val, i8imm:$c),
155 "mov.v2.f32 \t${dst:vecfull}, ${src:vecfull};"
156 "\n\tmov.f32 \t$dst${c:vecelem}, $val;",
157 [(set V2F32Regs:$dst,
158 (vector_insert V2F32Regs:$src, Float32Regs:$val, imm:$c))],
159 FMOV32rr>;
160
161// Insert v2i64
162def V2i64Insert : NVPTXVecInst<(outs V2I64Regs:$dst),
163 (ins V2I64Regs:$src, Int64Regs:$val, i8imm:$c),
164 "mov.v2.u64 \t${dst:vecfull}, ${src:vecfull};"
165 "\n\tmov.u64 \t$dst${c:vecelem}, $val;",
166 [(set V2I64Regs:$dst,
167 (vector_insert V2I64Regs:$src, Int64Regs:$val, imm:$c))],
168 IMOV64rr>;
169
170// Insert v2f64
171def V2f64Insert : NVPTXVecInst<(outs V2F64Regs:$dst),
172 (ins V2F64Regs:$src, Float64Regs:$val, i8imm:$c),
173 "mov.v2.f64 \t${dst:vecfull}, ${src:vecfull};"
174 "\n\tmov.f64 \t$dst${c:vecelem}, $val;",
175 [(set V2F64Regs:$dst,
176 (vector_insert V2F64Regs:$src, Float64Regs:$val, imm:$c))],
177 FMOV64rr>;
178
179// Insert v4i32
180def V4i32Insert : NVPTXVecInst<(outs V4I32Regs:$dst),
181 (ins V4I32Regs:$src, Int32Regs:$val, i8imm:$c),
182 "mov.v4.u32 \t${dst:vecfull}, ${src:vecfull};"
183 "\n\tmov.u32 \t$dst${c:vecelem}, $val;",
184 [(set V4I32Regs:$dst,
185 (vector_insert V4I32Regs:$src, Int32Regs:$val, imm:$c))],
186 IMOV32rr>;
187
188// Insert v4f32
189def V4f32Insert : NVPTXVecInst<(outs V4F32Regs:$dst),
190 (ins V4F32Regs:$src, Float32Regs:$val, i8imm:$c),
191 "mov.v4.f32 \t${dst:vecfull}, ${src:vecfull};"
192 "\n\tmov.f32 \t$dst${c:vecelem}, $val;",
193 [(set V4F32Regs:$dst,
194 (vector_insert V4F32Regs:$src, Float32Regs:$val, imm:$c))],
195 FMOV32rr>;
196}
197
198class BinOpAsmString<string c> {
199 string s = c;
200}
201
202class V4AsmStr<string opcode> : BinOpAsmString<
203 !strconcat(!strconcat(!strconcat(!strconcat(
204 !strconcat(!strconcat(!strconcat(
205 opcode, " \t${dst}_0, ${a}_0, ${b}_0;\n\t"),
206 opcode), " \t${dst}_1, ${a}_1, ${b}_1;\n\t"),
207 opcode), " \t${dst}_2, ${a}_2, ${b}_2;\n\t"),
208 opcode), " \t${dst}_3, ${a}_3, ${b}_3;")>;
209
210class V2AsmStr<string opcode> : BinOpAsmString<
211 !strconcat(!strconcat(!strconcat(
212 opcode, " \t${dst}_0, ${a}_0, ${b}_0;\n\t"),
213 opcode), " \t${dst}_1, ${a}_1, ${b}_1;")>;
214
215class V4MADStr<string opcode> : BinOpAsmString<
216 !strconcat(!strconcat(!strconcat(!strconcat(
217 !strconcat(!strconcat(!strconcat(
218 opcode, " \t${dst}_0, ${a}_0, ${b}_0, ${c}_0;\n\t"),
219 opcode), " \t${dst}_1, ${a}_1, ${b}_1, ${c}_1;\n\t"),
220 opcode), " \t${dst}_2, ${a}_2, ${b}_2, ${c}_2;\n\t"),
221 opcode), " \t${dst}_3, ${a}_3, ${b}_3, ${c}_3;")>;
222
223class V2MADStr<string opcode> : BinOpAsmString<
224 !strconcat(!strconcat(!strconcat(
225 opcode, " \t${dst}_0, ${a}_0, ${b}_0, ${c}_0;\n\t"),
226 opcode), " \t${dst}_1, ${a}_1, ${b}_1, ${c}_1;")>;
227
228class V4UnaryStr<string opcode> : BinOpAsmString<
229 !strconcat(!strconcat(!strconcat(!strconcat(
230 !strconcat(!strconcat(!strconcat(
231 opcode, " \t${dst}_0, ${a}_0;\n\t"),
232 opcode), " \t${dst}_1, ${a}_1;\n\t"),
233 opcode), " \t${dst}_2, ${a}_2;\n\t"),
234 opcode), " \t${dst}_3, ${a}_3;")>;
235
236class V2UnaryStr<string opcode> : BinOpAsmString<
237 !strconcat(!strconcat(!strconcat(
238 opcode, " \t${dst}_0, ${a}_0;\n\t"),
239 opcode), " \t${dst}_1, ${a}_1;")>;
240
241class VecBinaryOp<BinOpAsmString asmstr, SDNode OpNode, NVPTXRegClass regclass,
242 NVPTXInst sInst=NOP> :
243 NVPTXVecInst<(outs regclass:$dst), (ins regclass:$a, regclass:$b),
244 asmstr.s,
245 [(set regclass:$dst, (OpNode regclass:$a, regclass:$b))],
246 sInst>;
247
248class VecShiftOp<BinOpAsmString asmstr, SDNode OpNode, NVPTXRegClass regclass1,
249 NVPTXRegClass regclass2, NVPTXInst sInst=NOP> :
250 NVPTXVecInst<(outs regclass1:$dst), (ins regclass1:$a, regclass2:$b),
251 asmstr.s,
252 [(set regclass1:$dst, (OpNode regclass1:$a, regclass2:$b))],
253 sInst>;
254
255class VecUnaryOp<BinOpAsmString asmstr, PatFrag OpNode, NVPTXRegClass regclass,
256 NVPTXInst sInst=NOP> :
257 NVPTXVecInst<(outs regclass:$dst), (ins regclass:$a),
258 asmstr.s,
259 [(set regclass:$dst, (OpNode regclass:$a))], sInst>;
260
261multiclass IntBinVOp<string asmstr, SDNode OpNode,
262 NVPTXInst i64op=NOP, NVPTXInst i32op=NOP, NVPTXInst
263 i16op=NOP, NVPTXInst i8op=NOP> {
264 def V2I64 : VecBinaryOp<V2AsmStr<!strconcat(asmstr, "64")>, OpNode, V2I64Regs,
265 i64op>;
266 def V4I32 : VecBinaryOp<V4AsmStr<!strconcat(asmstr, "32")>, OpNode, V4I32Regs,
267 i32op>;
268 def V2I32 : VecBinaryOp<V2AsmStr<!strconcat(asmstr, "32")>, OpNode, V2I32Regs,
269 i32op>;
270 def V4I16 : VecBinaryOp<V4AsmStr<!strconcat(asmstr, "16")>, OpNode, V4I16Regs,
271 i16op>;
272 def V2I16 : VecBinaryOp<V2AsmStr<!strconcat(asmstr, "16")>, OpNode, V2I16Regs,
273 i16op>;
274 def V4I8 : VecBinaryOp<V4AsmStr<!strconcat(asmstr, "16")>, OpNode, V4I8Regs,
275 i8op>;
276 def V2I8 : VecBinaryOp<V2AsmStr<!strconcat(asmstr, "16")>, OpNode, V2I8Regs,
277 i8op>;
278}
279
280multiclass FloatBinVOp<string asmstr, SDNode OpNode,
281 NVPTXInst f64=NOP, NVPTXInst f32=NOP,
282 NVPTXInst f32_ftz=NOP> {
283 def V2F64 : VecBinaryOp<V2AsmStr<!strconcat(asmstr, "f64")>, OpNode,
284 V2F64Regs, f64>;
285 def V4F32_ftz : VecBinaryOp<V4AsmStr<!strconcat(asmstr, "ftz.f32")>, OpNode,
286 V4F32Regs, f32_ftz>, Requires<[doF32FTZ]>;
287 def V2F32_ftz : VecBinaryOp<V2AsmStr<!strconcat(asmstr, "ftz.f32")>, OpNode,
288 V2F32Regs, f32_ftz>, Requires<[doF32FTZ]>;
289 def V4F32 : VecBinaryOp<V4AsmStr<!strconcat(asmstr, "f32")>, OpNode,
290 V4F32Regs, f32>;
291 def V2F32 : VecBinaryOp<V2AsmStr<!strconcat(asmstr, "f32")>, OpNode,
292 V2F32Regs, f32>;
293}
294
295multiclass IntUnaryVOp<string asmstr, PatFrag OpNode,
296 NVPTXInst i64op=NOP, NVPTXInst i32op=NOP,
297 NVPTXInst i16op=NOP, NVPTXInst i8op=NOP> {
298 def V2I64 : VecUnaryOp<V2UnaryStr<!strconcat(asmstr, "64")>, OpNode,
299 V2I64Regs, i64op>;
300 def V4I32 : VecUnaryOp<V4UnaryStr<!strconcat(asmstr, "32")>, OpNode,
301 V4I32Regs, i32op>;
302 def V2I32 : VecUnaryOp<V2UnaryStr<!strconcat(asmstr, "32")>, OpNode,
303 V2I32Regs, i32op>;
304 def V4I16 : VecUnaryOp<V4UnaryStr<!strconcat(asmstr, "16")>, OpNode,
305 V4I16Regs, i16op>;
306 def V2I16 : VecUnaryOp<V2UnaryStr<!strconcat(asmstr, "16")>, OpNode,
307 V2I16Regs, i16op>;
308 def V4I8 : VecUnaryOp<V4UnaryStr<!strconcat(asmstr, "16")>, OpNode,
309 V4I8Regs, i8op>;
310 def V2I8 : VecUnaryOp<V2UnaryStr<!strconcat(asmstr, "16")>, OpNode,
311 V2I8Regs, i8op>;
312}
313
314
315// Integer Arithmetic
316let VecInstType=isVecOther.Value in {
317defm VAdd : IntBinVOp<"add.s", add, ADDi64rr, ADDi32rr, ADDi16rr, ADDi8rr>;
318defm VSub : IntBinVOp<"sub.s", sub, SUBi64rr, SUBi32rr, SUBi16rr, SUBi8rr>;
319
320def AddCCV4I32 : VecBinaryOp<V4AsmStr<"add.cc.s32">, addc, V4I32Regs,
321 ADDCCi32rr>;
322def AddCCV2I32 : VecBinaryOp<V2AsmStr<"add.cc.s32">, addc, V2I32Regs,
323 ADDCCi32rr>;
324def SubCCV4I32 : VecBinaryOp<V4AsmStr<"sub.cc.s32">, subc, V4I32Regs,
325 SUBCCi32rr>;
326def SubCCV2I32 : VecBinaryOp<V2AsmStr<"sub.cc.s32">, subc, V2I32Regs,
327 SUBCCi32rr>;
328def AddCCCV4I32 : VecBinaryOp<V4AsmStr<"addc.cc.s32">, adde, V4I32Regs,
329 ADDCCCi32rr>;
330def AddCCCV2I32 : VecBinaryOp<V2AsmStr<"addc.cc.s32">, adde, V2I32Regs,
331 ADDCCCi32rr>;
332def SubCCCV4I32 : VecBinaryOp<V4AsmStr<"subc.cc.s32">, sube, V4I32Regs,
333 SUBCCCi32rr>;
334def SubCCCV2I32 : VecBinaryOp<V2AsmStr<"subc.cc.s32">, sube, V2I32Regs,
335 SUBCCCi32rr>;
336
337def ShiftLV2I64 : VecShiftOp<V2AsmStr<"shl.b64">, shl, V2I64Regs, V2I32Regs,
338 SHLi64rr>;
339def ShiftLV2I32 : VecShiftOp<V2AsmStr<"shl.b32">, shl, V2I32Regs, V2I32Regs,
340 SHLi32rr>;
341def ShiftLV4I32 : VecShiftOp<V4AsmStr<"shl.b32">, shl, V4I32Regs, V4I32Regs,
342 SHLi32rr>;
343def ShiftLV2I16 : VecShiftOp<V2AsmStr<"shl.b16">, shl, V2I16Regs, V2I32Regs,
344 SHLi16rr>;
345def ShiftLV4I16 : VecShiftOp<V4AsmStr<"shl.b16">, shl, V4I16Regs, V4I32Regs,
346 SHLi16rr>;
347def ShiftLV2I8 : VecShiftOp<V2AsmStr<"shl.b16">, shl, V2I8Regs, V2I32Regs,
348 SHLi8rr>;
349def ShiftLV4I8 : VecShiftOp<V4AsmStr<"shl.b16">, shl, V4I8Regs, V4I32Regs,
350 SHLi8rr>;
351}
352
353// cvt to v*i32, helpers for shift
354class CVTtoVeci32<NVPTXRegClass inclass, NVPTXRegClass outclass, string asmstr,
355 NVPTXInst sInst=NOP> :
356 NVPTXVecInst<(outs outclass:$d), (ins inclass:$s), asmstr, [], sInst>;
357
358class VecCVTStrHelper<string op, string dest, string src> {
359 string s=!strconcat(op, !strconcat("\t",
360 !strconcat(dest, !strconcat(", ", !strconcat(src, ";")))));
361}
362
363class Vec2CVTStr<string op> {
364 string s=!strconcat(VecCVTStrHelper<op, "${d}_0", "${s}_0">.s,
365 !strconcat("\n\t", VecCVTStrHelper<op, "${d}_1", "${s}_1">.s));
366}
367
368class Vec4CVTStr<string op> {
369 string s=!strconcat(VecCVTStrHelper<op, "${d}_0", "${s}_0">.s,
370 !strconcat("\n\t",
371 !strconcat(VecCVTStrHelper<op, "${d}_1", "${s}_1">.s,
372 !strconcat("\n\t",
373 !strconcat(VecCVTStrHelper<op, "${d}_2", "${s}_2">.s,
374 !strconcat("\n\t", VecCVTStrHelper<op, "${d}_3", "${s}_3">.s))))));
375}
376
377let VecInstType=isVecOther.Value in {
378def CVTv2i8tov2i32 : CVTtoVeci32<V2I8Regs, V2I32Regs,
379 Vec2CVTStr<"cvt.u32.u16">.s, Zint_extendext8to32>;
380def CVTv2i16tov2i32 : CVTtoVeci32<V2I16Regs, V2I32Regs,
381 Vec2CVTStr<"cvt.u32.u16">.s, Zint_extendext16to32>;
382def CVTv4i8tov4i32 : CVTtoVeci32<V4I8Regs, V4I32Regs,
383 Vec4CVTStr<"cvt.u32.u16">.s, Zint_extendext8to32>;
384def CVTv4i16tov4i32 : CVTtoVeci32<V4I16Regs, V4I32Regs,
385 Vec4CVTStr<"cvt.u32.u16">.s, Zint_extendext16to32>;
386def CVTv2i64tov2i32 : CVTtoVeci32<V2I64Regs, V2I32Regs,
387 Vec2CVTStr<"cvt.u32.u64">.s, TRUNC_64to32>;
388}
389
390def : Pat<(shl V2I16Regs:$src1, V2I16Regs:$src2),
391 (ShiftLV2I16 V2I16Regs:$src1, (CVTv2i16tov2i32 V2I16Regs:$src2))>;
392def : Pat<(shl V2I8Regs:$src1, V2I8Regs:$src2),
393 (ShiftLV2I8 V2I8Regs:$src1, (CVTv2i8tov2i32 V2I8Regs:$src2))>;
394def : Pat<(shl V2I64Regs:$src1, V2I64Regs:$src2),
395 (ShiftLV2I64 V2I64Regs:$src1, (CVTv2i64tov2i32 V2I64Regs:$src2))>;
396
397def : Pat<(shl V4I16Regs:$src1, V4I16Regs:$src2),
398 (ShiftLV4I16 V4I16Regs:$src1, (CVTv4i16tov4i32 V4I16Regs:$src2))>;
399def : Pat<(shl V4I8Regs:$src1, V4I8Regs:$src2),
400 (ShiftLV4I8 V4I8Regs:$src1, (CVTv4i8tov4i32 V4I8Regs:$src2))>;
401
402let VecInstType=isVecOther.Value in {
403def ShiftRAV2I64 : VecShiftOp<V2AsmStr<"shr.s64">, sra, V2I64Regs, V2I32Regs,
404 SRAi64rr>;
405def ShiftRAV2I32 : VecShiftOp<V2AsmStr<"shr.s32">, sra, V2I32Regs, V2I32Regs,
406 SRAi32rr>;
407def ShiftRAV4I32 : VecShiftOp<V4AsmStr<"shr.s32">, sra, V4I32Regs, V4I32Regs,
408 SRAi32rr>;
409def ShiftRAV2I16 : VecShiftOp<V2AsmStr<"shr.s16">, sra, V2I16Regs, V2I32Regs,
410 SRAi16rr>;
411def ShiftRAV4I16 : VecShiftOp<V4AsmStr<"shr.s16">, sra, V4I16Regs, V4I32Regs,
412 SRAi16rr>;
413def ShiftRAV2I8 : VecShiftOp<V2AsmStr<"shr.s16">, sra, V2I8Regs, V2I32Regs,
414 SRAi8rr>;
415def ShiftRAV4I8 : VecShiftOp<V4AsmStr<"shr.s16">, sra, V4I8Regs, V4I32Regs,
416 SRAi8rr>;
417
418def ShiftRLV2I64 : VecShiftOp<V2AsmStr<"shr.u64">, srl, V2I64Regs, V2I32Regs,
419 SRLi64rr>;
420def ShiftRLV2I32 : VecShiftOp<V2AsmStr<"shr.u32">, srl, V2I32Regs, V2I32Regs,
421 SRLi32rr>;
422def ShiftRLV4I32 : VecShiftOp<V4AsmStr<"shr.u32">, srl, V4I32Regs, V4I32Regs,
423 SRLi32rr>;
424def ShiftRLV2I16 : VecShiftOp<V2AsmStr<"shr.u16">, srl, V2I16Regs, V2I32Regs,
425 SRLi16rr>;
426def ShiftRLV4I16 : VecShiftOp<V4AsmStr<"shr.u16">, srl, V4I16Regs, V4I32Regs,
427 SRLi16rr>;
428def ShiftRLV2I8 : VecShiftOp<V2AsmStr<"shr.u16">, srl, V2I8Regs, V2I32Regs,
429 SRLi8rr>;
430def ShiftRLV4I8 : VecShiftOp<V4AsmStr<"shr.u16">, srl, V4I8Regs, V4I32Regs,
431 SRLi8rr>;
432
433defm VMult : IntBinVOp<"mul.lo.s", mul, MULTi64rr, MULTi32rr, MULTi16rr,
434 MULTi8rr>;
435defm VMultHS : IntBinVOp<"mul.hi.s", mulhs, MULTHSi64rr, MULTHSi32rr,
436 MULTHSi16rr,
437 MULTHSi8rr>;
438defm VMultHU : IntBinVOp<"mul.hi.u", mulhu, MULTHUi64rr, MULTHUi32rr,
439 MULTHUi16rr,
440 MULTHUi8rr>;
441defm VSDiv : IntBinVOp<"div.s", sdiv, SDIVi64rr, SDIVi32rr, SDIVi16rr,
442 SDIVi8rr>;
443defm VUDiv : IntBinVOp<"div.u", udiv, UDIVi64rr, UDIVi32rr, UDIVi16rr,
444 UDIVi8rr>;
445defm VSRem : IntBinVOp<"rem.s", srem, SREMi64rr, SREMi32rr, SREMi16rr,
446 SREMi8rr>;
447defm VURem : IntBinVOp<"rem.u", urem, UREMi64rr, UREMi32rr, UREMi16rr,
448 UREMi8rr>;
449}
450
451def : Pat<(sra V2I16Regs:$src1, V2I16Regs:$src2),
452 (ShiftRAV2I16 V2I16Regs:$src1, (CVTv2i16tov2i32 V2I16Regs:$src2))>;
453def : Pat<(sra V2I8Regs:$src1, V2I8Regs:$src2),
454 (ShiftRAV2I8 V2I8Regs:$src1, (CVTv2i8tov2i32 V2I8Regs:$src2))>;
455def : Pat<(sra V2I64Regs:$src1, V2I64Regs:$src2),
456 (ShiftRAV2I64 V2I64Regs:$src1, (CVTv2i64tov2i32 V2I64Regs:$src2))>;
457
458def : Pat<(sra V4I16Regs:$src1, V4I16Regs:$src2),
459 (ShiftRAV4I16 V4I16Regs:$src1, (CVTv4i16tov4i32 V4I16Regs:$src2))>;
460def : Pat<(sra V4I8Regs:$src1, V4I8Regs:$src2),
461 (ShiftRAV4I8 V4I8Regs:$src1, (CVTv4i8tov4i32 V4I8Regs:$src2))>;
462
463def : Pat<(srl V2I16Regs:$src1, V2I16Regs:$src2),
464 (ShiftRLV2I16 V2I16Regs:$src1, (CVTv2i16tov2i32 V2I16Regs:$src2))>;
465def : Pat<(srl V2I8Regs:$src1, V2I8Regs:$src2),
466 (ShiftRLV2I8 V2I8Regs:$src1, (CVTv2i8tov2i32 V2I8Regs:$src2))>;
467def : Pat<(srl V2I64Regs:$src1, V2I64Regs:$src2),
468 (ShiftRLV2I64 V2I64Regs:$src1, (CVTv2i64tov2i32 V2I64Regs:$src2))>;
469
470def : Pat<(srl V4I16Regs:$src1, V4I16Regs:$src2),
471 (ShiftRLV4I16 V4I16Regs:$src1, (CVTv4i16tov4i32 V4I16Regs:$src2))>;
472def : Pat<(srl V4I8Regs:$src1, V4I8Regs:$src2),
473 (ShiftRLV4I8 V4I8Regs:$src1, (CVTv4i8tov4i32 V4I8Regs:$src2))>;
474
475multiclass VMAD<string asmstr, NVPTXRegClass regclassv4,
476 NVPTXRegClass regclassv2,
477 SDNode an=add, SDNode mn=mul, NVPTXInst sop=NOP,
478 Predicate Pred> {
479 def V4 : NVPTXVecInst<(outs regclassv4:$dst),
480 (ins regclassv4:$a, regclassv4:$b, regclassv4:$c),
481 V4MADStr<asmstr>.s,
482 [(set regclassv4:$dst,
483 (an (mn regclassv4:$a, regclassv4:$b), regclassv4:$c))],
484 sop>,
485 Requires<[Pred]>;
486 def V2 : NVPTXVecInst<(outs regclassv2:$dst),
487 (ins regclassv2:$a, regclassv2:$b, regclassv2:$c),
488 V2MADStr<asmstr>.s,
489 [(set regclassv2:$dst,
490 (an (mn regclassv2:$a, regclassv2:$b), regclassv2:$c))],
491 sop>,
492 Requires<[Pred]>;
493}
494
495multiclass VMADV2Only<string asmstr, NVPTXRegClass regclass, NVPTXInst sop=NOP,
496 Predicate Pred> {
497 def V2 : NVPTXVecInst<(outs regclass:$dst),
498 (ins regclass:$a, regclass:$b, regclass:$c),
499 V2MADStr<asmstr>.s,
500 [(set regclass:$dst, (add
501 (mul regclass:$a, regclass:$b), regclass:$c))], sop>,
502 Requires<[Pred]>;
503}
504multiclass VFMADV2Only<string asmstr, NVPTXRegClass regclass, NVPTXInst sop=NOP,
505 Predicate Pred> {
506 def V2 : NVPTXVecInst<(outs regclass:$dst),
507 (ins regclass:$a, regclass:$b, regclass:$c),
508 V2MADStr<asmstr>.s,
509 [(set regclass:$dst, (fadd
510 (fmul regclass:$a, regclass:$b), regclass:$c))], sop>,
511 Requires<[Pred]>;
512}
513
514let VecInstType=isVecOther.Value in {
515defm I8MAD : VMAD<"mad.lo.s16", V4I8Regs, V2I8Regs, add, mul, MAD8rrr, true>;
516defm I16MAD : VMAD<"mad.lo.s16", V4I16Regs, V2I16Regs, add, mul, MAD16rrr,
517 true>;
518defm I32MAD : VMAD<"mad.lo.s32", V4I32Regs, V2I32Regs, add, mul, MAD32rrr,
519 true>;
520defm I64MAD : VMADV2Only<"mad.lo.s64", V2I64Regs, MAD64rrr, true>;
521
522defm VNeg : IntUnaryVOp<"neg.s", ineg, INEG64, INEG32, INEG16, INEG8>;
523
524defm VAddf : FloatBinVOp<"add.", fadd, FADDf64rr, FADDf32rr, FADDf32rr_ftz>;
525defm VSubf : FloatBinVOp<"sub.", fsub, FSUBf64rr, FSUBf32rr, FSUBf32rr_ftz>;
526defm VMulf : FloatBinVOp<"mul.", fmul, FMULf64rr, FMULf32rr, FMULf32rr_ftz>;
527
528defm F32MAD_ftz : VMAD<"mad.ftz.f32", V4F32Regs, V2F32Regs, fadd, fmul,
529 FMAD32_ftzrrr, doFMADF32_ftz>;
530defm F32FMA_ftz : VMAD<"fma.rn.ftz.f32", V4F32Regs, V2F32Regs, fadd, fmul,
531 FMA32_ftzrrr, doFMAF32_ftz>;
532defm F32MAD : VMAD<"mad.f32", V4F32Regs, V2F32Regs, fadd, fmul, FMAD32rrr,
533 doFMADF32>;
534defm F32FMA : VMAD<"fma.rn.f32", V4F32Regs, V2F32Regs, fadd, fmul, FMA32rrr,
535 doFMAF32>;
536defm F64FMA : VFMADV2Only<"fma.rn.f64", V2F64Regs, FMA64rrr, doFMAF64>;
537}
538
539let VecInstType=isVecOther.Value in {
540def V4F32Div_prec_ftz : VecBinaryOp<V4AsmStr<"div.rn.ftz.f32">, fdiv, V4F32Regs,
541 FDIV32rr_prec_ftz>, Requires<[doF32FTZ, reqPTX20]>;
542def V2F32Div_prec_ftz : VecBinaryOp<V2AsmStr<"div.rn.ftz.f32">, fdiv, V2F32Regs,
543 FDIV32rr_prec_ftz>, Requires<[doF32FTZ, reqPTX20]>;
544def V4F32Div_prec : VecBinaryOp<V4AsmStr<"div.rn.f32">, fdiv, V4F32Regs,
545 FDIV32rr_prec>, Requires<[reqPTX20]>;
546def V2F32Div_prec : VecBinaryOp<V2AsmStr<"div.rn.f32">, fdiv, V2F32Regs,
547 FDIV32rr_prec>, Requires<[reqPTX20]>;
548def V2F32Div_ftz : VecBinaryOp<V2AsmStr<"div.full.ftz.f32">, fdiv, V2F32Regs,
549 FDIV32rr_ftz>, Requires<[doF32FTZ]>;
550def V4F32Div_ftz : VecBinaryOp<V4AsmStr<"div.full.ftz.f32">, fdiv, V4F32Regs,
551 FDIV32rr_ftz>, Requires<[doF32FTZ]>;
552def V2F32Div : VecBinaryOp<V2AsmStr<"div.full.f32">, fdiv, V2F32Regs, FDIV32rr>;
553def V4F32Div : VecBinaryOp<V4AsmStr<"div.full.f32">, fdiv, V4F32Regs, FDIV32rr>;
554def V2F64Div : VecBinaryOp<V2AsmStr<"div.rn.f64">, fdiv, V2F64Regs, FDIV64rr>;
555}
556
557def fnegpat : PatFrag<(ops node:$in), (fneg node:$in)>;
558
559let VecInstType=isVecOther.Value in {
560def VNegv2f32_ftz : VecUnaryOp<V2UnaryStr<"neg.ftz.f32">, fnegpat, V2F32Regs,
561 FNEGf32_ftz>, Requires<[doF32FTZ]>;
562def VNegv4f32_ftz : VecUnaryOp<V4UnaryStr<"neg.ftz.f32">, fnegpat, V4F32Regs,
563 FNEGf32_ftz>, Requires<[doF32FTZ]>;
564def VNegv2f32 : VecUnaryOp<V2UnaryStr<"neg.f32">, fnegpat, V2F32Regs, FNEGf32>;
565def VNegv4f32 : VecUnaryOp<V4UnaryStr<"neg.f32">, fnegpat, V4F32Regs, FNEGf32>;
566def VNegv2f64 : VecUnaryOp<V2UnaryStr<"neg.f64">, fnegpat, V2F64Regs, FNEGf64>;
567
568// Logical Arithmetic
569defm VAnd : IntBinVOp<"and.b", and, ANDb64rr, ANDb32rr, ANDb16rr, ANDb8rr>;
570defm VOr : IntBinVOp<"or.b", or, ORb64rr, ORb32rr, ORb16rr, ORb8rr>;
571defm VXor : IntBinVOp<"xor.b", xor, XORb64rr, XORb32rr, XORb16rr, XORb8rr>;
572
573defm VNot : IntUnaryVOp<"not.b", not, NOT64, NOT32, NOT16, NOT8>;
574}
575
576
577multiclass V2FPCONTRACT32_SUB_PAT<NVPTXInst Inst, Predicate Pred> {
578 def : Pat<(fsub V2F32Regs:$a, (fmul V2F32Regs:$b, V2F32Regs:$c)),
579 (Inst (VNegv2f32 V2F32Regs:$b), V2F32Regs:$c, V2F32Regs:$a)>,
580 Requires<[Pred]>;
581
582 def : Pat<(fsub (fmul V2F32Regs:$a, V2F32Regs:$b), V2F32Regs:$c),
583 (Inst V2F32Regs:$a, V2F32Regs:$b, (VNegv2f32 V2F32Regs:$c))>,
584 Requires<[Pred]>;
585}
586
587defm V2FMAF32ext_ftz : V2FPCONTRACT32_SUB_PAT<F32FMA_ftzV2, doFMAF32AGG_ftz>;
588defm V2FMADF32ext_ftz : V2FPCONTRACT32_SUB_PAT<F32MAD_ftzV2, doFMADF32_ftz>;
589defm V2FMAF32ext : V2FPCONTRACT32_SUB_PAT<F32FMAV2, doFMAF32AGG>;
590defm V2FMADF32ext : V2FPCONTRACT32_SUB_PAT<F32MADV2, doFMADF32>;
591
592multiclass V4FPCONTRACT32_SUB_PAT<NVPTXInst Inst, Predicate Pred> {
593 def : Pat<(fsub V4F32Regs:$a, (fmul V4F32Regs:$b, V4F32Regs:$c)),
594 (Inst (VNegv4f32 V4F32Regs:$b), V4F32Regs:$c, V4F32Regs:$a)>,
595 Requires<[Pred]>;
596
597 def : Pat<(fsub (fmul V4F32Regs:$a, V4F32Regs:$b), V4F32Regs:$c),
598 (Inst V4F32Regs:$a, V4F32Regs:$b, (VNegv4f32 V4F32Regs:$c))>,
599 Requires<[Pred]>;
600}
601
602defm V4FMAF32ext_ftz : V4FPCONTRACT32_SUB_PAT<F32FMA_ftzV4, doFMAF32AGG_ftz>;
603defm V4FMADF32ext_ftz : V4FPCONTRACT32_SUB_PAT<F32MAD_ftzV4, doFMADF32_ftz>;
604defm V4FMAF32ext : V4FPCONTRACT32_SUB_PAT<F32FMAV4, doFMAF32AGG>;
605defm V4FMADF32ext : V4FPCONTRACT32_SUB_PAT<F32MADV4, doFMADF32>;
606
607multiclass V2FPCONTRACT64_SUB_PAT<NVPTXInst Inst, Predicate Pred> {
608 def : Pat<(fsub V2F64Regs:$a, (fmul V2F64Regs:$b, V2F64Regs:$c)),
609 (Inst (VNegv2f64 V2F64Regs:$b), V2F64Regs:$c, V2F64Regs:$a)>,
610 Requires<[Pred]>;
611
612 def : Pat<(fsub (fmul V2F64Regs:$a, V2F64Regs:$b), V2F64Regs:$c),
613 (Inst V2F64Regs:$a, V2F64Regs:$b, (VNegv2f64 V2F64Regs:$c))>,
614 Requires<[Pred]>;
615}
616
617defm V2FMAF64ext : V2FPCONTRACT64_SUB_PAT<F64FMAV2, doFMAF64AGG>;
618
619class VecModStr<string vecsize, string elem, string extra, string l="">
620{
621 string t1 = !strconcat("${c", elem);
622 string t2 = !strconcat(t1, ":vecv");
623 string t3 = !strconcat(t2, vecsize);
624 string t4 = !strconcat(t3, extra);
625 string t5 = !strconcat(t4, l);
626 string s = !strconcat(t5, "}");
627}
628class ShuffleOneLine<string vecsize, string elem, string type>
629{
630 string t1 = VecModStr<vecsize, elem, "comm", "1">.s;
631 string t2 = !strconcat(t1, "mov.");
632 string t3 = !strconcat(t2, type);
633 string t4 = !strconcat(t3, " \t${dst}_");
634 string t5 = !strconcat(t4, elem);
635 string t6 = !strconcat(t5, ", $src1");
636 string t7 = !strconcat(t6, VecModStr<vecsize, elem, "pos">.s);
637 string t8 = !strconcat(t7, ";\n\t");
638 string t9 = !strconcat(t8, VecModStr<vecsize, elem, "comm", "2">.s);
639 string t10 = !strconcat(t9, "mov.");
640 string t11 = !strconcat(t10, type);
641 string t12 = !strconcat(t11, " \t${dst}_");
642 string t13 = !strconcat(t12, elem);
643 string t14 = !strconcat(t13, ", $src2");
644 string t15 = !strconcat(t14, VecModStr<vecsize, elem, "pos">.s);
645 string s = !strconcat(t15, ";");
646}
647class ShuffleAsmStr2<string type>
648{
649 string t1 = ShuffleOneLine<"2", "0", type>.s;
650 string t2 = !strconcat(t1, "\n\t");
651 string s = !strconcat(t2, ShuffleOneLine<"2", "1", type>.s);
652}
653class ShuffleAsmStr4<string type>
654{
655 string t1 = ShuffleOneLine<"4", "0", type>.s;
656 string t2 = !strconcat(t1, "\n\t");
657 string t3 = !strconcat(t2, ShuffleOneLine<"4", "1", type>.s);
658 string t4 = !strconcat(t3, "\n\t");
659 string t5 = !strconcat(t4, ShuffleOneLine<"4", "2", type>.s);
660 string t6 = !strconcat(t5, "\n\t");
661 string s = !strconcat(t6, ShuffleOneLine<"4", "3", type>.s);
662}
663
664let neverHasSideEffects=1, VecInstType=isVecShuffle.Value in {
665def VecShuffle_v4f32 : NVPTXVecInst<(outs V4F32Regs:$dst),
666 (ins V4F32Regs:$src1, V4F32Regs:$src2,
667 i8imm:$c0, i8imm:$c1, i8imm:$c2, i8imm:$c3),
668 !strconcat("//Mov $dst, $src1, $src2, $c0, $c1, $c2, $c3;\n\t",
669 ShuffleAsmStr4<"f32">.s),
670 [], FMOV32rr>;
671
672def VecShuffle_v4i32 : NVPTXVecInst<(outs V4I32Regs:$dst),
673 (ins V4I32Regs:$src1, V4I32Regs:$src2,
674 i8imm:$c0, i8imm:$c1, i8imm:$c2, i8imm:$c3),
675 !strconcat("//Mov $dst, $src1, $src2, $c0, $c1, $c2, $c3;\n\t",
676 ShuffleAsmStr4<"u32">.s),
677 [], IMOV32rr>;
678
679def VecShuffle_v4i16 : NVPTXVecInst<(outs V4I16Regs:$dst),
680 (ins V4I16Regs:$src1, V4I16Regs:$src2,
681 i8imm:$c0, i8imm:$c1, i8imm:$c2, i8imm:$c3),
682 !strconcat("//Mov $dst, $src1, $src2, $c0, $c1, $c2, $c3;\n\t",
683 ShuffleAsmStr4<"u16">.s),
684 [], IMOV16rr>;
685
686def VecShuffle_v4i8 : NVPTXVecInst<(outs V4I8Regs:$dst),
687 (ins V4I8Regs:$src1, V4I8Regs:$src2,
688 i8imm:$c0, i8imm:$c1, i8imm:$c2, i8imm:$c3),
689 !strconcat("//Mov $dst, $src1, $src2, $c0, $c1, $c2, $c3;\n\t",
690 ShuffleAsmStr4<"u16">.s),
691 [], IMOV8rr>;
692
693def VecShuffle_v2f32 : NVPTXVecInst<(outs V2F32Regs:$dst),
694 (ins V2F32Regs:$src1, V2F32Regs:$src2,
695 i8imm:$c0, i8imm:$c1),
696 !strconcat("//Mov $dst, $src1, $src2, $c0, $c1;\n\t",
697 ShuffleAsmStr2<"f32">.s),
698 [], FMOV32rr>;
699
700def VecShuffle_v2i32 : NVPTXVecInst<(outs V2I32Regs:$dst),
701 (ins V2I32Regs:$src1, V2I32Regs:$src2,
702 i8imm:$c0, i8imm:$c1),
703 !strconcat("//Mov $dst, $src1, $src2, $c0, $c1;\n\t",
704 ShuffleAsmStr2<"u32">.s),
705 [], IMOV32rr>;
706
707def VecShuffle_v2i8 : NVPTXVecInst<(outs V2I8Regs:$dst),
708 (ins V2I8Regs:$src1, V2I8Regs:$src2,
709 i8imm:$c0, i8imm:$c1),
710 !strconcat("//Mov $dst, $src1, $src2, $c0, $c1;\n\t",
711 ShuffleAsmStr2<"u16">.s),
712 [], IMOV8rr>;
713
714def VecShuffle_v2i16 : NVPTXVecInst<(outs V2I16Regs:$dst),
715 (ins V2I16Regs:$src1, V2I16Regs:$src2,
716 i8imm:$c0, i8imm:$c1),
717 !strconcat("//Mov $dst, $src1, $src2, $c0, $c1;\n\t",
718 ShuffleAsmStr2<"u16">.s),
719 [], IMOV16rr>;
720
721def VecShuffle_v2f64 : NVPTXVecInst<(outs V2F64Regs:$dst),
722 (ins V2F64Regs:$src1, V2F64Regs:$src2,
723 i8imm:$c0, i8imm:$c1),
724 !strconcat("//Mov $dst, $src1, $src2, $c0, $c1;\n\t",
725 ShuffleAsmStr2<"f64">.s),
726 [], FMOV64rr>;
727
728def VecShuffle_v2i64 : NVPTXVecInst<(outs V2I64Regs:$dst),
729 (ins V2I64Regs:$src1, V2I64Regs:$src2,
730 i8imm:$c0, i8imm:$c1),
731 !strconcat("//Mov $dst, $src1, $src2, $c0, $c1;\n\t",
732 ShuffleAsmStr2<"u64">.s),
733 [], IMOV64rr>;
734}
735
736def ShuffleMask0 : SDNodeXForm<vector_shuffle, [{
737 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
738 return CurDAG->getTargetConstant(SVOp->getMaskElt(0), MVT::i32);
739}]>;
740def ShuffleMask1 : SDNodeXForm<vector_shuffle, [{
741 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
742 return CurDAG->getTargetConstant(SVOp->getMaskElt(1), MVT::i32);
743}]>;
744def ShuffleMask2 : SDNodeXForm<vector_shuffle, [{
745 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
746 return CurDAG->getTargetConstant(SVOp->getMaskElt(2), MVT::i32);
747}]>;
748def ShuffleMask3 : SDNodeXForm<vector_shuffle, [{
749 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
750 return CurDAG->getTargetConstant(SVOp->getMaskElt(3), MVT::i32);
751}]>;
752
753// The spurious call is here to silence a compiler warning about N being
754// unused.
755def vec_shuf : PatFrag<(ops node:$lhs, node:$rhs),
756 (vector_shuffle node:$lhs, node:$rhs),
757 [{ N->getGluedNode(); return true; }]>;
758
759def : Pat<(v2f64 (vec_shuf:$op V2F64Regs:$src1, V2F64Regs:$src2)),
760 (VecShuffle_v2f64 V2F64Regs:$src1, V2F64Regs:$src2,
761 (ShuffleMask0 node:$op), (ShuffleMask1 node:$op))>;
762
763def : Pat<(v4f32 (vec_shuf:$op V4F32Regs:$src1, V4F32Regs:$src2)),
764 (VecShuffle_v4f32 V4F32Regs:$src1, V4F32Regs:$src2,
765 (ShuffleMask0 node:$op), (ShuffleMask1 node:$op),
766 (ShuffleMask2 node:$op), (ShuffleMask3 node:$op))>;
767
768def : Pat<(v2f32 (vec_shuf:$op V2F32Regs:$src1, V2F32Regs:$src2)),
769 (VecShuffle_v2f32 V2F32Regs:$src1, V2F32Regs:$src2,
770 (ShuffleMask0 node:$op), (ShuffleMask1 node:$op))>;
771
772def : Pat<(v2i64 (vec_shuf:$op V2I64Regs:$src1, V2I64Regs:$src2)),
773 (VecShuffle_v2i64 V2I64Regs:$src1, V2I64Regs:$src2,
774 (ShuffleMask0 node:$op), (ShuffleMask1 node:$op))>;
775
776def : Pat<(v4i32 (vec_shuf:$op V4I32Regs:$src1, V4I32Regs:$src2)),
777 (VecShuffle_v4i32 V4I32Regs:$src1, V4I32Regs:$src2,
778 (ShuffleMask0 node:$op), (ShuffleMask1 node:$op),
779 (ShuffleMask2 node:$op), (ShuffleMask3 node:$op))>;
780
781def : Pat<(v2i32 (vec_shuf:$op V2I32Regs:$src1, V2I32Regs:$src2)),
782 (VecShuffle_v2i32 V2I32Regs:$src1, V2I32Regs:$src2,
783 (ShuffleMask0 node:$op), (ShuffleMask1 node:$op))>;
784
785def : Pat<(v4i16 (vec_shuf:$op V4I16Regs:$src1, V4I16Regs:$src2)),
786 (VecShuffle_v4i16 V4I16Regs:$src1, V4I16Regs:$src2,
787 (ShuffleMask0 node:$op), (ShuffleMask1 node:$op),
788 (ShuffleMask2 node:$op), (ShuffleMask3 node:$op))>;
789
790def : Pat<(v2i16 (vec_shuf:$op V2I16Regs:$src1, V2I16Regs:$src2)),
791 (VecShuffle_v2i16 V2I16Regs:$src1, V2I16Regs:$src2,
792 (ShuffleMask0 node:$op), (ShuffleMask1 node:$op))>;
793
794def : Pat<(v4i8 (vec_shuf:$op V4I8Regs:$src1, V4I8Regs:$src2)),
795 (VecShuffle_v4i8 V4I8Regs:$src1, V4I8Regs:$src2,
796 (ShuffleMask0 node:$op), (ShuffleMask1 node:$op),
797 (ShuffleMask2 node:$op), (ShuffleMask3 node:$op))>;
798
799def : Pat<(v2i8 (vec_shuf:$op V2I8Regs:$src1, V2I8Regs:$src2)),
800 (VecShuffle_v2i8 V2I8Regs:$src1, V2I8Regs:$src2,
801 (ShuffleMask0 node:$op), (ShuffleMask1 node:$op))>;
802
803class Build_Vector2<string asmstr, NVPTXRegClass vclass, NVPTXRegClass sclass,
804 NVPTXInst si>
805 : NVPTXVecInst<(outs vclass:$dst),
806 (ins sclass:$a1, sclass:$a2),
807 !strconcat(asmstr, "\t${dst:vecfull}, {{$a1, $a2}};"),
808 [(set vclass:$dst, (build_vector sclass:$a1, sclass:$a2))],
809 si>;
810class Build_Vector4<string asmstr, NVPTXRegClass vclass, NVPTXRegClass sclass,
811 NVPTXInst si>
812 : NVPTXVecInst<(outs vclass:$dst),
813 (ins sclass:$a1, sclass:$a2, sclass:$a3, sclass:$a4),
814 !strconcat(asmstr, "\t${dst:vecfull}, {{$a1, $a2, $a3, $a4}};"),
815 [(set vclass:$dst,
816 (build_vector sclass:$a1, sclass:$a2,
817 sclass:$a3, sclass:$a4))], si>;
818
819let isAsCheapAsAMove=1, VecInstType=isVecBuild.Value in {
820def Build_Vector2_f32 : Build_Vector2<"mov.v2.f32", V2F32Regs, Float32Regs,
821 FMOV32rr>;
822def Build_Vector2_f64 : Build_Vector2<"mov.v2.f64", V2F64Regs, Float64Regs,
823 FMOV64rr>;
824
825def Build_Vector2_i32 : Build_Vector2<"mov.v2.u32", V2I32Regs, Int32Regs,
826 IMOV32rr>;
827def Build_Vector2_i64 : Build_Vector2<"mov.v2.u64", V2I64Regs, Int64Regs,
828 IMOV64rr>;
829def Build_Vector2_i16 : Build_Vector2<"mov.v2.u16", V2I16Regs, Int16Regs,
830 IMOV16rr>;
831def Build_Vector2_i8 : Build_Vector2<"mov.v2.u16", V2I8Regs, Int8Regs,
832 IMOV8rr>;
833
834def Build_Vector4_f32 : Build_Vector4<"mov.v4.f32", V4F32Regs, Float32Regs,
835 FMOV32rr>;
836
837def Build_Vector4_i32 : Build_Vector4<"mov.v4.u32", V4I32Regs, Int32Regs,
838 IMOV32rr>;
839def Build_Vector4_i16 : Build_Vector4<"mov.v4.u16", V4I16Regs, Int16Regs,
840 IMOV16rr>;
841def Build_Vector4_i8 : Build_Vector4<"mov.v4.u16", V4I8Regs, Int8Regs,
842 IMOV8rr>;
843}
844
845class Vec_Move<string asmstr, NVPTXRegClass vclass, NVPTXInst sop=NOP>
846 : NVPTXVecInst<(outs vclass:$dst), (ins vclass:$src),
847 !strconcat(asmstr, "\t${dst:vecfull}, ${src:vecfull};"),
848 [], sop>;
849
850let isAsCheapAsAMove=1, neverHasSideEffects=1, IsSimpleMove=1,
851 VecInstType=isVecOther.Value in {
852def V4f32Mov : Vec_Move<"mov.v4.f32", V4F32Regs, FMOV32rr>;
853def V2f32Mov : Vec_Move<"mov.v2.f32", V2F32Regs, FMOV32rr>;
854
855def V4i32Mov : Vec_Move<"mov.v4.u32", V4I32Regs, IMOV32rr>;
856def V2i32Mov : Vec_Move<"mov.v2.u32", V2I32Regs, IMOV32rr>;
857
858def V4i16Mov : Vec_Move<"mov.v4.u16", V4I16Regs, IMOV16rr>;
859def V2i16Mov : Vec_Move<"mov.v2.u16", V2I16Regs, IMOV16rr>;
860
861def V4i8Mov : Vec_Move<"mov.v4.u16", V4I8Regs, IMOV8rr>;
862def V2i8Mov : Vec_Move<"mov.v2.u16", V2I8Regs, IMOV8rr>;
863
864def V2f64Mov : Vec_Move<"mov.v2.f64", V2F64Regs, FMOV64rr>;
865def V2i64Mov : Vec_Move<"mov.v2.u64", V2I64Regs, IMOV64rr>;
866}
867
868// extract subvector patterns
869def extract_subvec : SDNode<"ISD::EXTRACT_SUBVECTOR",
870 SDTypeProfile<1, 2, [SDTCisPtrTy<2>]>>;
871
872def : Pat<(v2f32 (extract_subvec V4F32Regs:$src, 0)),
873 (Build_Vector2_f32 (V4f32Extract V4F32Regs:$src, 0),
874 (V4f32Extract V4F32Regs:$src, 1))>;
875def : Pat<(v2f32 (extract_subvec V4F32Regs:$src, 2)),
876 (Build_Vector2_f32 (V4f32Extract V4F32Regs:$src, 2),
877 (V4f32Extract V4F32Regs:$src, 3))>;
878def : Pat<(v2i32 (extract_subvec V4I32Regs:$src, 0)),
879 (Build_Vector2_i32 (V4i32Extract V4I32Regs:$src, 0),
880 (V4i32Extract V4I32Regs:$src, 1))>;
881def : Pat<(v2i32 (extract_subvec V4I32Regs:$src, 2)),
882 (Build_Vector2_i32 (V4i32Extract V4I32Regs:$src, 2),
883 (V4i32Extract V4I32Regs:$src, 3))>;
884def : Pat<(v2i16 (extract_subvec V4I16Regs:$src, 0)),
885 (Build_Vector2_i16 (V4i16Extract V4I16Regs:$src, 0),
886 (V4i16Extract V4I16Regs:$src, 1))>;
887def : Pat<(v2i16 (extract_subvec V4I16Regs:$src, 2)),
888 (Build_Vector2_i16 (V4i16Extract V4I16Regs:$src, 2),
889 (V4i16Extract V4I16Regs:$src, 3))>;
890def : Pat<(v2i8 (extract_subvec V4I8Regs:$src, 0)),
891 (Build_Vector2_i8 (V4i8Extract V4I8Regs:$src, 0),
892 (V4i8Extract V4I8Regs:$src, 1))>;
893def : Pat<(v2i8 (extract_subvec V4I8Regs:$src, 2)),
894 (Build_Vector2_i8 (V4i8Extract V4I8Regs:$src, 2),
895 (V4i8Extract V4I8Regs:$src, 3))>;
896
897// Select instructions
898class Select_OneLine<string type, string pos> {
899 string t1 = !strconcat("selp.", type);
900 string t2 = !strconcat(t1, " \t${dst}_");
901 string t3 = !strconcat(t2, pos);
902 string t4 = !strconcat(t3, ", ${src1}_");
903 string t5 = !strconcat(t4, pos);
904 string t6 = !strconcat(t5, ", ${src2}_");
905 string t7 = !strconcat(t6, pos);
906 string s = !strconcat(t7, ", $p;");
907}
908
909class Select_Str2<string type> {
910 string t1 = Select_OneLine<type, "0">.s;
911 string t2 = !strconcat(t1, "\n\t");
912 string s = !strconcat(t2, Select_OneLine<type, "1">.s);
913}
914
915class Select_Str4<string type> {
916 string t1 = Select_OneLine<type, "0">.s;
917 string t2 = !strconcat(t1, "\n\t");
918 string t3 = !strconcat(t2, Select_OneLine<type, "1">.s);
919 string t4 = !strconcat(t3, "\n\t");
920 string t5 = !strconcat(t4, Select_OneLine<type, "2">.s);
921 string t6 = !strconcat(t5, "\n\t");
922 string s = !strconcat(t6, Select_OneLine<type, "3">.s);
923
924}
925
926class Vec_Select<NVPTXRegClass vclass, string asmstr, NVPTXInst sop>
927 : NVPTXVecInst<(outs vclass:$dst),
928 (ins vclass:$src1, vclass:$src2, Int1Regs:$p),
929 asmstr,
930 [(set vclass:$dst, (select Int1Regs:$p, vclass:$src1,
931 vclass:$src2))],
932 sop>;
933
934let VecInstType=isVecOther.Value in {
935def V2I64_Select : Vec_Select<V2I64Regs, Select_Str2<"b64">.s, SELECTi64rr>;
936def V4I32_Select : Vec_Select<V4I32Regs, Select_Str4<"b32">.s, SELECTi32rr>;
937def V2I32_Select : Vec_Select<V2I32Regs, Select_Str2<"b32">.s, SELECTi32rr>;
938def V4I16_Select : Vec_Select<V4I16Regs, Select_Str4<"b16">.s, SELECTi16rr>;
939def V2I16_Select : Vec_Select<V2I16Regs, Select_Str2<"b16">.s, SELECTi16rr>;
940def V4I8_Select : Vec_Select<V4I8Regs, Select_Str4<"b16">.s, SELECTi8rr>;
941def V2I8_Select : Vec_Select<V2I8Regs, Select_Str2<"b16">.s, SELECTi8rr>;
942
943def V2F64_Select : Vec_Select<V2F64Regs, Select_Str2<"f64">.s, SELECTf64rr>;
944def V4F32_Select : Vec_Select<V4F32Regs, Select_Str4<"f32">.s, SELECTf32rr>;
945def V2F32_Select : Vec_Select<V2F32Regs, Select_Str2<"f32">.s, SELECTf32rr>;
946}
947
948// Comparison instructions
949
950// setcc convenience fragments.
951def vsetoeq : PatFrag<(ops node:$lhs, node:$rhs),
952 (setcc node:$lhs, node:$rhs, SETOEQ)>;
953def vsetogt : PatFrag<(ops node:$lhs, node:$rhs),
954 (setcc node:$lhs, node:$rhs, SETOGT)>;
955def vsetoge : PatFrag<(ops node:$lhs, node:$rhs),
956 (setcc node:$lhs, node:$rhs, SETOGE)>;
957def vsetolt : PatFrag<(ops node:$lhs, node:$rhs),
958 (setcc node:$lhs, node:$rhs, SETOLT)>;
959def vsetole : PatFrag<(ops node:$lhs, node:$rhs),
960 (setcc node:$lhs, node:$rhs, SETOLE)>;
961def vsetone : PatFrag<(ops node:$lhs, node:$rhs),
962 (setcc node:$lhs, node:$rhs, SETONE)>;
963def vseto : PatFrag<(ops node:$lhs, node:$rhs),
964 (setcc node:$lhs, node:$rhs, SETO)>;
965def vsetuo : PatFrag<(ops node:$lhs, node:$rhs),
966 (setcc node:$lhs, node:$rhs, SETUO)>;
967def vsetueq : PatFrag<(ops node:$lhs, node:$rhs),
968 (setcc node:$lhs, node:$rhs, SETUEQ)>;
969def vsetugt : PatFrag<(ops node:$lhs, node:$rhs),
970 (setcc node:$lhs, node:$rhs, SETUGT)>;
971def vsetuge : PatFrag<(ops node:$lhs, node:$rhs),
972 (setcc node:$lhs, node:$rhs, SETUGE)>;
973def vsetult : PatFrag<(ops node:$lhs, node:$rhs),
974 (setcc node:$lhs, node:$rhs, SETULT)>;
975def vsetule : PatFrag<(ops node:$lhs, node:$rhs),
976 (setcc node:$lhs, node:$rhs, SETULE)>;
977def vsetune : PatFrag<(ops node:$lhs, node:$rhs),
978 (setcc node:$lhs, node:$rhs, SETUNE)>;
979def vseteq : PatFrag<(ops node:$lhs, node:$rhs),
980 (setcc node:$lhs, node:$rhs, SETEQ)>;
981def vsetgt : PatFrag<(ops node:$lhs, node:$rhs),
982 (setcc node:$lhs, node:$rhs, SETGT)>;
983def vsetge : PatFrag<(ops node:$lhs, node:$rhs),
984 (setcc node:$lhs, node:$rhs, SETGE)>;
985def vsetlt : PatFrag<(ops node:$lhs, node:$rhs),
986 (setcc node:$lhs, node:$rhs, SETLT)>;
987def vsetle : PatFrag<(ops node:$lhs, node:$rhs),
988 (setcc node:$lhs, node:$rhs, SETLE)>;
989def vsetne : PatFrag<(ops node:$lhs, node:$rhs),
990 (setcc node:$lhs, node:$rhs, SETNE)>;
991
992class Vec_Compare<PatFrag op, NVPTXRegClass outrclass, NVPTXRegClass inrclass,
993 NVPTXInst sop>
994 : NVPTXVecInst<(outs outrclass:$dst),
995 (ins inrclass:$a, inrclass:$b),
996 "Unsupported",
997 [(set outrclass:$dst, (op inrclass:$a, inrclass:$b))],
998 sop>;
999
1000multiclass Vec_Compare_All<PatFrag op,
1001 NVPTXInst inst8,
1002 NVPTXInst inst16,
1003 NVPTXInst inst32,
1004 NVPTXInst inst64>
1005{
1006 def V2I8 : Vec_Compare<op, V2I8Regs, V2I8Regs, inst8>;
1007 def V4I8 : Vec_Compare<op, V4I8Regs, V4I8Regs, inst8>;
1008 def V2I16 : Vec_Compare<op, V2I16Regs, V2I16Regs, inst16>;
1009 def V4I16 : Vec_Compare<op, V4I16Regs, V4I16Regs, inst16>;
1010 def V2I32 : Vec_Compare<op, V2I32Regs, V2I32Regs, inst32>;
1011 def V4I32 : Vec_Compare<op, V4I32Regs, V4I32Regs, inst32>;
1012 def V2I64 : Vec_Compare<op, V2I64Regs, V2I64Regs, inst64>;
1013}
1014
1015let VecInstType=isVecOther.Value in {
1016 defm VecSGT : Vec_Compare_All<vsetgt, ISetSGTi8rr_toi8, ISetSGTi16rr_toi16,
1017 ISetSGTi32rr_toi32, ISetSGTi64rr_toi64>;
1018 defm VecUGT : Vec_Compare_All<vsetugt, ISetUGTi8rr_toi8, ISetUGTi16rr_toi16,
1019 ISetUGTi32rr_toi32, ISetUGTi64rr_toi64>;
1020 defm VecSLT : Vec_Compare_All<vsetlt, ISetSLTi8rr_toi8, ISetSLTi16rr_toi16,
1021 ISetSLTi32rr_toi32, ISetSLTi64rr_toi64>;
1022 defm VecULT : Vec_Compare_All<vsetult, ISetULTi8rr_toi8, ISetULTi16rr_toi16,
1023 ISetULTi32rr_toi32, ISetULTi64rr_toi64>;
1024 defm VecSGE : Vec_Compare_All<vsetge, ISetSGEi8rr_toi8, ISetSGEi16rr_toi16,
1025 ISetSGEi32rr_toi32, ISetSGEi64rr_toi64>;
1026 defm VecUGE : Vec_Compare_All<vsetuge, ISetUGEi8rr_toi8, ISetUGEi16rr_toi16,
1027 ISetUGEi32rr_toi32, ISetUGEi64rr_toi64>;
1028 defm VecSLE : Vec_Compare_All<vsetle, ISetSLEi8rr_toi8, ISetSLEi16rr_toi16,
1029 ISetSLEi32rr_toi32, ISetSLEi64rr_toi64>;
1030 defm VecULE : Vec_Compare_All<vsetule, ISetULEi8rr_toi8, ISetULEi16rr_toi16,
1031 ISetULEi32rr_toi32, ISetULEi64rr_toi64>;
1032 defm VecSEQ : Vec_Compare_All<vseteq, ISetSEQi8rr_toi8, ISetSEQi16rr_toi16,
1033 ISetSEQi32rr_toi32, ISetSEQi64rr_toi64>;
1034 defm VecUEQ : Vec_Compare_All<vsetueq, ISetUEQi8rr_toi8, ISetUEQi16rr_toi16,
1035 ISetUEQi32rr_toi32, ISetUEQi64rr_toi64>;
1036 defm VecSNE : Vec_Compare_All<vsetne, ISetSNEi8rr_toi8, ISetSNEi16rr_toi16,
1037 ISetSNEi32rr_toi32, ISetSNEi64rr_toi64>;
1038 defm VecUNE : Vec_Compare_All<vsetune, ISetUNEi8rr_toi8, ISetUNEi16rr_toi16,
1039 ISetUNEi32rr_toi32, ISetUNEi64rr_toi64>;
1040}
1041
1042multiclass FVec_Compare_All<PatFrag op,
1043 NVPTXInst instf32,
1044 NVPTXInst instf64>
1045{
1046 def V2F32 : Vec_Compare<op, V2I32Regs, V2F32Regs, instf32>;
1047 def V4F32 : Vec_Compare<op, V4I32Regs, V4F32Regs, instf32>;
1048 def V2F64 : Vec_Compare<op, V2I64Regs, V2F64Regs, instf64>;
1049}
1050
1051let VecInstType=isVecOther.Value in {
1052 defm FVecGT : FVec_Compare_All<vsetogt, FSetGTf32rr_toi32,
1053 FSetGTf64rr_toi64>;
1054 defm FVecLT : FVec_Compare_All<vsetolt, FSetLTf32rr_toi32,
1055 FSetLTf64rr_toi64>;
1056 defm FVecGE : FVec_Compare_All<vsetoge, FSetGEf32rr_toi32,
1057 FSetGEf64rr_toi64>;
1058 defm FVecLE : FVec_Compare_All<vsetole, FSetLEf32rr_toi32,
1059 FSetLEf64rr_toi64>;
1060 defm FVecEQ : FVec_Compare_All<vsetoeq, FSetEQf32rr_toi32,
1061 FSetEQf64rr_toi64>;
1062 defm FVecNE : FVec_Compare_All<vsetone, FSetNEf32rr_toi32,
1063 FSetNEf64rr_toi64>;
1064
1065 defm FVecUGT : FVec_Compare_All<vsetugt, FSetUGTf32rr_toi32,
1066 FSetUGTf64rr_toi64>;
1067 defm FVecULT : FVec_Compare_All<vsetult, FSetULTf32rr_toi32,
1068 FSetULTf64rr_toi64>;
1069 defm FVecUGE : FVec_Compare_All<vsetuge, FSetUGEf32rr_toi32,
1070 FSetUGEf64rr_toi64>;
1071 defm FVecULE : FVec_Compare_All<vsetule, FSetULEf32rr_toi32,
1072 FSetULEf64rr_toi64>;
1073 defm FVecUEQ : FVec_Compare_All<vsetueq, FSetUEQf32rr_toi32,
1074 FSetUEQf64rr_toi64>;
1075 defm FVecUNE : FVec_Compare_All<vsetune, FSetUNEf32rr_toi32,
1076 FSetUNEf64rr_toi64>;
1077
1078 defm FVecNUM : FVec_Compare_All<vseto, FSetNUMf32rr_toi32,
1079 FSetNUMf64rr_toi64>;
1080 defm FVecNAN : FVec_Compare_All<vsetuo, FSetNANf32rr_toi32,
1081 FSetNANf64rr_toi64>;
1082}
1083
1084class LoadParamScalar4Inst<NVPTXRegClass regclass, string opstr> :
1085 NVPTXInst<(outs regclass:$d1, regclass:$d2, regclass:$d3, regclass:$d4),
1086 (ins i32imm:$a, i32imm:$b),
1087 !strconcat(!strconcat("ld.param", opstr),
1088 "\t{{$d1, $d2, $d3, $d4}}, [retval0+$b];"), []>;
1089
1090class LoadParamScalar2Inst<NVPTXRegClass regclass, string opstr> :
1091 NVPTXInst<(outs regclass:$d1, regclass:$d2),
1092 (ins i32imm:$a, i32imm:$b),
1093 !strconcat(!strconcat("ld.param", opstr),
1094 "\t{{$d1, $d2}}, [retval0+$b];"), []>;
1095
1096
1097class StoreParamScalar4Inst<NVPTXRegClass regclass, string opstr> :
1098 NVPTXInst<(outs),
1099 (ins regclass:$s1, regclass:$s2, regclass:$s3, regclass:$s4,
1100 i32imm:$a, i32imm:$b),
1101 !strconcat(!strconcat("st.param", opstr),
1102 "\t[param$a+$b], {{$s1, $s2, $s3, $s4}};"), []>;
1103
1104class StoreParamScalar2Inst<NVPTXRegClass regclass, string opstr> :
1105 NVPTXInst<(outs),
1106 (ins regclass:$s1, regclass:$s2, i32imm:$a, i32imm:$b),
1107 !strconcat(!strconcat("st.param", opstr),
1108 "\t[param$a+$b], {{$s1, $s2}};"), []>;
1109
1110class StoreRetvalScalar4Inst<NVPTXRegClass regclass, string opstr> :
1111 NVPTXInst<(outs),
1112 (ins regclass:$s1, regclass:$s2, regclass:$s3, regclass:$s4,
1113 i32imm:$a),
1114 !strconcat(!strconcat("st.param", opstr),
1115 "\t[func_retval+$a], {{$s1, $s2, $s3, $s4}};"), []>;
1116
1117class StoreRetvalScalar2Inst<NVPTXRegClass regclass, string opstr> :
1118 NVPTXInst<(outs),
1119 (ins regclass:$s1, regclass:$s2, i32imm:$a),
1120 !strconcat(!strconcat("st.param", opstr),
1121 "\t[func_retval+$a], {{$s1, $s2}};"), []>;
1122
1123def LoadParamScalar4I32 : LoadParamScalar4Inst<Int32Regs, ".v4.b32">;
1124def LoadParamScalar4I16 : LoadParamScalar4Inst<Int16Regs, ".v4.b16">;
1125def LoadParamScalar4I8 : LoadParamScalar4Inst<Int8Regs, ".v4.b8">;
1126
1127def LoadParamScalar2I64 : LoadParamScalar2Inst<Int32Regs, ".v2.b64">;
1128def LoadParamScalar2I32 : LoadParamScalar2Inst<Int32Regs, ".v2.b32">;
1129def LoadParamScalar2I16 : LoadParamScalar2Inst<Int32Regs, ".v2.b16">;
1130def LoadParamScalar2I8 : LoadParamScalar2Inst<Int32Regs, ".v2.b8">;
1131
1132def LoadParamScalar4F32 : LoadParamScalar4Inst<Float32Regs, ".v4.f32">;
1133def LoadParamScalar2F32 : LoadParamScalar2Inst<Float32Regs, ".v2.f32">;
1134def LoadParamScalar2F64 : LoadParamScalar2Inst<Float64Regs, ".v2.f64">;
1135
1136def StoreParamScalar4I32 : StoreParamScalar4Inst<Int32Regs, ".v4.b32">;
1137def StoreParamScalar4I16 : StoreParamScalar4Inst<Int16Regs, ".v4.b16">;
1138def StoreParamScalar4I8 : StoreParamScalar4Inst<Int8Regs, ".v4.b8">;
1139
1140def StoreParamScalar2I64 : StoreParamScalar2Inst<Int64Regs, ".v2.b64">;
1141def StoreParamScalar2I32 : StoreParamScalar2Inst<Int32Regs, ".v2.b32">;
1142def StoreParamScalar2I16 : StoreParamScalar2Inst<Int16Regs, ".v2.b16">;
1143def StoreParamScalar2I8 : StoreParamScalar2Inst<Int8Regs, ".v2.b8">;
1144
1145def StoreParamScalar4F32 : StoreParamScalar4Inst<Float32Regs, ".v4.f32">;
1146def StoreParamScalar2F32 : StoreParamScalar2Inst<Float32Regs, ".v2.f32">;
1147def StoreParamScalar2F64 : StoreParamScalar2Inst<Float64Regs, ".v2.f64">;
1148
1149def StoreRetvalScalar4I32 : StoreRetvalScalar4Inst<Int32Regs, ".v4.b32">;
1150def StoreRetvalScalar4I16 : StoreRetvalScalar4Inst<Int16Regs, ".v4.b16">;
1151def StoreRetvalScalar4I8 : StoreRetvalScalar4Inst<Int8Regs, ".v4.b8">;
1152
1153def StoreRetvalScalar2I64 : StoreRetvalScalar2Inst<Int64Regs, ".v2.b64">;
1154def StoreRetvalScalar2I32 : StoreRetvalScalar2Inst<Int32Regs, ".v2.b32">;
1155def StoreRetvalScalar2I16 : StoreRetvalScalar2Inst<Int16Regs, ".v2.b16">;
1156def StoreRetvalScalar2I8 : StoreRetvalScalar2Inst<Int8Regs, ".v2.b8">;
1157
1158def StoreRetvalScalar4F32 : StoreRetvalScalar4Inst<Float32Regs, ".v4.f32">;
1159def StoreRetvalScalar2F32 : StoreRetvalScalar2Inst<Float32Regs, ".v2.f32">;
1160def StoreRetvalScalar2F64 : StoreRetvalScalar2Inst<Float64Regs, ".v2.f64">;
1161
1162class LoadParamVecInst<NVPTXRegClass regclass, string opstr, NVPTXInst sop=NOP>:
1163 NVPTXVecInst<(outs regclass:$dst), (ins i32imm:$a, i32imm:$b),
1164 "loadparam : $dst <- [$a, $b]",
1165 [(set regclass:$dst, (LoadParam (i32 imm:$a), (i32 imm:$b)))],
1166 sop>;
1167
1168class StoreParamVecInst<NVPTXRegClass regclass, string opstr, NVPTXInst sop=NOP>
1169 : NVPTXVecInst<(outs), (ins regclass:$val, i32imm:$a, i32imm:$b),
1170 "storeparam : [$a, $b] <- $val",
1171 [(StoreParam (i32 imm:$a), (i32 imm:$b), regclass:$val)], sop>;
1172
1173class StoreRetvalVecInst<NVPTXRegClass regclass, string opstr,
1174 NVPTXInst sop=NOP>
1175 : NVPTXVecInst<(outs), (ins regclass:$val, i32imm:$a),
1176 "storeretval : retval[$a] <- $val",
1177 [(StoreRetval (i32 imm:$a), regclass:$val)], sop>;
1178
1179let VecInstType=isVecLD.Value in {
1180def LoadParamV4I32 : LoadParamVecInst<V4I32Regs, ".v4.b32",
1181 LoadParamScalar4I32>;
1182def LoadParamV4I16 : LoadParamVecInst<V4I16Regs, ".v4.b16",
1183 LoadParamScalar4I16>;
1184def LoadParamV4I8 : LoadParamVecInst<V4I8Regs, ".v4.b8",
1185 LoadParamScalar4I8>;
1186
1187def LoadParamV2I64 : LoadParamVecInst<V2I64Regs, ".v2.b64",
1188 LoadParamScalar2I64>;
1189def LoadParamV2I32 : LoadParamVecInst<V2I32Regs, ".v2.b32",
1190 LoadParamScalar2I32>;
1191def LoadParamV2I16 : LoadParamVecInst<V2I16Regs, ".v2.b16",
1192 LoadParamScalar2I16>;
1193def LoadParamV2I8 : LoadParamVecInst<V2I8Regs, ".v2.b8",
1194 LoadParamScalar2I8>;
1195
1196def LoadParamV4F32 : LoadParamVecInst<V4F32Regs, ".v4.f32",
1197 LoadParamScalar4F32>;
1198def LoadParamV2F32 : LoadParamVecInst<V2F32Regs, ".v2.f32",
1199 LoadParamScalar2F32>;
1200def LoadParamV2F64 : LoadParamVecInst<V2F64Regs, ".v2.f64",
1201 LoadParamScalar2F64>;
1202}
1203
1204let VecInstType=isVecST.Value in {
1205def StoreParamV4I32 : StoreParamVecInst<V4I32Regs, ".v4.b32",
1206 StoreParamScalar4I32>;
1207def StoreParamV4I16 : StoreParamVecInst<V4I16Regs, ".v4.b16",
1208 StoreParamScalar4I16>;
1209def StoreParamV4I8 : StoreParamVecInst<V4I8Regs, ".v4.b8",
1210 StoreParamScalar4I8>;
1211
1212def StoreParamV2I64 : StoreParamVecInst<V2I64Regs, ".v2.b64",
1213 StoreParamScalar2I64>;
1214def StoreParamV2I32 : StoreParamVecInst<V2I32Regs, ".v2.b32",
1215 StoreParamScalar2I32>;
1216def StoreParamV2I16 : StoreParamVecInst<V2I16Regs, ".v2.b16",
1217 StoreParamScalar2I16>;
1218def StoreParamV2I8 : StoreParamVecInst<V2I8Regs, ".v2.b8",
1219 StoreParamScalar2I8>;
1220
1221def StoreParamV4F32 : StoreParamVecInst<V4F32Regs, ".v4.f32",
1222 StoreParamScalar4F32>;
1223def StoreParamV2F32 : StoreParamVecInst<V2F32Regs, ".v2.f32",
1224 StoreParamScalar2F32>;
1225def StoreParamV2F64 : StoreParamVecInst<V2F64Regs, ".v2.f64",
1226 StoreParamScalar2F64>;
1227
1228def StoreRetvalV4I32 : StoreRetvalVecInst<V4I32Regs, ".v4.b32",
1229 StoreRetvalScalar4I32>;
1230def StoreRetvalV4I16 : StoreRetvalVecInst<V4I16Regs, ".v4.b16",
1231 StoreRetvalScalar4I16>;
1232def StoreRetvalV4I8 : StoreRetvalVecInst<V4I8Regs, ".v4.b8",
1233 StoreRetvalScalar4I8>;
1234
1235def StoreRetvalV2I64 : StoreRetvalVecInst<V2I64Regs, ".v2.b64",
1236 StoreRetvalScalar2I64>;
1237def StoreRetvalV2I32 : StoreRetvalVecInst<V2I32Regs, ".v2.b32",
1238 StoreRetvalScalar2I32>;
1239def StoreRetvalV2I16 : StoreRetvalVecInst<V2I16Regs, ".v2.b16",
1240 StoreRetvalScalar2I16>;
1241def StoreRetvalV2I8 : StoreRetvalVecInst<V2I8Regs, ".v2.b8",
1242 StoreRetvalScalar2I8>;
1243
1244def StoreRetvalV4F32 : StoreRetvalVecInst<V4F32Regs, ".v4.f32",
1245 StoreRetvalScalar4F32>;
1246def StoreRetvalV2F32 : StoreRetvalVecInst<V2F32Regs, ".v2.f32",
1247 StoreRetvalScalar2F32>;
1248def StoreRetvalV2F64 : StoreRetvalVecInst<V2F64Regs, ".v2.f64",
1249 StoreRetvalScalar2F64>;
1250
1251}
1252
1253
1254// Int vector to int scalar bit convert
1255// v4i8 -> i32
1256def : Pat<(i32 (bitconvert V4I8Regs:$s)),
1257 (V4I8toI32 (V4i8Extract V4I8Regs:$s,0), (V4i8Extract V4I8Regs:$s,1),
1258 (V4i8Extract V4I8Regs:$s,2), (V4i8Extract V4I8Regs:$s,3))>;
1259// v4i16 -> i64
1260def : Pat<(i64 (bitconvert V4I16Regs:$s)),
1261 (V4I16toI64 (V4i16Extract V4I16Regs:$s,0),
1262 (V4i16Extract V4I16Regs:$s,1),
1263 (V4i16Extract V4I16Regs:$s,2),
1264 (V4i16Extract V4I16Regs:$s,3))>;
1265// v2i8 -> i16
1266def : Pat<(i16 (bitconvert V2I8Regs:$s)),
1267 (V2I8toI16 (V2i8Extract V2I8Regs:$s,0), (V2i8Extract V2I8Regs:$s,1))>;
1268// v2i16 -> i32
1269def : Pat<(i32 (bitconvert V2I16Regs:$s)),
1270 (V2I16toI32 (V2i16Extract V2I16Regs:$s,0),
1271 (V2i16Extract V2I16Regs:$s,1))>;
1272// v2i32 -> i64
1273def : Pat<(i64 (bitconvert V2I32Regs:$s)),
1274 (V2I32toI64 (V2i32Extract V2I32Regs:$s,0),
1275 (V2i32Extract V2I32Regs:$s,1))>;
1276
1277// Int scalar to int vector bit convert
1278let VecInstType=isVecDest.Value in {
1279// i32 -> v4i8
1280def VecI32toV4I8 : NVPTXVecInst<(outs V4I8Regs:$d), (ins Int32Regs:$s),
1281 "Error!",
1282 [(set V4I8Regs:$d, (bitconvert Int32Regs:$s))],
1283 I32toV4I8>;
1284// i64 -> v4i16
1285def VecI64toV4I16 : NVPTXVecInst<(outs V4I16Regs:$d), (ins Int64Regs:$s),
1286 "Error!",
1287 [(set V4I16Regs:$d, (bitconvert Int64Regs:$s))],
1288 I64toV4I16>;
1289// i16 -> v2i8
1290def VecI16toV2I8 : NVPTXVecInst<(outs V2I8Regs:$d), (ins Int16Regs:$s),
1291 "Error!",
1292 [(set V2I8Regs:$d, (bitconvert Int16Regs:$s))],
1293 I16toV2I8>;
1294// i32 -> v2i16
1295def VecI32toV2I16 : NVPTXVecInst<(outs V2I16Regs:$d), (ins Int32Regs:$s),
1296 "Error!",
1297 [(set V2I16Regs:$d, (bitconvert Int32Regs:$s))],
1298 I32toV2I16>;
1299// i64 -> v2i32
1300def VecI64toV2I32 : NVPTXVecInst<(outs V2I32Regs:$d), (ins Int64Regs:$s),
1301 "Error!",
1302 [(set V2I32Regs:$d, (bitconvert Int64Regs:$s))],
1303 I64toV2I32>;
1304}
1305
1306// Int vector to int vector bit convert
1307// v4i8 -> v2i16
1308def : Pat<(v2i16 (bitconvert V4I8Regs:$s)),
1309 (VecI32toV2I16
1310 (V4I8toI32 (V4i8Extract V4I8Regs:$s,0), (V4i8Extract V4I8Regs:$s,1),
1311 (V4i8Extract V4I8Regs:$s,2), (V4i8Extract V4I8Regs:$s,3)))>;
1312// v4i16 -> v2i32
1313def : Pat<(v2i32 (bitconvert V4I16Regs:$s)),
1314 (VecI64toV2I32
1315 (V4I16toI64 (V4i16Extract V4I16Regs:$s,0), (V4i16Extract V4I16Regs:$s,1),
1316 (V4i16Extract V4I16Regs:$s,2), (V4i16Extract V4I16Regs:$s,3)))>;
1317// v2i16 -> v4i8
1318def : Pat<(v4i8 (bitconvert V2I16Regs:$s)),
1319 (VecI32toV4I8
1320 (V2I16toI32 (V2i16Extract V2I16Regs:$s,0), (V2i16Extract V2I16Regs:$s,1)))>;
1321// v2i32 -> v4i16
1322def : Pat<(v4i16 (bitconvert V2I32Regs:$s)),
1323 (VecI64toV4I16
1324 (V2I32toI64 (V2i32Extract V2I32Regs:$s,0), (V2i32Extract V2I32Regs:$s,1)))>;
1325// v2i64 -> v4i32
1326def : Pat<(v4i32 (bitconvert V2I64Regs:$s)),
1327 (Build_Vector4_i32
1328 (V2i32Extract (VecI64toV2I32 (V2i64Extract V2I64Regs:$s, 0)), 0),
1329 (V2i32Extract (VecI64toV2I32 (V2i64Extract V2I64Regs:$s, 0)), 1),
1330 (V2i32Extract (VecI64toV2I32 (V2i64Extract V2I64Regs:$s, 1)), 0),
1331 (V2i32Extract (VecI64toV2I32 (V2i64Extract V2I64Regs:$s, 1)), 1))>;
1332// v4i32 -> v2i64
1333def : Pat<(v2i64 (bitconvert V4I32Regs:$s)),
1334 (Build_Vector2_i64
1335 (V2I32toI64 (V4i32Extract V4I32Regs:$s,0), (V4i32Extract V4I32Regs:$s,1)),
1336 (V2I32toI64 (V4i32Extract V4I32Regs:$s,2), (V4i32Extract V4I32Regs:$s,3)))>;
1337
1338// Fp scalar to fp vector convert
1339// f64 -> v2f32
1340let VecInstType=isVecDest.Value in {
1341def VecF64toV2F32 : NVPTXVecInst<(outs V2F32Regs:$d), (ins Float64Regs:$s),
1342 "Error!",
1343 [(set V2F32Regs:$d, (bitconvert Float64Regs:$s))],
1344 F64toV2F32>;
1345}
1346
1347// Fp vector to fp scalar convert
1348// v2f32 -> f64
1349def : Pat<(f64 (bitconvert V2F32Regs:$s)),
1350 (V2F32toF64 (V2f32Extract V2F32Regs:$s,0), (V2f32Extract V2F32Regs:$s,1))>;
1351
1352// Fp scalar to int vector convert
1353// f32 -> v4i8
1354def : Pat<(v4i8 (bitconvert Float32Regs:$s)),
1355 (VecI32toV4I8 (BITCONVERT_32_F2I Float32Regs:$s))>;
1356// f32 -> v2i16
1357def : Pat<(v2i16 (bitconvert Float32Regs:$s)),
1358 (VecI32toV2I16 (BITCONVERT_32_F2I Float32Regs:$s))>;
1359// f64 -> v4i16
1360def : Pat<(v4i16 (bitconvert Float64Regs:$s)),
1361 (VecI64toV4I16 (BITCONVERT_64_F2I Float64Regs:$s))>;
1362// f64 -> v2i32
1363def : Pat<(v2i32 (bitconvert Float64Regs:$s)),
1364 (VecI64toV2I32 (BITCONVERT_64_F2I Float64Regs:$s))>;
1365
1366// Int vector to fp scalar convert
1367// v4i8 -> f32
1368def : Pat<(f32 (bitconvert V4I8Regs:$s)),
1369 (BITCONVERT_32_I2F
1370 (V4I8toI32 (V4i8Extract V4I8Regs:$s,0), (V4i8Extract V4I8Regs:$s,1),
1371 (V4i8Extract V4I8Regs:$s,2), (V4i8Extract V4I8Regs:$s,3)))>;
1372// v4i16 -> f64
1373def : Pat<(f64 (bitconvert V4I16Regs:$s)),
1374 (BITCONVERT_64_I2F
1375 (V4I16toI64 (V4i16Extract V4I16Regs:$s,0), (V4i16Extract V4I16Regs:$s,1),
1376 (V4i16Extract V4I16Regs:$s,2), (V4i16Extract V4I16Regs:$s,3)))>;
1377// v2i16 -> f32
1378def : Pat<(f32 (bitconvert V2I16Regs:$s)),
1379 (BITCONVERT_32_I2F
1380 (V2I16toI32 (V2i16Extract V2I16Regs:$s,0), (V2i16Extract V2I16Regs:$s,1)))>;
1381// v2i32 -> f64
1382def : Pat<(f64 (bitconvert V2I32Regs:$s)),
1383 (BITCONVERT_64_I2F
1384 (V2I32toI64 (V2i32Extract V2I32Regs:$s,0), (V2i32Extract V2I32Regs:$s,1)))>;
1385
1386// Int scalar to fp vector convert
1387// i64 -> v2f32
1388def : Pat<(v2f32 (bitconvert Int64Regs:$s)),
1389 (VecF64toV2F32 (BITCONVERT_64_I2F Int64Regs:$s))>;
1390
1391// Fp vector to int scalar convert
1392// v2f32 -> i64
1393def : Pat<(i64 (bitconvert V2F32Regs:$s)),
1394 (BITCONVERT_64_F2I
1395 (V2F32toF64 (V2f32Extract V2F32Regs:$s,0), (V2f32Extract V2F32Regs:$s,1)))>;
1396
1397// Int vector to fp vector convert
1398// v2i64 -> v4f32
1399def : Pat<(v4f32 (bitconvert V2I64Regs:$s)),
1400 (Build_Vector4_f32
1401 (BITCONVERT_32_I2F (V2i32Extract (VecI64toV2I32
1402 (V2i64Extract V2I64Regs:$s, 0)), 0)),
1403 (BITCONVERT_32_I2F (V2i32Extract (VecI64toV2I32
1404 (V2i64Extract V2I64Regs:$s, 0)), 1)),
1405 (BITCONVERT_32_I2F (V2i32Extract (VecI64toV2I32
1406 (V2i64Extract V2I64Regs:$s, 1)), 0)),
1407 (BITCONVERT_32_I2F (V2i32Extract (VecI64toV2I32
1408 (V2i64Extract V2I64Regs:$s, 1)), 1)))>;
1409// v2i64 -> v2f64
1410def : Pat<(v2f64 (bitconvert V2I64Regs:$s)),
1411 (Build_Vector2_f64
1412 (BITCONVERT_64_I2F (V2i64Extract V2I64Regs:$s,0)),
1413 (BITCONVERT_64_I2F (V2i64Extract V2I64Regs:$s,1)))>;
1414// v2i32 -> v2f32
1415def : Pat<(v2f32 (bitconvert V2I32Regs:$s)),
1416 (Build_Vector2_f32
1417 (BITCONVERT_32_I2F (V2i32Extract V2I32Regs:$s,0)),
1418 (BITCONVERT_32_I2F (V2i32Extract V2I32Regs:$s,1)))>;
1419// v4i32 -> v2f64
1420def : Pat<(v2f64 (bitconvert V4I32Regs:$s)),
1421 (Build_Vector2_f64
1422 (BITCONVERT_64_I2F (V2I32toI64 (V4i32Extract V4I32Regs:$s,0),
1423 (V4i32Extract V4I32Regs:$s,1))),
1424 (BITCONVERT_64_I2F (V2I32toI64 (V4i32Extract V4I32Regs:$s,2),
1425 (V4i32Extract V4I32Regs:$s,3))))>;
1426// v4i32 -> v4f32
1427def : Pat<(v4f32 (bitconvert V4I32Regs:$s)),
1428 (Build_Vector4_f32
1429 (BITCONVERT_32_I2F (V4i32Extract V4I32Regs:$s,0)),
1430 (BITCONVERT_32_I2F (V4i32Extract V4I32Regs:$s,1)),
1431 (BITCONVERT_32_I2F (V4i32Extract V4I32Regs:$s,2)),
1432 (BITCONVERT_32_I2F (V4i32Extract V4I32Regs:$s,3)))>;
1433// v4i16 -> v2f32
1434def : Pat<(v2f32 (bitconvert V4I16Regs:$s)),
1435 (VecF64toV2F32 (BITCONVERT_64_I2F
1436 (V4I16toI64 (V4i16Extract V4I16Regs:$s,0),
1437 (V4i16Extract V4I16Regs:$s,1),
1438 (V4i16Extract V4I16Regs:$s,2),
1439 (V4i16Extract V4I16Regs:$s,3))))>;
1440
1441// Fp vector to int vector convert
1442// v2i64 <- v4f32
1443def : Pat<(v2i64 (bitconvert V4F32Regs:$s)),
1444 (Build_Vector2_i64
1445 (BITCONVERT_64_F2I (V2F32toF64 (V4f32Extract V4F32Regs:$s,0),
1446 (V4f32Extract V4F32Regs:$s,1))),
1447 (BITCONVERT_64_F2I (V2F32toF64 (V4f32Extract V4F32Regs:$s,2),
1448 (V4f32Extract V4F32Regs:$s,3))))>;
1449// v2i64 <- v2f64
1450def : Pat<(v2i64 (bitconvert V2F64Regs:$s)),
1451 (Build_Vector2_i64
1452 (BITCONVERT_64_F2I (V2f64Extract V2F64Regs:$s,0)),
1453 (BITCONVERT_64_F2I (V2f64Extract V2F64Regs:$s,1)))>;
1454// v2i32 <- v2f32
1455def : Pat<(v2i32 (bitconvert V2F32Regs:$s)),
1456 (Build_Vector2_i32
1457 (BITCONVERT_32_F2I (V2f32Extract V2F32Regs:$s,0)),
1458 (BITCONVERT_32_F2I (V2f32Extract V2F32Regs:$s,1)))>;
1459// v4i32 <- v2f64
1460def : Pat<(v4i32 (bitconvert V2F64Regs:$s)),
1461 (Build_Vector4_i32
1462 (BITCONVERT_32_F2I (V2f32Extract (VecF64toV2F32
1463 (V2f64Extract V2F64Regs:$s, 0)), 0)),
1464 (BITCONVERT_32_F2I (V2f32Extract (VecF64toV2F32
1465 (V2f64Extract V2F64Regs:$s, 0)), 1)),
1466 (BITCONVERT_32_F2I (V2f32Extract (VecF64toV2F32
1467 (V2f64Extract V2F64Regs:$s, 1)), 0)),
1468 (BITCONVERT_32_F2I (V2f32Extract (VecF64toV2F32
1469 (V2f64Extract V2F64Regs:$s, 1)), 1)))>;
1470// v4i32 <- v4f32
1471def : Pat<(v4i32 (bitconvert V4F32Regs:$s)),
1472 (Build_Vector4_i32
1473 (BITCONVERT_32_F2I (V4f32Extract V4F32Regs:$s,0)),
1474 (BITCONVERT_32_F2I (V4f32Extract V4F32Regs:$s,1)),
1475 (BITCONVERT_32_F2I (V4f32Extract V4F32Regs:$s,2)),
1476 (BITCONVERT_32_F2I (V4f32Extract V4F32Regs:$s,3)))>;
1477// v4i16 <- v2f32
1478def : Pat<(v4i16 (bitconvert V2F32Regs:$s)),
1479 (VecI64toV4I16 (BITCONVERT_64_F2I
1480 (V2F32toF64 (V2f32Extract V2F32Regs:$s,0),
1481 (V2f32Extract V2F32Regs:$s,1))))>;