blob: cf1c096b5af4fd12d0714edfaa1dfa50f5a1e282 [file] [log] [blame]
Eric Christopher06b32cd2015-02-20 00:36:53 +00001//===-- X86InstrAVX512.td - AVX512 Instruction Set ---------*- tablegen -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file describes the X86 AVX512 instruction set, defining the
11// instructions, and properties of the instructions which are needed for code
12// generation, machine code emission, and analysis.
13//
14//===----------------------------------------------------------------------===//
15
Adam Nemet5ed17da2014-08-21 19:50:07 +000016// Group template arguments that can be derived from the vector type (EltNum x
17// EltVT). These are things like the register class for the writemask, etc.
18// The idea is to pass one of these as the template argument rather than the
19// individual arguments.
Elena Demikhovskyfa4a6c12014-12-09 07:06:32 +000020// The template is also used for scalar types, in this case numelts is 1.
Robert Khasanov4204c1a2014-12-12 14:21:30 +000021class X86VectorVTInfo<int numelts, ValueType eltvt, RegisterClass rc,
Adam Nemet5ed17da2014-08-21 19:50:07 +000022 string suffix = ""> {
23 RegisterClass RC = rc;
Robert Khasanov4204c1a2014-12-12 14:21:30 +000024 ValueType EltVT = eltvt;
Adam Nemet449b3f02014-10-15 23:42:09 +000025 int NumElts = numelts;
Adam Nemet5ed17da2014-08-21 19:50:07 +000026
27 // Corresponding mask register class.
28 RegisterClass KRC = !cast<RegisterClass>("VK" # NumElts);
29
30 // Corresponding write-mask register class.
31 RegisterClass KRCWM = !cast<RegisterClass>("VK" # NumElts # "WM");
32
Igor Bregerfca0a342016-01-28 13:19:25 +000033 // The mask VT.
Simon Pilgrimb13961d2016-06-11 14:34:10 +000034 ValueType KVT = !cast<ValueType>(!if (!eq (NumElts, 1), "i1",
35 "v" # NumElts # "i1"));
36
Adam Nemet5ed17da2014-08-21 19:50:07 +000037 // The GPR register class that can hold the write mask. Use GR8 for fewer
38 // than 8 elements. Use shift-right and equal to work around the lack of
39 // !lt in tablegen.
40 RegisterClass MRC =
41 !cast<RegisterClass>("GR" #
42 !if (!eq (!srl(NumElts, 3), 0), 8, NumElts));
43
44 // Suffix used in the instruction mnemonic.
45 string Suffix = suffix;
46
Elena Demikhovskyfa4a6c12014-12-09 07:06:32 +000047 // VTName is a string name for vector VT. For vector types it will be
48 // v # NumElts # EltVT, so for vector of 8 elements of i32 it will be v8i32
49 // It is a little bit complex for scalar types, where NumElts = 1.
50 // In this case we build v4f32 or v2f64
51 string VTName = "v" # !if (!eq (NumElts, 1),
52 !if (!eq (EltVT.Size, 32), 4,
53 !if (!eq (EltVT.Size, 64), 2, NumElts)), NumElts) # EltVT;
Robert Khasanov2ea081d2014-08-25 14:49:34 +000054
Adam Nemet5ed17da2014-08-21 19:50:07 +000055 // The vector VT.
Robert Khasanov2ea081d2014-08-25 14:49:34 +000056 ValueType VT = !cast<ValueType>(VTName);
Adam Nemet5ed17da2014-08-21 19:50:07 +000057
58 string EltTypeName = !cast<string>(EltVT);
59 // Size of the element type in bits, e.g. 32 for v16i32.
Robert Khasanov2ea081d2014-08-25 14:49:34 +000060 string EltSizeName = !subst("i", "", !subst("f", "", EltTypeName));
61 int EltSize = EltVT.Size;
Adam Nemet5ed17da2014-08-21 19:50:07 +000062
63 // "i" for integer types and "f" for floating-point types
Robert Khasanov2ea081d2014-08-25 14:49:34 +000064 string TypeVariantName = !subst(EltSizeName, "", EltTypeName);
Adam Nemet5ed17da2014-08-21 19:50:07 +000065
66 // Size of RC in bits, e.g. 512 for VR512.
67 int Size = VT.Size;
68
69 // The corresponding memory operand, e.g. i512mem for VR512.
70 X86MemOperand MemOp = !cast<X86MemOperand>(TypeVariantName # Size # "mem");
Robert Khasanov2ea081d2014-08-25 14:49:34 +000071 X86MemOperand ScalarMemOp = !cast<X86MemOperand>(EltVT # "mem");
72
73 // Load patterns
74 // Note: For 128/256-bit integer VT we choose loadv2i64/loadv4i64
75 // due to load promotion during legalization
76 PatFrag LdFrag = !cast<PatFrag>("load" #
77 !if (!eq (TypeVariantName, "i"),
78 !if (!eq (Size, 128), "v2i64",
79 !if (!eq (Size, 256), "v4i64",
80 VTName)), VTName));
Elena Demikhovsky2689d782015-03-02 12:46:21 +000081
82 PatFrag AlignedLdFrag = !cast<PatFrag>("alignedload" #
83 !if (!eq (TypeVariantName, "i"),
84 !if (!eq (Size, 128), "v2i64",
85 !if (!eq (Size, 256), "v4i64",
Michael Liao66233b72015-08-06 09:06:20 +000086 !if (!eq (Size, 512),
Elena Demikhovsky2689d782015-03-02 12:46:21 +000087 !if (!eq (EltSize, 64), "v8i64", "v16i32"),
88 VTName))), VTName));
89
Robert Khasanov2ea081d2014-08-25 14:49:34 +000090 PatFrag ScalarLdFrag = !cast<PatFrag>("load" # EltVT);
Adam Nemet5ed17da2014-08-21 19:50:07 +000091
92 // The corresponding float type, e.g. v16f32 for v16i32
Robert Khasanov2ea081d2014-08-25 14:49:34 +000093 // Note: For EltSize < 32, FloatVT is illegal and TableGen
94 // fails to compile, so we choose FloatVT = VT
95 ValueType FloatVT = !cast<ValueType>(
96 !if (!eq (!srl(EltSize,5),0),
97 VTName,
98 !if (!eq(TypeVariantName, "i"),
99 "v" # NumElts # "f" # EltSize,
100 VTName)));
Adam Nemet5ed17da2014-08-21 19:50:07 +0000101
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +0000102 ValueType IntVT = !cast<ValueType>(
103 !if (!eq (!srl(EltSize,5),0),
104 VTName,
105 !if (!eq(TypeVariantName, "f"),
106 "v" # NumElts # "i" # EltSize,
107 VTName)));
Adam Nemet5ed17da2014-08-21 19:50:07 +0000108 // The string to specify embedded broadcast in assembly.
109 string BroadcastStr = "{1to" # NumElts # "}";
Adam Nemet55536c62014-09-25 23:48:45 +0000110
Adam Nemet449b3f02014-10-15 23:42:09 +0000111 // 8-bit compressed displacement tuple/subvector format. This is only
112 // defined for NumElts <= 8.
113 CD8VForm CD8TupleForm = !if (!eq (!srl(NumElts, 4), 0),
114 !cast<CD8VForm>("CD8VT" # NumElts), ?);
115
Adam Nemet55536c62014-09-25 23:48:45 +0000116 SubRegIndex SubRegIdx = !if (!eq (Size, 128), sub_xmm,
117 !if (!eq (Size, 256), sub_ymm, ?));
118
119 Domain ExeDomain = !if (!eq (EltTypeName, "f32"), SSEPackedSingle,
120 !if (!eq (EltTypeName, "f64"), SSEPackedDouble,
121 SSEPackedInt));
Adam Nemet09377232014-10-08 23:25:31 +0000122
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +0000123 RegisterClass FRC = !if (!eq (EltTypeName, "f32"), FR32X, FR64X);
124
Adam Nemet09377232014-10-08 23:25:31 +0000125 // A vector type of the same width with element type i32. This is used to
126 // create the canonical constant zero node ImmAllZerosV.
127 ValueType i32VT = !cast<ValueType>("v" # !srl(Size, 5) # "i32");
128 dag ImmAllZerosV = (VT (bitconvert (i32VT immAllZerosV)));
Elena Demikhovskyd207f172015-03-03 15:03:35 +0000129
130 string ZSuffix = !if (!eq (Size, 128), "Z128",
131 !if (!eq (Size, 256), "Z256", "Z"));
Adam Nemet5ed17da2014-08-21 19:50:07 +0000132}
133
Robert Khasanov2ea081d2014-08-25 14:49:34 +0000134def v64i8_info : X86VectorVTInfo<64, i8, VR512, "b">;
135def v32i16_info : X86VectorVTInfo<32, i16, VR512, "w">;
Adam Nemet5ed17da2014-08-21 19:50:07 +0000136def v16i32_info : X86VectorVTInfo<16, i32, VR512, "d">;
137def v8i64_info : X86VectorVTInfo<8, i64, VR512, "q">;
Adam Nemet6bddb8c2014-09-29 22:54:41 +0000138def v16f32_info : X86VectorVTInfo<16, f32, VR512, "ps">;
139def v8f64_info : X86VectorVTInfo<8, f64, VR512, "pd">;
Adam Nemet5ed17da2014-08-21 19:50:07 +0000140
Robert Khasanov2ea081d2014-08-25 14:49:34 +0000141// "x" in v32i8x_info means RC = VR256X
142def v32i8x_info : X86VectorVTInfo<32, i8, VR256X, "b">;
143def v16i16x_info : X86VectorVTInfo<16, i16, VR256X, "w">;
144def v8i32x_info : X86VectorVTInfo<8, i32, VR256X, "d">;
145def v4i64x_info : X86VectorVTInfo<4, i64, VR256X, "q">;
Robert Khasanov3e534c92014-10-28 16:37:13 +0000146def v8f32x_info : X86VectorVTInfo<8, f32, VR256X, "ps">;
147def v4f64x_info : X86VectorVTInfo<4, f64, VR256X, "pd">;
Robert Khasanov2ea081d2014-08-25 14:49:34 +0000148
149def v16i8x_info : X86VectorVTInfo<16, i8, VR128X, "b">;
150def v8i16x_info : X86VectorVTInfo<8, i16, VR128X, "w">;
151def v4i32x_info : X86VectorVTInfo<4, i32, VR128X, "d">;
152def v2i64x_info : X86VectorVTInfo<2, i64, VR128X, "q">;
Robert Khasanov3e534c92014-10-28 16:37:13 +0000153def v4f32x_info : X86VectorVTInfo<4, f32, VR128X, "ps">;
154def v2f64x_info : X86VectorVTInfo<2, f64, VR128X, "pd">;
Robert Khasanov2ea081d2014-08-25 14:49:34 +0000155
Elena Demikhovskyfa4a6c12014-12-09 07:06:32 +0000156// We map scalar types to the smallest (128-bit) vector type
157// with the appropriate element type. This allows to use the same masking logic.
Asaf Badouh2744d212015-09-20 14:31:19 +0000158def i32x_info : X86VectorVTInfo<1, i32, GR32, "si">;
159def i64x_info : X86VectorVTInfo<1, i64, GR64, "sq">;
Elena Demikhovsky905a5a62014-11-26 10:46:49 +0000160def f32x_info : X86VectorVTInfo<1, f32, VR128X, "ss">;
161def f64x_info : X86VectorVTInfo<1, f64, VR128X, "sd">;
162
Robert Khasanov2ea081d2014-08-25 14:49:34 +0000163class AVX512VLVectorVTInfo<X86VectorVTInfo i512, X86VectorVTInfo i256,
164 X86VectorVTInfo i128> {
165 X86VectorVTInfo info512 = i512;
166 X86VectorVTInfo info256 = i256;
167 X86VectorVTInfo info128 = i128;
168}
169
170def avx512vl_i8_info : AVX512VLVectorVTInfo<v64i8_info, v32i8x_info,
171 v16i8x_info>;
172def avx512vl_i16_info : AVX512VLVectorVTInfo<v32i16_info, v16i16x_info,
173 v8i16x_info>;
174def avx512vl_i32_info : AVX512VLVectorVTInfo<v16i32_info, v8i32x_info,
175 v4i32x_info>;
176def avx512vl_i64_info : AVX512VLVectorVTInfo<v8i64_info, v4i64x_info,
177 v2i64x_info>;
Robert Khasanovaf318f72014-10-30 14:21:47 +0000178def avx512vl_f32_info : AVX512VLVectorVTInfo<v16f32_info, v8f32x_info,
179 v4f32x_info>;
180def avx512vl_f64_info : AVX512VLVectorVTInfo<v8f64_info, v4f64x_info,
181 v2f64x_info>;
Robert Khasanov2ea081d2014-08-25 14:49:34 +0000182
Adam Nemet52bb6cf2014-10-08 23:25:23 +0000183// This multiclass generates the masking variants from the non-masking
184// variant. It only provides the assembly pieces for the masking variants.
185// It assumes custom ISel patterns for masking which can be provided as
186// template arguments.
Adam Nemet34801422014-10-08 23:25:39 +0000187multiclass AVX512_maskable_custom<bits<8> O, Format F,
188 dag Outs,
189 dag Ins, dag MaskingIns, dag ZeroMaskingIns,
190 string OpcodeStr,
191 string AttSrcAsm, string IntelSrcAsm,
192 list<dag> Pattern,
193 list<dag> MaskingPattern,
194 list<dag> ZeroMaskingPattern,
195 string MaskingConstraint = "",
196 InstrItinClass itin = NoItinerary,
197 bit IsCommutable = 0> {
Adam Nemet52bb6cf2014-10-08 23:25:23 +0000198 let isCommutable = IsCommutable in
199 def NAME: AVX512<O, F, Outs, Ins,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000200 OpcodeStr#"\t{"#AttSrcAsm#", $dst|"#
Craig Topper9d2cab72016-01-11 01:03:40 +0000201 "$dst, "#IntelSrcAsm#"}",
Adam Nemet52bb6cf2014-10-08 23:25:23 +0000202 Pattern, itin>;
203
204 // Prefer over VMOV*rrk Pat<>
205 let AddedComplexity = 20 in
206 def NAME#k: AVX512<O, F, Outs, MaskingIns,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000207 OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"#
208 "$dst {${mask}}, "#IntelSrcAsm#"}",
Adam Nemet52bb6cf2014-10-08 23:25:23 +0000209 MaskingPattern, itin>,
210 EVEX_K {
211 // In case of the 3src subclass this is overridden with a let.
212 string Constraints = MaskingConstraint;
213 }
214 let AddedComplexity = 30 in // Prefer over VMOV*rrkz Pat<>
215 def NAME#kz: AVX512<O, F, Outs, ZeroMaskingIns,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000216 OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}} {z}|"#
217 "$dst {${mask}} {z}, "#IntelSrcAsm#"}",
Adam Nemet52bb6cf2014-10-08 23:25:23 +0000218 ZeroMaskingPattern,
219 itin>,
220 EVEX_KZ;
221}
222
Robert Khasanov2ea081d2014-08-25 14:49:34 +0000223
Adam Nemet34801422014-10-08 23:25:39 +0000224// Common base class of AVX512_maskable and AVX512_maskable_3src.
225multiclass AVX512_maskable_common<bits<8> O, Format F, X86VectorVTInfo _,
226 dag Outs,
227 dag Ins, dag MaskingIns, dag ZeroMaskingIns,
228 string OpcodeStr,
229 string AttSrcAsm, string IntelSrcAsm,
230 dag RHS, dag MaskingRHS,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000231 SDNode Select = vselect,
Adam Nemet34801422014-10-08 23:25:39 +0000232 string MaskingConstraint = "",
233 InstrItinClass itin = NoItinerary,
234 bit IsCommutable = 0> :
235 AVX512_maskable_custom<O, F, Outs, Ins, MaskingIns, ZeroMaskingIns, OpcodeStr,
236 AttSrcAsm, IntelSrcAsm,
237 [(set _.RC:$dst, RHS)],
238 [(set _.RC:$dst, MaskingRHS)],
239 [(set _.RC:$dst,
Elena Demikhovsky905a5a62014-11-26 10:46:49 +0000240 (Select _.KRCWM:$mask, RHS, _.ImmAllZerosV))],
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000241 MaskingConstraint, NoItinerary, IsCommutable>;
Adam Nemet2e2537f2014-08-07 17:53:55 +0000242
Adam Nemet2e91ee52014-08-14 17:13:19 +0000243// This multiclass generates the unconditional/non-masking, the masking and
Elena Demikhovsky905a5a62014-11-26 10:46:49 +0000244// the zero-masking variant of the vector instruction. In the masking case, the
Adam Nemet2e91ee52014-08-14 17:13:19 +0000245// perserved vector elements come from a new dummy input operand tied to $dst.
Adam Nemet34801422014-10-08 23:25:39 +0000246multiclass AVX512_maskable<bits<8> O, Format F, X86VectorVTInfo _,
247 dag Outs, dag Ins, string OpcodeStr,
248 string AttSrcAsm, string IntelSrcAsm,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000249 dag RHS,
Elena Demikhovskybe8808d2014-11-12 07:31:03 +0000250 InstrItinClass itin = NoItinerary,
Igor Breger73ee8ba2016-05-31 08:04:21 +0000251 bit IsCommutable = 0, SDNode Select = vselect> :
Adam Nemet34801422014-10-08 23:25:39 +0000252 AVX512_maskable_common<O, F, _, Outs, Ins,
253 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),
254 !con((ins _.KRCWM:$mask), Ins),
255 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
Igor Breger73ee8ba2016-05-31 08:04:21 +0000256 (Select _.KRCWM:$mask, RHS, _.RC:$src0), Select,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000257 "$src0 = $dst", itin, IsCommutable>;
Elena Demikhovsky905a5a62014-11-26 10:46:49 +0000258
259// This multiclass generates the unconditional/non-masking, the masking and
260// the zero-masking variant of the scalar instruction.
261multiclass AVX512_maskable_scalar<bits<8> O, Format F, X86VectorVTInfo _,
262 dag Outs, dag Ins, string OpcodeStr,
263 string AttSrcAsm, string IntelSrcAsm,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000264 dag RHS,
Elena Demikhovsky905a5a62014-11-26 10:46:49 +0000265 InstrItinClass itin = NoItinerary,
266 bit IsCommutable = 0> :
267 AVX512_maskable_common<O, F, _, Outs, Ins,
268 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),
269 !con((ins _.KRCWM:$mask), Ins),
270 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
Craig Topper74ed0872016-05-18 06:55:59 +0000271 (X86selects _.KRCWM:$mask, RHS, _.RC:$src0),
272 X86selects, "$src0 = $dst", itin, IsCommutable>;
Adam Nemet2e91ee52014-08-14 17:13:19 +0000273
Adam Nemet34801422014-10-08 23:25:39 +0000274// Similar to AVX512_maskable but in this case one of the source operands
Adam Nemet2e91ee52014-08-14 17:13:19 +0000275// ($src1) is already tied to $dst so we just use that for the preserved
276// vector elements. NOTE that the NonTiedIns (the ins dag) should exclude
277// $src1.
Adam Nemet34801422014-10-08 23:25:39 +0000278multiclass AVX512_maskable_3src<bits<8> O, Format F, X86VectorVTInfo _,
279 dag Outs, dag NonTiedIns, string OpcodeStr,
280 string AttSrcAsm, string IntelSrcAsm,
281 dag RHS> :
282 AVX512_maskable_common<O, F, _, Outs,
283 !con((ins _.RC:$src1), NonTiedIns),
284 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),
285 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),
286 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
287 (vselect _.KRCWM:$mask, RHS, _.RC:$src1)>;
Adam Nemet2e91ee52014-08-14 17:13:19 +0000288
Craig Topperaad5f112015-11-30 00:13:24 +0000289// Similar to AVX512_maskable_3rc but in this case the input VT for the tied
290// operand differs from the output VT. This requires a bitconvert on
291// the preserved vector going into the vselect.
292multiclass AVX512_maskable_3src_cast<bits<8> O, Format F, X86VectorVTInfo OutVT,
293 X86VectorVTInfo InVT,
294 dag Outs, dag NonTiedIns, string OpcodeStr,
295 string AttSrcAsm, string IntelSrcAsm,
296 dag RHS> :
297 AVX512_maskable_common<O, F, OutVT, Outs,
298 !con((ins InVT.RC:$src1), NonTiedIns),
299 !con((ins InVT.RC:$src1, InVT.KRCWM:$mask), NonTiedIns),
300 !con((ins InVT.RC:$src1, InVT.KRCWM:$mask), NonTiedIns),
301 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
302 (vselect InVT.KRCWM:$mask, RHS,
303 (bitconvert InVT.RC:$src1))>;
304
Igor Breger15820b02015-07-01 13:24:28 +0000305multiclass AVX512_maskable_3src_scalar<bits<8> O, Format F, X86VectorVTInfo _,
306 dag Outs, dag NonTiedIns, string OpcodeStr,
307 string AttSrcAsm, string IntelSrcAsm,
308 dag RHS> :
309 AVX512_maskable_common<O, F, _, Outs,
310 !con((ins _.RC:$src1), NonTiedIns),
311 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),
312 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),
313 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
Craig Topper74ed0872016-05-18 06:55:59 +0000314 (X86selects _.KRCWM:$mask, RHS, _.RC:$src1),
315 X86selects>;
Adam Nemet2b5cdbb2014-10-08 23:25:33 +0000316
Adam Nemet34801422014-10-08 23:25:39 +0000317multiclass AVX512_maskable_in_asm<bits<8> O, Format F, X86VectorVTInfo _,
318 dag Outs, dag Ins,
319 string OpcodeStr,
320 string AttSrcAsm, string IntelSrcAsm,
321 list<dag> Pattern> :
322 AVX512_maskable_custom<O, F, Outs, Ins,
323 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),
324 !con((ins _.KRCWM:$mask), Ins),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000325 OpcodeStr, AttSrcAsm, IntelSrcAsm, Pattern, [], [],
Adam Nemet34801422014-10-08 23:25:39 +0000326 "$src0 = $dst">;
Adam Nemet2b5cdbb2014-10-08 23:25:33 +0000327
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000328
329// Instruction with mask that puts result in mask register,
330// like "compare" and "vptest"
331multiclass AVX512_maskable_custom_cmp<bits<8> O, Format F,
332 dag Outs,
333 dag Ins, dag MaskingIns,
334 string OpcodeStr,
335 string AttSrcAsm, string IntelSrcAsm,
336 list<dag> Pattern,
Craig Topper156622a2016-01-11 00:44:56 +0000337 list<dag> MaskingPattern> {
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000338 def NAME: AVX512<O, F, Outs, Ins,
Craig Topper156622a2016-01-11 00:44:56 +0000339 OpcodeStr#"\t{"#AttSrcAsm#", $dst|"#
340 "$dst, "#IntelSrcAsm#"}",
341 Pattern, NoItinerary>;
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000342
343 def NAME#k: AVX512<O, F, Outs, MaskingIns,
Craig Topper156622a2016-01-11 00:44:56 +0000344 OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"#
345 "$dst {${mask}}, "#IntelSrcAsm#"}",
346 MaskingPattern, NoItinerary>, EVEX_K;
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000347}
348
349multiclass AVX512_maskable_common_cmp<bits<8> O, Format F, X86VectorVTInfo _,
350 dag Outs,
351 dag Ins, dag MaskingIns,
352 string OpcodeStr,
353 string AttSrcAsm, string IntelSrcAsm,
Craig Topper156622a2016-01-11 00:44:56 +0000354 dag RHS, dag MaskingRHS> :
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000355 AVX512_maskable_custom_cmp<O, F, Outs, Ins, MaskingIns, OpcodeStr,
356 AttSrcAsm, IntelSrcAsm,
357 [(set _.KRC:$dst, RHS)],
Craig Topper156622a2016-01-11 00:44:56 +0000358 [(set _.KRC:$dst, MaskingRHS)]>;
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000359
360multiclass AVX512_maskable_cmp<bits<8> O, Format F, X86VectorVTInfo _,
361 dag Outs, dag Ins, string OpcodeStr,
362 string AttSrcAsm, string IntelSrcAsm,
Craig Topper156622a2016-01-11 00:44:56 +0000363 dag RHS> :
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000364 AVX512_maskable_common_cmp<O, F, _, Outs, Ins,
365 !con((ins _.KRCWM:$mask), Ins),
366 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
Craig Topper156622a2016-01-11 00:44:56 +0000367 (and _.KRCWM:$mask, RHS)>;
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000368
Elena Demikhovsky29792e92015-05-07 11:24:42 +0000369multiclass AVX512_maskable_cmp_alt<bits<8> O, Format F, X86VectorVTInfo _,
370 dag Outs, dag Ins, string OpcodeStr,
371 string AttSrcAsm, string IntelSrcAsm> :
372 AVX512_maskable_custom_cmp<O, F, Outs,
373 Ins, !con((ins _.KRCWM:$mask),Ins), OpcodeStr,
Craig Topper156622a2016-01-11 00:44:56 +0000374 AttSrcAsm, IntelSrcAsm, [],[]>;
Elena Demikhovsky29792e92015-05-07 11:24:42 +0000375
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000376// Bitcasts between 512-bit vector types. Return the original type since
Craig Topper2388b462016-06-03 04:15:27 +0000377// no instruction is needed for the conversion.
378def : Pat<(v8f64 (bitconvert (v8i64 VR512:$src))), (v8f64 VR512:$src)>;
379def : Pat<(v8f64 (bitconvert (v16i32 VR512:$src))), (v8f64 VR512:$src)>;
380def : Pat<(v8f64 (bitconvert (v32i16 VR512:$src))), (v8f64 VR512:$src)>;
381def : Pat<(v8f64 (bitconvert (v64i8 VR512:$src))), (v8f64 VR512:$src)>;
382def : Pat<(v8f64 (bitconvert (v16f32 VR512:$src))), (v8f64 VR512:$src)>;
383def : Pat<(v16f32 (bitconvert (v8i64 VR512:$src))), (v16f32 VR512:$src)>;
384def : Pat<(v16f32 (bitconvert (v16i32 VR512:$src))), (v16f32 VR512:$src)>;
385def : Pat<(v16f32 (bitconvert (v32i16 VR512:$src))), (v16f32 VR512:$src)>;
386def : Pat<(v16f32 (bitconvert (v64i8 VR512:$src))), (v16f32 VR512:$src)>;
387def : Pat<(v16f32 (bitconvert (v8f64 VR512:$src))), (v16f32 VR512:$src)>;
388def : Pat<(v8i64 (bitconvert (v16i32 VR512:$src))), (v8i64 VR512:$src)>;
389def : Pat<(v8i64 (bitconvert (v32i16 VR512:$src))), (v8i64 VR512:$src)>;
390def : Pat<(v8i64 (bitconvert (v64i8 VR512:$src))), (v8i64 VR512:$src)>;
391def : Pat<(v8i64 (bitconvert (v8f64 VR512:$src))), (v8i64 VR512:$src)>;
392def : Pat<(v8i64 (bitconvert (v16f32 VR512:$src))), (v8i64 VR512:$src)>;
393def : Pat<(v16i32 (bitconvert (v8i64 VR512:$src))), (v16i32 VR512:$src)>;
394def : Pat<(v16i32 (bitconvert (v16f32 VR512:$src))), (v16i32 VR512:$src)>;
395def : Pat<(v16i32 (bitconvert (v32i16 VR512:$src))), (v16i32 VR512:$src)>;
396def : Pat<(v16i32 (bitconvert (v64i8 VR512:$src))), (v16i32 VR512:$src)>;
397def : Pat<(v16i32 (bitconvert (v8f64 VR512:$src))), (v16i32 VR512:$src)>;
398def : Pat<(v32i16 (bitconvert (v8i64 VR512:$src))), (v32i16 VR512:$src)>;
399def : Pat<(v32i16 (bitconvert (v16i32 VR512:$src))), (v32i16 VR512:$src)>;
400def : Pat<(v32i16 (bitconvert (v64i8 VR512:$src))), (v32i16 VR512:$src)>;
401def : Pat<(v32i16 (bitconvert (v8f64 VR512:$src))), (v32i16 VR512:$src)>;
402def : Pat<(v32i16 (bitconvert (v16f32 VR512:$src))), (v32i16 VR512:$src)>;
403def : Pat<(v32i16 (bitconvert (v16f32 VR512:$src))), (v32i16 VR512:$src)>;
404def : Pat<(v64i8 (bitconvert (v8i64 VR512:$src))), (v64i8 VR512:$src)>;
405def : Pat<(v64i8 (bitconvert (v16i32 VR512:$src))), (v64i8 VR512:$src)>;
406def : Pat<(v64i8 (bitconvert (v32i16 VR512:$src))), (v64i8 VR512:$src)>;
407def : Pat<(v64i8 (bitconvert (v8f64 VR512:$src))), (v64i8 VR512:$src)>;
408def : Pat<(v64i8 (bitconvert (v16f32 VR512:$src))), (v64i8 VR512:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000409
Craig Topper9d9251b2016-05-08 20:10:20 +0000410// Alias instruction that maps zero vector to pxor / xorp* for AVX-512.
411// This is expanded by ExpandPostRAPseudos to an xorps / vxorps, and then
412// swizzled by ExecutionDepsFix to pxor.
413// We set canFoldAsLoad because this can be converted to a constant-pool
414// load of an all-zeros value if folding it would be beneficial.
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000415let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
416 isPseudo = 1, Predicates = [HasAVX512] in {
417def AVX512_512_SET0 : I<0, Pseudo, (outs VR512:$dst), (ins), "",
Craig Topper9d9251b2016-05-08 20:10:20 +0000418 [(set VR512:$dst, (v16i32 immAllZerosV))]>;
Craig Topperfb1746b2014-01-30 06:03:19 +0000419}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000420
Craig Toppere5ce84a2016-05-08 21:33:53 +0000421let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
422 isPseudo = 1, Predicates = [HasVLX] in {
423def AVX512_128_SET0 : I<0, Pseudo, (outs VR128X:$dst), (ins), "",
424 [(set VR128X:$dst, (v4i32 immAllZerosV))]>;
425def AVX512_256_SET0 : I<0, Pseudo, (outs VR256X:$dst), (ins), "",
426 [(set VR256X:$dst, (v8i32 immAllZerosV))]>;
427}
428
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000429//===----------------------------------------------------------------------===//
430// AVX-512 - VECTOR INSERT
431//
Igor Breger0ede3cb2015-09-20 06:52:42 +0000432multiclass vinsert_for_size<int Opcode, X86VectorVTInfo From, X86VectorVTInfo To,
433 PatFrag vinsert_insert> {
Craig Toppere1cac152016-06-07 07:27:54 +0000434 let ExeDomain = To.ExeDomain in {
Igor Breger0ede3cb2015-09-20 06:52:42 +0000435 defm rr : AVX512_maskable<Opcode, MRMSrcReg, To, (outs To.RC:$dst),
436 (ins To.RC:$src1, From.RC:$src2, i32u8imm:$src3),
437 "vinsert" # From.EltTypeName # "x" # From.NumElts,
438 "$src3, $src2, $src1", "$src1, $src2, $src3",
439 (vinsert_insert:$src3 (To.VT To.RC:$src1),
440 (From.VT From.RC:$src2),
441 (iPTR imm))>, AVX512AIi8Base, EVEX_4V;
Adam Nemet4e2ef472014-10-02 23:18:28 +0000442
Igor Breger0ede3cb2015-09-20 06:52:42 +0000443 defm rm : AVX512_maskable<Opcode, MRMSrcMem, To, (outs To.RC:$dst),
444 (ins To.RC:$src1, From.MemOp:$src2, i32u8imm:$src3),
445 "vinsert" # From.EltTypeName # "x" # From.NumElts,
446 "$src3, $src2, $src1", "$src1, $src2, $src3",
447 (vinsert_insert:$src3 (To.VT To.RC:$src1),
448 (From.VT (bitconvert (From.LdFrag addr:$src2))),
449 (iPTR imm))>, AVX512AIi8Base, EVEX_4V,
450 EVEX_CD8<From.EltSize, From.CD8TupleForm>;
Adam Nemet4e2ef472014-10-02 23:18:28 +0000451 }
Adam Nemet4285c1f2014-10-15 23:42:17 +0000452}
Adam Nemet4e2ef472014-10-02 23:18:28 +0000453
Igor Breger0ede3cb2015-09-20 06:52:42 +0000454multiclass vinsert_for_size_lowering<string InstrStr, X86VectorVTInfo From,
455 X86VectorVTInfo To, PatFrag vinsert_insert,
456 SDNodeXForm INSERT_get_vinsert_imm , list<Predicate> p> {
457 let Predicates = p in {
Adam Nemet4285c1f2014-10-15 23:42:17 +0000458 def : Pat<(vinsert_insert:$ins
Igor Breger0ede3cb2015-09-20 06:52:42 +0000459 (To.VT To.RC:$src1), (From.VT From.RC:$src2), (iPTR imm)),
460 (To.VT (!cast<Instruction>(InstrStr#"rr")
461 To.RC:$src1, From.RC:$src2,
462 (INSERT_get_vinsert_imm To.RC:$ins)))>;
463
464 def : Pat<(vinsert_insert:$ins
465 (To.VT To.RC:$src1),
466 (From.VT (bitconvert (From.LdFrag addr:$src2))),
467 (iPTR imm)),
468 (To.VT (!cast<Instruction>(InstrStr#"rm")
469 To.RC:$src1, addr:$src2,
470 (INSERT_get_vinsert_imm To.RC:$ins)))>;
471 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000472}
473
Adam Nemetb1c3ef42014-10-15 23:42:04 +0000474multiclass vinsert_for_type<ValueType EltVT32, int Opcode128,
475 ValueType EltVT64, int Opcode256> {
Igor Breger0ede3cb2015-09-20 06:52:42 +0000476
477 let Predicates = [HasVLX] in
478 defm NAME # "32x4Z256" : vinsert_for_size<Opcode128,
479 X86VectorVTInfo< 4, EltVT32, VR128X>,
480 X86VectorVTInfo< 8, EltVT32, VR256X>,
481 vinsert128_insert>, EVEX_V256;
482
483 defm NAME # "32x4Z" : vinsert_for_size<Opcode128,
Adam Nemet4e2ef472014-10-02 23:18:28 +0000484 X86VectorVTInfo< 4, EltVT32, VR128X>,
485 X86VectorVTInfo<16, EltVT32, VR512>,
Igor Breger0ede3cb2015-09-20 06:52:42 +0000486 vinsert128_insert>, EVEX_V512;
487
488 defm NAME # "64x4Z" : vinsert_for_size<Opcode256,
Adam Nemet4e2ef472014-10-02 23:18:28 +0000489 X86VectorVTInfo< 4, EltVT64, VR256X>,
490 X86VectorVTInfo< 8, EltVT64, VR512>,
Igor Breger0ede3cb2015-09-20 06:52:42 +0000491 vinsert256_insert>, VEX_W, EVEX_V512;
492
493 let Predicates = [HasVLX, HasDQI] in
494 defm NAME # "64x2Z256" : vinsert_for_size<Opcode128,
495 X86VectorVTInfo< 2, EltVT64, VR128X>,
496 X86VectorVTInfo< 4, EltVT64, VR256X>,
497 vinsert128_insert>, VEX_W, EVEX_V256;
498
499 let Predicates = [HasDQI] in {
500 defm NAME # "64x2Z" : vinsert_for_size<Opcode128,
501 X86VectorVTInfo< 2, EltVT64, VR128X>,
502 X86VectorVTInfo< 8, EltVT64, VR512>,
503 vinsert128_insert>, VEX_W, EVEX_V512;
504
505 defm NAME # "32x8Z" : vinsert_for_size<Opcode256,
506 X86VectorVTInfo< 8, EltVT32, VR256X>,
507 X86VectorVTInfo<16, EltVT32, VR512>,
508 vinsert256_insert>, EVEX_V512;
509 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000510}
511
Adam Nemet4e2ef472014-10-02 23:18:28 +0000512defm VINSERTF : vinsert_for_type<f32, 0x18, f64, 0x1a>;
513defm VINSERTI : vinsert_for_type<i32, 0x38, i64, 0x3a>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000514
Igor Breger0ede3cb2015-09-20 06:52:42 +0000515// Codegen pattern with the alternative types,
516// Only add this if 64x2 and its friends are not supported natively via AVX512DQ.
517defm : vinsert_for_size_lowering<"VINSERTF32x4Z256", v2f64x_info, v4f64x_info,
518 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX, NoDQI]>;
519defm : vinsert_for_size_lowering<"VINSERTI32x4Z256", v2i64x_info, v4i64x_info,
520 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX, NoDQI]>;
521
522defm : vinsert_for_size_lowering<"VINSERTF32x4Z", v2f64x_info, v8f64_info,
523 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512, NoDQI]>;
524defm : vinsert_for_size_lowering<"VINSERTI32x4Z", v2i64x_info, v8i64_info,
525 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512, NoDQI]>;
526
527defm : vinsert_for_size_lowering<"VINSERTF64x4Z", v8f32x_info, v16f32_info,
528 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512, NoDQI]>;
529defm : vinsert_for_size_lowering<"VINSERTI64x4Z", v8i32x_info, v16i32_info,
530 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512, NoDQI]>;
531
532// Codegen pattern with the alternative types insert VEC128 into VEC256
533defm : vinsert_for_size_lowering<"VINSERTI32x4Z256", v8i16x_info, v16i16x_info,
534 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>;
535defm : vinsert_for_size_lowering<"VINSERTI32x4Z256", v16i8x_info, v32i8x_info,
536 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>;
537// Codegen pattern with the alternative types insert VEC128 into VEC512
538defm : vinsert_for_size_lowering<"VINSERTI32x4Z", v8i16x_info, v32i16_info,
539 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>;
540defm : vinsert_for_size_lowering<"VINSERTI32x4Z", v16i8x_info, v64i8_info,
541 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>;
542// Codegen pattern with the alternative types insert VEC256 into VEC512
543defm : vinsert_for_size_lowering<"VINSERTI64x4Z", v16i16x_info, v32i16_info,
544 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>;
545defm : vinsert_for_size_lowering<"VINSERTI64x4Z", v32i8x_info, v64i8_info,
546 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>;
547
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000548// vinsertps - insert f32 to XMM
549def VINSERTPSzrr : AVX512AIi8<0x21, MRMSrcReg, (outs VR128X:$dst),
Craig Topper7ff6ab32015-01-21 08:43:49 +0000550 (ins VR128X:$src1, VR128X:$src2, u8imm:$src3),
Elena Demikhovskycf088092013-12-11 14:31:04 +0000551 "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
Filipe Cabecinhas20352212014-04-21 20:07:29 +0000552 [(set VR128X:$dst, (X86insertps VR128X:$src1, VR128X:$src2, imm:$src3))]>,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000553 EVEX_4V;
554def VINSERTPSzrm: AVX512AIi8<0x21, MRMSrcMem, (outs VR128X:$dst),
Craig Topper7ff6ab32015-01-21 08:43:49 +0000555 (ins VR128X:$src1, f32mem:$src2, u8imm:$src3),
Elena Demikhovskycf088092013-12-11 14:31:04 +0000556 "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
Filipe Cabecinhas20352212014-04-21 20:07:29 +0000557 [(set VR128X:$dst, (X86insertps VR128X:$src1,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000558 (v4f32 (scalar_to_vector (loadf32 addr:$src2))),
559 imm:$src3))]>, EVEX_4V, EVEX_CD8<32, CD8VT1>;
560
561//===----------------------------------------------------------------------===//
562// AVX-512 VECTOR EXTRACT
563//---
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000564
Igor Breger7f69a992015-09-10 12:54:54 +0000565multiclass vextract_for_size<int Opcode,
566 X86VectorVTInfo From, X86VectorVTInfo To,
Craig Topper5f3fef82016-05-22 07:40:58 +0000567 PatFrag vextract_extract> {
Igor Breger7f69a992015-09-10 12:54:54 +0000568
569 let hasSideEffects = 0, ExeDomain = To.ExeDomain in {
570 // use AVX512_maskable_in_asm (AVX512_maskable can't be used due to
571 // vextract_extract), we interesting only in patterns without mask,
572 // intrinsics pattern match generated bellow.
573 defm rr : AVX512_maskable_in_asm<Opcode, MRMDestReg, To, (outs To.RC:$dst),
574 (ins From.RC:$src1, i32u8imm:$idx),
575 "vextract" # To.EltTypeName # "x" # To.NumElts,
576 "$idx, $src1", "$src1, $idx",
577 [(set To.RC:$dst, (vextract_extract:$idx (From.VT From.RC:$src1),
578 (iPTR imm)))]>,
579 AVX512AIi8Base, EVEX;
Craig Toppere1cac152016-06-07 07:27:54 +0000580 def mr : AVX512AIi8<Opcode, MRMDestMem, (outs),
581 (ins To.MemOp:$dst, From.RC:$src1, i32u8imm:$idx),
582 "vextract" # To.EltTypeName # "x" # To.NumElts #
583 "\t{$idx, $src1, $dst|$dst, $src1, $idx}",
584 [(store (To.VT (vextract_extract:$idx
585 (From.VT From.RC:$src1), (iPTR imm))),
586 addr:$dst)]>, EVEX;
Igor Breger7f69a992015-09-10 12:54:54 +0000587
Craig Toppere1cac152016-06-07 07:27:54 +0000588 let mayStore = 1, hasSideEffects = 0 in
589 def mrk : AVX512AIi8<Opcode, MRMDestMem, (outs),
590 (ins To.MemOp:$dst, To.KRCWM:$mask,
591 From.RC:$src1, i32u8imm:$idx),
592 "vextract" # To.EltTypeName # "x" # To.NumElts #
593 "\t{$idx, $src1, $dst {${mask}}|"
594 "$dst {${mask}}, $src1, $idx}",
595 []>, EVEX_K, EVEX;
Igor Breger7f69a992015-09-10 12:54:54 +0000596 }
Renato Golindb7ea862015-09-09 19:44:40 +0000597
598 // Intrinsic call with masking.
599 def : Pat<(!cast<Intrinsic>("int_x86_avx512_mask_vextract" # To.EltTypeName #
Igor Breger7f69a992015-09-10 12:54:54 +0000600 "x" # To.NumElts # "_" # From.Size)
601 From.RC:$src1, (iPTR imm:$idx), To.RC:$src0, To.MRC:$mask),
602 (!cast<Instruction>(NAME # To.EltSize # "x" # To.NumElts #
603 From.ZSuffix # "rrk")
604 To.RC:$src0,
605 (COPY_TO_REGCLASS To.MRC:$mask, To.KRCWM),
606 From.RC:$src1, imm:$idx)>;
Renato Golindb7ea862015-09-09 19:44:40 +0000607
608 // Intrinsic call with zero-masking.
609 def : Pat<(!cast<Intrinsic>("int_x86_avx512_mask_vextract" # To.EltTypeName #
Igor Breger7f69a992015-09-10 12:54:54 +0000610 "x" # To.NumElts # "_" # From.Size)
611 From.RC:$src1, (iPTR imm:$idx), To.ImmAllZerosV, To.MRC:$mask),
612 (!cast<Instruction>(NAME # To.EltSize # "x" # To.NumElts #
613 From.ZSuffix # "rrkz")
614 (COPY_TO_REGCLASS To.MRC:$mask, To.KRCWM),
615 From.RC:$src1, imm:$idx)>;
Renato Golindb7ea862015-09-09 19:44:40 +0000616
617 // Intrinsic call without masking.
618 def : Pat<(!cast<Intrinsic>("int_x86_avx512_mask_vextract" # To.EltTypeName #
Igor Breger7f69a992015-09-10 12:54:54 +0000619 "x" # To.NumElts # "_" # From.Size)
620 From.RC:$src1, (iPTR imm:$idx), To.ImmAllZerosV, (i8 -1)),
621 (!cast<Instruction>(NAME # To.EltSize # "x" # To.NumElts #
622 From.ZSuffix # "rr")
623 From.RC:$src1, imm:$idx)>;
Igor Bregerac29a822015-09-09 14:35:09 +0000624}
625
Igor Bregerdefab3c2015-10-08 12:55:01 +0000626// Codegen pattern for the alternative types
627multiclass vextract_for_size_lowering<string InstrStr, X86VectorVTInfo From,
628 X86VectorVTInfo To, PatFrag vextract_extract,
Craig Topper5f3fef82016-05-22 07:40:58 +0000629 SDNodeXForm EXTRACT_get_vextract_imm, list<Predicate> p> {
Craig Topperdb960ed2016-05-21 22:50:14 +0000630 let Predicates = p in {
Igor Bregerdefab3c2015-10-08 12:55:01 +0000631 def : Pat<(vextract_extract:$ext (From.VT From.RC:$src1), (iPTR imm)),
632 (To.VT (!cast<Instruction>(InstrStr#"rr")
633 From.RC:$src1,
634 (EXTRACT_get_vextract_imm To.RC:$ext)))>;
Craig Topperdb960ed2016-05-21 22:50:14 +0000635 def : Pat<(store (To.VT (vextract_extract:$ext (From.VT From.RC:$src1),
636 (iPTR imm))), addr:$dst),
637 (!cast<Instruction>(InstrStr#"mr") addr:$dst, From.RC:$src1,
638 (EXTRACT_get_vextract_imm To.RC:$ext))>;
639 }
Igor Breger7f69a992015-09-10 12:54:54 +0000640}
641
642multiclass vextract_for_type<ValueType EltVT32, int Opcode128,
Igor Bregerdefab3c2015-10-08 12:55:01 +0000643 ValueType EltVT64, int Opcode256> {
644 defm NAME # "32x4Z" : vextract_for_size<Opcode128,
Adam Nemet55536c62014-09-25 23:48:45 +0000645 X86VectorVTInfo<16, EltVT32, VR512>,
646 X86VectorVTInfo< 4, EltVT32, VR128X>,
Igor Bregerdefab3c2015-10-08 12:55:01 +0000647 vextract128_extract>,
Igor Breger7f69a992015-09-10 12:54:54 +0000648 EVEX_V512, EVEX_CD8<32, CD8VT4>;
Igor Bregerdefab3c2015-10-08 12:55:01 +0000649 defm NAME # "64x4Z" : vextract_for_size<Opcode256,
Adam Nemet55536c62014-09-25 23:48:45 +0000650 X86VectorVTInfo< 8, EltVT64, VR512>,
651 X86VectorVTInfo< 4, EltVT64, VR256X>,
Igor Bregerdefab3c2015-10-08 12:55:01 +0000652 vextract256_extract>,
Igor Breger7f69a992015-09-10 12:54:54 +0000653 VEX_W, EVEX_V512, EVEX_CD8<64, CD8VT4>;
654 let Predicates = [HasVLX] in
Igor Bregerdefab3c2015-10-08 12:55:01 +0000655 defm NAME # "32x4Z256" : vextract_for_size<Opcode128,
Igor Breger7f69a992015-09-10 12:54:54 +0000656 X86VectorVTInfo< 8, EltVT32, VR256X>,
657 X86VectorVTInfo< 4, EltVT32, VR128X>,
Igor Bregerdefab3c2015-10-08 12:55:01 +0000658 vextract128_extract>,
Igor Breger7f69a992015-09-10 12:54:54 +0000659 EVEX_V256, EVEX_CD8<32, CD8VT4>;
660 let Predicates = [HasVLX, HasDQI] in
661 defm NAME # "64x2Z256" : vextract_for_size<Opcode128,
662 X86VectorVTInfo< 4, EltVT64, VR256X>,
663 X86VectorVTInfo< 2, EltVT64, VR128X>,
664 vextract128_extract>,
665 VEX_W, EVEX_V256, EVEX_CD8<64, CD8VT2>;
666 let Predicates = [HasDQI] in {
667 defm NAME # "64x2Z" : vextract_for_size<Opcode128,
668 X86VectorVTInfo< 8, EltVT64, VR512>,
669 X86VectorVTInfo< 2, EltVT64, VR128X>,
670 vextract128_extract>,
671 VEX_W, EVEX_V512, EVEX_CD8<64, CD8VT2>;
672 defm NAME # "32x8Z" : vextract_for_size<Opcode256,
673 X86VectorVTInfo<16, EltVT32, VR512>,
674 X86VectorVTInfo< 8, EltVT32, VR256X>,
675 vextract256_extract>,
676 EVEX_V512, EVEX_CD8<32, CD8VT8>;
677 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000678}
679
Adam Nemet55536c62014-09-25 23:48:45 +0000680defm VEXTRACTF : vextract_for_type<f32, 0x19, f64, 0x1b>;
681defm VEXTRACTI : vextract_for_type<i32, 0x39, i64, 0x3b>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000682
Igor Bregerdefab3c2015-10-08 12:55:01 +0000683// extract_subvector codegen patterns with the alternative types.
684// Only add this if 64x2 and its friends are not supported natively via AVX512DQ.
685defm : vextract_for_size_lowering<"VEXTRACTF32x4Z", v8f64_info, v2f64x_info,
686 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512, NoDQI]>;
687defm : vextract_for_size_lowering<"VEXTRACTI32x4Z", v8i64_info, v2i64x_info,
688 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512, NoDQI]>;
689
690defm : vextract_for_size_lowering<"VEXTRACTF64x4Z", v16f32_info, v8f32x_info,
Igor Breger684af812015-10-26 12:26:34 +0000691 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512, NoDQI]>;
Igor Bregerdefab3c2015-10-08 12:55:01 +0000692defm : vextract_for_size_lowering<"VEXTRACTI64x4Z", v16i32_info, v8i32x_info,
693 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512, NoDQI]>;
694
695defm : vextract_for_size_lowering<"VEXTRACTF32x4Z256", v4f64x_info, v2f64x_info,
696 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX, NoDQI]>;
697defm : vextract_for_size_lowering<"VEXTRACTI32x4Z256", v4i64x_info, v2i64x_info,
698 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX, NoDQI]>;
699
Craig Topper08a68572016-05-21 22:50:04 +0000700// Codegen pattern with the alternative types extract VEC128 from VEC256
Craig Topper02626c02016-05-21 07:08:56 +0000701defm : vextract_for_size_lowering<"VEXTRACTI32x4Z256", v16i16x_info, v8i16x_info,
702 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>;
703defm : vextract_for_size_lowering<"VEXTRACTI32x4Z256", v32i8x_info, v16i8x_info,
704 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>;
705
706// Codegen pattern with the alternative types extract VEC128 from VEC512
Igor Bregerdefab3c2015-10-08 12:55:01 +0000707defm : vextract_for_size_lowering<"VEXTRACTI32x4Z", v32i16_info, v8i16x_info,
708 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>;
709defm : vextract_for_size_lowering<"VEXTRACTI32x4Z", v64i8_info, v16i8x_info,
710 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>;
711// Codegen pattern with the alternative types extract VEC256 from VEC512
712defm : vextract_for_size_lowering<"VEXTRACTI64x4Z", v32i16_info, v16i16x_info,
713 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>;
714defm : vextract_for_size_lowering<"VEXTRACTI64x4Z", v64i8_info, v32i8x_info,
715 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>;
716
Craig Topper5f3fef82016-05-22 07:40:58 +0000717// A 128-bit subvector extract from the first 256-bit vector position
718// is a subregister copy that needs no instruction.
719def : Pat<(v2i64 (extract_subvector (v8i64 VR512:$src), (iPTR 0))),
720 (v2i64 (EXTRACT_SUBREG (v8i64 VR512:$src), sub_xmm))>;
721def : Pat<(v2f64 (extract_subvector (v8f64 VR512:$src), (iPTR 0))),
722 (v2f64 (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm))>;
723def : Pat<(v4i32 (extract_subvector (v16i32 VR512:$src), (iPTR 0))),
724 (v4i32 (EXTRACT_SUBREG (v16i32 VR512:$src), sub_xmm))>;
725def : Pat<(v4f32 (extract_subvector (v16f32 VR512:$src), (iPTR 0))),
726 (v4f32 (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm))>;
727def : Pat<(v8i16 (extract_subvector (v32i16 VR512:$src), (iPTR 0))),
728 (v8i16 (EXTRACT_SUBREG (v32i16 VR512:$src), sub_xmm))>;
729def : Pat<(v16i8 (extract_subvector (v64i8 VR512:$src), (iPTR 0))),
730 (v16i8 (EXTRACT_SUBREG (v64i8 VR512:$src), sub_xmm))>;
731
732// A 256-bit subvector extract from the first 256-bit vector position
733// is a subregister copy that needs no instruction.
734def : Pat<(v4i64 (extract_subvector (v8i64 VR512:$src), (iPTR 0))),
735 (v4i64 (EXTRACT_SUBREG (v8i64 VR512:$src), sub_ymm))>;
736def : Pat<(v4f64 (extract_subvector (v8f64 VR512:$src), (iPTR 0))),
737 (v4f64 (EXTRACT_SUBREG (v8f64 VR512:$src), sub_ymm))>;
738def : Pat<(v8i32 (extract_subvector (v16i32 VR512:$src), (iPTR 0))),
739 (v8i32 (EXTRACT_SUBREG (v16i32 VR512:$src), sub_ymm))>;
740def : Pat<(v8f32 (extract_subvector (v16f32 VR512:$src), (iPTR 0))),
741 (v8f32 (EXTRACT_SUBREG (v16f32 VR512:$src), sub_ymm))>;
742def : Pat<(v16i16 (extract_subvector (v32i16 VR512:$src), (iPTR 0))),
743 (v16i16 (EXTRACT_SUBREG (v32i16 VR512:$src), sub_ymm))>;
744def : Pat<(v32i8 (extract_subvector (v64i8 VR512:$src), (iPTR 0))),
745 (v32i8 (EXTRACT_SUBREG (v64i8 VR512:$src), sub_ymm))>;
746
747let AddedComplexity = 25 in { // to give priority over vinsertf128rm
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000748// A 128-bit subvector insert to the first 512-bit vector position
749// is a subregister copy that needs no instruction.
Igor Bregerfca0a342016-01-28 13:19:25 +0000750def : Pat<(v8i64 (insert_subvector undef, (v2i64 VR128X:$src), (iPTR 0))),
751 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)>;
752def : Pat<(v8f64 (insert_subvector undef, (v2f64 VR128X:$src), (iPTR 0))),
753 (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)>;
754def : Pat<(v16i32 (insert_subvector undef, (v4i32 VR128X:$src), (iPTR 0))),
755 (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)>;
756def : Pat<(v16f32 (insert_subvector undef, (v4f32 VR128X:$src), (iPTR 0))),
757 (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)>;
758def : Pat<(v32i16 (insert_subvector undef, (v8i16 VR128X:$src), (iPTR 0))),
759 (INSERT_SUBREG (v32i16 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)>;
760def : Pat<(v64i8 (insert_subvector undef, (v16i8 VR128X:$src), (iPTR 0))),
761 (INSERT_SUBREG (v64i8 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000762
Craig Topper5f3fef82016-05-22 07:40:58 +0000763// A 256-bit subvector insert to the first 512-bit vector position
764// is a subregister copy that needs no instruction.
Igor Bregerfca0a342016-01-28 13:19:25 +0000765def : Pat<(v8i64 (insert_subvector undef, (v4i64 VR256X:$src), (iPTR 0))),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000766 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
Igor Bregerfca0a342016-01-28 13:19:25 +0000767def : Pat<(v8f64 (insert_subvector undef, (v4f64 VR256X:$src), (iPTR 0))),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000768 (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
Igor Bregerfca0a342016-01-28 13:19:25 +0000769def : Pat<(v16i32 (insert_subvector undef, (v8i32 VR256X:$src), (iPTR 0))),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000770 (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
Igor Bregerfca0a342016-01-28 13:19:25 +0000771def : Pat<(v16f32 (insert_subvector undef, (v8f32 VR256X:$src), (iPTR 0))),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000772 (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
Igor Bregerfca0a342016-01-28 13:19:25 +0000773def : Pat<(v32i16 (insert_subvector undef, (v16i16 VR256X:$src), (iPTR 0))),
Igor Bregercbb95502015-10-18 09:56:39 +0000774 (INSERT_SUBREG (v32i16 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
Igor Bregerfca0a342016-01-28 13:19:25 +0000775def : Pat<(v64i8 (insert_subvector undef, (v32i8 VR256X:$src), (iPTR 0))),
Igor Bregercbb95502015-10-18 09:56:39 +0000776 (INSERT_SUBREG (v64i8 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
Craig Toppera1041ff2016-05-22 07:40:40 +0000777}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000778
779// vextractps - extract 32 bits from XMM
Craig Topper03b849e2016-05-21 22:50:11 +0000780def VEXTRACTPSZrr : AVX512AIi8<0x17, MRMDestReg, (outs GR32:$dst),
Craig Topperfc946a02015-01-25 02:21:13 +0000781 (ins VR128X:$src1, u8imm:$src2),
Elena Demikhovskycf088092013-12-11 14:31:04 +0000782 "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000783 [(set GR32:$dst, (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2))]>,
784 EVEX;
785
Craig Topper03b849e2016-05-21 22:50:11 +0000786def VEXTRACTPSZmr : AVX512AIi8<0x17, MRMDestMem, (outs),
Craig Topperfc946a02015-01-25 02:21:13 +0000787 (ins f32mem:$dst, VR128X:$src1, u8imm:$src2),
Elena Demikhovskycf088092013-12-11 14:31:04 +0000788 "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000789 [(store (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2),
Elena Demikhovsky2aafc222014-02-11 07:25:59 +0000790 addr:$dst)]>, EVEX, EVEX_CD8<32, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000791
792//===---------------------------------------------------------------------===//
793// AVX-512 BROADCAST
794//---
Igor Breger131008f2016-05-01 08:40:00 +0000795// broadcast with a scalar argument.
796multiclass avx512_broadcast_scalar<bits<8> opc, string OpcodeStr,
797 X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo> {
Simon Pilgrimb13961d2016-06-11 14:34:10 +0000798
Igor Breger131008f2016-05-01 08:40:00 +0000799 let isCodeGenOnly = 1 in {
800 def r_s : I< opc, MRMSrcReg, (outs DestInfo.RC:$dst),
801 (ins SrcInfo.FRC:$src), OpcodeStr#"\t{$src, $dst|$dst, $src}",
802 [(set DestInfo.RC:$dst, (DestInfo.VT (X86VBroadcast SrcInfo.FRC:$src)))]>,
803 Requires<[HasAVX512]>, T8PD, EVEX;
Simon Pilgrimb13961d2016-06-11 14:34:10 +0000804
Igor Breger131008f2016-05-01 08:40:00 +0000805 let Constraints = "$src0 = $dst" in
806 def rk_s : I< opc, MRMSrcReg, (outs DestInfo.RC:$dst),
807 (ins DestInfo.RC:$src0, DestInfo.KRCWM:$mask, SrcInfo.FRC:$src),
808 OpcodeStr#"\t{$src, $dst {${mask}} |$dst {${mask}}, $src}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +0000809 [(set DestInfo.RC:$dst,
Igor Breger131008f2016-05-01 08:40:00 +0000810 (vselect DestInfo.KRCWM:$mask,
811 (DestInfo.VT (X86VBroadcast SrcInfo.FRC:$src)),
812 DestInfo.RC:$src0))]>,
Simon Pilgrimb13961d2016-06-11 14:34:10 +0000813 Requires<[HasAVX512]>, T8PD, EVEX, EVEX_K;
Igor Breger131008f2016-05-01 08:40:00 +0000814
815 def rkz_s : I< opc, MRMSrcReg, (outs DestInfo.RC:$dst),
816 (ins DestInfo.KRCWM:$mask, SrcInfo.FRC:$src),
817 OpcodeStr#"\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +0000818 [(set DestInfo.RC:$dst,
Igor Breger131008f2016-05-01 08:40:00 +0000819 (vselect DestInfo.KRCWM:$mask,
820 (DestInfo.VT (X86VBroadcast SrcInfo.FRC:$src)),
821 DestInfo.ImmAllZerosV))]>,
Simon Pilgrimb13961d2016-06-11 14:34:10 +0000822 Requires<[HasAVX512]>, T8PD, EVEX, EVEX_KZ;
Igor Breger131008f2016-05-01 08:40:00 +0000823 } // let isCodeGenOnly = 1 in
824}
Robert Khasanovaf318f72014-10-30 14:21:47 +0000825
Igor Breger21296d22015-10-20 11:56:42 +0000826multiclass avx512_broadcast_rm<bits<8> opc, string OpcodeStr,
827 X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo> {
828
829 defm r : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),
830 (ins SrcInfo.RC:$src), OpcodeStr, "$src", "$src",
831 (DestInfo.VT (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src)))>,
832 T8PD, EVEX;
Craig Toppere1cac152016-06-07 07:27:54 +0000833 defm m : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
Igor Breger52bd1d52016-05-31 07:43:39 +0000834 (ins SrcInfo.ScalarMemOp:$src), OpcodeStr, "$src", "$src",
Craig Toppere1cac152016-06-07 07:27:54 +0000835 (DestInfo.VT (X86VBroadcast
836 (SrcInfo.ScalarLdFrag addr:$src)))>,
837 T8PD, EVEX, EVEX_CD8<SrcInfo.EltSize, CD8VT1>;
838
839 let isCodeGenOnly = 1 in
840 defm m_Int : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
841 (ins SrcInfo.ScalarMemOp:$src), OpcodeStr, "$src", "$src",
Simon Pilgrimb13961d2016-06-11 14:34:10 +0000842 (DestInfo.VT
843 (X86VBroadcast
844 (SrcInfo.VT (scalar_to_vector
Craig Toppere1cac152016-06-07 07:27:54 +0000845 (SrcInfo.ScalarLdFrag addr:$src)))))>,
846 T8PD, EVEX, EVEX_CD8<SrcInfo.EltSize, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000847}
Robert Khasanovaf318f72014-10-30 14:21:47 +0000848
Igor Breger21296d22015-10-20 11:56:42 +0000849multiclass avx512_fp_broadcast_vl<bits<8> opc, string OpcodeStr,
850 AVX512VLVectorVTInfo _> {
851 defm Z : avx512_broadcast_rm<opc, OpcodeStr, _.info512, _.info128>,
Igor Breger131008f2016-05-01 08:40:00 +0000852 avx512_broadcast_scalar<opc, OpcodeStr, _.info512, _.info128>,
Robert Khasanovaf318f72014-10-30 14:21:47 +0000853 EVEX_V512;
854
855 let Predicates = [HasVLX] in {
Igor Breger21296d22015-10-20 11:56:42 +0000856 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, _.info256, _.info128>,
Igor Breger131008f2016-05-01 08:40:00 +0000857 avx512_broadcast_scalar<opc, OpcodeStr, _.info256, _.info128>,
Igor Breger21296d22015-10-20 11:56:42 +0000858 EVEX_V256;
Robert Khasanovaf318f72014-10-30 14:21:47 +0000859 }
860}
861
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000862let ExeDomain = SSEPackedSingle in {
Igor Breger21296d22015-10-20 11:56:42 +0000863 defm VBROADCASTSS : avx512_fp_broadcast_vl<0x18, "vbroadcastss",
864 avx512vl_f32_info>;
Robert Khasanovaf318f72014-10-30 14:21:47 +0000865 let Predicates = [HasVLX] in {
Simon Pilgrimb13961d2016-06-11 14:34:10 +0000866 defm VBROADCASTSSZ128 :
867 avx512_broadcast_rm<0x18, "vbroadcastss", v4f32x_info, v4f32x_info>,
868 avx512_broadcast_scalar<0x18, "vbroadcastss", v4f32x_info, v4f32x_info>,
Igor Breger131008f2016-05-01 08:40:00 +0000869 EVEX_V128;
Robert Khasanovaf318f72014-10-30 14:21:47 +0000870 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000871}
872
873let ExeDomain = SSEPackedDouble in {
Igor Breger21296d22015-10-20 11:56:42 +0000874 defm VBROADCASTSD : avx512_fp_broadcast_vl<0x19, "vbroadcastsd",
875 avx512vl_f64_info>, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000876}
877
Quentin Colombet4bf1c282013-10-25 17:47:18 +0000878def : Pat<(int_x86_avx512_vbroadcast_ss_512 addr:$src),
Robert Khasanovaf318f72014-10-30 14:21:47 +0000879 (VBROADCASTSSZm addr:$src)>;
Quentin Colombet4bf1c282013-10-25 17:47:18 +0000880def : Pat<(int_x86_avx512_vbroadcast_sd_512 addr:$src),
Robert Khasanovaf318f72014-10-30 14:21:47 +0000881 (VBROADCASTSDZm addr:$src)>;
Quentin Colombet4bf1c282013-10-25 17:47:18 +0000882
Robert Khasanovcbc57032014-12-09 16:38:41 +0000883multiclass avx512_int_broadcast_reg<bits<8> opc, X86VectorVTInfo _,
884 RegisterClass SrcRC> {
Igor Breger0aeda372016-02-07 08:30:50 +0000885 defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
Simon Pilgrimb13961d2016-06-11 14:34:10 +0000886 (ins SrcRC:$src),
887 "vpbroadcast"##_.Suffix, "$src", "$src",
Igor Breger0aeda372016-02-07 08:30:50 +0000888 (_.VT (X86VBroadcast SrcRC:$src))>, T8PD, EVEX;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000889}
890
Robert Khasanovcbc57032014-12-09 16:38:41 +0000891multiclass avx512_int_broadcast_reg_vl<bits<8> opc, AVX512VLVectorVTInfo _,
892 RegisterClass SrcRC, Predicate prd> {
893 let Predicates = [prd] in
894 defm Z : avx512_int_broadcast_reg<opc, _.info512, SrcRC>, EVEX_V512;
895 let Predicates = [prd, HasVLX] in {
896 defm Z256 : avx512_int_broadcast_reg<opc, _.info256, SrcRC>, EVEX_V256;
897 defm Z128 : avx512_int_broadcast_reg<opc, _.info128, SrcRC>, EVEX_V128;
898 }
899}
900
Igor Breger0aeda372016-02-07 08:30:50 +0000901let isCodeGenOnly = 1 in {
902defm VPBROADCASTBr : avx512_int_broadcast_reg_vl<0x7A, avx512vl_i8_info, GR8,
Robert Khasanovcbc57032014-12-09 16:38:41 +0000903 HasBWI>;
Igor Breger0aeda372016-02-07 08:30:50 +0000904defm VPBROADCASTWr : avx512_int_broadcast_reg_vl<0x7B, avx512vl_i16_info, GR16,
Robert Khasanovcbc57032014-12-09 16:38:41 +0000905 HasBWI>;
Igor Breger0aeda372016-02-07 08:30:50 +0000906}
907let isAsmParserOnly = 1 in {
908 defm VPBROADCASTBr_Alt : avx512_int_broadcast_reg_vl<0x7A, avx512vl_i8_info,
909 GR32, HasBWI>;
910 defm VPBROADCASTWr_Alt : avx512_int_broadcast_reg_vl<0x7B, avx512vl_i16_info,
Simon Pilgrimb13961d2016-06-11 14:34:10 +0000911 GR32, HasBWI>;
Igor Breger0aeda372016-02-07 08:30:50 +0000912}
Robert Khasanovcbc57032014-12-09 16:38:41 +0000913defm VPBROADCASTDr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i32_info, GR32,
914 HasAVX512>;
915defm VPBROADCASTQr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i64_info, GR64,
916 HasAVX512>, VEX_W;
Michael Liao5bf95782014-12-04 05:20:33 +0000917
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000918def : Pat <(v16i32 (X86vzext VK16WM:$mask)),
Robert Khasanovcbc57032014-12-09 16:38:41 +0000919 (VPBROADCASTDrZrkz VK16WM:$mask, (i32 (MOV32ri 0x1)))>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000920def : Pat <(v8i64 (X86vzext VK8WM:$mask)),
Robert Khasanovcbc57032014-12-09 16:38:41 +0000921 (VPBROADCASTQrZrkz VK8WM:$mask, (i64 (MOV64ri 0x1)))>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000922
Igor Breger21296d22015-10-20 11:56:42 +0000923// Provide aliases for broadcast from the same register class that
924// automatically does the extract.
925multiclass avx512_int_broadcast_rm_lowering<X86VectorVTInfo DestInfo,
926 X86VectorVTInfo SrcInfo> {
927 def : Pat<(DestInfo.VT (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src))),
928 (!cast<Instruction>(NAME#DestInfo.ZSuffix#"r")
929 (EXTRACT_SUBREG (SrcInfo.VT SrcInfo.RC:$src), sub_xmm))>;
930}
931
932multiclass avx512_int_broadcast_rm_vl<bits<8> opc, string OpcodeStr,
933 AVX512VLVectorVTInfo _, Predicate prd> {
934 let Predicates = [prd] in {
935 defm Z : avx512_broadcast_rm<opc, OpcodeStr, _.info512, _.info128>,
936 avx512_int_broadcast_rm_lowering<_.info512, _.info256>,
937 EVEX_V512;
938 // Defined separately to avoid redefinition.
939 defm Z_Alt : avx512_int_broadcast_rm_lowering<_.info512, _.info512>;
940 }
941 let Predicates = [prd, HasVLX] in {
942 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, _.info256, _.info128>,
943 avx512_int_broadcast_rm_lowering<_.info256, _.info256>,
944 EVEX_V256;
945 defm Z128 : avx512_broadcast_rm<opc, OpcodeStr, _.info128, _.info128>,
946 EVEX_V128;
Elena Demikhovskydd0794e2013-10-24 07:16:35 +0000947 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000948}
949
Igor Breger21296d22015-10-20 11:56:42 +0000950defm VPBROADCASTB : avx512_int_broadcast_rm_vl<0x78, "vpbroadcastb",
951 avx512vl_i8_info, HasBWI>;
952defm VPBROADCASTW : avx512_int_broadcast_rm_vl<0x79, "vpbroadcastw",
953 avx512vl_i16_info, HasBWI>;
954defm VPBROADCASTD : avx512_int_broadcast_rm_vl<0x58, "vpbroadcastd",
955 avx512vl_i32_info, HasAVX512>;
956defm VPBROADCASTQ : avx512_int_broadcast_rm_vl<0x59, "vpbroadcastq",
957 avx512vl_i64_info, HasAVX512>, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000958
Elena Demikhovskyad9c3962015-05-18 06:42:57 +0000959multiclass avx512_subvec_broadcast_rm<bits<8> opc, string OpcodeStr,
960 X86VectorVTInfo _Dst, X86VectorVTInfo _Src> {
Simon Pilgrimb13961d2016-06-11 14:34:10 +0000961 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
Craig Toppere1cac152016-06-07 07:27:54 +0000962 (ins _Src.MemOp:$src), OpcodeStr, "$src", "$src",
963 (_Dst.VT (X86SubVBroadcast
Simon Pilgrimb13961d2016-06-11 14:34:10 +0000964 (_Src.VT (bitconvert (_Src.LdFrag addr:$src)))))>,
Craig Toppere1cac152016-06-07 07:27:54 +0000965 AVX5128IBase, EVEX;
Adam Nemet73f72e12014-06-27 00:43:38 +0000966}
967
Elena Demikhovskyad9c3962015-05-18 06:42:57 +0000968defm VBROADCASTI32X4 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4",
969 v16i32_info, v4i32x_info>,
Adam Nemet73f72e12014-06-27 00:43:38 +0000970 EVEX_V512, EVEX_CD8<32, CD8VT4>;
Elena Demikhovskyad9c3962015-05-18 06:42:57 +0000971defm VBROADCASTF32X4 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4",
972 v16f32_info, v4f32x_info>,
973 EVEX_V512, EVEX_CD8<32, CD8VT4>;
974defm VBROADCASTI64X4 : avx512_subvec_broadcast_rm<0x5b, "vbroadcasti64x4",
975 v8i64_info, v4i64x_info>, VEX_W,
Adam Nemet73f72e12014-06-27 00:43:38 +0000976 EVEX_V512, EVEX_CD8<64, CD8VT4>;
Elena Demikhovskyad9c3962015-05-18 06:42:57 +0000977defm VBROADCASTF64X4 : avx512_subvec_broadcast_rm<0x1b, "vbroadcastf64x4",
978 v8f64_info, v4f64x_info>, VEX_W,
979 EVEX_V512, EVEX_CD8<64, CD8VT4>;
980
981let Predicates = [HasVLX] in {
982defm VBROADCASTI32X4Z256 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4",
983 v8i32x_info, v4i32x_info>,
984 EVEX_V256, EVEX_CD8<32, CD8VT4>;
985defm VBROADCASTF32X4Z256 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4",
986 v8f32x_info, v4f32x_info>,
987 EVEX_V256, EVEX_CD8<32, CD8VT4>;
988}
989let Predicates = [HasVLX, HasDQI] in {
990defm VBROADCASTI64X2Z128 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti64x2",
991 v4i64x_info, v2i64x_info>, VEX_W,
992 EVEX_V256, EVEX_CD8<64, CD8VT2>;
993defm VBROADCASTF64X2Z128 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf64x2",
994 v4f64x_info, v2f64x_info>, VEX_W,
995 EVEX_V256, EVEX_CD8<64, CD8VT2>;
996}
997let Predicates = [HasDQI] in {
998defm VBROADCASTI64X2 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti64x2",
999 v8i64_info, v2i64x_info>, VEX_W,
1000 EVEX_V512, EVEX_CD8<64, CD8VT2>;
1001defm VBROADCASTI32X8 : avx512_subvec_broadcast_rm<0x5b, "vbroadcasti32x8",
1002 v16i32_info, v8i32x_info>,
1003 EVEX_V512, EVEX_CD8<32, CD8VT8>;
1004defm VBROADCASTF64X2 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf64x2",
1005 v8f64_info, v2f64x_info>, VEX_W,
1006 EVEX_V512, EVEX_CD8<64, CD8VT2>;
1007defm VBROADCASTF32X8 : avx512_subvec_broadcast_rm<0x1b, "vbroadcastf32x8",
1008 v16f32_info, v8f32x_info>,
1009 EVEX_V512, EVEX_CD8<32, CD8VT8>;
1010}
Adam Nemet73f72e12014-06-27 00:43:38 +00001011
Igor Bregerfa798a92015-11-02 07:39:36 +00001012multiclass avx512_common_broadcast_32x2<bits<8> opc, string OpcodeStr,
Igor Breger52bd1d52016-05-31 07:43:39 +00001013 AVX512VLVectorVTInfo _Dst, AVX512VLVectorVTInfo _Src> {
Igor Bregerfa798a92015-11-02 07:39:36 +00001014 let Predicates = [HasDQI] in
Igor Breger52bd1d52016-05-31 07:43:39 +00001015 defm Z : avx512_broadcast_rm<opc, OpcodeStr, _Dst.info512, _Src.info128>,
Igor Bregerfa798a92015-11-02 07:39:36 +00001016 EVEX_V512;
1017 let Predicates = [HasDQI, HasVLX] in
Igor Breger52bd1d52016-05-31 07:43:39 +00001018 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, _Dst.info256, _Src.info128>,
Igor Bregerfa798a92015-11-02 07:39:36 +00001019 EVEX_V256;
1020}
1021
1022multiclass avx512_common_broadcast_i32x2<bits<8> opc, string OpcodeStr,
Igor Breger52bd1d52016-05-31 07:43:39 +00001023 AVX512VLVectorVTInfo _Dst, AVX512VLVectorVTInfo _Src> :
1024 avx512_common_broadcast_32x2<opc, OpcodeStr, _Dst, _Src> {
Igor Bregerfa798a92015-11-02 07:39:36 +00001025
1026 let Predicates = [HasDQI, HasVLX] in
Igor Breger52bd1d52016-05-31 07:43:39 +00001027 defm Z128 : avx512_broadcast_rm<opc, OpcodeStr, _Dst.info128, _Src.info128>,
1028 EVEX_V128;
Igor Bregerfa798a92015-11-02 07:39:36 +00001029}
1030
1031defm VPBROADCASTI32X2 : avx512_common_broadcast_i32x2<0x59, "vbroadcasti32x2",
Igor Breger52bd1d52016-05-31 07:43:39 +00001032 avx512vl_i32_info, avx512vl_i64_info>;
Igor Bregerfa798a92015-11-02 07:39:36 +00001033defm VPBROADCASTF32X2 : avx512_common_broadcast_32x2<0x19, "vbroadcastf32x2",
Igor Breger52bd1d52016-05-31 07:43:39 +00001034 avx512vl_f32_info, avx512vl_f64_info>;
Igor Bregerfa798a92015-11-02 07:39:36 +00001035
Robert Khasanovdd09a8f2014-10-28 12:28:51 +00001036def : Pat<(v16f32 (X86VBroadcast (v16f32 VR512:$src))),
Robert Khasanovaf318f72014-10-30 14:21:47 +00001037 (VBROADCASTSSZr (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm))>;
Elena Demikhovsky08ce53c2015-05-18 07:06:23 +00001038def : Pat<(v16f32 (X86VBroadcast (v8f32 VR256X:$src))),
1039 (VBROADCASTSSZr (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm))>;
1040
Robert Khasanovdd09a8f2014-10-28 12:28:51 +00001041def : Pat<(v8f64 (X86VBroadcast (v8f64 VR512:$src))),
Robert Khasanovaf318f72014-10-30 14:21:47 +00001042 (VBROADCASTSDZr (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm))>;
Elena Demikhovsky08ce53c2015-05-18 07:06:23 +00001043def : Pat<(v8f64 (X86VBroadcast (v4f64 VR256X:$src))),
1044 (VBROADCASTSDZr (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm))>;
Robert Khasanovdd09a8f2014-10-28 12:28:51 +00001045
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001046//===----------------------------------------------------------------------===//
1047// AVX-512 BROADCAST MASK TO VECTOR REGISTER
1048//---
Asaf Badouh0d957b82015-11-18 09:42:45 +00001049multiclass avx512_mask_broadcastm<bits<8> opc, string OpcodeStr,
1050 X86VectorVTInfo _, RegisterClass KRC> {
1051 def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.RC:$dst), (ins KRC:$src),
Craig Topperedb09112014-11-25 20:11:23 +00001052 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
Asaf Badouh0d957b82015-11-18 09:42:45 +00001053 [(set _.RC:$dst, (_.VT (X86VBroadcastm KRC:$src)))]>, EVEX;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001054}
1055
Simon Pilgrimb13961d2016-06-11 14:34:10 +00001056multiclass avx512_mask_broadcast<bits<8> opc, string OpcodeStr,
Asaf Badouh0d957b82015-11-18 09:42:45 +00001057 AVX512VLVectorVTInfo VTInfo, RegisterClass KRC> {
1058 let Predicates = [HasCDI] in
1059 defm Z : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info512, KRC>, EVEX_V512;
1060 let Predicates = [HasCDI, HasVLX] in {
1061 defm Z256 : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info256, KRC>, EVEX_V256;
1062 defm Z128 : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info128, KRC>, EVEX_V128;
1063 }
1064}
1065
Elena Demikhovsky4b01b732014-10-26 09:52:24 +00001066defm VPBROADCASTMW2D : avx512_mask_broadcast<0x3A, "vpbroadcastmw2d",
Asaf Badouh0d957b82015-11-18 09:42:45 +00001067 avx512vl_i32_info, VK16>;
Elena Demikhovsky4b01b732014-10-26 09:52:24 +00001068defm VPBROADCASTMB2Q : avx512_mask_broadcast<0x2A, "vpbroadcastmb2q",
Asaf Badouh0d957b82015-11-18 09:42:45 +00001069 avx512vl_i64_info, VK8>, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001070
1071//===----------------------------------------------------------------------===//
Craig Topperaad5f112015-11-30 00:13:24 +00001072// -- VPERMI2 - 3 source operands form --
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001073multiclass avx512_perm_i<bits<8> opc, string OpcodeStr,
Craig Topperaad5f112015-11-30 00:13:24 +00001074 X86VectorVTInfo _, X86VectorVTInfo IdxVT> {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001075let Constraints = "$src1 = $dst" in {
Craig Topperaad5f112015-11-30 00:13:24 +00001076 defm rr: AVX512_maskable_3src_cast<opc, MRMSrcReg, _, IdxVT, (outs _.RC:$dst),
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001077 (ins _.RC:$src2, _.RC:$src3),
1078 OpcodeStr, "$src3, $src2", "$src2, $src3",
Craig Topperaad5f112015-11-30 00:13:24 +00001079 (_.VT (X86VPermi2X IdxVT.RC:$src1, _.RC:$src2, _.RC:$src3))>, EVEX_4V,
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001080 AVX5128IBase;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001081
Craig Topperaad5f112015-11-30 00:13:24 +00001082 defm rm: AVX512_maskable_3src_cast<opc, MRMSrcMem, _, IdxVT, (outs _.RC:$dst),
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001083 (ins _.RC:$src2, _.MemOp:$src3),
1084 OpcodeStr, "$src3, $src2", "$src2, $src3",
Craig Topperaad5f112015-11-30 00:13:24 +00001085 (_.VT (X86VPermi2X IdxVT.RC:$src1, _.RC:$src2,
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001086 (_.VT (bitconvert (_.LdFrag addr:$src3)))))>,
1087 EVEX_4V, AVX5128IBase;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001088 }
1089}
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001090multiclass avx512_perm_i_mb<bits<8> opc, string OpcodeStr,
Craig Topperaad5f112015-11-30 00:13:24 +00001091 X86VectorVTInfo _, X86VectorVTInfo IdxVT> {
Craig Toppere1cac152016-06-07 07:27:54 +00001092 let Constraints = "$src1 = $dst" in
Craig Topperaad5f112015-11-30 00:13:24 +00001093 defm rmb: AVX512_maskable_3src_cast<opc, MRMSrcMem, _, IdxVT, (outs _.RC:$dst),
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001094 (ins _.RC:$src2, _.ScalarMemOp:$src3),
1095 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),
1096 !strconcat("$src2, ${src3}", _.BroadcastStr ),
Craig Topperaad5f112015-11-30 00:13:24 +00001097 (_.VT (X86VPermi2X IdxVT.RC:$src1,
Michael Liao66233b72015-08-06 09:06:20 +00001098 _.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3)))))>,
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001099 AVX5128IBase, EVEX_4V, EVEX_B;
Adam Nemetefe9c982014-07-02 21:25:58 +00001100}
1101
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001102multiclass avx512_perm_i_sizes<bits<8> opc, string OpcodeStr,
Craig Topperaad5f112015-11-30 00:13:24 +00001103 AVX512VLVectorVTInfo VTInfo,
1104 AVX512VLVectorVTInfo ShuffleMask> {
1105 defm NAME: avx512_perm_i<opc, OpcodeStr, VTInfo.info512,
1106 ShuffleMask.info512>,
1107 avx512_perm_i_mb<opc, OpcodeStr, VTInfo.info512,
1108 ShuffleMask.info512>, EVEX_V512;
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001109 let Predicates = [HasVLX] in {
Craig Topperaad5f112015-11-30 00:13:24 +00001110 defm NAME#128: avx512_perm_i<opc, OpcodeStr, VTInfo.info128,
1111 ShuffleMask.info128>,
1112 avx512_perm_i_mb<opc, OpcodeStr, VTInfo.info128,
1113 ShuffleMask.info128>, EVEX_V128;
1114 defm NAME#256: avx512_perm_i<opc, OpcodeStr, VTInfo.info256,
1115 ShuffleMask.info256>,
1116 avx512_perm_i_mb<opc, OpcodeStr, VTInfo.info256,
1117 ShuffleMask.info256>, EVEX_V256;
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001118 }
1119}
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001120
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001121multiclass avx512_perm_i_sizes_bw<bits<8> opc, string OpcodeStr,
Craig Topperaad5f112015-11-30 00:13:24 +00001122 AVX512VLVectorVTInfo VTInfo,
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001123 AVX512VLVectorVTInfo Idx,
1124 Predicate Prd> {
1125 let Predicates = [Prd] in
Craig Topperaad5f112015-11-30 00:13:24 +00001126 defm NAME: avx512_perm_i<opc, OpcodeStr, VTInfo.info512,
1127 Idx.info512>, EVEX_V512;
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001128 let Predicates = [Prd, HasVLX] in {
Craig Topperaad5f112015-11-30 00:13:24 +00001129 defm NAME#128: avx512_perm_i<opc, OpcodeStr, VTInfo.info128,
1130 Idx.info128>, EVEX_V128;
1131 defm NAME#256: avx512_perm_i<opc, OpcodeStr, VTInfo.info256,
1132 Idx.info256>, EVEX_V256;
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001133 }
1134}
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001135
Craig Topperaad5f112015-11-30 00:13:24 +00001136defm VPERMI2D : avx512_perm_i_sizes<0x76, "vpermi2d",
1137 avx512vl_i32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
1138defm VPERMI2Q : avx512_perm_i_sizes<0x76, "vpermi2q",
1139 avx512vl_i64_info, avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001140defm VPERMI2W : avx512_perm_i_sizes_bw<0x75, "vpermi2w",
1141 avx512vl_i16_info, avx512vl_i16_info, HasBWI>,
1142 VEX_W, EVEX_CD8<16, CD8VF>;
1143defm VPERMI2B : avx512_perm_i_sizes_bw<0x75, "vpermi2b",
1144 avx512vl_i8_info, avx512vl_i8_info, HasVBMI>,
1145 EVEX_CD8<8, CD8VF>;
Craig Topperaad5f112015-11-30 00:13:24 +00001146defm VPERMI2PS : avx512_perm_i_sizes<0x77, "vpermi2ps",
1147 avx512vl_f32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
1148defm VPERMI2PD : avx512_perm_i_sizes<0x77, "vpermi2pd",
1149 avx512vl_f64_info, avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001150
Craig Topperaad5f112015-11-30 00:13:24 +00001151// VPERMT2
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001152multiclass avx512_perm_t<bits<8> opc, string OpcodeStr,
Craig Toppera47576f2015-11-26 20:21:29 +00001153 X86VectorVTInfo _, X86VectorVTInfo IdxVT> {
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001154let Constraints = "$src1 = $dst" in {
1155 defm rr: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
1156 (ins IdxVT.RC:$src2, _.RC:$src3),
1157 OpcodeStr, "$src3, $src2", "$src2, $src3",
Craig Toppera47576f2015-11-26 20:21:29 +00001158 (_.VT (X86VPermt2 _.RC:$src1, IdxVT.RC:$src2, _.RC:$src3))>, EVEX_4V,
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001159 AVX5128IBase;
1160
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001161 defm rm: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
1162 (ins IdxVT.RC:$src2, _.MemOp:$src3),
1163 OpcodeStr, "$src3, $src2", "$src2, $src3",
Craig Toppera47576f2015-11-26 20:21:29 +00001164 (_.VT (X86VPermt2 _.RC:$src1, IdxVT.RC:$src2,
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001165 (bitconvert (_.LdFrag addr:$src3))))>,
1166 EVEX_4V, AVX5128IBase;
1167 }
1168}
1169multiclass avx512_perm_t_mb<bits<8> opc, string OpcodeStr,
Craig Toppera47576f2015-11-26 20:21:29 +00001170 X86VectorVTInfo _, X86VectorVTInfo IdxVT> {
Craig Toppere1cac152016-06-07 07:27:54 +00001171 let Constraints = "$src1 = $dst" in
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001172 defm rmb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
1173 (ins IdxVT.RC:$src2, _.ScalarMemOp:$src3),
1174 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),
1175 !strconcat("$src2, ${src3}", _.BroadcastStr ),
Craig Toppera47576f2015-11-26 20:21:29 +00001176 (_.VT (X86VPermt2 _.RC:$src1,
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001177 IdxVT.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3)))))>,
1178 AVX5128IBase, EVEX_4V, EVEX_B;
1179}
1180
1181multiclass avx512_perm_t_sizes<bits<8> opc, string OpcodeStr,
Craig Toppera47576f2015-11-26 20:21:29 +00001182 AVX512VLVectorVTInfo VTInfo,
1183 AVX512VLVectorVTInfo ShuffleMask> {
1184 defm NAME: avx512_perm_t<opc, OpcodeStr, VTInfo.info512,
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001185 ShuffleMask.info512>,
Craig Toppera47576f2015-11-26 20:21:29 +00001186 avx512_perm_t_mb<opc, OpcodeStr, VTInfo.info512,
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001187 ShuffleMask.info512>, EVEX_V512;
1188 let Predicates = [HasVLX] in {
Craig Toppera47576f2015-11-26 20:21:29 +00001189 defm NAME#128: avx512_perm_t<opc, OpcodeStr, VTInfo.info128,
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001190 ShuffleMask.info128>,
Craig Toppera47576f2015-11-26 20:21:29 +00001191 avx512_perm_t_mb<opc, OpcodeStr, VTInfo.info128,
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001192 ShuffleMask.info128>, EVEX_V128;
Craig Toppera47576f2015-11-26 20:21:29 +00001193 defm NAME#256: avx512_perm_t<opc, OpcodeStr, VTInfo.info256,
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001194 ShuffleMask.info256>,
Craig Toppera47576f2015-11-26 20:21:29 +00001195 avx512_perm_t_mb<opc, OpcodeStr, VTInfo.info256,
1196 ShuffleMask.info256>, EVEX_V256;
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001197 }
1198}
1199
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001200multiclass avx512_perm_t_sizes_bw<bits<8> opc, string OpcodeStr,
Craig Toppera47576f2015-11-26 20:21:29 +00001201 AVX512VLVectorVTInfo VTInfo,
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001202 AVX512VLVectorVTInfo Idx,
1203 Predicate Prd> {
1204 let Predicates = [Prd] in
Craig Toppera47576f2015-11-26 20:21:29 +00001205 defm NAME: avx512_perm_t<opc, OpcodeStr, VTInfo.info512,
1206 Idx.info512>, EVEX_V512;
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001207 let Predicates = [Prd, HasVLX] in {
Craig Toppera47576f2015-11-26 20:21:29 +00001208 defm NAME#128: avx512_perm_t<opc, OpcodeStr, VTInfo.info128,
1209 Idx.info128>, EVEX_V128;
1210 defm NAME#256: avx512_perm_t<opc, OpcodeStr, VTInfo.info256,
1211 Idx.info256>, EVEX_V256;
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001212 }
1213}
1214
Craig Toppera47576f2015-11-26 20:21:29 +00001215defm VPERMT2D : avx512_perm_t_sizes<0x7E, "vpermt2d",
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001216 avx512vl_i32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
Craig Toppera47576f2015-11-26 20:21:29 +00001217defm VPERMT2Q : avx512_perm_t_sizes<0x7E, "vpermt2q",
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001218 avx512vl_i64_info, avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001219defm VPERMT2W : avx512_perm_t_sizes_bw<0x7D, "vpermt2w",
1220 avx512vl_i16_info, avx512vl_i16_info, HasBWI>,
1221 VEX_W, EVEX_CD8<16, CD8VF>;
1222defm VPERMT2B : avx512_perm_t_sizes_bw<0x7D, "vpermt2b",
1223 avx512vl_i8_info, avx512vl_i8_info, HasVBMI>,
1224 EVEX_CD8<8, CD8VF>;
Craig Toppera47576f2015-11-26 20:21:29 +00001225defm VPERMT2PS : avx512_perm_t_sizes<0x7F, "vpermt2ps",
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001226 avx512vl_f32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
Craig Toppera47576f2015-11-26 20:21:29 +00001227defm VPERMT2PD : avx512_perm_t_sizes<0x7F, "vpermt2pd",
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001228 avx512vl_f64_info, avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
Elena Demikhovsky299cf5112014-04-29 09:09:15 +00001229
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001230//===----------------------------------------------------------------------===//
1231// AVX-512 - BLEND using mask
1232//
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001233multiclass avx512_blendmask<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> {
1234 let ExeDomain = _.ExeDomain in {
Craig Toppere1cac152016-06-07 07:27:54 +00001235 let hasSideEffects = 0 in
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001236 def rr : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),
1237 (ins _.RC:$src1, _.RC:$src2),
1238 !strconcat(OpcodeStr,
Craig Topper9feea572016-01-11 00:44:58 +00001239 "\t{$src2, $src1, ${dst}|${dst}, $src1, $src2}"),
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001240 []>, EVEX_4V;
1241 def rrk : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),
1242 (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),
Elena Demikhovsky172a27c2014-01-08 10:54:22 +00001243 !strconcat(OpcodeStr,
Craig Topperedb09112014-11-25 20:11:23 +00001244 "\t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"),
Igor Breger64cfd3a2016-06-15 07:30:38 +00001245 [(set _.RC:$dst, (vselect _.KRCWM:$mask,
1246 (_.VT _.RC:$src2),
1247 (_.VT _.RC:$src1)))]>, EVEX_4V, EVEX_K;
Craig Toppere1cac152016-06-07 07:27:54 +00001248 let hasSideEffects = 0 in
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001249 def rrkz : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),
1250 (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),
1251 !strconcat(OpcodeStr,
1252 "\t{$src2, $src1, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src1, $src2}"),
1253 []>, EVEX_4V, EVEX_KZ;
Craig Toppere1cac152016-06-07 07:27:54 +00001254 let mayLoad = 1, hasSideEffects = 0 in
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001255 def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1256 (ins _.RC:$src1, _.MemOp:$src2),
1257 !strconcat(OpcodeStr,
Craig Topper9feea572016-01-11 00:44:58 +00001258 "\t{$src2, $src1, ${dst}|${dst}, $src1, $src2}"),
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001259 []>, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>;
1260 def rmk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1261 (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2),
Elena Demikhovsky172a27c2014-01-08 10:54:22 +00001262 !strconcat(OpcodeStr,
Craig Topperedb09112014-11-25 20:11:23 +00001263 "\t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"),
Igor Breger64cfd3a2016-06-15 07:30:38 +00001264 [(set _.RC:$dst, (vselect _.KRCWM:$mask,
1265 (_.VT (bitconvert (_.LdFrag addr:$src2))),
1266 (_.VT _.RC:$src1)))]>,
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001267 EVEX_4V, EVEX_K, EVEX_CD8<_.EltSize, CD8VF>;
Craig Toppere1cac152016-06-07 07:27:54 +00001268 let mayLoad = 1, hasSideEffects = 0 in
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001269 def rmkz : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1270 (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2),
1271 !strconcat(OpcodeStr,
1272 "\t{$src2, $src1, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src1, $src2}"),
1273 []>, EVEX_4V, EVEX_KZ, EVEX_CD8<_.EltSize, CD8VF>;
1274 }
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001275}
1276multiclass avx512_blendmask_rmb<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> {
1277
1278 def rmbk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1279 (ins _.KRCWM:$mask, _.RC:$src1, _.ScalarMemOp:$src2),
1280 !strconcat(OpcodeStr,
1281 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",
1282 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"),
Igor Breger64cfd3a2016-06-15 07:30:38 +00001283 [(set _.RC:$dst,(vselect _.KRCWM:$mask,
1284 (X86VBroadcast (_.ScalarLdFrag addr:$src2)),
1285 (_.VT _.RC:$src1)))]>,
Elena Demikhovsky31214492014-12-23 09:36:28 +00001286 EVEX_4V, EVEX_K, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>;
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001287
Craig Toppere1cac152016-06-07 07:27:54 +00001288 let mayLoad = 1, hasSideEffects = 0 in
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001289 def rmb : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1290 (ins _.RC:$src1, _.ScalarMemOp:$src2),
1291 !strconcat(OpcodeStr,
1292 "\t{${src2}", _.BroadcastStr, ", $src1, $dst|",
1293 "$dst, $src1, ${src2}", _.BroadcastStr, "}"),
Elena Demikhovsky31214492014-12-23 09:36:28 +00001294 []>, EVEX_4V, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>;
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001295
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001296}
1297
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001298multiclass blendmask_dq <bits<8> opc, string OpcodeStr,
1299 AVX512VLVectorVTInfo VTInfo> {
1300 defm Z : avx512_blendmask <opc, OpcodeStr, VTInfo.info512>,
1301 avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info512>, EVEX_V512;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001302
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001303 let Predicates = [HasVLX] in {
1304 defm Z256 : avx512_blendmask<opc, OpcodeStr, VTInfo.info256>,
1305 avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info256>, EVEX_V256;
1306 defm Z128 : avx512_blendmask<opc, OpcodeStr, VTInfo.info128>,
1307 avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info128>, EVEX_V128;
1308 }
1309}
Elena Demikhovsky172a27c2014-01-08 10:54:22 +00001310
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001311multiclass blendmask_bw <bits<8> opc, string OpcodeStr,
1312 AVX512VLVectorVTInfo VTInfo> {
1313 let Predicates = [HasBWI] in
1314 defm Z : avx512_blendmask <opc, OpcodeStr, VTInfo.info512>, EVEX_V512;
Elena Demikhovsky172a27c2014-01-08 10:54:22 +00001315
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001316 let Predicates = [HasBWI, HasVLX] in {
1317 defm Z256 : avx512_blendmask <opc, OpcodeStr, VTInfo.info256>, EVEX_V256;
1318 defm Z128 : avx512_blendmask <opc, OpcodeStr, VTInfo.info128>, EVEX_V128;
1319 }
1320}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001321
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001322
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001323defm VBLENDMPS : blendmask_dq <0x65, "vblendmps", avx512vl_f32_info>;
1324defm VBLENDMPD : blendmask_dq <0x65, "vblendmpd", avx512vl_f64_info>, VEX_W;
1325defm VPBLENDMD : blendmask_dq <0x64, "vpblendmd", avx512vl_i32_info>;
1326defm VPBLENDMQ : blendmask_dq <0x64, "vpblendmq", avx512vl_i64_info>, VEX_W;
1327defm VPBLENDMB : blendmask_bw <0x66, "vpblendmb", avx512vl_i8_info>;
1328defm VPBLENDMW : blendmask_bw <0x66, "vpblendmw", avx512vl_i16_info>, VEX_W;
Elena Demikhovsky172a27c2014-01-08 10:54:22 +00001329
Elena Demikhovsky172a27c2014-01-08 10:54:22 +00001330
Craig Topper0fcf9252016-06-07 07:27:51 +00001331let Predicates = [HasAVX512, NoVLX] in {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001332def : Pat<(v8f32 (vselect (v8i1 VK8WM:$mask), (v8f32 VR256X:$src1),
1333 (v8f32 VR256X:$src2))),
Michael Liao5bf95782014-12-04 05:20:33 +00001334 (EXTRACT_SUBREG
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001335 (v16f32 (VBLENDMPSZrrk (COPY_TO_REGCLASS VK8WM:$mask, VK16WM),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001336 (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)),
1337 (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>;
1338
1339def : Pat<(v8i32 (vselect (v8i1 VK8WM:$mask), (v8i32 VR256X:$src1),
1340 (v8i32 VR256X:$src2))),
Michael Liao5bf95782014-12-04 05:20:33 +00001341 (EXTRACT_SUBREG
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001342 (v16i32 (VPBLENDMDZrrk (COPY_TO_REGCLASS VK8WM:$mask, VK16WM),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001343 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)),
1344 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>;
1345}
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00001346//===----------------------------------------------------------------------===//
1347// Compare Instructions
1348//===----------------------------------------------------------------------===//
1349
1350// avx512_cmp_scalar - AVX512 CMPSS and CMPSD
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001351
1352multiclass avx512_cmp_scalar<X86VectorVTInfo _, SDNode OpNode, SDNode OpNodeRnd>{
1353
1354 defm rr_Int : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
1355 (outs _.KRC:$dst),
1356 (ins _.RC:$src1, _.RC:$src2, AVXCC:$cc),
1357 "vcmp${cc}"#_.Suffix,
1358 "$src2, $src1", "$src1, $src2",
1359 (OpNode (_.VT _.RC:$src1),
1360 (_.VT _.RC:$src2),
1361 imm:$cc)>, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00001362 defm rm_Int : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,
1363 (outs _.KRC:$dst),
1364 (ins _.RC:$src1, _.ScalarMemOp:$src2, AVXCC:$cc),
1365 "vcmp${cc}"#_.Suffix,
1366 "$src2, $src1", "$src1, $src2",
1367 (OpNode (_.VT _.RC:$src1),
1368 (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))),
1369 imm:$cc)>, EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>;
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001370
1371 defm rrb_Int : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
1372 (outs _.KRC:$dst),
1373 (ins _.RC:$src1, _.RC:$src2, AVXCC:$cc),
1374 "vcmp${cc}"#_.Suffix,
Craig Topperbfe13ff2016-01-11 00:44:52 +00001375 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001376 (OpNodeRnd (_.VT _.RC:$src1),
1377 (_.VT _.RC:$src2),
1378 imm:$cc,
1379 (i32 FROUND_NO_EXC))>, EVEX_4V, EVEX_B;
1380 // Accept explicit immediate argument form instead of comparison code.
Craig Topper0550ce72014-01-05 04:55:55 +00001381 let isAsmParserOnly = 1, hasSideEffects = 0 in {
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001382 defm rri_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _,
1383 (outs VK1:$dst),
1384 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
1385 "vcmp"#_.Suffix,
1386 "$cc, $src2, $src1", "$src1, $src2, $cc">, EVEX_4V;
1387 defm rmi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _,
1388 (outs _.KRC:$dst),
Igor Breger4511e762016-02-22 11:48:27 +00001389 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$cc),
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001390 "vcmp"#_.Suffix,
1391 "$cc, $src2, $src1", "$src1, $src2, $cc">,
1392 EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>;
1393
1394 defm rrb_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _,
1395 (outs _.KRC:$dst),
1396 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
1397 "vcmp"#_.Suffix,
Craig Topperbfe13ff2016-01-11 00:44:52 +00001398 "$cc, {sae}, $src2, $src1","$src1, $src2, {sae}, $cc">,
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001399 EVEX_4V, EVEX_B;
1400 }// let isAsmParserOnly = 1, hasSideEffects = 0
1401
1402 let isCodeGenOnly = 1 in {
1403 def rr : AVX512Ii8<0xC2, MRMSrcReg,
1404 (outs _.KRC:$dst), (ins _.FRC:$src1, _.FRC:$src2, AVXCC:$cc),
1405 !strconcat("vcmp${cc}", _.Suffix,
1406 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
1407 [(set _.KRC:$dst, (OpNode _.FRC:$src1,
1408 _.FRC:$src2,
1409 imm:$cc))],
1410 IIC_SSE_ALU_F32S_RR>, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00001411 def rm : AVX512Ii8<0xC2, MRMSrcMem,
1412 (outs _.KRC:$dst),
1413 (ins _.FRC:$src1, _.ScalarMemOp:$src2, AVXCC:$cc),
1414 !strconcat("vcmp${cc}", _.Suffix,
1415 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
1416 [(set _.KRC:$dst, (OpNode _.FRC:$src1,
1417 (_.ScalarLdFrag addr:$src2),
1418 imm:$cc))],
1419 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>;
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00001420 }
1421}
1422
1423let Predicates = [HasAVX512] in {
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001424 defm VCMPSSZ : avx512_cmp_scalar<f32x_info, X86cmpms, X86cmpmsRnd>,
1425 AVX512XSIi8Base;
1426 defm VCMPSDZ : avx512_cmp_scalar<f64x_info, X86cmpms, X86cmpmsRnd>,
1427 AVX512XDIi8Base, VEX_W;
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00001428}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001429
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001430multiclass avx512_icmp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
1431 X86VectorVTInfo _> {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001432 def rr : AVX512BI<opc, MRMSrcReg,
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001433 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2),
1434 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
1435 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2)))],
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001436 IIC_SSE_ALU_F32P_RR>, EVEX_4V;
1437 def rm : AVX512BI<opc, MRMSrcMem,
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001438 (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2),
1439 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
1440 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
1441 (_.VT (bitconvert (_.LdFrag addr:$src2)))))],
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001442 IIC_SSE_ALU_F32P_RM>, EVEX_4V;
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001443 def rrk : AVX512BI<opc, MRMSrcReg,
1444 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),
1445 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst {${mask}}|",
1446 "$dst {${mask}}, $src1, $src2}"),
1447 [(set _.KRC:$dst, (and _.KRCWM:$mask,
1448 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))))],
1449 IIC_SSE_ALU_F32P_RR>, EVEX_4V, EVEX_K;
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001450 def rmk : AVX512BI<opc, MRMSrcMem,
1451 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2),
1452 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst {${mask}}|",
1453 "$dst {${mask}}, $src1, $src2}"),
1454 [(set _.KRC:$dst, (and _.KRCWM:$mask,
1455 (OpNode (_.VT _.RC:$src1),
1456 (_.VT (bitconvert
1457 (_.LdFrag addr:$src2))))))],
1458 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001459}
1460
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001461multiclass avx512_icmp_packed_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
Robert Khasanovf70f7982014-09-18 14:06:55 +00001462 X86VectorVTInfo _> :
1463 avx512_icmp_packed<opc, OpcodeStr, OpNode, _> {
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001464 def rmb : AVX512BI<opc, MRMSrcMem,
1465 (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2),
1466 !strconcat(OpcodeStr, "\t{${src2}", _.BroadcastStr, ", $src1, $dst",
1467 "|$dst, $src1, ${src2}", _.BroadcastStr, "}"),
1468 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
1469 (X86VBroadcast (_.ScalarLdFrag addr:$src2))))],
1470 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_B;
1471 def rmbk : AVX512BI<opc, MRMSrcMem,
1472 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1,
1473 _.ScalarMemOp:$src2),
1474 !strconcat(OpcodeStr,
1475 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",
1476 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"),
1477 [(set _.KRC:$dst, (and _.KRCWM:$mask,
1478 (OpNode (_.VT _.RC:$src1),
1479 (X86VBroadcast
1480 (_.ScalarLdFrag addr:$src2)))))],
1481 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K, EVEX_B;
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001482}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001483
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001484multiclass avx512_icmp_packed_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
1485 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
1486 let Predicates = [prd] in
1487 defm Z : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info512>,
1488 EVEX_V512;
1489
1490 let Predicates = [prd, HasVLX] in {
1491 defm Z256 : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info256>,
1492 EVEX_V256;
1493 defm Z128 : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info128>,
1494 EVEX_V128;
1495 }
1496}
1497
1498multiclass avx512_icmp_packed_rmb_vl<bits<8> opc, string OpcodeStr,
1499 SDNode OpNode, AVX512VLVectorVTInfo VTInfo,
1500 Predicate prd> {
1501 let Predicates = [prd] in
1502 defm Z : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info512>,
1503 EVEX_V512;
1504
1505 let Predicates = [prd, HasVLX] in {
1506 defm Z256 : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info256>,
1507 EVEX_V256;
1508 defm Z128 : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info128>,
1509 EVEX_V128;
1510 }
1511}
1512
1513defm VPCMPEQB : avx512_icmp_packed_vl<0x74, "vpcmpeqb", X86pcmpeqm,
1514 avx512vl_i8_info, HasBWI>,
1515 EVEX_CD8<8, CD8VF>;
1516
1517defm VPCMPEQW : avx512_icmp_packed_vl<0x75, "vpcmpeqw", X86pcmpeqm,
1518 avx512vl_i16_info, HasBWI>,
1519 EVEX_CD8<16, CD8VF>;
1520
Robert Khasanovf70f7982014-09-18 14:06:55 +00001521defm VPCMPEQD : avx512_icmp_packed_rmb_vl<0x76, "vpcmpeqd", X86pcmpeqm,
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001522 avx512vl_i32_info, HasAVX512>,
1523 EVEX_CD8<32, CD8VF>;
1524
Robert Khasanovf70f7982014-09-18 14:06:55 +00001525defm VPCMPEQQ : avx512_icmp_packed_rmb_vl<0x29, "vpcmpeqq", X86pcmpeqm,
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001526 avx512vl_i64_info, HasAVX512>,
1527 T8PD, VEX_W, EVEX_CD8<64, CD8VF>;
1528
1529defm VPCMPGTB : avx512_icmp_packed_vl<0x64, "vpcmpgtb", X86pcmpgtm,
1530 avx512vl_i8_info, HasBWI>,
1531 EVEX_CD8<8, CD8VF>;
1532
1533defm VPCMPGTW : avx512_icmp_packed_vl<0x65, "vpcmpgtw", X86pcmpgtm,
1534 avx512vl_i16_info, HasBWI>,
1535 EVEX_CD8<16, CD8VF>;
1536
Robert Khasanovf70f7982014-09-18 14:06:55 +00001537defm VPCMPGTD : avx512_icmp_packed_rmb_vl<0x66, "vpcmpgtd", X86pcmpgtm,
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001538 avx512vl_i32_info, HasAVX512>,
1539 EVEX_CD8<32, CD8VF>;
1540
Robert Khasanovf70f7982014-09-18 14:06:55 +00001541defm VPCMPGTQ : avx512_icmp_packed_rmb_vl<0x37, "vpcmpgtq", X86pcmpgtm,
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001542 avx512vl_i64_info, HasAVX512>,
1543 T8PD, VEX_W, EVEX_CD8<64, CD8VF>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001544
1545def : Pat<(v8i1 (X86pcmpgtm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001546 (COPY_TO_REGCLASS (VPCMPGTDZrr
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001547 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)),
1548 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm))), VK8)>;
1549
1550def : Pat<(v8i1 (X86pcmpeqm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001551 (COPY_TO_REGCLASS (VPCMPEQDZrr
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001552 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)),
1553 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm))), VK8)>;
1554
Robert Khasanov29e3b962014-08-27 09:34:37 +00001555multiclass avx512_icmp_cc<bits<8> opc, string Suffix, SDNode OpNode,
1556 X86VectorVTInfo _> {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001557 def rri : AVX512AIi8<opc, MRMSrcReg,
Craig Topper7d3c6d32015-01-28 10:09:56 +00001558 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, AVX512ICC:$cc),
Adam Nemet1efcb902014-07-01 18:03:43 +00001559 !strconcat("vpcmp${cc}", Suffix,
1560 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001561 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
1562 imm:$cc))],
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001563 IIC_SSE_ALU_F32P_RR>, EVEX_4V;
1564 def rmi : AVX512AIi8<opc, MRMSrcMem,
Craig Topper7d3c6d32015-01-28 10:09:56 +00001565 (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2, AVX512ICC:$cc),
Adam Nemet1efcb902014-07-01 18:03:43 +00001566 !strconcat("vpcmp${cc}", Suffix,
1567 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001568 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
1569 (_.VT (bitconvert (_.LdFrag addr:$src2))),
Craig Topper6e3a5822014-12-27 20:08:45 +00001570 imm:$cc))],
Robert Khasanov29e3b962014-08-27 09:34:37 +00001571 IIC_SSE_ALU_F32P_RM>, EVEX_4V;
1572 def rrik : AVX512AIi8<opc, MRMSrcReg,
1573 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2,
Craig Topper7d3c6d32015-01-28 10:09:56 +00001574 AVX512ICC:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001575 !strconcat("vpcmp${cc}", Suffix,
1576 "\t{$src2, $src1, $dst {${mask}}|",
1577 "$dst {${mask}}, $src1, $src2}"),
1578 [(set _.KRC:$dst, (and _.KRCWM:$mask,
1579 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
Craig Topper6e3a5822014-12-27 20:08:45 +00001580 imm:$cc)))],
Robert Khasanov29e3b962014-08-27 09:34:37 +00001581 IIC_SSE_ALU_F32P_RR>, EVEX_4V, EVEX_K;
Robert Khasanov29e3b962014-08-27 09:34:37 +00001582 def rmik : AVX512AIi8<opc, MRMSrcMem,
1583 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2,
Craig Topper7d3c6d32015-01-28 10:09:56 +00001584 AVX512ICC:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001585 !strconcat("vpcmp${cc}", Suffix,
1586 "\t{$src2, $src1, $dst {${mask}}|",
1587 "$dst {${mask}}, $src1, $src2}"),
1588 [(set _.KRC:$dst, (and _.KRCWM:$mask,
1589 (OpNode (_.VT _.RC:$src1),
1590 (_.VT (bitconvert (_.LdFrag addr:$src2))),
Craig Topper6e3a5822014-12-27 20:08:45 +00001591 imm:$cc)))],
Robert Khasanov29e3b962014-08-27 09:34:37 +00001592 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K;
1593
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001594 // Accept explicit immediate argument form instead of comparison code.
Craig Topper0550ce72014-01-05 04:55:55 +00001595 let isAsmParserOnly = 1, hasSideEffects = 0 in {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001596 def rri_alt : AVX512AIi8<opc, MRMSrcReg,
Craig Topper7ff6ab32015-01-21 08:43:49 +00001597 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001598 !strconcat("vpcmp", Suffix, "\t{$cc, $src2, $src1, $dst|",
1599 "$dst, $src1, $src2, $cc}"),
Adam Nemet1efcb902014-07-01 18:03:43 +00001600 [], IIC_SSE_ALU_F32P_RR>, EVEX_4V;
Craig Topper9f4d4852015-01-20 12:15:30 +00001601 let mayLoad = 1 in
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001602 def rmi_alt : AVX512AIi8<opc, MRMSrcMem,
Craig Topper7ff6ab32015-01-21 08:43:49 +00001603 (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001604 !strconcat("vpcmp", Suffix, "\t{$cc, $src2, $src1, $dst|",
1605 "$dst, $src1, $src2, $cc}"),
Adam Nemet1efcb902014-07-01 18:03:43 +00001606 [], IIC_SSE_ALU_F32P_RM>, EVEX_4V;
Robert Khasanov29e3b962014-08-27 09:34:37 +00001607 def rrik_alt : AVX512AIi8<opc, MRMSrcReg,
1608 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2,
Craig Topper7ff6ab32015-01-21 08:43:49 +00001609 u8imm:$cc),
Adam Nemet16de2482014-07-01 18:03:45 +00001610 !strconcat("vpcmp", Suffix,
Robert Khasanov29e3b962014-08-27 09:34:37 +00001611 "\t{$cc, $src2, $src1, $dst {${mask}}|",
1612 "$dst {${mask}}, $src1, $src2, $cc}"),
1613 [], IIC_SSE_ALU_F32P_RR>, EVEX_4V, EVEX_K;
Craig Topper9f4d4852015-01-20 12:15:30 +00001614 let mayLoad = 1 in
Robert Khasanov29e3b962014-08-27 09:34:37 +00001615 def rmik_alt : AVX512AIi8<opc, MRMSrcMem,
1616 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2,
Craig Topper7ff6ab32015-01-21 08:43:49 +00001617 u8imm:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001618 !strconcat("vpcmp", Suffix,
1619 "\t{$cc, $src2, $src1, $dst {${mask}}|",
1620 "$dst {${mask}}, $src1, $src2, $cc}"),
Adam Nemet16de2482014-07-01 18:03:45 +00001621 [], IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001622 }
1623}
1624
Robert Khasanov29e3b962014-08-27 09:34:37 +00001625multiclass avx512_icmp_cc_rmb<bits<8> opc, string Suffix, SDNode OpNode,
Robert Khasanovf70f7982014-09-18 14:06:55 +00001626 X86VectorVTInfo _> :
1627 avx512_icmp_cc<opc, Suffix, OpNode, _> {
Robert Khasanov29e3b962014-08-27 09:34:37 +00001628 def rmib : AVX512AIi8<opc, MRMSrcMem,
1629 (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2,
Craig Topper7d3c6d32015-01-28 10:09:56 +00001630 AVX512ICC:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001631 !strconcat("vpcmp${cc}", Suffix,
1632 "\t{${src2}", _.BroadcastStr, ", $src1, $dst|",
1633 "$dst, $src1, ${src2}", _.BroadcastStr, "}"),
1634 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
1635 (X86VBroadcast (_.ScalarLdFrag addr:$src2)),
Craig Topper6e3a5822014-12-27 20:08:45 +00001636 imm:$cc))],
Robert Khasanov29e3b962014-08-27 09:34:37 +00001637 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_B;
1638 def rmibk : AVX512AIi8<opc, MRMSrcMem,
1639 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1,
Craig Topper7d3c6d32015-01-28 10:09:56 +00001640 _.ScalarMemOp:$src2, AVX512ICC:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001641 !strconcat("vpcmp${cc}", Suffix,
1642 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",
1643 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"),
1644 [(set _.KRC:$dst, (and _.KRCWM:$mask,
1645 (OpNode (_.VT _.RC:$src1),
1646 (X86VBroadcast (_.ScalarLdFrag addr:$src2)),
Craig Topper6e3a5822014-12-27 20:08:45 +00001647 imm:$cc)))],
Robert Khasanov29e3b962014-08-27 09:34:37 +00001648 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K, EVEX_B;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001649
Robert Khasanov29e3b962014-08-27 09:34:37 +00001650 // Accept explicit immediate argument form instead of comparison code.
Craig Topper9f4d4852015-01-20 12:15:30 +00001651 let isAsmParserOnly = 1, hasSideEffects = 0, mayLoad = 1 in {
Robert Khasanov29e3b962014-08-27 09:34:37 +00001652 def rmib_alt : AVX512AIi8<opc, MRMSrcMem,
1653 (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2,
Craig Topper7ff6ab32015-01-21 08:43:49 +00001654 u8imm:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001655 !strconcat("vpcmp", Suffix,
1656 "\t{$cc, ${src2}", _.BroadcastStr, ", $src1, $dst|",
1657 "$dst, $src1, ${src2}", _.BroadcastStr, ", $cc}"),
1658 [], IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_B;
1659 def rmibk_alt : AVX512AIi8<opc, MRMSrcMem,
1660 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1,
Craig Topper7ff6ab32015-01-21 08:43:49 +00001661 _.ScalarMemOp:$src2, u8imm:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001662 !strconcat("vpcmp", Suffix,
1663 "\t{$cc, ${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",
1664 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, ", $cc}"),
1665 [], IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K, EVEX_B;
1666 }
1667}
1668
1669multiclass avx512_icmp_cc_vl<bits<8> opc, string Suffix, SDNode OpNode,
1670 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
1671 let Predicates = [prd] in
1672 defm Z : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info512>, EVEX_V512;
1673
1674 let Predicates = [prd, HasVLX] in {
1675 defm Z256 : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info256>, EVEX_V256;
1676 defm Z128 : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info128>, EVEX_V128;
1677 }
1678}
1679
1680multiclass avx512_icmp_cc_rmb_vl<bits<8> opc, string Suffix, SDNode OpNode,
1681 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
1682 let Predicates = [prd] in
1683 defm Z : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info512>,
1684 EVEX_V512;
1685
1686 let Predicates = [prd, HasVLX] in {
1687 defm Z256 : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info256>,
1688 EVEX_V256;
1689 defm Z128 : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info128>,
1690 EVEX_V128;
1691 }
1692}
1693
1694defm VPCMPB : avx512_icmp_cc_vl<0x3F, "b", X86cmpm, avx512vl_i8_info,
1695 HasBWI>, EVEX_CD8<8, CD8VF>;
1696defm VPCMPUB : avx512_icmp_cc_vl<0x3E, "ub", X86cmpmu, avx512vl_i8_info,
1697 HasBWI>, EVEX_CD8<8, CD8VF>;
1698
1699defm VPCMPW : avx512_icmp_cc_vl<0x3F, "w", X86cmpm, avx512vl_i16_info,
1700 HasBWI>, VEX_W, EVEX_CD8<16, CD8VF>;
1701defm VPCMPUW : avx512_icmp_cc_vl<0x3E, "uw", X86cmpmu, avx512vl_i16_info,
1702 HasBWI>, VEX_W, EVEX_CD8<16, CD8VF>;
1703
Robert Khasanovf70f7982014-09-18 14:06:55 +00001704defm VPCMPD : avx512_icmp_cc_rmb_vl<0x1F, "d", X86cmpm, avx512vl_i32_info,
Robert Khasanov29e3b962014-08-27 09:34:37 +00001705 HasAVX512>, EVEX_CD8<32, CD8VF>;
Robert Khasanovf70f7982014-09-18 14:06:55 +00001706defm VPCMPUD : avx512_icmp_cc_rmb_vl<0x1E, "ud", X86cmpmu, avx512vl_i32_info,
Robert Khasanov29e3b962014-08-27 09:34:37 +00001707 HasAVX512>, EVEX_CD8<32, CD8VF>;
1708
Robert Khasanovf70f7982014-09-18 14:06:55 +00001709defm VPCMPQ : avx512_icmp_cc_rmb_vl<0x1F, "q", X86cmpm, avx512vl_i64_info,
Robert Khasanov29e3b962014-08-27 09:34:37 +00001710 HasAVX512>, VEX_W, EVEX_CD8<64, CD8VF>;
Robert Khasanovf70f7982014-09-18 14:06:55 +00001711defm VPCMPUQ : avx512_icmp_cc_rmb_vl<0x1E, "uq", X86cmpmu, avx512vl_i64_info,
Robert Khasanov29e3b962014-08-27 09:34:37 +00001712 HasAVX512>, VEX_W, EVEX_CD8<64, CD8VF>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001713
Elena Demikhovsky29792e92015-05-07 11:24:42 +00001714multiclass avx512_vcmp_common<X86VectorVTInfo _> {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001715
Elena Demikhovsky29792e92015-05-07 11:24:42 +00001716 defm rri : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
1717 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2,AVXCC:$cc),
1718 "vcmp${cc}"#_.Suffix,
1719 "$src2, $src1", "$src1, $src2",
1720 (X86cmpm (_.VT _.RC:$src1),
1721 (_.VT _.RC:$src2),
1722 imm:$cc)>;
1723
Craig Toppere1cac152016-06-07 07:27:54 +00001724 defm rmi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,
1725 (outs _.KRC:$dst),(ins _.RC:$src1, _.MemOp:$src2, AVXCC:$cc),
1726 "vcmp${cc}"#_.Suffix,
1727 "$src2, $src1", "$src1, $src2",
1728 (X86cmpm (_.VT _.RC:$src1),
1729 (_.VT (bitconvert (_.LdFrag addr:$src2))),
1730 imm:$cc)>;
Elena Demikhovsky29792e92015-05-07 11:24:42 +00001731
Craig Toppere1cac152016-06-07 07:27:54 +00001732 defm rmbi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,
1733 (outs _.KRC:$dst),
1734 (ins _.RC:$src1, _.ScalarMemOp:$src2, AVXCC:$cc),
1735 "vcmp${cc}"#_.Suffix,
1736 "${src2}"##_.BroadcastStr##", $src1",
1737 "$src1, ${src2}"##_.BroadcastStr,
1738 (X86cmpm (_.VT _.RC:$src1),
1739 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
1740 imm:$cc)>,EVEX_B;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001741 // Accept explicit immediate argument form instead of comparison code.
Craig Topper0550ce72014-01-05 04:55:55 +00001742 let isAsmParserOnly = 1, hasSideEffects = 0 in {
Elena Demikhovsky29792e92015-05-07 11:24:42 +00001743 defm rri_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _,
1744 (outs _.KRC:$dst),
1745 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
1746 "vcmp"#_.Suffix,
1747 "$cc, $src2, $src1", "$src1, $src2, $cc">;
1748
1749 let mayLoad = 1 in {
1750 defm rmi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _,
1751 (outs _.KRC:$dst),
1752 (ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc),
1753 "vcmp"#_.Suffix,
1754 "$cc, $src2, $src1", "$src1, $src2, $cc">;
1755
1756 defm rmbi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _,
1757 (outs _.KRC:$dst),
1758 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$cc),
1759 "vcmp"#_.Suffix,
1760 "$cc, ${src2}"##_.BroadcastStr##", $src1",
1761 "$src1, ${src2}"##_.BroadcastStr##", $cc">,EVEX_B;
1762 }
1763 }
1764}
1765
1766multiclass avx512_vcmp_sae<X86VectorVTInfo _> {
1767 // comparison code form (VCMP[EQ/LT/LE/...]
1768 defm rrib : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
1769 (outs _.KRC:$dst),(ins _.RC:$src1, _.RC:$src2, AVXCC:$cc),
1770 "vcmp${cc}"#_.Suffix,
Craig Topperbfe13ff2016-01-11 00:44:52 +00001771 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
Elena Demikhovsky29792e92015-05-07 11:24:42 +00001772 (X86cmpmRnd (_.VT _.RC:$src1),
1773 (_.VT _.RC:$src2),
1774 imm:$cc,
1775 (i32 FROUND_NO_EXC))>, EVEX_B;
1776
1777 let isAsmParserOnly = 1, hasSideEffects = 0 in {
1778 defm rrib_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _,
1779 (outs _.KRC:$dst),
1780 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
1781 "vcmp"#_.Suffix,
Craig Topperbfe13ff2016-01-11 00:44:52 +00001782 "$cc, {sae}, $src2, $src1",
1783 "$src1, $src2, {sae}, $cc">, EVEX_B;
Elena Demikhovsky29792e92015-05-07 11:24:42 +00001784 }
1785}
1786
1787multiclass avx512_vcmp<AVX512VLVectorVTInfo _> {
1788 let Predicates = [HasAVX512] in {
1789 defm Z : avx512_vcmp_common<_.info512>,
1790 avx512_vcmp_sae<_.info512>, EVEX_V512;
1791
1792 }
1793 let Predicates = [HasAVX512,HasVLX] in {
1794 defm Z128 : avx512_vcmp_common<_.info128>, EVEX_V128;
1795 defm Z256 : avx512_vcmp_common<_.info256>, EVEX_V256;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001796 }
1797}
1798
Elena Demikhovsky29792e92015-05-07 11:24:42 +00001799defm VCMPPD : avx512_vcmp<avx512vl_f64_info>,
1800 AVX512PDIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
1801defm VCMPPS : avx512_vcmp<avx512vl_f32_info>,
1802 AVX512PSIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001803
1804def : Pat<(v8i1 (X86cmpm (v8f32 VR256X:$src1), (v8f32 VR256X:$src2), imm:$cc)),
1805 (COPY_TO_REGCLASS (VCMPPSZrri
1806 (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)),
1807 (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)),
1808 imm:$cc), VK8)>;
1809def : Pat<(v8i1 (X86cmpm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)),
1810 (COPY_TO_REGCLASS (VPCMPDZrri
1811 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)),
1812 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)),
1813 imm:$cc), VK8)>;
1814def : Pat<(v8i1 (X86cmpmu (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)),
1815 (COPY_TO_REGCLASS (VPCMPUDZrri
1816 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)),
1817 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)),
1818 imm:$cc), VK8)>;
Elena Demikhovskyde3f7512014-01-01 15:12:34 +00001819
Asaf Badouh572bbce2015-09-20 08:46:07 +00001820// ----------------------------------------------------------------
1821// FPClass
Asaf Badouh696e8e02015-10-18 11:04:38 +00001822//handle fpclass instruction mask = op(reg_scalar,imm)
1823// op(mem_scalar,imm)
1824multiclass avx512_scalar_fpclass<bits<8> opc, string OpcodeStr, SDNode OpNode,
1825 X86VectorVTInfo _, Predicate prd> {
1826 let Predicates = [prd] in {
1827 def rr : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),//_.KRC:$dst),
1828 (ins _.RC:$src1, i32u8imm:$src2),
Craig Topper048e7002016-01-08 06:09:20 +00001829 OpcodeStr##_.Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Asaf Badouh696e8e02015-10-18 11:04:38 +00001830 [(set _.KRC:$dst,(OpNode (_.VT _.RC:$src1),
1831 (i32 imm:$src2)))], NoItinerary>;
1832 def rrk : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
1833 (ins _.KRCWM:$mask, _.RC:$src1, i32u8imm:$src2),
1834 OpcodeStr##_.Suffix#
Craig Topper048e7002016-01-08 06:09:20 +00001835 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00001836 [(set _.KRC:$dst,(or _.KRCWM:$mask,
Asaf Badouh696e8e02015-10-18 11:04:38 +00001837 (OpNode (_.VT _.RC:$src1),
1838 (i32 imm:$src2))))], NoItinerary>, EVEX_K;
Craig Toppere1cac152016-06-07 07:27:54 +00001839 let AddedComplexity = 20 in {
Asaf Badouh696e8e02015-10-18 11:04:38 +00001840 def rm : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
1841 (ins _.MemOp:$src1, i32u8imm:$src2),
1842 OpcodeStr##_.Suffix##
Craig Topper048e7002016-01-08 06:09:20 +00001843 "\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Asaf Badouh696e8e02015-10-18 11:04:38 +00001844 [(set _.KRC:$dst,
1845 (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
1846 (i32 imm:$src2)))], NoItinerary>;
1847 def rmk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
1848 (ins _.KRCWM:$mask, _.MemOp:$src1, i32u8imm:$src2),
1849 OpcodeStr##_.Suffix##
Craig Topper048e7002016-01-08 06:09:20 +00001850 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00001851 [(set _.KRC:$dst,(or _.KRCWM:$mask,
Asaf Badouh696e8e02015-10-18 11:04:38 +00001852 (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
1853 (i32 imm:$src2))))], NoItinerary>, EVEX_K;
1854 }
1855 }
1856}
1857
Asaf Badouh572bbce2015-09-20 08:46:07 +00001858//handle fpclass instruction mask = fpclass(reg_vec, reg_vec, imm)
1859// fpclass(reg_vec, mem_vec, imm)
1860// fpclass(reg_vec, broadcast(eltVt), imm)
1861multiclass avx512_vector_fpclass<bits<8> opc, string OpcodeStr, SDNode OpNode,
1862 X86VectorVTInfo _, string mem, string broadcast>{
1863 def rr : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
1864 (ins _.RC:$src1, i32u8imm:$src2),
Craig Topper048e7002016-01-08 06:09:20 +00001865 OpcodeStr##_.Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Asaf Badouh572bbce2015-09-20 08:46:07 +00001866 [(set _.KRC:$dst,(OpNode (_.VT _.RC:$src1),
1867 (i32 imm:$src2)))], NoItinerary>;
1868 def rrk : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
1869 (ins _.KRCWM:$mask, _.RC:$src1, i32u8imm:$src2),
1870 OpcodeStr##_.Suffix#
Craig Topper048e7002016-01-08 06:09:20 +00001871 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00001872 [(set _.KRC:$dst,(or _.KRCWM:$mask,
Asaf Badouh572bbce2015-09-20 08:46:07 +00001873 (OpNode (_.VT _.RC:$src1),
1874 (i32 imm:$src2))))], NoItinerary>, EVEX_K;
Craig Toppere1cac152016-06-07 07:27:54 +00001875 def rm : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
1876 (ins _.MemOp:$src1, i32u8imm:$src2),
1877 OpcodeStr##_.Suffix##mem#
1878 "\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00001879 [(set _.KRC:$dst,(OpNode
Craig Toppere1cac152016-06-07 07:27:54 +00001880 (_.VT (bitconvert (_.LdFrag addr:$src1))),
1881 (i32 imm:$src2)))], NoItinerary>;
1882 def rmk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
1883 (ins _.KRCWM:$mask, _.MemOp:$src1, i32u8imm:$src2),
1884 OpcodeStr##_.Suffix##mem#
1885 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00001886 [(set _.KRC:$dst, (or _.KRCWM:$mask, (OpNode
Craig Toppere1cac152016-06-07 07:27:54 +00001887 (_.VT (bitconvert (_.LdFrag addr:$src1))),
1888 (i32 imm:$src2))))], NoItinerary>, EVEX_K;
1889 def rmb : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
1890 (ins _.ScalarMemOp:$src1, i32u8imm:$src2),
1891 OpcodeStr##_.Suffix##broadcast##"\t{$src2, ${src1}"##
1892 _.BroadcastStr##", $dst|$dst, ${src1}"
1893 ##_.BroadcastStr##", $src2}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00001894 [(set _.KRC:$dst,(OpNode
1895 (_.VT (X86VBroadcast
Craig Toppere1cac152016-06-07 07:27:54 +00001896 (_.ScalarLdFrag addr:$src1))),
1897 (i32 imm:$src2)))], NoItinerary>,EVEX_B;
1898 def rmbk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
1899 (ins _.KRCWM:$mask, _.ScalarMemOp:$src1, i32u8imm:$src2),
1900 OpcodeStr##_.Suffix##broadcast##"\t{$src2, ${src1}"##
1901 _.BroadcastStr##", $dst {${mask}}|$dst {${mask}}, ${src1}"##
1902 _.BroadcastStr##", $src2}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00001903 [(set _.KRC:$dst,(or _.KRCWM:$mask, (OpNode
1904 (_.VT (X86VBroadcast
Craig Toppere1cac152016-06-07 07:27:54 +00001905 (_.ScalarLdFrag addr:$src1))),
1906 (i32 imm:$src2))))], NoItinerary>,
1907 EVEX_B, EVEX_K;
Asaf Badouh572bbce2015-09-20 08:46:07 +00001908}
1909
Asaf Badouh572bbce2015-09-20 08:46:07 +00001910multiclass avx512_vector_fpclass_all<string OpcodeStr,
Simon Pilgrimb13961d2016-06-11 14:34:10 +00001911 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd,
Asaf Badouh572bbce2015-09-20 08:46:07 +00001912 string broadcast>{
1913 let Predicates = [prd] in {
Simon Pilgrimb13961d2016-06-11 14:34:10 +00001914 defm Z : avx512_vector_fpclass<opc, OpcodeStr, OpNode, _.info512, "{z}",
Asaf Badouh572bbce2015-09-20 08:46:07 +00001915 broadcast>, EVEX_V512;
1916 }
1917 let Predicates = [prd, HasVLX] in {
1918 defm Z128 : avx512_vector_fpclass<opc, OpcodeStr, OpNode, _.info128, "{x}",
1919 broadcast>, EVEX_V128;
1920 defm Z256 : avx512_vector_fpclass<opc, OpcodeStr, OpNode, _.info256, "{y}",
1921 broadcast>, EVEX_V256;
1922 }
1923}
1924
1925multiclass avx512_fp_fpclass_all<string OpcodeStr, bits<8> opcVec,
Asaf Badouh696e8e02015-10-18 11:04:38 +00001926 bits<8> opcScalar, SDNode VecOpNode, SDNode ScalarOpNode, Predicate prd>{
Simon Pilgrim18bcf932016-02-03 09:41:59 +00001927 defm PS : avx512_vector_fpclass_all<OpcodeStr, avx512vl_f32_info, opcVec,
Asaf Badouh696e8e02015-10-18 11:04:38 +00001928 VecOpNode, prd, "{l}">, EVEX_CD8<32, CD8VF>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00001929 defm PD : avx512_vector_fpclass_all<OpcodeStr, avx512vl_f64_info, opcVec,
Asaf Badouh696e8e02015-10-18 11:04:38 +00001930 VecOpNode, prd, "{q}">,EVEX_CD8<64, CD8VF> , VEX_W;
1931 defm SS : avx512_scalar_fpclass<opcScalar, OpcodeStr, ScalarOpNode,
1932 f32x_info, prd>, EVEX_CD8<32, CD8VT1>;
1933 defm SD : avx512_scalar_fpclass<opcScalar, OpcodeStr, ScalarOpNode,
1934 f64x_info, prd>, EVEX_CD8<64, CD8VT1>, VEX_W;
Asaf Badouh572bbce2015-09-20 08:46:07 +00001935}
1936
Asaf Badouh696e8e02015-10-18 11:04:38 +00001937defm VFPCLASS : avx512_fp_fpclass_all<"vfpclass", 0x66, 0x67, X86Vfpclass,
1938 X86Vfpclasss, HasDQI>, AVX512AIi8Base,EVEX;
Asaf Badouh572bbce2015-09-20 08:46:07 +00001939
Elena Demikhovsky29792e92015-05-07 11:24:42 +00001940//-----------------------------------------------------------------
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001941// Mask register copy, including
1942// - copy between mask registers
1943// - load/store mask registers
1944// - copy from GPR to mask register and vice versa
1945//
1946multiclass avx512_mask_mov<bits<8> opc_kk, bits<8> opc_km, bits<8> opc_mk,
1947 string OpcodeStr, RegisterClass KRC,
Elena Demikhovskyba846722015-02-17 09:20:12 +00001948 ValueType vvt, X86MemOperand x86memop> {
Craig Toppere1cac152016-06-07 07:27:54 +00001949 let hasSideEffects = 0 in
1950 def kk : I<opc_kk, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src),
1951 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>;
1952 def km : I<opc_km, MRMSrcMem, (outs KRC:$dst), (ins x86memop:$src),
1953 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
1954 [(set KRC:$dst, (vvt (load addr:$src)))]>;
1955 def mk : I<opc_mk, MRMDestMem, (outs), (ins x86memop:$dst, KRC:$src),
1956 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
1957 [(store KRC:$src, addr:$dst)]>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001958}
1959
1960multiclass avx512_mask_mov_gpr<bits<8> opc_kr, bits<8> opc_rk,
1961 string OpcodeStr,
1962 RegisterClass KRC, RegisterClass GRC> {
Elena Demikhovskyf404e052014-01-05 14:21:07 +00001963 let hasSideEffects = 0 in {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001964 def kr : I<opc_kr, MRMSrcReg, (outs KRC:$dst), (ins GRC:$src),
Craig Topperedb09112014-11-25 20:11:23 +00001965 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001966 def rk : I<opc_rk, MRMSrcReg, (outs GRC:$dst), (ins KRC:$src),
Craig Topperedb09112014-11-25 20:11:23 +00001967 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001968 }
1969}
1970
Robert Khasanov74acbb72014-07-23 14:49:42 +00001971let Predicates = [HasDQI] in
Elena Demikhovskyba846722015-02-17 09:20:12 +00001972 defm KMOVB : avx512_mask_mov<0x90, 0x90, 0x91, "kmovb", VK8, v8i1, i8mem>,
Robert Khasanov74acbb72014-07-23 14:49:42 +00001973 avx512_mask_mov_gpr<0x92, 0x93, "kmovb", VK8, GR32>,
1974 VEX, PD;
1975
1976let Predicates = [HasAVX512] in
Elena Demikhovskyba846722015-02-17 09:20:12 +00001977 defm KMOVW : avx512_mask_mov<0x90, 0x90, 0x91, "kmovw", VK16, v16i1, i16mem>,
Robert Khasanov74acbb72014-07-23 14:49:42 +00001978 avx512_mask_mov_gpr<0x92, 0x93, "kmovw", VK16, GR32>,
Craig Topper5ccb6172014-02-18 00:21:49 +00001979 VEX, PS;
Robert Khasanov74acbb72014-07-23 14:49:42 +00001980
1981let Predicates = [HasBWI] in {
Elena Demikhovskyba846722015-02-17 09:20:12 +00001982 defm KMOVD : avx512_mask_mov<0x90, 0x90, 0x91, "kmovd", VK32, v32i1,i32mem>,
1983 VEX, PD, VEX_W;
Robert Khasanov74acbb72014-07-23 14:49:42 +00001984 defm KMOVD : avx512_mask_mov_gpr<0x92, 0x93, "kmovd", VK32, GR32>,
1985 VEX, XD;
Elena Demikhovskyba846722015-02-17 09:20:12 +00001986 defm KMOVQ : avx512_mask_mov<0x90, 0x90, 0x91, "kmovq", VK64, v64i1, i64mem>,
1987 VEX, PS, VEX_W;
Robert Khasanov74acbb72014-07-23 14:49:42 +00001988 defm KMOVQ : avx512_mask_mov_gpr<0x92, 0x93, "kmovq", VK64, GR64>,
1989 VEX, XD, VEX_W;
1990}
1991
1992// GR from/to mask register
1993let Predicates = [HasDQI] in {
1994 def : Pat<(v8i1 (bitconvert (i8 GR8:$src))),
1995 (KMOVBkr (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit))>;
1996 def : Pat<(i8 (bitconvert (v8i1 VK8:$src))),
1997 (EXTRACT_SUBREG (KMOVBrk VK8:$src), sub_8bit)>;
Craig Topperddab3952016-06-14 03:12:54 +00001998 def : Pat<(i32 (zext (i8 (bitconvert (v8i1 VK8:$src))))),
1999 (KMOVBrk VK8:$src)>;
Craig Topper283418f2016-06-21 07:37:32 +00002000 def : Pat<(i32 (anyext (i8 (bitconvert (v8i1 VK8:$src))))),
2001 (KMOVBrk VK8:$src)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002002}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002003let Predicates = [HasAVX512] in {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002004 def : Pat<(v16i1 (bitconvert (i16 GR16:$src))),
2005 (KMOVWkr (SUBREG_TO_REG (i32 0), GR16:$src, sub_16bit))>;
2006 def : Pat<(i16 (bitconvert (v16i1 VK16:$src))),
2007 (EXTRACT_SUBREG (KMOVWrk VK16:$src), sub_16bit)>;
Craig Topperddab3952016-06-14 03:12:54 +00002008 def : Pat<(i32 (zext (i16 (bitconvert (v16i1 VK16:$src))))),
2009 (KMOVWrk VK16:$src)>;
Craig Topper283418f2016-06-21 07:37:32 +00002010 def : Pat<(i32 (anyext (i16 (bitconvert (v16i1 VK16:$src))))),
2011 (KMOVWrk VK16:$src)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002012}
2013let Predicates = [HasBWI] in {
2014 def : Pat<(v32i1 (bitconvert (i32 GR32:$src))), (KMOVDkr GR32:$src)>;
2015 def : Pat<(i32 (bitconvert (v32i1 VK32:$src))), (KMOVDrk VK32:$src)>;
2016}
2017let Predicates = [HasBWI] in {
2018 def : Pat<(v64i1 (bitconvert (i64 GR64:$src))), (KMOVQkr GR64:$src)>;
2019 def : Pat<(i64 (bitconvert (v64i1 VK64:$src))), (KMOVQrk VK64:$src)>;
2020}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002021
Robert Khasanov74acbb72014-07-23 14:49:42 +00002022// Load/store kreg
2023let Predicates = [HasDQI] in {
2024 def : Pat<(store (i8 (bitconvert (v8i1 VK8:$src))), addr:$dst),
2025 (KMOVBmk addr:$dst, VK8:$src)>;
Elena Demikhovskyba846722015-02-17 09:20:12 +00002026 def : Pat<(v8i1 (bitconvert (i8 (load addr:$src)))),
2027 (KMOVBkm addr:$src)>;
Elena Demikhovsky9f83c732015-09-02 09:20:58 +00002028
2029 def : Pat<(store VK4:$src, addr:$dst),
2030 (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK4:$src, VK8))>;
2031 def : Pat<(store VK2:$src, addr:$dst),
2032 (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK2:$src, VK8))>;
Igor Bregerd6c187b2016-01-27 08:43:25 +00002033 def : Pat<(store VK1:$src, addr:$dst),
2034 (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK1:$src, VK8))>;
Elena Demikhovsky5e426f72016-04-03 08:41:12 +00002035
2036 def : Pat<(v2i1 (load addr:$src)),
2037 (COPY_TO_REGCLASS (KMOVBkm addr:$src), VK2)>;
2038 def : Pat<(v4i1 (load addr:$src)),
2039 (COPY_TO_REGCLASS (KMOVBkm addr:$src), VK4)>;
Elena Demikhovskyba846722015-02-17 09:20:12 +00002040}
2041let Predicates = [HasAVX512, NoDQI] in {
Igor Bregerd6c187b2016-01-27 08:43:25 +00002042 def : Pat<(store VK1:$src, addr:$dst),
2043 (MOV8mr addr:$dst,
2044 (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)),
2045 sub_8bit))>;
2046 def : Pat<(store VK2:$src, addr:$dst),
2047 (MOV8mr addr:$dst,
2048 (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK2:$src, VK16)),
2049 sub_8bit))>;
2050 def : Pat<(store VK4:$src, addr:$dst),
2051 (MOV8mr addr:$dst,
2052 (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK4:$src, VK16)),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00002053 sub_8bit))>;
Igor Bregerd6c187b2016-01-27 08:43:25 +00002054 def : Pat<(store VK8:$src, addr:$dst),
2055 (MOV8mr addr:$dst,
2056 (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16)),
2057 sub_8bit))>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00002058
Elena Demikhovsky5e426f72016-04-03 08:41:12 +00002059 def : Pat<(v8i1 (load addr:$src)),
Craig Topper99e30e62016-06-14 03:13:00 +00002060 (COPY_TO_REGCLASS (MOVZX32rm8 addr:$src), VK8)>;
Elena Demikhovsky5e426f72016-04-03 08:41:12 +00002061 def : Pat<(v2i1 (load addr:$src)),
Craig Topper99e30e62016-06-14 03:13:00 +00002062 (COPY_TO_REGCLASS (MOVZX32rm8 addr:$src), VK2)>;
Elena Demikhovsky5e426f72016-04-03 08:41:12 +00002063 def : Pat<(v4i1 (load addr:$src)),
Craig Topper99e30e62016-06-14 03:13:00 +00002064 (COPY_TO_REGCLASS (MOVZX32rm8 addr:$src), VK4)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002065}
Elena Demikhovsky5e426f72016-04-03 08:41:12 +00002066
Robert Khasanov74acbb72014-07-23 14:49:42 +00002067let Predicates = [HasAVX512] in {
2068 def : Pat<(store (i16 (bitconvert (v16i1 VK16:$src))), addr:$dst),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002069 (KMOVWmk addr:$dst, VK16:$src)>;
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00002070 def : Pat<(i1 (load addr:$src)),
Craig Topper34d97072016-06-14 03:13:03 +00002071 (COPY_TO_REGCLASS (AND32ri8 (MOVZX32rm8 addr:$src), (i32 1)), VK1)>;
Elena Demikhovskyba846722015-02-17 09:20:12 +00002072 def : Pat<(v16i1 (bitconvert (i16 (load addr:$src)))),
2073 (KMOVWkm addr:$src)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002074}
2075let Predicates = [HasBWI] in {
2076 def : Pat<(store (i32 (bitconvert (v32i1 VK32:$src))), addr:$dst),
2077 (KMOVDmk addr:$dst, VK32:$src)>;
Elena Demikhovskyba846722015-02-17 09:20:12 +00002078 def : Pat<(v32i1 (bitconvert (i32 (load addr:$src)))),
2079 (KMOVDkm addr:$src)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002080 def : Pat<(store (i64 (bitconvert (v64i1 VK64:$src))), addr:$dst),
2081 (KMOVQmk addr:$dst, VK64:$src)>;
Elena Demikhovskyba846722015-02-17 09:20:12 +00002082 def : Pat<(v64i1 (bitconvert (i64 (load addr:$src)))),
2083 (KMOVQkm addr:$src)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002084}
Elena Demikhovskyc5f67262013-12-17 08:33:15 +00002085
Elena Demikhovskyad0a56f2016-07-06 14:15:43 +00002086def assertzext_i1 : PatFrag<(ops node:$src), (assertzext node:$src), [{
2087 return cast<VTSDNode>(N->getOperand(1))->getVT() == MVT::i1;
2088}]>;
2089
Robert Khasanov74acbb72014-07-23 14:49:42 +00002090let Predicates = [HasAVX512] in {
Elena Demikhovsky34d2d762014-08-18 11:59:06 +00002091 def : Pat<(i1 (trunc (i64 GR64:$src))),
Elena Demikhovskyad0a56f2016-07-06 14:15:43 +00002092 (COPY_TO_REGCLASS (i16 (EXTRACT_SUBREG (AND64ri8 $src, (i64 1)),
2093 sub_16bit)), VK1)>;
2094
2095 def : Pat<(i1 (trunc (i64 (assertzext_i1 GR64:$src)))),
2096 (COPY_TO_REGCLASS (i16 (EXTRACT_SUBREG $src, sub_16bit)), VK1)>;
Elena Demikhovsky34d2d762014-08-18 11:59:06 +00002097
Elena Demikhovsky64c95482013-12-24 14:24:07 +00002098 def : Pat<(i1 (trunc (i32 GR32:$src))),
Elena Demikhovskyad0a56f2016-07-06 14:15:43 +00002099 (COPY_TO_REGCLASS (i16 (EXTRACT_SUBREG (AND32ri8 $src, (i32 1)),
2100 sub_16bit)), VK1)>;
2101
2102 def : Pat<(i1 (trunc (i32 (assertzext_i1 GR32:$src)))),
2103 (COPY_TO_REGCLASS (i16 (EXTRACT_SUBREG $src, sub_16bit)), VK1)>;
Elena Demikhovsky64c95482013-12-24 14:24:07 +00002104
2105 def : Pat<(i1 (trunc (i8 GR8:$src))),
Elena Demikhovskyad0a56f2016-07-06 14:15:43 +00002106 (COPY_TO_REGCLASS (i16 (SUBREG_TO_REG (i64 0), (AND8ri8 $src, (i8 1)),
2107 sub_8bit)), VK1)>;
2108
2109 def : Pat<(i1 (trunc (i8 (assertzext_i1 GR8:$src)))),
2110 (COPY_TO_REGCLASS (i16 (SUBREG_TO_REG (i64 0), $src, sub_8bit)), VK1)>;
2111
Elena Demikhovskyc9657012014-02-20 06:34:39 +00002112 def : Pat<(i1 (trunc (i16 GR16:$src))),
Elena Demikhovskyad0a56f2016-07-06 14:15:43 +00002113 (COPY_TO_REGCLASS (AND16ri GR16:$src, (i16 1)), VK1)>;
2114
2115 def : Pat<(i1 (trunc (i16 (assertzext_i1 GR16:$src)))),
2116 (COPY_TO_REGCLASS $src, VK1)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002117
Elena Demikhovsky3ebfe112014-02-23 14:28:35 +00002118 def : Pat<(i32 (zext VK1:$src)),
Elena Demikhovskyad0a56f2016-07-06 14:15:43 +00002119 (i32 (SUBREG_TO_REG (i64 0), (i16 (COPY_TO_REGCLASS $src, GR16)),
2120 sub_16bit))>;
2121
Elena Demikhovsky86c7b462015-05-27 14:09:33 +00002122 def : Pat<(i32 (anyext VK1:$src)),
Elena Demikhovskyad0a56f2016-07-06 14:15:43 +00002123 (i32 (SUBREG_TO_REG (i64 0), (i16 (COPY_TO_REGCLASS $src, GR16)),
2124 sub_16bit))>;
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00002125
Elena Demikhovsky64c95482013-12-24 14:24:07 +00002126 def : Pat<(i8 (zext VK1:$src)),
Elena Demikhovskyad0a56f2016-07-06 14:15:43 +00002127 (i8 (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS VK1:$src, GR16)), sub_8bit))>;
2128
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00002129 def : Pat<(i8 (anyext VK1:$src)),
Elena Demikhovskyad0a56f2016-07-06 14:15:43 +00002130 (i8 (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS $src, GR16)), sub_8bit))>;
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00002131
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00002132 def : Pat<(i64 (zext VK1:$src)),
Elena Demikhovskyad0a56f2016-07-06 14:15:43 +00002133 (i64 (SUBREG_TO_REG (i64 0), (i16 (COPY_TO_REGCLASS $src, GR16)),
2134 sub_16bit))>;
2135
Michael Kuperstein18d6d3d2016-06-17 20:21:17 +00002136 def : Pat<(i64 (anyext VK1:$src)),
Elena Demikhovskyad0a56f2016-07-06 14:15:43 +00002137 (i64 (SUBREG_TO_REG (i64 0), (i16 (COPY_TO_REGCLASS $src, GR16)),
2138 sub_16bit))>;
Michael Kuperstein18d6d3d2016-06-17 20:21:17 +00002139
Elena Demikhovsky750498c2014-02-17 07:29:33 +00002140 def : Pat<(i16 (zext VK1:$src)),
Elena Demikhovskyad0a56f2016-07-06 14:15:43 +00002141 (COPY_TO_REGCLASS $src, GR16)>;
2142
Michael Kuperstein18d6d3d2016-06-17 20:21:17 +00002143 def : Pat<(i16 (anyext VK1:$src)),
Elena Demikhovskyad0a56f2016-07-06 14:15:43 +00002144 (i16 (COPY_TO_REGCLASS $src, GR16))>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002145}
Elena Demikhovsky6015f5c2015-12-15 08:40:41 +00002146def : Pat<(v16i1 (scalar_to_vector VK1:$src)),
2147 (COPY_TO_REGCLASS VK1:$src, VK16)>;
2148def : Pat<(v8i1 (scalar_to_vector VK1:$src)),
2149 (COPY_TO_REGCLASS VK1:$src, VK8)>;
2150def : Pat<(v4i1 (scalar_to_vector VK1:$src)),
2151 (COPY_TO_REGCLASS VK1:$src, VK4)>;
2152def : Pat<(v2i1 (scalar_to_vector VK1:$src)),
2153 (COPY_TO_REGCLASS VK1:$src, VK2)>;
2154def : Pat<(v32i1 (scalar_to_vector VK1:$src)),
2155 (COPY_TO_REGCLASS VK1:$src, VK32)>;
2156def : Pat<(v64i1 (scalar_to_vector VK1:$src)),
2157 (COPY_TO_REGCLASS VK1:$src, VK64)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002158
Igor Bregerd6c187b2016-01-27 08:43:25 +00002159def : Pat<(store (i1 -1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>;
2160def : Pat<(store (i1 1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>;
2161def : Pat<(store (i1 0), addr:$dst), (MOV8mi addr:$dst, (i8 0))>;
2162
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002163// With AVX-512 only, 8-bit mask is promoted to 16-bit mask.
Elena Demikhovsky75d14892015-05-10 10:33:32 +00002164let Predicates = [HasAVX512, NoDQI] in {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002165 // GR from/to 8-bit mask without native support
2166 def : Pat<(v8i1 (bitconvert (i8 GR8:$src))),
2167 (COPY_TO_REGCLASS
Igor Bregerdd6522c2016-01-18 12:02:45 +00002168 (KMOVWkr (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit)), VK8)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002169 def : Pat<(i8 (bitconvert (v8i1 VK8:$src))),
2170 (EXTRACT_SUBREG
2171 (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16)),
2172 sub_8bit)>;
Craig Topperddab3952016-06-14 03:12:54 +00002173 def : Pat<(i32 (zext (i8 (bitconvert (v8i1 VK8:$src))))),
2174 (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16))>;
Craig Topper283418f2016-06-21 07:37:32 +00002175 def : Pat<(i32 (anyext (i8 (bitconvert (v8i1 VK8:$src))))),
2176 (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16))>;
Elena Demikhovsky75d14892015-05-10 10:33:32 +00002177}
Elena Demikhovskyf61727d2015-05-20 14:32:03 +00002178
Elena Demikhovsky75d14892015-05-10 10:33:32 +00002179let Predicates = [HasAVX512] in {
Elena Demikhovsky9f423d62014-02-10 07:02:39 +00002180 def : Pat<(i1 (X86Vextract VK16:$src, (iPTR 0))),
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00002181 (COPY_TO_REGCLASS VK16:$src, VK1)>;
Elena Demikhovsky9f423d62014-02-10 07:02:39 +00002182 def : Pat<(i1 (X86Vextract VK8:$src, (iPTR 0))),
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00002183 (COPY_TO_REGCLASS VK8:$src, VK1)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002184}
2185let Predicates = [HasBWI] in {
2186 def : Pat<(i1 (X86Vextract VK32:$src, (iPTR 0))),
2187 (COPY_TO_REGCLASS VK32:$src, VK1)>;
2188 def : Pat<(i1 (X86Vextract VK64:$src, (iPTR 0))),
2189 (COPY_TO_REGCLASS VK64:$src, VK1)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002190}
2191
2192// Mask unary operation
2193// - KNOT
2194multiclass avx512_mask_unop<bits<8> opc, string OpcodeStr,
Robert Khasanov74acbb72014-07-23 14:49:42 +00002195 RegisterClass KRC, SDPatternOperator OpNode,
2196 Predicate prd> {
2197 let Predicates = [prd] in
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002198 def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src),
Craig Topperedb09112014-11-25 20:11:23 +00002199 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002200 [(set KRC:$dst, (OpNode KRC:$src))]>;
2201}
2202
Robert Khasanov74acbb72014-07-23 14:49:42 +00002203multiclass avx512_mask_unop_all<bits<8> opc, string OpcodeStr,
2204 SDPatternOperator OpNode> {
2205 defm B : avx512_mask_unop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,
2206 HasDQI>, VEX, PD;
2207 defm W : avx512_mask_unop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,
2208 HasAVX512>, VEX, PS;
2209 defm D : avx512_mask_unop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,
2210 HasBWI>, VEX, PD, VEX_W;
2211 defm Q : avx512_mask_unop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,
2212 HasBWI>, VEX, PS, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002213}
2214
Robert Khasanov74acbb72014-07-23 14:49:42 +00002215defm KNOT : avx512_mask_unop_all<0x44, "knot", not>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002216
Elena Demikhovskye382c3f2013-12-10 13:53:10 +00002217multiclass avx512_mask_unop_int<string IntName, string InstName> {
2218 let Predicates = [HasAVX512] in
2219 def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_w")
2220 (i16 GR16:$src)),
2221 (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"Wrr")
2222 (v16i1 (COPY_TO_REGCLASS GR16:$src, VK16))), GR16)>;
2223}
2224defm : avx512_mask_unop_int<"knot", "KNOT">;
2225
Robert Khasanov74acbb72014-07-23 14:49:42 +00002226let Predicates = [HasDQI] in
2227def : Pat<(xor VK8:$src1, (v8i1 immAllOnesV)), (KNOTBrr VK8:$src1)>;
2228let Predicates = [HasAVX512] in
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002229def : Pat<(xor VK16:$src1, (v16i1 immAllOnesV)), (KNOTWrr VK16:$src1)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002230let Predicates = [HasBWI] in
2231def : Pat<(xor VK32:$src1, (v32i1 immAllOnesV)), (KNOTDrr VK32:$src1)>;
2232let Predicates = [HasBWI] in
2233def : Pat<(xor VK64:$src1, (v64i1 immAllOnesV)), (KNOTQrr VK64:$src1)>;
2234
2235// KNL does not support KMOVB, 8-bit mask is promoted to 16-bit
Elena Demikhovskyd2cb3c82015-02-12 08:40:34 +00002236let Predicates = [HasAVX512, NoDQI] in {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002237def : Pat<(xor VK8:$src1, (v8i1 immAllOnesV)),
2238 (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$src1, VK16)), VK8)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002239def : Pat<(not VK8:$src),
2240 (COPY_TO_REGCLASS
2241 (KNOTWrr (COPY_TO_REGCLASS VK8:$src, VK16)), VK8)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002242}
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002243def : Pat<(xor VK4:$src1, (v4i1 immAllOnesV)),
2244 (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK4:$src1, VK16)), VK4)>;
2245def : Pat<(xor VK2:$src1, (v2i1 immAllOnesV)),
2246 (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK2:$src1, VK16)), VK2)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002247
2248// Mask binary operation
Elena Demikhovskye382c3f2013-12-10 13:53:10 +00002249// - KAND, KANDN, KOR, KXNOR, KXOR
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002250multiclass avx512_mask_binop<bits<8> opc, string OpcodeStr,
Robert Khasanov595683d2014-07-28 13:46:45 +00002251 RegisterClass KRC, SDPatternOperator OpNode,
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002252 Predicate prd, bit IsCommutable> {
2253 let Predicates = [prd], isCommutable = IsCommutable in
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002254 def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2),
2255 !strconcat(OpcodeStr,
Craig Topperedb09112014-11-25 20:11:23 +00002256 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002257 [(set KRC:$dst, (OpNode KRC:$src1, KRC:$src2))]>;
2258}
2259
Robert Khasanov595683d2014-07-28 13:46:45 +00002260multiclass avx512_mask_binop_all<bits<8> opc, string OpcodeStr,
Igor Breger59ac3392015-08-31 11:50:23 +00002261 SDPatternOperator OpNode, bit IsCommutable,
2262 Predicate prdW = HasAVX512> {
Robert Khasanov595683d2014-07-28 13:46:45 +00002263 defm B : avx512_mask_binop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002264 HasDQI, IsCommutable>, VEX_4V, VEX_L, PD;
Robert Khasanov595683d2014-07-28 13:46:45 +00002265 defm W : avx512_mask_binop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,
Igor Breger59ac3392015-08-31 11:50:23 +00002266 prdW, IsCommutable>, VEX_4V, VEX_L, PS;
Robert Khasanov595683d2014-07-28 13:46:45 +00002267 defm D : avx512_mask_binop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002268 HasBWI, IsCommutable>, VEX_4V, VEX_L, VEX_W, PD;
Robert Khasanov595683d2014-07-28 13:46:45 +00002269 defm Q : avx512_mask_binop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002270 HasBWI, IsCommutable>, VEX_4V, VEX_L, VEX_W, PS;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002271}
2272
2273def andn : PatFrag<(ops node:$i0, node:$i1), (and (not node:$i0), node:$i1)>;
2274def xnor : PatFrag<(ops node:$i0, node:$i1), (not (xor node:$i0, node:$i1))>;
2275
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002276defm KAND : avx512_mask_binop_all<0x41, "kand", and, 1>;
2277defm KOR : avx512_mask_binop_all<0x45, "kor", or, 1>;
2278defm KXNOR : avx512_mask_binop_all<0x46, "kxnor", xnor, 1>;
2279defm KXOR : avx512_mask_binop_all<0x47, "kxor", xor, 1>;
2280defm KANDN : avx512_mask_binop_all<0x42, "kandn", andn, 0>;
Igor Breger59ac3392015-08-31 11:50:23 +00002281defm KADD : avx512_mask_binop_all<0x4A, "kadd", add, 1, HasDQI>;
Elena Demikhovskyb64d7e82013-12-25 10:06:40 +00002282
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002283multiclass avx512_mask_binop_int<string IntName, string InstName> {
2284 let Predicates = [HasAVX512] in
Elena Demikhovskye382c3f2013-12-10 13:53:10 +00002285 def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_w")
2286 (i16 GR16:$src1), (i16 GR16:$src2)),
2287 (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"Wrr")
2288 (v16i1 (COPY_TO_REGCLASS GR16:$src1, VK16)),
2289 (v16i1 (COPY_TO_REGCLASS GR16:$src2, VK16))), GR16)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002290}
2291
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002292defm : avx512_mask_binop_int<"kand", "KAND">;
2293defm : avx512_mask_binop_int<"kandn", "KANDN">;
2294defm : avx512_mask_binop_int<"kor", "KOR">;
2295defm : avx512_mask_binop_int<"kxnor", "KXNOR">;
2296defm : avx512_mask_binop_int<"kxor", "KXOR">;
Elena Demikhovskye382c3f2013-12-10 13:53:10 +00002297
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002298multiclass avx512_binop_pat<SDPatternOperator OpNode, Instruction Inst> {
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002299 // With AVX512F, 8-bit mask is promoted to 16-bit mask,
2300 // for the DQI set, this type is legal and KxxxB instruction is used
2301 let Predicates = [NoDQI] in
2302 def : Pat<(OpNode VK8:$src1, VK8:$src2),
2303 (COPY_TO_REGCLASS
2304 (Inst (COPY_TO_REGCLASS VK8:$src1, VK16),
2305 (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>;
2306
2307 // All types smaller than 8 bits require conversion anyway
2308 def : Pat<(OpNode VK1:$src1, VK1:$src2),
2309 (COPY_TO_REGCLASS (Inst
2310 (COPY_TO_REGCLASS VK1:$src1, VK16),
2311 (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>;
2312 def : Pat<(OpNode VK2:$src1, VK2:$src2),
2313 (COPY_TO_REGCLASS (Inst
2314 (COPY_TO_REGCLASS VK2:$src1, VK16),
2315 (COPY_TO_REGCLASS VK2:$src2, VK16)), VK1)>;
2316 def : Pat<(OpNode VK4:$src1, VK4:$src2),
2317 (COPY_TO_REGCLASS (Inst
2318 (COPY_TO_REGCLASS VK4:$src1, VK16),
2319 (COPY_TO_REGCLASS VK4:$src2, VK16)), VK1)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002320}
2321
2322defm : avx512_binop_pat<and, KANDWrr>;
2323defm : avx512_binop_pat<andn, KANDNWrr>;
2324defm : avx512_binop_pat<or, KORWrr>;
2325defm : avx512_binop_pat<xnor, KXNORWrr>;
2326defm : avx512_binop_pat<xor, KXORWrr>;
2327
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002328def : Pat<(xor (xor VK16:$src1, VK16:$src2), (v16i1 immAllOnesV)),
2329 (KXNORWrr VK16:$src1, VK16:$src2)>;
2330def : Pat<(xor (xor VK8:$src1, VK8:$src2), (v8i1 immAllOnesV)),
Elena Demikhovsky00c9ad52015-06-10 06:49:28 +00002331 (KXNORBrr VK8:$src1, VK8:$src2)>, Requires<[HasDQI]>;
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002332def : Pat<(xor (xor VK32:$src1, VK32:$src2), (v32i1 immAllOnesV)),
Elena Demikhovsky00c9ad52015-06-10 06:49:28 +00002333 (KXNORDrr VK32:$src1, VK32:$src2)>, Requires<[HasBWI]>;
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002334def : Pat<(xor (xor VK64:$src1, VK64:$src2), (v64i1 immAllOnesV)),
Elena Demikhovsky00c9ad52015-06-10 06:49:28 +00002335 (KXNORQrr VK64:$src1, VK64:$src2)>, Requires<[HasBWI]>;
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002336
2337let Predicates = [NoDQI] in
2338def : Pat<(xor (xor VK8:$src1, VK8:$src2), (v8i1 immAllOnesV)),
2339 (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK8:$src1, VK16),
2340 (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>;
2341
2342def : Pat<(xor (xor VK4:$src1, VK4:$src2), (v4i1 immAllOnesV)),
2343 (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK4:$src1, VK16),
2344 (COPY_TO_REGCLASS VK4:$src2, VK16)), VK4)>;
2345
2346def : Pat<(xor (xor VK2:$src1, VK2:$src2), (v2i1 immAllOnesV)),
2347 (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK2:$src1, VK16),
2348 (COPY_TO_REGCLASS VK2:$src2, VK16)), VK2)>;
2349
2350def : Pat<(xor (xor VK1:$src1, VK1:$src2), (i1 1)),
2351 (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK1:$src1, VK16),
2352 (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>;
2353
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002354// Mask unpacking
Igor Bregera54a1a82015-09-08 13:10:00 +00002355multiclass avx512_mask_unpck<string Suffix,RegisterClass KRC, ValueType VT,
2356 RegisterClass KRCSrc, Predicate prd> {
2357 let Predicates = [prd] in {
Craig Topperad2ce362016-01-05 07:44:08 +00002358 let hasSideEffects = 0 in
Igor Bregera54a1a82015-09-08 13:10:00 +00002359 def rr : I<0x4b, MRMSrcReg, (outs KRC:$dst),
2360 (ins KRC:$src1, KRC:$src2),
2361 "kunpck"#Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
2362 VEX_4V, VEX_L;
2363
2364 def : Pat<(VT (concat_vectors KRCSrc:$src1, KRCSrc:$src2)),
2365 (!cast<Instruction>(NAME##rr)
2366 (COPY_TO_REGCLASS KRCSrc:$src2, KRC),
2367 (COPY_TO_REGCLASS KRCSrc:$src1, KRC))>;
2368 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002369}
2370
Igor Bregera54a1a82015-09-08 13:10:00 +00002371defm KUNPCKBW : avx512_mask_unpck<"bw", VK16, v16i1, VK8, HasAVX512>, PD;
2372defm KUNPCKWD : avx512_mask_unpck<"wd", VK32, v32i1, VK16, HasBWI>, PS;
2373defm KUNPCKDQ : avx512_mask_unpck<"dq", VK64, v64i1, VK32, HasBWI>, PS, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002374
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002375// Mask bit testing
2376multiclass avx512_mask_testop<bits<8> opc, string OpcodeStr, RegisterClass KRC,
Igor Breger5ea0a6812015-08-31 13:30:19 +00002377 SDNode OpNode, Predicate prd> {
2378 let Predicates = [prd], Defs = [EFLAGS] in
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002379 def rr : I<opc, MRMSrcReg, (outs), (ins KRC:$src1, KRC:$src2),
Craig Topperedb09112014-11-25 20:11:23 +00002380 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002381 [(set EFLAGS, (OpNode KRC:$src1, KRC:$src2))]>;
2382}
2383
Igor Breger5ea0a6812015-08-31 13:30:19 +00002384multiclass avx512_mask_testop_w<bits<8> opc, string OpcodeStr, SDNode OpNode,
2385 Predicate prdW = HasAVX512> {
2386 defm B : avx512_mask_testop<opc, OpcodeStr#"b", VK8, OpNode, HasDQI>,
2387 VEX, PD;
2388 defm W : avx512_mask_testop<opc, OpcodeStr#"w", VK16, OpNode, prdW>,
2389 VEX, PS;
2390 defm Q : avx512_mask_testop<opc, OpcodeStr#"q", VK64, OpNode, HasBWI>,
2391 VEX, PS, VEX_W;
2392 defm D : avx512_mask_testop<opc, OpcodeStr#"d", VK32, OpNode, HasBWI>,
2393 VEX, PD, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002394}
2395
2396defm KORTEST : avx512_mask_testop_w<0x98, "kortest", X86kortest>;
Igor Breger5ea0a6812015-08-31 13:30:19 +00002397defm KTEST : avx512_mask_testop_w<0x99, "ktest", X86ktest, HasDQI>;
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00002398
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002399// Mask shift
2400multiclass avx512_mask_shiftop<bits<8> opc, string OpcodeStr, RegisterClass KRC,
2401 SDNode OpNode> {
2402 let Predicates = [HasAVX512] in
Craig Topper7ff6ab32015-01-21 08:43:49 +00002403 def ri : Ii8<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src, u8imm:$imm),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002404 !strconcat(OpcodeStr,
Craig Topperedb09112014-11-25 20:11:23 +00002405 "\t{$imm, $src, $dst|$dst, $src, $imm}"),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002406 [(set KRC:$dst, (OpNode KRC:$src, (i8 imm:$imm)))]>;
2407}
2408
2409multiclass avx512_mask_shiftop_w<bits<8> opc1, bits<8> opc2, string OpcodeStr,
2410 SDNode OpNode> {
2411 defm W : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "w"), VK16, OpNode>,
Elena Demikhovsky1a603b32015-01-25 12:47:15 +00002412 VEX, TAPD, VEX_W;
2413 let Predicates = [HasDQI] in
2414 defm B : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "b"), VK8, OpNode>,
2415 VEX, TAPD;
2416 let Predicates = [HasBWI] in {
2417 defm Q : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "q"), VK64, OpNode>,
2418 VEX, TAPD, VEX_W;
Elena Demikhovsky1a603b32015-01-25 12:47:15 +00002419 defm D : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "d"), VK32, OpNode>,
2420 VEX, TAPD;
Michael Liao66233b72015-08-06 09:06:20 +00002421 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002422}
2423
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00002424defm KSHIFTL : avx512_mask_shiftop_w<0x32, 0x33, "kshiftl", X86vshli>;
2425defm KSHIFTR : avx512_mask_shiftop_w<0x30, 0x31, "kshiftr", X86vsrli>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002426
2427// Mask setting all 0s or 1s
2428multiclass avx512_mask_setop<RegisterClass KRC, ValueType VT, PatFrag Val> {
2429 let Predicates = [HasAVX512] in
2430 let isReMaterializable = 1, isAsCheapAsAMove = 1, isPseudo = 1 in
2431 def #NAME# : I<0, Pseudo, (outs KRC:$dst), (ins), "",
2432 [(set KRC:$dst, (VT Val))]>;
2433}
2434
2435multiclass avx512_mask_setop_w<PatFrag Val> {
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00002436 defm B : avx512_mask_setop<VK8, v8i1, Val>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002437 defm W : avx512_mask_setop<VK16, v16i1, Val>;
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002438 defm D : avx512_mask_setop<VK32, v32i1, Val>;
2439 defm Q : avx512_mask_setop<VK64, v64i1, Val>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002440}
2441
2442defm KSET0 : avx512_mask_setop_w<immAllZerosV>;
2443defm KSET1 : avx512_mask_setop_w<immAllOnesV>;
2444
2445// With AVX-512 only, 8-bit mask is promoted to 16-bit mask.
2446let Predicates = [HasAVX512] in {
2447 def : Pat<(v8i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK8)>;
2448 def : Pat<(v8i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK8)>;
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002449 def : Pat<(v4i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK4)>;
2450 def : Pat<(v2i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK2)>;
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00002451 def : Pat<(i1 0), (COPY_TO_REGCLASS (KSET0W), VK1)>;
Elena Demikhovsky1d6a4952015-05-17 07:28:51 +00002452 def : Pat<(i1 1), (COPY_TO_REGCLASS (KSHIFTRWri (KSET1W), (i8 15)), VK1)>;
2453 def : Pat<(i1 -1), (COPY_TO_REGCLASS (KSHIFTRWri (KSET1W), (i8 15)), VK1)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002454}
Igor Bregerf1bd7612016-03-06 07:46:03 +00002455
2456// Patterns for kmask insert_subvector/extract_subvector to/from index=0
2457multiclass operation_subvector_mask_lowering<RegisterClass subRC, ValueType subVT,
2458 RegisterClass RC, ValueType VT> {
2459 def : Pat<(subVT (extract_subvector (VT RC:$src), (iPTR 0))),
2460 (subVT (COPY_TO_REGCLASS RC:$src, subRC))>;
Simon Pilgrimb13961d2016-06-11 14:34:10 +00002461
Igor Bregerf1bd7612016-03-06 07:46:03 +00002462 def : Pat<(VT (insert_subvector undef, subRC:$src, (iPTR 0))),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00002463 (VT (COPY_TO_REGCLASS subRC:$src, RC))>;
Igor Bregerf1bd7612016-03-06 07:46:03 +00002464}
2465
2466defm : operation_subvector_mask_lowering<VK2, v2i1, VK4, v4i1>;
2467defm : operation_subvector_mask_lowering<VK2, v2i1, VK8, v8i1>;
2468defm : operation_subvector_mask_lowering<VK2, v2i1, VK16, v16i1>;
2469defm : operation_subvector_mask_lowering<VK2, v2i1, VK32, v32i1>;
2470defm : operation_subvector_mask_lowering<VK2, v2i1, VK64, v64i1>;
2471
2472defm : operation_subvector_mask_lowering<VK4, v4i1, VK8, v8i1>;
2473defm : operation_subvector_mask_lowering<VK4, v4i1, VK16, v16i1>;
2474defm : operation_subvector_mask_lowering<VK4, v4i1, VK32, v32i1>;
2475defm : operation_subvector_mask_lowering<VK4, v4i1, VK64, v64i1>;
2476
2477defm : operation_subvector_mask_lowering<VK8, v8i1, VK16, v16i1>;
2478defm : operation_subvector_mask_lowering<VK8, v8i1, VK32, v32i1>;
2479defm : operation_subvector_mask_lowering<VK8, v8i1, VK64, v64i1>;
2480
2481defm : operation_subvector_mask_lowering<VK16, v16i1, VK32, v32i1>;
2482defm : operation_subvector_mask_lowering<VK16, v16i1, VK64, v64i1>;
2483
2484defm : operation_subvector_mask_lowering<VK32, v32i1, VK64, v64i1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002485
Igor Breger999ac752016-03-08 15:21:25 +00002486def : Pat<(v2i1 (extract_subvector (v4i1 VK4:$src), (iPTR 2))),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00002487 (v2i1 (COPY_TO_REGCLASS
Igor Breger999ac752016-03-08 15:21:25 +00002488 (KSHIFTRWri (COPY_TO_REGCLASS VK4:$src, VK16), (i8 2)),
2489 VK2))>;
2490def : Pat<(v4i1 (extract_subvector (v8i1 VK8:$src), (iPTR 4))),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00002491 (v4i1 (COPY_TO_REGCLASS
Igor Breger999ac752016-03-08 15:21:25 +00002492 (KSHIFTRWri (COPY_TO_REGCLASS VK8:$src, VK16), (i8 4)),
2493 VK4))>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002494def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 8))),
2495 (v8i1 (COPY_TO_REGCLASS (KSHIFTRWri VK16:$src, (i8 8)), VK8))>;
Elena Demikhovsky6015f5c2015-12-15 08:40:41 +00002496def : Pat<(v16i1 (extract_subvector (v32i1 VK32:$src), (iPTR 16))),
2497 (v16i1 (COPY_TO_REGCLASS (KSHIFTRDri VK32:$src, (i8 16)), VK16))>;
Elena Demikhovsky86c7b462015-05-27 14:09:33 +00002498def : Pat<(v32i1 (extract_subvector (v64i1 VK64:$src), (iPTR 32))),
2499 (v32i1 (COPY_TO_REGCLASS (KSHIFTRQri VK64:$src, (i8 32)), VK32))>;
2500
Elena Demikhovsky9737e382014-03-02 09:19:44 +00002501def : Pat<(v8i1 (X86vshli VK8:$src, (i8 imm:$imm))),
Elena Demikhovsky1a603b32015-01-25 12:47:15 +00002502 (v8i1 (COPY_TO_REGCLASS
2503 (KSHIFTLWri (COPY_TO_REGCLASS VK8:$src, VK16),
2504 (I8Imm $imm)), VK8))>, Requires<[HasAVX512, NoDQI]>;
Elena Demikhovsky9737e382014-03-02 09:19:44 +00002505
Elena Demikhovskyde05f102015-03-05 15:11:35 +00002506def : Pat<(v4i1 (X86vshli VK4:$src, (i8 imm:$imm))),
2507 (v4i1 (COPY_TO_REGCLASS
2508 (KSHIFTLWri (COPY_TO_REGCLASS VK4:$src, VK16),
2509 (I8Imm $imm)), VK4))>, Requires<[HasAVX512]>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002510//===----------------------------------------------------------------------===//
2511// AVX-512 - Aligned and unaligned load and store
2512//
2513
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002514
2515multiclass avx512_load<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
Elena Demikhovskyd207f172015-03-03 15:03:35 +00002516 PatFrag ld_frag, PatFrag mload,
Craig Topperc9293492016-02-26 06:50:29 +00002517 bit IsReMaterializable = 1,
2518 SDPatternOperator SelectOprr = vselect> {
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002519 let hasSideEffects = 0 in {
2520 def rr : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst), (ins _.RC:$src),
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002521 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [],
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002522 _.ExeDomain>, EVEX;
2523 def rrkz : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst),
2524 (ins _.KRCWM:$mask, _.RC:$src),
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002525 !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}} {z}|",
Simon Pilgrim18bcf932016-02-03 09:41:59 +00002526 "${dst} {${mask}} {z}, $src}"),
Igor Breger7a000f52016-01-21 14:18:11 +00002527 [(set _.RC:$dst, (_.VT (vselect _.KRCWM:$mask,
2528 (_.VT _.RC:$src),
2529 _.ImmAllZerosV)))], _.ExeDomain>,
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002530 EVEX, EVEX_KZ;
2531
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002532 let canFoldAsLoad = 1, isReMaterializable = IsReMaterializable,
2533 SchedRW = [WriteLoad] in
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002534 def rm : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst), (ins _.MemOp:$src),
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002535 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002536 [(set _.RC:$dst, (_.VT (bitconvert (ld_frag addr:$src))))],
2537 _.ExeDomain>, EVEX;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002538
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002539 let Constraints = "$src0 = $dst" in {
2540 def rrk : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst),
2541 (ins _.RC:$src0, _.KRCWM:$mask, _.RC:$src1),
2542 !strconcat(OpcodeStr, "\t{$src1, ${dst} {${mask}}|",
2543 "${dst} {${mask}}, $src1}"),
Craig Topperc9293492016-02-26 06:50:29 +00002544 [(set _.RC:$dst, (_.VT (SelectOprr _.KRCWM:$mask,
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002545 (_.VT _.RC:$src1),
2546 (_.VT _.RC:$src0))))], _.ExeDomain>,
2547 EVEX, EVEX_K;
Craig Toppere1cac152016-06-07 07:27:54 +00002548 let SchedRW = [WriteLoad] in
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002549 def rmk : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst),
2550 (ins _.RC:$src0, _.KRCWM:$mask, _.MemOp:$src1),
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002551 !strconcat(OpcodeStr, "\t{$src1, ${dst} {${mask}}|",
2552 "${dst} {${mask}}, $src1}"),
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002553 [(set _.RC:$dst, (_.VT
2554 (vselect _.KRCWM:$mask,
2555 (_.VT (bitconvert (ld_frag addr:$src1))),
2556 (_.VT _.RC:$src0))))], _.ExeDomain>, EVEX, EVEX_K;
Elena Demikhovskyfd056672014-03-13 12:05:52 +00002557 }
Craig Toppere1cac152016-06-07 07:27:54 +00002558 let SchedRW = [WriteLoad] in
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002559 def rmkz : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst),
2560 (ins _.KRCWM:$mask, _.MemOp:$src),
2561 OpcodeStr #"\t{$src, ${dst} {${mask}} {z}|"#
2562 "${dst} {${mask}} {z}, $src}",
2563 [(set _.RC:$dst, (_.VT (vselect _.KRCWM:$mask,
2564 (_.VT (bitconvert (ld_frag addr:$src))), _.ImmAllZerosV)))],
2565 _.ExeDomain>, EVEX, EVEX_KZ;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002566 }
Elena Demikhovskyd207f172015-03-03 15:03:35 +00002567 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, undef)),
2568 (!cast<Instruction>(NAME#_.ZSuffix##rmkz) _.KRCWM:$mask, addr:$ptr)>;
2569
2570 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, _.ImmAllZerosV)),
2571 (!cast<Instruction>(NAME#_.ZSuffix##rmkz) _.KRCWM:$mask, addr:$ptr)>;
2572
2573 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src0))),
2574 (!cast<Instruction>(NAME#_.ZSuffix##rmk) _.RC:$src0,
2575 _.KRCWM:$mask, addr:$ptr)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002576}
2577
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002578multiclass avx512_alignedload_vl<bits<8> opc, string OpcodeStr,
2579 AVX512VLVectorVTInfo _,
2580 Predicate prd,
2581 bit IsReMaterializable = 1> {
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002582 let Predicates = [prd] in
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002583 defm Z : avx512_load<opc, OpcodeStr, _.info512, _.info512.AlignedLdFrag,
Elena Demikhovskyd207f172015-03-03 15:03:35 +00002584 masked_load_aligned512, IsReMaterializable>, EVEX_V512;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002585
2586 let Predicates = [prd, HasVLX] in {
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002587 defm Z256 : avx512_load<opc, OpcodeStr, _.info256, _.info256.AlignedLdFrag,
Elena Demikhovskyd207f172015-03-03 15:03:35 +00002588 masked_load_aligned256, IsReMaterializable>, EVEX_V256;
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002589 defm Z128 : avx512_load<opc, OpcodeStr, _.info128, _.info128.AlignedLdFrag,
Elena Demikhovskyd207f172015-03-03 15:03:35 +00002590 masked_load_aligned128, IsReMaterializable>, EVEX_V128;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002591 }
2592}
2593
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002594multiclass avx512_load_vl<bits<8> opc, string OpcodeStr,
2595 AVX512VLVectorVTInfo _,
2596 Predicate prd,
Craig Topperc9293492016-02-26 06:50:29 +00002597 bit IsReMaterializable = 1,
2598 SDPatternOperator SelectOprr = vselect> {
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002599 let Predicates = [prd] in
2600 defm Z : avx512_load<opc, OpcodeStr, _.info512, _.info512.LdFrag,
Craig Topperc9293492016-02-26 06:50:29 +00002601 masked_load_unaligned, IsReMaterializable,
2602 SelectOprr>, EVEX_V512;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002603
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002604 let Predicates = [prd, HasVLX] in {
2605 defm Z256 : avx512_load<opc, OpcodeStr, _.info256, _.info256.LdFrag,
Craig Topperc9293492016-02-26 06:50:29 +00002606 masked_load_unaligned, IsReMaterializable,
2607 SelectOprr>, EVEX_V256;
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002608 defm Z128 : avx512_load<opc, OpcodeStr, _.info128, _.info128.LdFrag,
Craig Topperc9293492016-02-26 06:50:29 +00002609 masked_load_unaligned, IsReMaterializable,
2610 SelectOprr>, EVEX_V128;
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002611 }
2612}
2613
2614multiclass avx512_store<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
Elena Demikhovskyd207f172015-03-03 15:03:35 +00002615 PatFrag st_frag, PatFrag mstore> {
Igor Breger81b79de2015-11-19 07:43:43 +00002616
Craig Topper99f6b622016-05-01 01:03:56 +00002617 let hasSideEffects = 0 in {
Igor Breger81b79de2015-11-19 07:43:43 +00002618 def rr_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst), (ins _.RC:$src),
2619 OpcodeStr # ".s\t{$src, $dst|$dst, $src}",
2620 [], _.ExeDomain>, EVEX;
2621 def rrk_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst),
2622 (ins _.KRCWM:$mask, _.RC:$src),
2623 OpcodeStr # ".s\t{$src, ${dst} {${mask}}|"#
2624 "${dst} {${mask}}, $src}",
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002625 [], _.ExeDomain>, EVEX, EVEX_K;
Igor Breger81b79de2015-11-19 07:43:43 +00002626 def rrkz_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst),
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002627 (ins _.KRCWM:$mask, _.RC:$src),
Igor Breger81b79de2015-11-19 07:43:43 +00002628 OpcodeStr # ".s\t{$src, ${dst} {${mask}} {z}|" #
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002629 "${dst} {${mask}} {z}, $src}",
2630 [], _.ExeDomain>, EVEX, EVEX_KZ;
Craig Topper99f6b622016-05-01 01:03:56 +00002631 }
Igor Breger81b79de2015-11-19 07:43:43 +00002632
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002633 def mr : AVX512PI<opc, MRMDestMem, (outs), (ins _.MemOp:$dst, _.RC:$src),
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002634 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002635 [(st_frag (_.VT _.RC:$src), addr:$dst)], _.ExeDomain>, EVEX;
Elena Demikhovskyfd056672014-03-13 12:05:52 +00002636 def mrk : AVX512PI<opc, MRMDestMem, (outs),
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002637 (ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src),
2638 OpcodeStr # "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}",
2639 [], _.ExeDomain>, EVEX, EVEX_K;
Elena Demikhovskyd207f172015-03-03 15:03:35 +00002640
2641 def: Pat<(mstore addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src)),
2642 (!cast<Instruction>(NAME#_.ZSuffix##mrk) addr:$ptr,
2643 _.KRCWM:$mask, _.RC:$src)>;
Elena Demikhovskyfd056672014-03-13 12:05:52 +00002644}
2645
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002646
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002647multiclass avx512_store_vl< bits<8> opc, string OpcodeStr,
2648 AVX512VLVectorVTInfo _, Predicate prd> {
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002649 let Predicates = [prd] in
Elena Demikhovskyd207f172015-03-03 15:03:35 +00002650 defm Z : avx512_store<opc, OpcodeStr, _.info512, store,
2651 masked_store_unaligned>, EVEX_V512;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002652
2653 let Predicates = [prd, HasVLX] in {
Elena Demikhovskyd207f172015-03-03 15:03:35 +00002654 defm Z256 : avx512_store<opc, OpcodeStr, _.info256, store,
2655 masked_store_unaligned>, EVEX_V256;
2656 defm Z128 : avx512_store<opc, OpcodeStr, _.info128, store,
2657 masked_store_unaligned>, EVEX_V128;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002658 }
2659}
2660
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002661multiclass avx512_alignedstore_vl<bits<8> opc, string OpcodeStr,
2662 AVX512VLVectorVTInfo _, Predicate prd> {
2663 let Predicates = [prd] in
Elena Demikhovskyd207f172015-03-03 15:03:35 +00002664 defm Z : avx512_store<opc, OpcodeStr, _.info512, alignedstore512,
2665 masked_store_aligned512>, EVEX_V512;
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002666
2667 let Predicates = [prd, HasVLX] in {
Elena Demikhovskyd207f172015-03-03 15:03:35 +00002668 defm Z256 : avx512_store<opc, OpcodeStr, _.info256, alignedstore256,
2669 masked_store_aligned256>, EVEX_V256;
2670 defm Z128 : avx512_store<opc, OpcodeStr, _.info128, alignedstore,
2671 masked_store_aligned128>, EVEX_V128;
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002672 }
2673}
2674
2675defm VMOVAPS : avx512_alignedload_vl<0x28, "vmovaps", avx512vl_f32_info,
2676 HasAVX512>,
2677 avx512_alignedstore_vl<0x29, "vmovaps", avx512vl_f32_info,
2678 HasAVX512>, PS, EVEX_CD8<32, CD8VF>;
2679
2680defm VMOVAPD : avx512_alignedload_vl<0x28, "vmovapd", avx512vl_f64_info,
2681 HasAVX512>,
2682 avx512_alignedstore_vl<0x29, "vmovapd", avx512vl_f64_info,
2683 HasAVX512>, PD, VEX_W, EVEX_CD8<64, CD8VF>;
2684
Craig Topperc9293492016-02-26 06:50:29 +00002685defm VMOVUPS : avx512_load_vl<0x10, "vmovups", avx512vl_f32_info, HasAVX512,
2686 1, null_frag>,
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002687 avx512_store_vl<0x11, "vmovups", avx512vl_f32_info, HasAVX512>,
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002688 PS, EVEX_CD8<32, CD8VF>;
2689
Craig Topperc9293492016-02-26 06:50:29 +00002690defm VMOVUPD : avx512_load_vl<0x10, "vmovupd", avx512vl_f64_info, HasAVX512, 0,
2691 null_frag>,
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002692 avx512_store_vl<0x11, "vmovupd", avx512vl_f64_info, HasAVX512>,
2693 PD, VEX_W, EVEX_CD8<64, CD8VF>;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002694
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002695defm VMOVDQA32 : avx512_alignedload_vl<0x6F, "vmovdqa32", avx512vl_i32_info,
2696 HasAVX512>,
2697 avx512_alignedstore_vl<0x7F, "vmovdqa32", avx512vl_i32_info,
2698 HasAVX512>, PD, EVEX_CD8<32, CD8VF>;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002699
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002700defm VMOVDQA64 : avx512_alignedload_vl<0x6F, "vmovdqa64", avx512vl_i64_info,
2701 HasAVX512>,
2702 avx512_alignedstore_vl<0x7F, "vmovdqa64", avx512vl_i64_info,
2703 HasAVX512>, PD, VEX_W, EVEX_CD8<64, CD8VF>;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002704
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002705defm VMOVDQU8 : avx512_load_vl<0x6F, "vmovdqu8", avx512vl_i8_info, HasBWI>,
2706 avx512_store_vl<0x7F, "vmovdqu8", avx512vl_i8_info,
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002707 HasBWI>, XD, EVEX_CD8<8, CD8VF>;
2708
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002709defm VMOVDQU16 : avx512_load_vl<0x6F, "vmovdqu16", avx512vl_i16_info, HasBWI>,
2710 avx512_store_vl<0x7F, "vmovdqu16", avx512vl_i16_info,
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002711 HasBWI>, XD, VEX_W, EVEX_CD8<16, CD8VF>;
2712
Craig Topperc9293492016-02-26 06:50:29 +00002713defm VMOVDQU32 : avx512_load_vl<0x6F, "vmovdqu32", avx512vl_i32_info, HasAVX512,
2714 1, null_frag>,
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002715 avx512_store_vl<0x7F, "vmovdqu32", avx512vl_i32_info,
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002716 HasAVX512>, XS, EVEX_CD8<32, CD8VF>;
2717
Craig Topperc9293492016-02-26 06:50:29 +00002718defm VMOVDQU64 : avx512_load_vl<0x6F, "vmovdqu64", avx512vl_i64_info, HasAVX512,
2719 1, null_frag>,
Elena Demikhovsky2689d782015-03-02 12:46:21 +00002720 avx512_store_vl<0x7F, "vmovdqu64", avx512vl_i64_info,
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002721 HasAVX512>, XS, VEX_W, EVEX_CD8<64, CD8VF>;
Elena Demikhovsky1f3ed412013-10-22 09:19:28 +00002722
Elena Demikhovskya30e4372014-02-05 07:05:03 +00002723def : Pat<(v8i64 (vselect VK8WM:$mask, (bc_v8i64 (v16i32 immAllZerosV)),
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002724 (v8i64 VR512:$src))),
Igor Breger7a000f52016-01-21 14:18:11 +00002725 (VMOVDQA64Zrrkz (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$mask, VK16)),
Elena Demikhovskya30e4372014-02-05 07:05:03 +00002726 VK8), VR512:$src)>;
2727
Elena Demikhovskya30e4372014-02-05 07:05:03 +00002728def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 immAllZerosV),
Robert Khasanov7ca7df02014-08-04 14:35:15 +00002729 (v16i32 VR512:$src))),
Igor Breger7a000f52016-01-21 14:18:11 +00002730 (VMOVDQA32Zrrkz (KNOTWrr VK16WM:$mask), VR512:$src)>;
Elena Demikhovskyf1de34b2014-12-04 09:40:44 +00002731
Craig Topper33c550c2016-05-22 00:39:30 +00002732// These patterns exist to prevent the above patterns from introducing a second
2733// mask inversion when one already exists.
2734def : Pat<(v8i64 (vselect (xor VK8:$mask, (v8i1 immAllOnesV)),
2735 (bc_v8i64 (v16i32 immAllZerosV)),
2736 (v8i64 VR512:$src))),
2737 (VMOVDQA64Zrrkz VK8:$mask, VR512:$src)>;
2738def : Pat<(v16i32 (vselect (xor VK16:$mask, (v16i1 immAllOnesV)),
2739 (v16i32 immAllZerosV),
2740 (v16i32 VR512:$src))),
2741 (VMOVDQA32Zrrkz VK16WM:$mask, VR512:$src)>;
2742
Craig Topper95bdabd2016-05-22 23:44:33 +00002743let Predicates = [HasVLX] in {
2744 // Special patterns for storing subvector extracts of lower 128-bits of 256.
2745 // Its cheaper to just use VMOVAPS/VMOVUPS instead of VEXTRACTF128mr
2746 def : Pat<(alignedstore (v2f64 (extract_subvector
2747 (v4f64 VR256X:$src), (iPTR 0))), addr:$dst),
2748 (VMOVAPDZ128mr addr:$dst, (v2f64 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
2749 def : Pat<(alignedstore (v4f32 (extract_subvector
2750 (v8f32 VR256X:$src), (iPTR 0))), addr:$dst),
2751 (VMOVAPSZ128mr addr:$dst, (v4f32 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
2752 def : Pat<(alignedstore (v2i64 (extract_subvector
2753 (v4i64 VR256X:$src), (iPTR 0))), addr:$dst),
2754 (VMOVDQA64Z128mr addr:$dst, (v2i64 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
2755 def : Pat<(alignedstore (v4i32 (extract_subvector
2756 (v8i32 VR256X:$src), (iPTR 0))), addr:$dst),
2757 (VMOVDQA32Z128mr addr:$dst, (v4i32 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
2758 def : Pat<(alignedstore (v8i16 (extract_subvector
2759 (v16i16 VR256X:$src), (iPTR 0))), addr:$dst),
2760 (VMOVDQA32Z128mr addr:$dst, (v8i16 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
2761 def : Pat<(alignedstore (v16i8 (extract_subvector
2762 (v32i8 VR256X:$src), (iPTR 0))), addr:$dst),
2763 (VMOVDQA32Z128mr addr:$dst, (v16i8 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
2764
2765 def : Pat<(store (v2f64 (extract_subvector
2766 (v4f64 VR256X:$src), (iPTR 0))), addr:$dst),
2767 (VMOVUPDZ128mr addr:$dst, (v2f64 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
2768 def : Pat<(store (v4f32 (extract_subvector
2769 (v8f32 VR256X:$src), (iPTR 0))), addr:$dst),
2770 (VMOVUPSZ128mr addr:$dst, (v4f32 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
2771 def : Pat<(store (v2i64 (extract_subvector
2772 (v4i64 VR256X:$src), (iPTR 0))), addr:$dst),
2773 (VMOVDQU64Z128mr addr:$dst, (v2i64 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
2774 def : Pat<(store (v4i32 (extract_subvector
2775 (v8i32 VR256X:$src), (iPTR 0))), addr:$dst),
2776 (VMOVDQU32Z128mr addr:$dst, (v4i32 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
2777 def : Pat<(store (v8i16 (extract_subvector
2778 (v16i16 VR256X:$src), (iPTR 0))), addr:$dst),
2779 (VMOVDQU32Z128mr addr:$dst, (v8i16 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
2780 def : Pat<(store (v16i8 (extract_subvector
2781 (v32i8 VR256X:$src), (iPTR 0))), addr:$dst),
2782 (VMOVDQU32Z128mr addr:$dst, (v16i8 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
2783
2784 // Special patterns for storing subvector extracts of lower 128-bits of 512.
2785 // Its cheaper to just use VMOVAPS/VMOVUPS instead of VEXTRACTF128mr
2786 def : Pat<(alignedstore (v2f64 (extract_subvector
2787 (v8f64 VR512:$src), (iPTR 0))), addr:$dst),
2788 (VMOVAPDZ128mr addr:$dst, (v2f64 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
2789 def : Pat<(alignedstore (v4f32 (extract_subvector
2790 (v16f32 VR512:$src), (iPTR 0))), addr:$dst),
2791 (VMOVAPSZ128mr addr:$dst, (v4f32 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
2792 def : Pat<(alignedstore (v2i64 (extract_subvector
2793 (v8i64 VR512:$src), (iPTR 0))), addr:$dst),
2794 (VMOVDQA64Z128mr addr:$dst, (v2i64 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
2795 def : Pat<(alignedstore (v4i32 (extract_subvector
2796 (v16i32 VR512:$src), (iPTR 0))), addr:$dst),
2797 (VMOVDQA32Z128mr addr:$dst, (v4i32 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
2798 def : Pat<(alignedstore (v8i16 (extract_subvector
2799 (v32i16 VR512:$src), (iPTR 0))), addr:$dst),
2800 (VMOVDQA32Z128mr addr:$dst, (v8i16 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
2801 def : Pat<(alignedstore (v16i8 (extract_subvector
2802 (v64i8 VR512:$src), (iPTR 0))), addr:$dst),
2803 (VMOVDQA32Z128mr addr:$dst, (v16i8 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
2804
2805 def : Pat<(store (v2f64 (extract_subvector
2806 (v8f64 VR512:$src), (iPTR 0))), addr:$dst),
2807 (VMOVUPDZ128mr addr:$dst, (v2f64 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
2808 def : Pat<(store (v4f32 (extract_subvector
2809 (v16f32 VR512:$src), (iPTR 0))), addr:$dst),
2810 (VMOVUPSZ128mr addr:$dst, (v4f32 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
2811 def : Pat<(store (v2i64 (extract_subvector
2812 (v8i64 VR512:$src), (iPTR 0))), addr:$dst),
2813 (VMOVDQU64Z128mr addr:$dst, (v2i64 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
2814 def : Pat<(store (v4i32 (extract_subvector
2815 (v16i32 VR512:$src), (iPTR 0))), addr:$dst),
2816 (VMOVDQU32Z128mr addr:$dst, (v4i32 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
2817 def : Pat<(store (v8i16 (extract_subvector
2818 (v32i16 VR512:$src), (iPTR 0))), addr:$dst),
2819 (VMOVDQU32Z128mr addr:$dst, (v8i16 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
2820 def : Pat<(store (v16i8 (extract_subvector
2821 (v64i8 VR512:$src), (iPTR 0))), addr:$dst),
2822 (VMOVDQU32Z128mr addr:$dst, (v16i8 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
2823
2824 // Special patterns for storing subvector extracts of lower 256-bits of 512.
2825 // Its cheaper to just use VMOVAPS/VMOVUPS instead of VEXTRACTF128mr
2826 def : Pat<(alignedstore (v4f64 (extract_subvector
2827 (v8f64 VR512:$src), (iPTR 0))), addr:$dst),
2828 (VMOVAPDZ256mr addr:$dst, (v4f64 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
2829 def : Pat<(alignedstore (v8f32 (extract_subvector
2830 (v16f32 VR512:$src), (iPTR 0))), addr:$dst),
2831 (VMOVAPSZ256mr addr:$dst, (v8f32 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
2832 def : Pat<(alignedstore (v4i64 (extract_subvector
2833 (v8i64 VR512:$src), (iPTR 0))), addr:$dst),
2834 (VMOVDQA64Z256mr addr:$dst, (v4i64 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
2835 def : Pat<(alignedstore (v8i32 (extract_subvector
2836 (v16i32 VR512:$src), (iPTR 0))), addr:$dst),
2837 (VMOVDQA32Z256mr addr:$dst, (v8i32 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
2838 def : Pat<(alignedstore (v16i16 (extract_subvector
2839 (v32i16 VR512:$src), (iPTR 0))), addr:$dst),
2840 (VMOVDQA32Z256mr addr:$dst, (v16i16 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
2841 def : Pat<(alignedstore (v32i8 (extract_subvector
2842 (v64i8 VR512:$src), (iPTR 0))), addr:$dst),
2843 (VMOVDQA32Z256mr addr:$dst, (v32i8 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
2844
2845 def : Pat<(store (v4f64 (extract_subvector
2846 (v8f64 VR512:$src), (iPTR 0))), addr:$dst),
2847 (VMOVUPDZ256mr addr:$dst, (v4f64 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
2848 def : Pat<(store (v8f32 (extract_subvector
2849 (v16f32 VR512:$src), (iPTR 0))), addr:$dst),
2850 (VMOVUPSZ256mr addr:$dst, (v8f32 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
2851 def : Pat<(store (v4i64 (extract_subvector
2852 (v8i64 VR512:$src), (iPTR 0))), addr:$dst),
2853 (VMOVDQU64Z256mr addr:$dst, (v4i64 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
2854 def : Pat<(store (v8i32 (extract_subvector
2855 (v16i32 VR512:$src), (iPTR 0))), addr:$dst),
2856 (VMOVDQU32Z256mr addr:$dst, (v8i32 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
2857 def : Pat<(store (v16i16 (extract_subvector
2858 (v32i16 VR512:$src), (iPTR 0))), addr:$dst),
2859 (VMOVDQU32Z256mr addr:$dst, (v16i16 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
2860 def : Pat<(store (v32i8 (extract_subvector
2861 (v64i8 VR512:$src), (iPTR 0))), addr:$dst),
2862 (VMOVDQU32Z256mr addr:$dst, (v32i8 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
2863}
2864
2865
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002866// Move Int Doubleword to Packed Double Int
2867//
Elena Demikhovsky767fc962014-01-14 15:10:08 +00002868def VMOVDI2PDIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR32:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00002869 "vmovd\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002870 [(set VR128X:$dst,
2871 (v4i32 (scalar_to_vector GR32:$src)))], IIC_SSE_MOVDQ>,
Craig Topper401675c2015-12-28 06:32:47 +00002872 EVEX;
Elena Demikhovsky767fc962014-01-14 15:10:08 +00002873def VMOVDI2PDIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst), (ins i32mem:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00002874 "vmovd\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002875 [(set VR128X:$dst,
2876 (v4i32 (scalar_to_vector (loadi32 addr:$src))))],
Craig Topper401675c2015-12-28 06:32:47 +00002877 IIC_SSE_MOVDQ>, EVEX, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky767fc962014-01-14 15:10:08 +00002878def VMOV64toPQIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR64:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00002879 "vmovq\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002880 [(set VR128X:$dst,
2881 (v2i64 (scalar_to_vector GR64:$src)))],
Craig Topper401675c2015-12-28 06:32:47 +00002882 IIC_SSE_MOVDQ>, EVEX, VEX_W;
Craig Topperc648c9b2015-12-28 06:11:42 +00002883let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayLoad = 1 in
2884def VMOV64toPQIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst),
2885 (ins i64mem:$src),
2886 "vmovq\t{$src, $dst|$dst, $src}", []>,
Craig Topper401675c2015-12-28 06:32:47 +00002887 EVEX, VEX_W, EVEX_CD8<64, CD8VT1>;
Craig Topper88adf2a2013-10-12 05:41:08 +00002888let isCodeGenOnly = 1 in {
Craig Topperaf88afb2015-12-28 06:11:45 +00002889def VMOV64toSDZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR64X:$dst), (ins GR64:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00002890 "vmovq\t{$src, $dst|$dst, $src}",
Craig Topperaf88afb2015-12-28 06:11:45 +00002891 [(set FR64X:$dst, (bitconvert GR64:$src))],
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002892 IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteMove]>;
Craig Topperaf88afb2015-12-28 06:11:45 +00002893def VMOVSDto64Zrr : AVX512BI<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64X:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00002894 "vmovq\t{$src, $dst|$dst, $src}",
Craig Topperaf88afb2015-12-28 06:11:45 +00002895 [(set GR64:$dst, (bitconvert FR64X:$src))],
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002896 IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteMove]>;
Craig Topperaf88afb2015-12-28 06:11:45 +00002897def VMOVSDto64Zmr : AVX512BI<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64X:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00002898 "vmovq\t{$src, $dst|$dst, $src}",
Craig Topperaf88afb2015-12-28 06:11:45 +00002899 [(store (i64 (bitconvert FR64X:$src)), addr:$dst)],
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002900 IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteStore]>,
2901 EVEX_CD8<64, CD8VT1>;
Craig Topperc648c9b2015-12-28 06:11:42 +00002902}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002903
2904// Move Int Doubleword to Single Scalar
2905//
Craig Topper88adf2a2013-10-12 05:41:08 +00002906let isCodeGenOnly = 1 in {
Elena Demikhovsky767fc962014-01-14 15:10:08 +00002907def VMOVDI2SSZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR32X:$dst), (ins GR32:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00002908 "vmovd\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002909 [(set FR32X:$dst, (bitconvert GR32:$src))],
Craig Topper401675c2015-12-28 06:32:47 +00002910 IIC_SSE_MOVDQ>, EVEX;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002911
Elena Demikhovsky767fc962014-01-14 15:10:08 +00002912def VMOVDI2SSZrm : AVX512BI<0x6E, MRMSrcMem, (outs FR32X:$dst), (ins i32mem:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00002913 "vmovd\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002914 [(set FR32X:$dst, (bitconvert (loadi32 addr:$src)))],
Craig Topper401675c2015-12-28 06:32:47 +00002915 IIC_SSE_MOVDQ>, EVEX, EVEX_CD8<32, CD8VT1>;
Craig Topper88adf2a2013-10-12 05:41:08 +00002916}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002917
Elena Demikhovsky767fc962014-01-14 15:10:08 +00002918// Move doubleword from xmm register to r/m32
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002919//
Elena Demikhovsky767fc962014-01-14 15:10:08 +00002920def VMOVPDI2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128X:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00002921 "vmovd\t{$src, $dst|$dst, $src}",
Matt Arsenaultfbd9bbf2015-12-11 19:20:16 +00002922 [(set GR32:$dst, (extractelt (v4i32 VR128X:$src),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002923 (iPTR 0)))], IIC_SSE_MOVD_ToGP>,
Craig Topper401675c2015-12-28 06:32:47 +00002924 EVEX;
Elena Demikhovsky767fc962014-01-14 15:10:08 +00002925def VMOVPDI2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002926 (ins i32mem:$dst, VR128X:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00002927 "vmovd\t{$src, $dst|$dst, $src}",
Matt Arsenaultfbd9bbf2015-12-11 19:20:16 +00002928 [(store (i32 (extractelt (v4i32 VR128X:$src),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002929 (iPTR 0))), addr:$dst)], IIC_SSE_MOVDQ>,
Craig Topper401675c2015-12-28 06:32:47 +00002930 EVEX, EVEX_CD8<32, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002931
Elena Demikhovsky767fc962014-01-14 15:10:08 +00002932// Move quadword from xmm1 register to r/m64
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002933//
2934def VMOVPQIto64Zrr : I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128X:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00002935 "vmovq\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002936 [(set GR64:$dst, (extractelt (v2i64 VR128X:$src),
2937 (iPTR 0)))],
Craig Topper401675c2015-12-28 06:32:47 +00002938 IIC_SSE_MOVD_ToGP>, PD, EVEX, VEX_W,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002939 Requires<[HasAVX512, In64BitMode]>;
2940
Craig Topperc648c9b2015-12-28 06:11:42 +00002941let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayStore = 1 in
2942def VMOVPQIto64Zmr : I<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, VR128X:$src),
2943 "vmovq\t{$src, $dst|$dst, $src}",
Craig Topper401675c2015-12-28 06:32:47 +00002944 [], IIC_SSE_MOVD_ToGP>, PD, EVEX, VEX_W,
Craig Topperc648c9b2015-12-28 06:11:42 +00002945 Requires<[HasAVX512, In64BitMode]>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002946
Craig Topperc648c9b2015-12-28 06:11:42 +00002947def VMOVPQI2QIZmr : I<0xD6, MRMDestMem, (outs),
2948 (ins i64mem:$dst, VR128X:$src),
2949 "vmovq\t{$src, $dst|$dst, $src}",
2950 [(store (extractelt (v2i64 VR128X:$src), (iPTR 0)),
2951 addr:$dst)], IIC_SSE_MOVDQ>,
Craig Topper401675c2015-12-28 06:32:47 +00002952 EVEX, PD, VEX_W, EVEX_CD8<64, CD8VT1>,
Craig Topperc648c9b2015-12-28 06:11:42 +00002953 Sched<[WriteStore]>, Requires<[HasAVX512, In64BitMode]>;
2954
2955let hasSideEffects = 0 in
2956def VMOVPQI2QIZrr : AVX512BI<0xD6, MRMDestReg, (outs VR128X:$dst),
2957 (ins VR128X:$src),
2958 "vmovq.s\t{$src, $dst|$dst, $src}",[]>,
Craig Topper401675c2015-12-28 06:32:47 +00002959 EVEX, VEX_W;
Igor Bregere293e832015-11-29 07:41:26 +00002960
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002961// Move Scalar Single to Double Int
2962//
Craig Topper88adf2a2013-10-12 05:41:08 +00002963let isCodeGenOnly = 1 in {
Elena Demikhovsky767fc962014-01-14 15:10:08 +00002964def VMOVSS2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002965 (ins FR32X:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00002966 "vmovd\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002967 [(set GR32:$dst, (bitconvert FR32X:$src))],
Craig Topper401675c2015-12-28 06:32:47 +00002968 IIC_SSE_MOVD_ToGP>, EVEX;
Elena Demikhovsky767fc962014-01-14 15:10:08 +00002969def VMOVSS2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002970 (ins i32mem:$dst, FR32X:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00002971 "vmovd\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002972 [(store (i32 (bitconvert FR32X:$src)), addr:$dst)],
Craig Topper401675c2015-12-28 06:32:47 +00002973 IIC_SSE_MOVDQ>, EVEX, EVEX_CD8<32, CD8VT1>;
Craig Topper88adf2a2013-10-12 05:41:08 +00002974}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002975
2976// Move Quadword Int to Packed Quadword Int
2977//
Craig Topperc648c9b2015-12-28 06:11:42 +00002978def VMOVQI2PQIZrm : AVX512XSI<0x7E, MRMSrcMem, (outs VR128X:$dst),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002979 (ins i64mem:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00002980 "vmovq\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002981 [(set VR128X:$dst,
2982 (v2i64 (scalar_to_vector (loadi64 addr:$src))))]>,
Craig Topperc648c9b2015-12-28 06:11:42 +00002983 EVEX, VEX_W, EVEX_CD8<8, CD8VT8>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002984
2985//===----------------------------------------------------------------------===//
2986// AVX-512 MOVSS, MOVSD
2987//===----------------------------------------------------------------------===//
2988
Simon Pilgrim18bcf932016-02-03 09:41:59 +00002989multiclass avx512_move_scalar <string asm, SDNode OpNode,
Asaf Badouh41ecf462015-12-06 13:26:56 +00002990 X86VectorVTInfo _> {
Simon Pilgrim18bcf932016-02-03 09:41:59 +00002991 defm rr_Int : AVX512_maskable_scalar<0x10, MRMSrcReg, _, (outs _.RC:$dst),
Asaf Badouh41ecf462015-12-06 13:26:56 +00002992 (ins _.RC:$src1, _.RC:$src2),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00002993 asm, "$src2, $src1","$src1, $src2",
Asaf Badouh41ecf462015-12-06 13:26:56 +00002994 (_.VT (OpNode (_.VT _.RC:$src1),
2995 (_.VT _.RC:$src2))),
2996 IIC_SSE_MOV_S_RR>, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00002997 let Constraints = "$src1 = $dst" in
Asaf Badouh41ecf462015-12-06 13:26:56 +00002998 defm rm_Int : AVX512_maskable_3src_scalar<0x10, MRMSrcMem, _,
Simon Pilgrim18bcf932016-02-03 09:41:59 +00002999 (outs _.RC:$dst),
Asaf Badouh41ecf462015-12-06 13:26:56 +00003000 (ins _.ScalarMemOp:$src),
3001 asm,"$src","$src",
Simon Pilgrim18bcf932016-02-03 09:41:59 +00003002 (_.VT (OpNode (_.VT _.RC:$src1),
3003 (_.VT (scalar_to_vector
Asaf Badouh41ecf462015-12-06 13:26:56 +00003004 (_.ScalarLdFrag addr:$src)))))>, EVEX;
3005 let isCodeGenOnly = 1 in {
Simon Pilgrim18bcf932016-02-03 09:41:59 +00003006 def rr : AVX512PI<0x10, MRMSrcReg, (outs _.RC:$dst),
Asaf Badouh41ecf462015-12-06 13:26:56 +00003007 (ins _.RC:$src1, _.FRC:$src2),
3008 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
3009 [(set _.RC:$dst, (_.VT (OpNode _.RC:$src1,
3010 (scalar_to_vector _.FRC:$src2))))],
3011 _.ExeDomain,IIC_SSE_MOV_S_RR>, EVEX_4V;
Asaf Badouh41ecf462015-12-06 13:26:56 +00003012 def rm : AVX512PI<0x10, MRMSrcMem, (outs _.FRC:$dst), (ins _.ScalarMemOp:$src),
3013 !strconcat(asm, "\t{$src, $dst|$dst, $src}"),
3014 [(set _.FRC:$dst, (_.ScalarLdFrag addr:$src))],
3015 _.ExeDomain, IIC_SSE_MOV_S_RM>, EVEX;
3016 }
Craig Toppere1cac152016-06-07 07:27:54 +00003017 def mr: AVX512PI<0x11, MRMDestMem, (outs), (ins _.ScalarMemOp:$dst, _.FRC:$src),
3018 !strconcat(asm, "\t{$src, $dst|$dst, $src}"),
3019 [(store _.FRC:$src, addr:$dst)], _.ExeDomain, IIC_SSE_MOV_S_MR>,
3020 EVEX;
3021 let mayStore = 1 in
3022 def mrk: AVX512PI<0x11, MRMDestMem, (outs),
3023 (ins _.ScalarMemOp:$dst, VK1WM:$mask, _.FRC:$src),
3024 !strconcat(asm, "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}"),
3025 [], _.ExeDomain, IIC_SSE_MOV_S_MR>, EVEX, EVEX_K;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003026}
3027
Asaf Badouh41ecf462015-12-06 13:26:56 +00003028defm VMOVSSZ : avx512_move_scalar<"vmovss", X86Movss, f32x_info>,
3029 VEX_LIG, XS, EVEX_CD8<32, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003030
Asaf Badouh41ecf462015-12-06 13:26:56 +00003031defm VMOVSDZ : avx512_move_scalar<"vmovsd", X86Movsd, f64x_info>,
3032 VEX_LIG, XD, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003033
Craig Topper74ed0872016-05-18 06:55:59 +00003034def : Pat<(f32 (X86selects VK1WM:$mask, (f32 FR32X:$src1), (f32 FR32X:$src2))),
Asaf Badouh41ecf462015-12-06 13:26:56 +00003035 (COPY_TO_REGCLASS (VMOVSSZrr_Intk (COPY_TO_REGCLASS FR32X:$src2, VR128X),
3036 VK1WM:$mask, (v4f32 (IMPLICIT_DEF)),(COPY_TO_REGCLASS FR32X:$src1, VR128X)), FR32X)>;
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00003037
Craig Topper74ed0872016-05-18 06:55:59 +00003038def : Pat<(f64 (X86selects VK1WM:$mask, (f64 FR64X:$src1), (f64 FR64X:$src2))),
Asaf Badouh41ecf462015-12-06 13:26:56 +00003039 (COPY_TO_REGCLASS (VMOVSDZrr_Intk (COPY_TO_REGCLASS FR64X:$src2, VR128X),
3040 VK1WM:$mask, (v2f64 (IMPLICIT_DEF)), (COPY_TO_REGCLASS FR64X:$src1, VR128X)), FR64X)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003041
Elena Demikhovskyff620ed2014-08-27 07:38:43 +00003042def : Pat<(int_x86_avx512_mask_store_ss addr:$dst, VR128X:$src, GR8:$mask),
3043 (VMOVSSZmrk addr:$dst, (i1 (COPY_TO_REGCLASS GR8:$mask, VK1WM)),
3044 (COPY_TO_REGCLASS VR128X:$src, FR32X))>;
3045
Craig Topper99f6b622016-05-01 01:03:56 +00003046let hasSideEffects = 0 in
Igor Breger4424aaa2015-11-19 07:58:33 +00003047defm VMOVSSZrr_REV : AVX512_maskable_in_asm<0x11, MRMDestReg, f32x_info,
3048 (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2),
3049 "vmovss.s", "$src2, $src1", "$src1, $src2", []>,
3050 XS, EVEX_4V, VEX_LIG;
3051
Craig Topper99f6b622016-05-01 01:03:56 +00003052let hasSideEffects = 0 in
Igor Breger4424aaa2015-11-19 07:58:33 +00003053defm VMOVSSDrr_REV : AVX512_maskable_in_asm<0x11, MRMDestReg, f64x_info,
3054 (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2),
3055 "vmovsd.s", "$src2, $src1", "$src1, $src2", []>,
3056 XD, EVEX_4V, VEX_LIG, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003057
3058let Predicates = [HasAVX512] in {
3059 let AddedComplexity = 15 in {
3060 // Move scalar to XMM zero-extended, zeroing a VR128X then do a
3061 // MOVS{S,D} to the lower bits.
3062 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector FR32X:$src)))),
3063 (VMOVSSZrr (v4f32 (V_SET0)), FR32X:$src)>;
3064 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128X:$src))),
3065 (VMOVSSZrr (v4f32 (V_SET0)), (COPY_TO_REGCLASS VR128X:$src, FR32X))>;
3066 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128X:$src))),
3067 (VMOVSSZrr (v4i32 (V_SET0)), (COPY_TO_REGCLASS VR128X:$src, FR32X))>;
3068 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64X:$src)))),
3069 (VMOVSDZrr (v2f64 (V_SET0)), FR64X:$src)>;
3070
3071 // Move low f32 and clear high bits.
3072 def : Pat<(v8f32 (X86vzmovl (v8f32 VR256X:$src))),
3073 (SUBREG_TO_REG (i32 0),
Michael Liao5bf95782014-12-04 05:20:33 +00003074 (VMOVSSZrr (v4f32 (V_SET0)),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003075 (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm)), sub_xmm)>;
3076 def : Pat<(v8i32 (X86vzmovl (v8i32 VR256X:$src))),
3077 (SUBREG_TO_REG (i32 0),
3078 (VMOVSSZrr (v4i32 (V_SET0)),
3079 (EXTRACT_SUBREG (v8i32 VR256X:$src), sub_xmm)), sub_xmm)>;
3080 }
3081
3082 let AddedComplexity = 20 in {
3083 // MOVSSrm zeros the high parts of the register; represent this
3084 // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0
3085 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))),
3086 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>;
3087 def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))),
3088 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>;
3089 def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))),
3090 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>;
3091
3092 // MOVSDrm zeros the high parts of the register; represent this
3093 // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0
3094 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))),
3095 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
3096 def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))),
3097 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
3098 def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))),
3099 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
3100 def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))),
3101 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
3102 def : Pat<(v2f64 (X86vzload addr:$src)),
3103 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
3104
3105 // Represent the same patterns above but in the form they appear for
3106 // 256-bit types
3107 def : Pat<(v8i32 (X86vzmovl (insert_subvector undef,
3108 (v4i32 (scalar_to_vector (loadi32 addr:$src))), (iPTR 0)))),
Elena Demikhovsky34586e72013-10-02 12:20:42 +00003109 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrm addr:$src), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003110 def : Pat<(v8f32 (X86vzmovl (insert_subvector undef,
3111 (v4f32 (scalar_to_vector (loadf32 addr:$src))), (iPTR 0)))),
3112 (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>;
3113 def : Pat<(v4f64 (X86vzmovl (insert_subvector undef,
3114 (v2f64 (scalar_to_vector (loadf64 addr:$src))), (iPTR 0)))),
3115 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;
Simon Pilgrim7823fd22016-02-04 19:27:51 +00003116 def : Pat<(v4f64 (X86vzload addr:$src)),
3117 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;
Simon Pilgrim6788f332016-02-04 16:12:56 +00003118
3119 // Represent the same patterns above but in the form they appear for
3120 // 512-bit types
3121 def : Pat<(v16i32 (X86vzmovl (insert_subvector undef,
3122 (v4i32 (scalar_to_vector (loadi32 addr:$src))), (iPTR 0)))),
3123 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrm addr:$src), sub_xmm)>;
3124 def : Pat<(v16f32 (X86vzmovl (insert_subvector undef,
3125 (v4f32 (scalar_to_vector (loadf32 addr:$src))), (iPTR 0)))),
3126 (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>;
3127 def : Pat<(v8f64 (X86vzmovl (insert_subvector undef,
3128 (v2f64 (scalar_to_vector (loadf64 addr:$src))), (iPTR 0)))),
3129 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;
Simon Pilgrim7823fd22016-02-04 19:27:51 +00003130 def : Pat<(v8f64 (X86vzload addr:$src)),
3131 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003132 }
3133 def : Pat<(v8f32 (X86vzmovl (insert_subvector undef,
3134 (v4f32 (scalar_to_vector FR32X:$src)), (iPTR 0)))),
3135 (SUBREG_TO_REG (i32 0), (v4f32 (VMOVSSZrr (v4f32 (V_SET0)),
3136 FR32X:$src)), sub_xmm)>;
3137 def : Pat<(v4f64 (X86vzmovl (insert_subvector undef,
3138 (v2f64 (scalar_to_vector FR64X:$src)), (iPTR 0)))),
3139 (SUBREG_TO_REG (i64 0), (v2f64 (VMOVSDZrr (v2f64 (V_SET0)),
3140 FR64X:$src)), sub_xmm)>;
3141 def : Pat<(v4i64 (X86vzmovl (insert_subvector undef,
3142 (v2i64 (scalar_to_vector (loadi64 addr:$src))), (iPTR 0)))),
Elena Demikhovsky34586e72013-10-02 12:20:42 +00003143 (SUBREG_TO_REG (i64 0), (VMOVQI2PQIZrm addr:$src), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003144
3145 // Move low f64 and clear high bits.
3146 def : Pat<(v4f64 (X86vzmovl (v4f64 VR256X:$src))),
3147 (SUBREG_TO_REG (i32 0),
3148 (VMOVSDZrr (v2f64 (V_SET0)),
3149 (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm)), sub_xmm)>;
3150
3151 def : Pat<(v4i64 (X86vzmovl (v4i64 VR256X:$src))),
3152 (SUBREG_TO_REG (i32 0), (VMOVSDZrr (v2i64 (V_SET0)),
3153 (EXTRACT_SUBREG (v4i64 VR256X:$src), sub_xmm)), sub_xmm)>;
3154
3155 // Extract and store.
Matt Arsenaultfbd9bbf2015-12-11 19:20:16 +00003156 def : Pat<(store (f32 (extractelt (v4f32 VR128X:$src), (iPTR 0))),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003157 addr:$dst),
3158 (VMOVSSZmr addr:$dst, (COPY_TO_REGCLASS (v4f32 VR128X:$src), FR32X))>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003159
3160 // Shuffle with VMOVSS
3161 def : Pat<(v4i32 (X86Movss VR128X:$src1, VR128X:$src2)),
3162 (VMOVSSZrr (v4i32 VR128X:$src1),
3163 (COPY_TO_REGCLASS (v4i32 VR128X:$src2), FR32X))>;
3164 def : Pat<(v4f32 (X86Movss VR128X:$src1, VR128X:$src2)),
3165 (VMOVSSZrr (v4f32 VR128X:$src1),
3166 (COPY_TO_REGCLASS (v4f32 VR128X:$src2), FR32X))>;
3167
3168 // 256-bit variants
3169 def : Pat<(v8i32 (X86Movss VR256X:$src1, VR256X:$src2)),
3170 (SUBREG_TO_REG (i32 0),
3171 (VMOVSSZrr (EXTRACT_SUBREG (v8i32 VR256X:$src1), sub_xmm),
3172 (EXTRACT_SUBREG (v8i32 VR256X:$src2), sub_xmm)),
3173 sub_xmm)>;
3174 def : Pat<(v8f32 (X86Movss VR256X:$src1, VR256X:$src2)),
3175 (SUBREG_TO_REG (i32 0),
3176 (VMOVSSZrr (EXTRACT_SUBREG (v8f32 VR256X:$src1), sub_xmm),
3177 (EXTRACT_SUBREG (v8f32 VR256X:$src2), sub_xmm)),
3178 sub_xmm)>;
3179
3180 // Shuffle with VMOVSD
3181 def : Pat<(v2i64 (X86Movsd VR128X:$src1, VR128X:$src2)),
3182 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
3183 def : Pat<(v2f64 (X86Movsd VR128X:$src1, VR128X:$src2)),
3184 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
3185 def : Pat<(v4f32 (X86Movsd VR128X:$src1, VR128X:$src2)),
3186 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
3187 def : Pat<(v4i32 (X86Movsd VR128X:$src1, VR128X:$src2)),
3188 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
3189
3190 // 256-bit variants
3191 def : Pat<(v4i64 (X86Movsd VR256X:$src1, VR256X:$src2)),
3192 (SUBREG_TO_REG (i32 0),
3193 (VMOVSDZrr (EXTRACT_SUBREG (v4i64 VR256X:$src1), sub_xmm),
3194 (EXTRACT_SUBREG (v4i64 VR256X:$src2), sub_xmm)),
3195 sub_xmm)>;
3196 def : Pat<(v4f64 (X86Movsd VR256X:$src1, VR256X:$src2)),
3197 (SUBREG_TO_REG (i32 0),
3198 (VMOVSDZrr (EXTRACT_SUBREG (v4f64 VR256X:$src1), sub_xmm),
3199 (EXTRACT_SUBREG (v4f64 VR256X:$src2), sub_xmm)),
3200 sub_xmm)>;
3201
3202 def : Pat<(v2f64 (X86Movlpd VR128X:$src1, VR128X:$src2)),
3203 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
3204 def : Pat<(v2i64 (X86Movlpd VR128X:$src1, VR128X:$src2)),
3205 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
3206 def : Pat<(v4f32 (X86Movlps VR128X:$src1, VR128X:$src2)),
3207 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
3208 def : Pat<(v4i32 (X86Movlps VR128X:$src1, VR128X:$src2)),
3209 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
3210}
3211
3212let AddedComplexity = 15 in
3213def VMOVZPQILo2PQIZrr : AVX512XSI<0x7E, MRMSrcReg, (outs VR128X:$dst),
3214 (ins VR128X:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00003215 "vmovq\t{$src, $dst|$dst, $src}",
Michael Liao5bf95782014-12-04 05:20:33 +00003216 [(set VR128X:$dst, (v2i64 (X86vzmovl
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003217 (v2i64 VR128X:$src))))],
3218 IIC_SSE_MOVQ_RR>, EVEX, VEX_W;
3219
Igor Breger4ec5abf2015-11-03 07:30:17 +00003220let AddedComplexity = 20 , isCodeGenOnly = 1 in
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003221def VMOVZPQILo2PQIZrm : AVX512XSI<0x7E, MRMSrcMem, (outs VR128X:$dst),
3222 (ins i128mem:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00003223 "vmovq\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003224 [(set VR128X:$dst, (v2i64 (X86vzmovl
3225 (loadv2i64 addr:$src))))],
3226 IIC_SSE_MOVDQ>, EVEX, VEX_W,
3227 EVEX_CD8<8, CD8VT8>;
3228
3229let Predicates = [HasAVX512] in {
Craig Topperde549852016-05-22 06:09:34 +00003230 let AddedComplexity = 15 in {
3231 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))),
3232 (VMOVDI2PDIZrr GR32:$src)>;
3233
3234 def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector GR64:$src)))),
3235 (VMOV64toPQIZrr GR64:$src)>;
3236
3237 def : Pat<(v4i64 (X86vzmovl (insert_subvector undef,
3238 (v2i64 (scalar_to_vector GR64:$src)),(iPTR 0)))),
3239 (SUBREG_TO_REG (i64 0), (VMOV64toPQIZrr GR64:$src), sub_xmm)>;
3240 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003241 // AVX 128-bit movd/movq instruction write zeros in the high 128-bit part.
3242 let AddedComplexity = 20 in {
3243 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector (loadi32 addr:$src))))),
3244 (VMOVDI2PDIZrm addr:$src)>;
Michael Liao5bf95782014-12-04 05:20:33 +00003245
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003246 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))),
3247 (VMOVDI2PDIZrm addr:$src)>;
3248 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))),
3249 (VMOVDI2PDIZrm addr:$src)>;
3250 def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))),
3251 (VMOVZPQILo2PQIZrm addr:$src)>;
3252 def : Pat<(v2f64 (X86vzmovl (v2f64 VR128X:$src))),
3253 (VMOVZPQILo2PQIZrr VR128X:$src)>;
Cameron McInally30bbb212013-12-05 00:11:25 +00003254 def : Pat<(v2i64 (X86vzload addr:$src)),
3255 (VMOVZPQILo2PQIZrm addr:$src)>;
Craig Topperde549852016-05-22 06:09:34 +00003256 def : Pat<(v4i64 (X86vzload addr:$src)),
3257 (SUBREG_TO_REG (i64 0), (VMOVZPQILo2PQIZrm addr:$src), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003258 }
Elena Demikhovsky3b75f5d2013-10-01 08:38:02 +00003259
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003260 // Use regular 128-bit instructions to match 256-bit scalar_to_vec+zext.
3261 def : Pat<(v8i32 (X86vzmovl (insert_subvector undef,
3262 (v4i32 (scalar_to_vector GR32:$src)),(iPTR 0)))),
3263 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src), sub_xmm)>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00003264
3265 // Use regular 128-bit instructions to match 512-bit scalar_to_vec+zext.
3266 def : Pat<(v8i64 (X86vzload addr:$src)),
3267 (SUBREG_TO_REG (i64 0), (VMOVZPQILo2PQIZrm addr:$src), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003268}
3269
3270def : Pat<(v16i32 (X86Vinsert (v16i32 immAllZerosV), GR32:$src2, (iPTR 0))),
3271 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src2), sub_xmm)>;
3272
3273def : Pat<(v8i64 (X86Vinsert (bc_v8i64 (v16i32 immAllZerosV)), GR64:$src2, (iPTR 0))),
3274 (SUBREG_TO_REG (i32 0), (VMOV64toPQIZrr GR64:$src2), sub_xmm)>;
3275
3276def : Pat<(v16i32 (X86Vinsert undef, GR32:$src2, (iPTR 0))),
3277 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src2), sub_xmm)>;
3278
3279def : Pat<(v8i64 (X86Vinsert undef, GR64:$src2, (iPTR 0))),
3280 (SUBREG_TO_REG (i32 0), (VMOV64toPQIZrr GR64:$src2), sub_xmm)>;
3281
3282//===----------------------------------------------------------------------===//
Adam Nemet7f62b232014-06-10 16:39:53 +00003283// AVX-512 - Non-temporals
3284//===----------------------------------------------------------------------===//
Robert Khasanoved882972014-08-13 10:46:00 +00003285let SchedRW = [WriteLoad] in {
3286 def VMOVNTDQAZrm : AVX512PI<0x2A, MRMSrcMem, (outs VR512:$dst),
3287 (ins i512mem:$src), "vmovntdqa\t{$src, $dst|$dst, $src}",
3288 [(set VR512:$dst, (int_x86_avx512_movntdqa addr:$src))],
3289 SSEPackedInt>, EVEX, T8PD, EVEX_V512,
3290 EVEX_CD8<64, CD8VF>;
Adam Nemet7f62b232014-06-10 16:39:53 +00003291
Craig Topper2f90c1f2016-06-07 07:27:57 +00003292 let Predicates = [HasVLX] in {
Robert Khasanoved882972014-08-13 10:46:00 +00003293 def VMOVNTDQAZ256rm : AVX512PI<0x2A, MRMSrcMem, (outs VR256X:$dst),
Craig Topper2f90c1f2016-06-07 07:27:57 +00003294 (ins i256mem:$src),
3295 "vmovntdqa\t{$src, $dst|$dst, $src}",
3296 [(set VR256X:$dst, (int_x86_avx2_movntdqa addr:$src))],
3297 SSEPackedInt>, EVEX, T8PD, EVEX_V256,
3298 EVEX_CD8<64, CD8VF>;
Adam Nemet7f62b232014-06-10 16:39:53 +00003299
Robert Khasanoved882972014-08-13 10:46:00 +00003300 def VMOVNTDQAZ128rm : AVX512PI<0x2A, MRMSrcMem, (outs VR128X:$dst),
Craig Topper2f90c1f2016-06-07 07:27:57 +00003301 (ins i128mem:$src),
3302 "vmovntdqa\t{$src, $dst|$dst, $src}",
3303 [(set VR128X:$dst, (int_x86_sse41_movntdqa addr:$src))],
3304 SSEPackedInt>, EVEX, T8PD, EVEX_V128,
3305 EVEX_CD8<64, CD8VF>;
Robert Khasanoved882972014-08-13 10:46:00 +00003306 }
Adam Nemetefd07852014-06-18 16:51:10 +00003307}
3308
Igor Bregerd3341f52016-01-20 13:11:47 +00003309multiclass avx512_movnt<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
3310 PatFrag st_frag = alignednontemporalstore,
3311 InstrItinClass itin = IIC_SSE_MOVNT> {
Craig Toppere1cac152016-06-07 07:27:54 +00003312 let SchedRW = [WriteStore], AddedComplexity = 400 in
Igor Bregerd3341f52016-01-20 13:11:47 +00003313 def mr : AVX512PI<opc, MRMDestMem, (outs), (ins _.MemOp:$dst, _.RC:$src),
Robert Khasanoved882972014-08-13 10:46:00 +00003314 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
Igor Bregerd3341f52016-01-20 13:11:47 +00003315 [(st_frag (_.VT _.RC:$src), addr:$dst)],
3316 _.ExeDomain, itin>, EVEX, EVEX_CD8<_.EltSize, CD8VF>;
Robert Khasanoved882972014-08-13 10:46:00 +00003317}
3318
Igor Bregerd3341f52016-01-20 13:11:47 +00003319multiclass avx512_movnt_vl<bits<8> opc, string OpcodeStr,
3320 AVX512VLVectorVTInfo VTInfo> {
3321 let Predicates = [HasAVX512] in
3322 defm Z : avx512_movnt<opc, OpcodeStr, VTInfo.info512>, EVEX_V512;
Robert Khasanoved882972014-08-13 10:46:00 +00003323
Igor Bregerd3341f52016-01-20 13:11:47 +00003324 let Predicates = [HasAVX512, HasVLX] in {
3325 defm Z256 : avx512_movnt<opc, OpcodeStr, VTInfo.info256>, EVEX_V256;
3326 defm Z128 : avx512_movnt<opc, OpcodeStr, VTInfo.info128>, EVEX_V128;
Robert Khasanoved882972014-08-13 10:46:00 +00003327 }
3328}
3329
Igor Bregerd3341f52016-01-20 13:11:47 +00003330defm VMOVNTDQ : avx512_movnt_vl<0xE7, "vmovntdq", avx512vl_i64_info>, PD;
3331defm VMOVNTPD : avx512_movnt_vl<0x2B, "vmovntpd", avx512vl_f64_info>, PD, VEX_W;
3332defm VMOVNTPS : avx512_movnt_vl<0x2B, "vmovntps", avx512vl_f32_info>, PS;
Robert Khasanoved882972014-08-13 10:46:00 +00003333
Craig Topper707c89c2016-05-08 23:43:17 +00003334let Predicates = [HasAVX512], AddedComplexity = 400 in {
3335 def : Pat<(alignednontemporalstore (v16i32 VR512:$src), addr:$dst),
3336 (VMOVNTDQZmr addr:$dst, VR512:$src)>;
3337 def : Pat<(alignednontemporalstore (v32i16 VR512:$src), addr:$dst),
3338 (VMOVNTDQZmr addr:$dst, VR512:$src)>;
3339 def : Pat<(alignednontemporalstore (v64i8 VR512:$src), addr:$dst),
3340 (VMOVNTDQZmr addr:$dst, VR512:$src)>;
Simon Pilgrim9a896232016-06-07 13:34:24 +00003341
3342 def : Pat<(v8f64 (alignednontemporalload addr:$src)),
3343 (VMOVNTDQAZrm addr:$src)>;
3344 def : Pat<(v16f32 (alignednontemporalload addr:$src)),
3345 (VMOVNTDQAZrm addr:$src)>;
3346 def : Pat<(v8i64 (alignednontemporalload addr:$src)),
3347 (VMOVNTDQAZrm addr:$src)>;
3348 def : Pat<(v16i32 (alignednontemporalload addr:$src)),
3349 (VMOVNTDQAZrm addr:$src)>;
3350 def : Pat<(v32i16 (alignednontemporalload addr:$src)),
3351 (VMOVNTDQAZrm addr:$src)>;
3352 def : Pat<(v64i8 (alignednontemporalload addr:$src)),
3353 (VMOVNTDQAZrm addr:$src)>;
Craig Topper707c89c2016-05-08 23:43:17 +00003354}
3355
Craig Topperc41320d2016-05-08 23:08:45 +00003356let Predicates = [HasVLX], AddedComplexity = 400 in {
3357 def : Pat<(alignednontemporalstore (v8i32 VR256X:$src), addr:$dst),
3358 (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;
3359 def : Pat<(alignednontemporalstore (v16i16 VR256X:$src), addr:$dst),
3360 (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;
3361 def : Pat<(alignednontemporalstore (v32i8 VR256X:$src), addr:$dst),
3362 (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;
3363
Simon Pilgrim9a896232016-06-07 13:34:24 +00003364 def : Pat<(v4f64 (alignednontemporalload addr:$src)),
3365 (VMOVNTDQAZ256rm addr:$src)>;
3366 def : Pat<(v8f32 (alignednontemporalload addr:$src)),
3367 (VMOVNTDQAZ256rm addr:$src)>;
3368 def : Pat<(v4i64 (alignednontemporalload addr:$src)),
3369 (VMOVNTDQAZ256rm addr:$src)>;
3370 def : Pat<(v8i32 (alignednontemporalload addr:$src)),
3371 (VMOVNTDQAZ256rm addr:$src)>;
3372 def : Pat<(v16i16 (alignednontemporalload addr:$src)),
3373 (VMOVNTDQAZ256rm addr:$src)>;
3374 def : Pat<(v32i8 (alignednontemporalload addr:$src)),
3375 (VMOVNTDQAZ256rm addr:$src)>;
3376
Craig Topperc41320d2016-05-08 23:08:45 +00003377 def : Pat<(alignednontemporalstore (v4i32 VR128X:$src), addr:$dst),
3378 (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;
3379 def : Pat<(alignednontemporalstore (v8i16 VR128X:$src), addr:$dst),
3380 (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;
3381 def : Pat<(alignednontemporalstore (v16i8 VR128X:$src), addr:$dst),
3382 (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;
Simon Pilgrim9a896232016-06-07 13:34:24 +00003383
3384 def : Pat<(v2f64 (alignednontemporalload addr:$src)),
3385 (VMOVNTDQAZ128rm addr:$src)>;
3386 def : Pat<(v4f32 (alignednontemporalload addr:$src)),
3387 (VMOVNTDQAZ128rm addr:$src)>;
3388 def : Pat<(v2i64 (alignednontemporalload addr:$src)),
3389 (VMOVNTDQAZ128rm addr:$src)>;
3390 def : Pat<(v4i32 (alignednontemporalload addr:$src)),
3391 (VMOVNTDQAZ128rm addr:$src)>;
3392 def : Pat<(v8i16 (alignednontemporalload addr:$src)),
3393 (VMOVNTDQAZ128rm addr:$src)>;
3394 def : Pat<(v16i8 (alignednontemporalload addr:$src)),
3395 (VMOVNTDQAZ128rm addr:$src)>;
Craig Topperc41320d2016-05-08 23:08:45 +00003396}
3397
Adam Nemet7f62b232014-06-10 16:39:53 +00003398//===----------------------------------------------------------------------===//
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003399// AVX-512 - Integer arithmetic
3400//
3401multiclass avx512_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
Robert Khasanov44241442014-10-08 14:37:45 +00003402 X86VectorVTInfo _, OpndItins itins,
3403 bit IsCommutable = 0> {
Adam Nemet34801422014-10-08 23:25:39 +00003404 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
Igor Bregerf2460112015-07-26 14:41:44 +00003405 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
Robert Khasanov44241442014-10-08 14:37:45 +00003406 "$src2, $src1", "$src1, $src2",
3407 (_.VT (OpNode _.RC:$src1, _.RC:$src2)),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003408 itins.rr, IsCommutable>,
Robert Khasanov44241442014-10-08 14:37:45 +00003409 AVX512BIBase, EVEX_4V;
Elena Demikhovskybb2f6b72014-03-27 09:45:08 +00003410
Craig Toppere1cac152016-06-07 07:27:54 +00003411 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
3412 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
3413 "$src2, $src1", "$src1, $src2",
3414 (_.VT (OpNode _.RC:$src1,
3415 (bitconvert (_.LdFrag addr:$src2)))),
3416 itins.rm>,
3417 AVX512BIBase, EVEX_4V;
Robert Khasanov545d1b72014-10-14 14:36:19 +00003418}
3419
3420multiclass avx512_binop_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
3421 X86VectorVTInfo _, OpndItins itins,
3422 bit IsCommutable = 0> :
3423 avx512_binop_rm<opc, OpcodeStr, OpNode, _, itins, IsCommutable> {
Craig Toppere1cac152016-06-07 07:27:54 +00003424 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
3425 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
3426 "${src2}"##_.BroadcastStr##", $src1",
3427 "$src1, ${src2}"##_.BroadcastStr,
3428 (_.VT (OpNode _.RC:$src1,
3429 (X86VBroadcast
3430 (_.ScalarLdFrag addr:$src2)))),
3431 itins.rm>,
3432 AVX512BIBase, EVEX_4V, EVEX_B;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003433}
Elena Demikhovskybb2f6b72014-03-27 09:45:08 +00003434
Robert Khasanovd5b14f72014-10-09 08:38:48 +00003435multiclass avx512_binop_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
3436 AVX512VLVectorVTInfo VTInfo, OpndItins itins,
3437 Predicate prd, bit IsCommutable = 0> {
3438 let Predicates = [prd] in
3439 defm Z : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info512, itins,
3440 IsCommutable>, EVEX_V512;
3441
3442 let Predicates = [prd, HasVLX] in {
3443 defm Z256 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info256, itins,
3444 IsCommutable>, EVEX_V256;
3445 defm Z128 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info128, itins,
3446 IsCommutable>, EVEX_V128;
3447 }
3448}
3449
Robert Khasanov545d1b72014-10-14 14:36:19 +00003450multiclass avx512_binop_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
3451 AVX512VLVectorVTInfo VTInfo, OpndItins itins,
3452 Predicate prd, bit IsCommutable = 0> {
3453 let Predicates = [prd] in
3454 defm Z : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info512, itins,
3455 IsCommutable>, EVEX_V512;
3456
3457 let Predicates = [prd, HasVLX] in {
3458 defm Z256 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info256, itins,
3459 IsCommutable>, EVEX_V256;
3460 defm Z128 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info128, itins,
3461 IsCommutable>, EVEX_V128;
3462 }
3463}
3464
3465multiclass avx512_binop_rm_vl_q<bits<8> opc, string OpcodeStr, SDNode OpNode,
3466 OpndItins itins, Predicate prd,
3467 bit IsCommutable = 0> {
3468 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i64_info,
3469 itins, prd, IsCommutable>,
3470 VEX_W, EVEX_CD8<64, CD8VF>;
3471}
3472
3473multiclass avx512_binop_rm_vl_d<bits<8> opc, string OpcodeStr, SDNode OpNode,
3474 OpndItins itins, Predicate prd,
3475 bit IsCommutable = 0> {
3476 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i32_info,
3477 itins, prd, IsCommutable>, EVEX_CD8<32, CD8VF>;
3478}
3479
3480multiclass avx512_binop_rm_vl_w<bits<8> opc, string OpcodeStr, SDNode OpNode,
3481 OpndItins itins, Predicate prd,
3482 bit IsCommutable = 0> {
3483 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i16_info,
3484 itins, prd, IsCommutable>, EVEX_CD8<16, CD8VF>;
3485}
3486
3487multiclass avx512_binop_rm_vl_b<bits<8> opc, string OpcodeStr, SDNode OpNode,
3488 OpndItins itins, Predicate prd,
3489 bit IsCommutable = 0> {
3490 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i8_info,
3491 itins, prd, IsCommutable>, EVEX_CD8<8, CD8VF>;
3492}
3493
3494multiclass avx512_binop_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr,
3495 SDNode OpNode, OpndItins itins, Predicate prd,
3496 bit IsCommutable = 0> {
Igor Bregerf2460112015-07-26 14:41:44 +00003497 defm Q : avx512_binop_rm_vl_q<opc_q, OpcodeStr#"q", OpNode, itins, prd,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003498 IsCommutable>;
3499
Igor Bregerf2460112015-07-26 14:41:44 +00003500 defm D : avx512_binop_rm_vl_d<opc_d, OpcodeStr#"d", OpNode, itins, prd,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003501 IsCommutable>;
3502}
3503
3504multiclass avx512_binop_rm_vl_bw<bits<8> opc_b, bits<8> opc_w, string OpcodeStr,
3505 SDNode OpNode, OpndItins itins, Predicate prd,
3506 bit IsCommutable = 0> {
Igor Bregerf2460112015-07-26 14:41:44 +00003507 defm W : avx512_binop_rm_vl_w<opc_w, OpcodeStr#"w", OpNode, itins, prd,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003508 IsCommutable>;
3509
Igor Bregerf2460112015-07-26 14:41:44 +00003510 defm B : avx512_binop_rm_vl_b<opc_b, OpcodeStr#"b", OpNode, itins, prd,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003511 IsCommutable>;
3512}
3513
3514multiclass avx512_binop_rm_vl_all<bits<8> opc_b, bits<8> opc_w,
3515 bits<8> opc_d, bits<8> opc_q,
3516 string OpcodeStr, SDNode OpNode,
3517 OpndItins itins, bit IsCommutable = 0> {
3518 defm NAME : avx512_binop_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode,
3519 itins, HasAVX512, IsCommutable>,
3520 avx512_binop_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode,
3521 itins, HasBWI, IsCommutable>;
3522}
3523
Elena Demikhovsky1eeece12015-04-02 10:51:40 +00003524multiclass avx512_binop_rm2<bits<8> opc, string OpcodeStr, OpndItins itins,
Michael Liao66233b72015-08-06 09:06:20 +00003525 SDNode OpNode,X86VectorVTInfo _Src,
Asaf Badouh5a3a0232016-02-01 15:48:21 +00003526 X86VectorVTInfo _Dst, X86VectorVTInfo _Brdct,
3527 bit IsCommutable = 0> {
Michael Liao66233b72015-08-06 09:06:20 +00003528 defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst),
Elena Demikhovsky1eeece12015-04-02 10:51:40 +00003529 (ins _Src.RC:$src1, _Src.RC:$src2), OpcodeStr,
Michael Liao66233b72015-08-06 09:06:20 +00003530 "$src2, $src1","$src1, $src2",
3531 (_Dst.VT (OpNode
3532 (_Src.VT _Src.RC:$src1),
Elena Demikhovsky1eeece12015-04-02 10:51:40 +00003533 (_Src.VT _Src.RC:$src2))),
Michael Liao66233b72015-08-06 09:06:20 +00003534 itins.rr, IsCommutable>,
Elena Demikhovsky1eeece12015-04-02 10:51:40 +00003535 AVX512BIBase, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00003536 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
3537 (ins _Src.RC:$src1, _Src.MemOp:$src2), OpcodeStr,
3538 "$src2, $src1", "$src1, $src2",
3539 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1),
3540 (bitconvert (_Src.LdFrag addr:$src2)))),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003541 itins.rm>,
Craig Toppere1cac152016-06-07 07:27:54 +00003542 AVX512BIBase, EVEX_4V;
3543
3544 defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
3545 (ins _Src.RC:$src1, _Dst.ScalarMemOp:$src2),
3546 OpcodeStr,
3547 "${src2}"##_Brdct.BroadcastStr##", $src1",
3548 "$src1, ${src2}"##_Dst.BroadcastStr,
3549 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert
3550 (_Brdct.VT (X86VBroadcast
3551 (_Brdct.ScalarLdFrag addr:$src2)))))),
3552 itins.rm>,
3553 AVX512BIBase, EVEX_4V, EVEX_B;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003554}
3555
Robert Khasanov545d1b72014-10-14 14:36:19 +00003556defm VPADD : avx512_binop_rm_vl_all<0xFC, 0xFD, 0xFE, 0xD4, "vpadd", add,
3557 SSE_INTALU_ITINS_P, 1>;
3558defm VPSUB : avx512_binop_rm_vl_all<0xF8, 0xF9, 0xFA, 0xFB, "vpsub", sub,
3559 SSE_INTALU_ITINS_P, 0>;
Elena Demikhovsky52266382015-05-04 12:35:55 +00003560defm VPADDS : avx512_binop_rm_vl_bw<0xEC, 0xED, "vpadds", X86adds,
3561 SSE_INTALU_ITINS_P, HasBWI, 1>;
3562defm VPSUBS : avx512_binop_rm_vl_bw<0xE8, 0xE9, "vpsubs", X86subs,
3563 SSE_INTALU_ITINS_P, HasBWI, 0>;
3564defm VPADDUS : avx512_binop_rm_vl_bw<0xDC, 0xDD, "vpaddus", X86addus,
Michael Liao66233b72015-08-06 09:06:20 +00003565 SSE_INTALU_ITINS_P, HasBWI, 1>;
Elena Demikhovsky52266382015-05-04 12:35:55 +00003566defm VPSUBUS : avx512_binop_rm_vl_bw<0xD8, 0xD9, "vpsubus", X86subus,
Michael Liao66233b72015-08-06 09:06:20 +00003567 SSE_INTALU_ITINS_P, HasBWI, 0>;
Igor Bregerf2460112015-07-26 14:41:44 +00003568defm VPMULLD : avx512_binop_rm_vl_d<0x40, "vpmulld", mul,
Michael Liao66233b72015-08-06 09:06:20 +00003569 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
Igor Bregerf2460112015-07-26 14:41:44 +00003570defm VPMULLW : avx512_binop_rm_vl_w<0xD5, "vpmullw", mul,
Michael Liao66233b72015-08-06 09:06:20 +00003571 SSE_INTALU_ITINS_P, HasBWI, 1>;
Igor Bregerf2460112015-07-26 14:41:44 +00003572defm VPMULLQ : avx512_binop_rm_vl_q<0x40, "vpmullq", mul,
Michael Liao66233b72015-08-06 09:06:20 +00003573 SSE_INTALU_ITINS_P, HasDQI, 1>, T8PD;
Igor Bregerf2460112015-07-26 14:41:44 +00003574defm VPMULHW : avx512_binop_rm_vl_w<0xE5, "vpmulhw", mulhs, SSE_INTALU_ITINS_P,
Asaf Badouh73f26f82015-07-05 12:23:20 +00003575 HasBWI, 1>;
Igor Bregerf2460112015-07-26 14:41:44 +00003576defm VPMULHUW : avx512_binop_rm_vl_w<0xE4, "vpmulhuw", mulhu, SSE_INTMUL_ITINS_P,
Michael Liao66233b72015-08-06 09:06:20 +00003577 HasBWI, 1>;
Igor Bregerf2460112015-07-26 14:41:44 +00003578defm VPMULHRSW : avx512_binop_rm_vl_w<0x0B, "vpmulhrsw", X86mulhrs, SSE_INTMUL_ITINS_P,
Michael Liao66233b72015-08-06 09:06:20 +00003579 HasBWI, 1>, T8PD;
Asaf Badouh81f03c32015-06-18 12:30:53 +00003580defm VPAVG : avx512_binop_rm_vl_bw<0xE0, 0xE3, "vpavg", X86avg,
Michael Liao66233b72015-08-06 09:06:20 +00003581 SSE_INTALU_ITINS_P, HasBWI, 1>;
3582
Simon Pilgrim18bcf932016-02-03 09:41:59 +00003583multiclass avx512_binop_all<bits<8> opc, string OpcodeStr, OpndItins itins,
Asaf Badouh5a3a0232016-02-01 15:48:21 +00003584 AVX512VLVectorVTInfo _SrcVTInfo, AVX512VLVectorVTInfo _DstVTInfo,
3585 SDNode OpNode, Predicate prd, bit IsCommutable = 0> {
3586 let Predicates = [prd] in
3587 defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
3588 _SrcVTInfo.info512, _DstVTInfo.info512,
3589 v8i64_info, IsCommutable>,
3590 EVEX_V512, EVEX_CD8<64, CD8VF>, VEX_W;
3591 let Predicates = [HasVLX, prd] in {
Elena Demikhovsky50b88dd2015-04-21 10:27:40 +00003592 defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
Simon Pilgrim18bcf932016-02-03 09:41:59 +00003593 _SrcVTInfo.info256, _DstVTInfo.info256,
Asaf Badouh5a3a0232016-02-01 15:48:21 +00003594 v4i64x_info, IsCommutable>,
3595 EVEX_V256, EVEX_CD8<64, CD8VF>, VEX_W;
Elena Demikhovsky50b88dd2015-04-21 10:27:40 +00003596 defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
Simon Pilgrim18bcf932016-02-03 09:41:59 +00003597 _SrcVTInfo.info128, _DstVTInfo.info128,
Asaf Badouh5a3a0232016-02-01 15:48:21 +00003598 v2i64x_info, IsCommutable>,
Elena Demikhovsky50b88dd2015-04-21 10:27:40 +00003599 EVEX_V128, EVEX_CD8<64, CD8VF>, VEX_W;
3600 }
Michael Liao66233b72015-08-06 09:06:20 +00003601}
Elena Demikhovsky50b88dd2015-04-21 10:27:40 +00003602
3603defm VPMULDQ : avx512_binop_all<0x28, "vpmuldq", SSE_INTALU_ITINS_P,
Asaf Badouh5a3a0232016-02-01 15:48:21 +00003604 avx512vl_i32_info, avx512vl_i64_info,
3605 X86pmuldq, HasAVX512, 1>,T8PD;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00003606defm VPMULUDQ : avx512_binop_all<0xF4, "vpmuludq", SSE_INTMUL_ITINS_P,
Asaf Badouh5a3a0232016-02-01 15:48:21 +00003607 avx512vl_i32_info, avx512vl_i64_info,
3608 X86pmuludq, HasAVX512, 1>;
3609defm VPMULTISHIFTQB : avx512_binop_all<0x83, "vpmultishiftqb", SSE_INTALU_ITINS_P,
3610 avx512vl_i8_info, avx512vl_i8_info,
3611 X86multishift, HasVBMI, 0>, T8PD;
Elena Demikhovsky172a27c2014-01-08 10:54:22 +00003612
Elena Demikhovsky2557a222015-05-04 09:14:02 +00003613multiclass avx512_packs_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
3614 X86VectorVTInfo _Src, X86VectorVTInfo _Dst> {
Craig Toppere1cac152016-06-07 07:27:54 +00003615 defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
3616 (ins _Src.RC:$src1, _Src.ScalarMemOp:$src2),
3617 OpcodeStr,
3618 "${src2}"##_Src.BroadcastStr##", $src1",
3619 "$src1, ${src2}"##_Src.BroadcastStr,
3620 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert
3621 (_Src.VT (X86VBroadcast
3622 (_Src.ScalarLdFrag addr:$src2))))))>,
3623 EVEX_4V, EVEX_B, EVEX_CD8<_Src.EltSize, CD8VF>;
Elena Demikhovsky2557a222015-05-04 09:14:02 +00003624}
3625
Michael Liao66233b72015-08-06 09:06:20 +00003626multiclass avx512_packs_rm<bits<8> opc, string OpcodeStr,
3627 SDNode OpNode,X86VectorVTInfo _Src,
Elena Demikhovsky2557a222015-05-04 09:14:02 +00003628 X86VectorVTInfo _Dst> {
Michael Liao66233b72015-08-06 09:06:20 +00003629 defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst),
Elena Demikhovsky2557a222015-05-04 09:14:02 +00003630 (ins _Src.RC:$src1, _Src.RC:$src2), OpcodeStr,
Michael Liao66233b72015-08-06 09:06:20 +00003631 "$src2, $src1","$src1, $src2",
3632 (_Dst.VT (OpNode
3633 (_Src.VT _Src.RC:$src1),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003634 (_Src.VT _Src.RC:$src2)))>,
3635 EVEX_CD8<_Src.EltSize, CD8VF>, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00003636 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
3637 (ins _Src.RC:$src1, _Src.MemOp:$src2), OpcodeStr,
3638 "$src2, $src1", "$src1, $src2",
3639 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1),
3640 (bitconvert (_Src.LdFrag addr:$src2))))>,
3641 EVEX_4V, EVEX_CD8<_Src.EltSize, CD8VF>;
Elena Demikhovsky2557a222015-05-04 09:14:02 +00003642}
3643
3644multiclass avx512_packs_all_i32_i16<bits<8> opc, string OpcodeStr,
3645 SDNode OpNode> {
Craig Topper5acb5a12016-05-01 06:24:57 +00003646 let Predicates = [HasBWI] in
Elena Demikhovsky2557a222015-05-04 09:14:02 +00003647 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i32_info,
3648 v32i16_info>,
3649 avx512_packs_rmb<opc, OpcodeStr, OpNode, v16i32_info,
3650 v32i16_info>, EVEX_V512;
Craig Topper5acb5a12016-05-01 06:24:57 +00003651 let Predicates = [HasBWI, HasVLX] in {
Elena Demikhovsky2557a222015-05-04 09:14:02 +00003652 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i32x_info,
3653 v16i16x_info>,
3654 avx512_packs_rmb<opc, OpcodeStr, OpNode, v8i32x_info,
3655 v16i16x_info>, EVEX_V256;
3656 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v4i32x_info,
3657 v8i16x_info>,
3658 avx512_packs_rmb<opc, OpcodeStr, OpNode, v4i32x_info,
3659 v8i16x_info>, EVEX_V128;
3660 }
3661}
3662multiclass avx512_packs_all_i16_i8<bits<8> opc, string OpcodeStr,
3663 SDNode OpNode> {
Craig Topper5acb5a12016-05-01 06:24:57 +00003664 let Predicates = [HasBWI] in
Elena Demikhovsky2557a222015-05-04 09:14:02 +00003665 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v32i16_info,
3666 v64i8_info>, EVEX_V512;
Craig Topper5acb5a12016-05-01 06:24:57 +00003667 let Predicates = [HasBWI, HasVLX] in {
Elena Demikhovsky2557a222015-05-04 09:14:02 +00003668 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i16x_info,
3669 v32i8x_info>, EVEX_V256;
3670 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i16x_info,
3671 v16i8x_info>, EVEX_V128;
3672 }
3673}
Igor Bregerf7fd5472015-07-21 07:11:28 +00003674
3675multiclass avx512_vpmadd<bits<8> opc, string OpcodeStr,
3676 SDNode OpNode, AVX512VLVectorVTInfo _Src,
3677 AVX512VLVectorVTInfo _Dst> {
Craig Topper5acb5a12016-05-01 06:24:57 +00003678 let Predicates = [HasBWI] in
Igor Bregerf7fd5472015-07-21 07:11:28 +00003679 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info512,
3680 _Dst.info512>, EVEX_V512;
Craig Topper5acb5a12016-05-01 06:24:57 +00003681 let Predicates = [HasBWI, HasVLX] in {
Igor Bregerf7fd5472015-07-21 07:11:28 +00003682 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info256,
3683 _Dst.info256>, EVEX_V256;
3684 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info128,
3685 _Dst.info128>, EVEX_V128;
3686 }
3687}
3688
Craig Topperb6da6542016-05-01 17:38:32 +00003689defm VPACKSSDW : avx512_packs_all_i32_i16<0x6B, "vpackssdw", X86Packss>, AVX512BIBase;
3690defm VPACKUSDW : avx512_packs_all_i32_i16<0x2b, "vpackusdw", X86Packus>, AVX5128IBase;
3691defm VPACKSSWB : avx512_packs_all_i16_i8 <0x63, "vpacksswb", X86Packss>, AVX512BIBase;
3692defm VPACKUSWB : avx512_packs_all_i16_i8 <0x67, "vpackuswb", X86Packus>, AVX512BIBase;
Igor Bregerf7fd5472015-07-21 07:11:28 +00003693
Craig Topper5acb5a12016-05-01 06:24:57 +00003694defm VPMADDUBSW : avx512_vpmadd<0x04, "vpmaddubsw", X86vpmaddubsw,
3695 avx512vl_i8_info, avx512vl_i16_info>, AVX512BIBase, T8PD;
3696defm VPMADDWD : avx512_vpmadd<0xF5, "vpmaddwd", X86vpmaddwd,
3697 avx512vl_i16_info, avx512vl_i32_info>, AVX512BIBase;
Elena Demikhovsky2557a222015-05-04 09:14:02 +00003698
Igor Bregerf2460112015-07-26 14:41:44 +00003699defm VPMAXSB : avx512_binop_rm_vl_b<0x3C, "vpmaxsb", smax,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003700 SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD;
Igor Bregerf2460112015-07-26 14:41:44 +00003701defm VPMAXSW : avx512_binop_rm_vl_w<0xEE, "vpmaxsw", smax,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003702 SSE_INTALU_ITINS_P, HasBWI, 1>;
Simon Pilgrim8b756592015-07-06 20:30:47 +00003703defm VPMAXS : avx512_binop_rm_vl_dq<0x3D, 0x3D, "vpmaxs", smax,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003704 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
Elena Demikhovsky199c8232013-10-27 08:18:37 +00003705
Igor Bregerf2460112015-07-26 14:41:44 +00003706defm VPMAXUB : avx512_binop_rm_vl_b<0xDE, "vpmaxub", umax,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003707 SSE_INTALU_ITINS_P, HasBWI, 1>;
Igor Bregerf2460112015-07-26 14:41:44 +00003708defm VPMAXUW : avx512_binop_rm_vl_w<0x3E, "vpmaxuw", umax,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003709 SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD;
Simon Pilgrim8b756592015-07-06 20:30:47 +00003710defm VPMAXU : avx512_binop_rm_vl_dq<0x3F, 0x3F, "vpmaxu", umax,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003711 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
Elena Demikhovsky199c8232013-10-27 08:18:37 +00003712
Igor Bregerf2460112015-07-26 14:41:44 +00003713defm VPMINSB : avx512_binop_rm_vl_b<0x38, "vpminsb", smin,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003714 SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD;
Igor Bregerf2460112015-07-26 14:41:44 +00003715defm VPMINSW : avx512_binop_rm_vl_w<0xEA, "vpminsw", smin,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003716 SSE_INTALU_ITINS_P, HasBWI, 1>;
Simon Pilgrim8b756592015-07-06 20:30:47 +00003717defm VPMINS : avx512_binop_rm_vl_dq<0x39, 0x39, "vpmins", smin,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003718 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
Elena Demikhovsky199c8232013-10-27 08:18:37 +00003719
Igor Bregerf2460112015-07-26 14:41:44 +00003720defm VPMINUB : avx512_binop_rm_vl_b<0xDA, "vpminub", umin,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003721 SSE_INTALU_ITINS_P, HasBWI, 1>;
Igor Bregerf2460112015-07-26 14:41:44 +00003722defm VPMINUW : avx512_binop_rm_vl_w<0x3A, "vpminuw", umin,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003723 SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD;
Simon Pilgrim8b756592015-07-06 20:30:47 +00003724defm VPMINU : avx512_binop_rm_vl_dq<0x3B, 0x3B, "vpminu", umin,
Robert Khasanov545d1b72014-10-14 14:36:19 +00003725 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003726//===----------------------------------------------------------------------===//
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003727// AVX-512 Logical Instructions
3728//===----------------------------------------------------------------------===//
3729
Robert Khasanov545d1b72014-10-14 14:36:19 +00003730defm VPAND : avx512_binop_rm_vl_dq<0xDB, 0xDB, "vpand", and,
3731 SSE_INTALU_ITINS_P, HasAVX512, 1>;
3732defm VPOR : avx512_binop_rm_vl_dq<0xEB, 0xEB, "vpor", or,
3733 SSE_INTALU_ITINS_P, HasAVX512, 1>;
3734defm VPXOR : avx512_binop_rm_vl_dq<0xEF, 0xEF, "vpxor", xor,
3735 SSE_INTALU_ITINS_P, HasAVX512, 1>;
3736defm VPANDN : avx512_binop_rm_vl_dq<0xDF, 0xDF, "vpandn", X86andnp,
Elena Demikhovsky72e3ccc2015-03-29 09:14:29 +00003737 SSE_INTALU_ITINS_P, HasAVX512, 0>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003738
3739//===----------------------------------------------------------------------===//
3740// AVX-512 FP arithmetic
3741//===----------------------------------------------------------------------===//
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00003742multiclass avx512_fp_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
3743 SDNode OpNode, SDNode VecNode, OpndItins itins,
3744 bit IsCommutable> {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003745
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00003746 defm rr_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
3747 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
3748 "$src2, $src1", "$src1, $src2",
3749 (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
3750 (i32 FROUND_CURRENT)),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003751 itins.rr, IsCommutable>;
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00003752
3753 defm rm_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
Igor Breger4511e762016-02-22 11:48:27 +00003754 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00003755 "$src2, $src1", "$src1, $src2",
3756 (VecNode (_.VT _.RC:$src1),
3757 (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))),
3758 (i32 FROUND_CURRENT)),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003759 itins.rm, IsCommutable>;
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00003760 let isCodeGenOnly = 1, isCommutable = IsCommutable,
3761 Predicates = [HasAVX512] in {
3762 def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst),
Michael Liao66233b72015-08-06 09:06:20 +00003763 (ins _.FRC:$src1, _.FRC:$src2),
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00003764 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
3765 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))],
3766 itins.rr>;
3767 def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst),
Michael Liao66233b72015-08-06 09:06:20 +00003768 (ins _.FRC:$src1, _.ScalarMemOp:$src2),
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00003769 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
3770 [(set _.FRC:$dst, (OpNode _.FRC:$src1,
3771 (_.ScalarLdFrag addr:$src2)))], itins.rr>;
3772 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003773}
3774
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00003775multiclass avx512_fp_scalar_round<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00003776 SDNode VecNode, OpndItins itins, bit IsCommutable = 0> {
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00003777
3778 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
3779 (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr,
3780 "$rc, $src2, $src1", "$src1, $src2, $rc",
3781 (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003782 (i32 imm:$rc)), itins.rr, IsCommutable>,
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00003783 EVEX_B, EVEX_RC;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003784}
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00003785multiclass avx512_fp_scalar_sae<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
3786 SDNode VecNode, OpndItins itins, bit IsCommutable> {
3787
3788 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
3789 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003790 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00003791 (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003792 (i32 FROUND_NO_EXC))>, EVEX_B;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003793}
3794
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00003795multiclass avx512_binop_s_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
3796 SDNode VecNode,
3797 SizeItins itins, bit IsCommutable> {
3798 defm SSZ : avx512_fp_scalar<opc, OpcodeStr#"ss", f32x_info, OpNode, VecNode,
3799 itins.s, IsCommutable>,
3800 avx512_fp_scalar_round<opc, OpcodeStr#"ss", f32x_info, VecNode,
3801 itins.s, IsCommutable>,
3802 XS, EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>;
3803 defm SDZ : avx512_fp_scalar<opc, OpcodeStr#"sd", f64x_info, OpNode, VecNode,
3804 itins.d, IsCommutable>,
3805 avx512_fp_scalar_round<opc, OpcodeStr#"sd", f64x_info, VecNode,
3806 itins.d, IsCommutable>,
3807 XD, VEX_W, EVEX_4V, VEX_LIG, EVEX_CD8<64, CD8VT1>;
3808}
3809
3810multiclass avx512_binop_s_sae<bits<8> opc, string OpcodeStr, SDNode OpNode,
3811 SDNode VecNode,
3812 SizeItins itins, bit IsCommutable> {
3813 defm SSZ : avx512_fp_scalar<opc, OpcodeStr#"ss", f32x_info, OpNode, VecNode,
3814 itins.s, IsCommutable>,
3815 avx512_fp_scalar_sae<opc, OpcodeStr#"ss", f32x_info, VecNode,
3816 itins.s, IsCommutable>,
3817 XS, EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>;
3818 defm SDZ : avx512_fp_scalar<opc, OpcodeStr#"sd", f64x_info, OpNode, VecNode,
3819 itins.d, IsCommutable>,
3820 avx512_fp_scalar_sae<opc, OpcodeStr#"sd", f64x_info, VecNode,
3821 itins.d, IsCommutable>,
3822 XD, VEX_W, EVEX_4V, VEX_LIG, EVEX_CD8<64, CD8VT1>;
3823}
3824defm VADD : avx512_binop_s_round<0x58, "vadd", fadd, X86faddRnd, SSE_ALU_ITINS_S, 1>;
3825defm VMUL : avx512_binop_s_round<0x59, "vmul", fmul, X86fmulRnd, SSE_ALU_ITINS_S, 1>;
3826defm VSUB : avx512_binop_s_round<0x5C, "vsub", fsub, X86fsubRnd, SSE_ALU_ITINS_S, 0>;
3827defm VDIV : avx512_binop_s_round<0x5E, "vdiv", fdiv, X86fdivRnd, SSE_ALU_ITINS_S, 0>;
3828defm VMIN : avx512_binop_s_sae <0x5D, "vmin", X86fmin, X86fminRnd, SSE_ALU_ITINS_S, 1>;
3829defm VMAX : avx512_binop_s_sae <0x5F, "vmax", X86fmax, X86fmaxRnd, SSE_ALU_ITINS_S, 1>;
3830
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003831multiclass avx512_fp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
Robert Khasanov595e5982014-10-29 15:43:02 +00003832 X86VectorVTInfo _, bit IsCommutable> {
3833 defm rr: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
3834 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
3835 "$src2, $src1", "$src1, $src2",
3836 (_.VT (OpNode _.RC:$src1, _.RC:$src2))>, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00003837 defm rm: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
3838 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix,
3839 "$src2, $src1", "$src1, $src2",
3840 (OpNode _.RC:$src1, (_.LdFrag addr:$src2))>, EVEX_4V;
3841 defm rmb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
3842 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix,
3843 "${src2}"##_.BroadcastStr##", $src1",
3844 "$src1, ${src2}"##_.BroadcastStr,
3845 (OpNode _.RC:$src1, (_.VT (X86VBroadcast
3846 (_.ScalarLdFrag addr:$src2))))>,
3847 EVEX_4V, EVEX_B;
Robert Khasanov595e5982014-10-29 15:43:02 +00003848}
Elena Demikhovskyf7c1b162014-03-06 08:45:30 +00003849
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00003850multiclass avx512_fp_round_packed<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd,
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00003851 X86VectorVTInfo _> {
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00003852 defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
3853 (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr##_.Suffix,
3854 "$rc, $src2, $src1", "$src1, $src2, $rc",
3855 (_.VT (OpNodeRnd _.RC:$src1, _.RC:$src2, (i32 imm:$rc)))>,
3856 EVEX_4V, EVEX_B, EVEX_RC;
3857}
3858
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003859
3860multiclass avx512_fp_sae_packed<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd,
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00003861 X86VectorVTInfo _> {
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003862 defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
3863 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
3864 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
3865 (_.VT (OpNodeRnd _.RC:$src1, _.RC:$src2, (i32 FROUND_NO_EXC)))>,
3866 EVEX_4V, EVEX_B;
3867}
3868
Michael Liao66233b72015-08-06 09:06:20 +00003869multiclass avx512_fp_binop_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topperdb290662016-05-01 05:57:06 +00003870 Predicate prd, bit IsCommutable = 0> {
3871 let Predicates = [prd] in {
Robert Khasanov595e5982014-10-29 15:43:02 +00003872 defm PSZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v16f32_info,
3873 IsCommutable>, EVEX_V512, PS,
3874 EVEX_CD8<32, CD8VF>;
3875 defm PDZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v8f64_info,
3876 IsCommutable>, EVEX_V512, PD, VEX_W,
3877 EVEX_CD8<64, CD8VF>;
Craig Topperdb290662016-05-01 05:57:06 +00003878 }
Elena Demikhovskyf7c1b162014-03-06 08:45:30 +00003879
Robert Khasanov595e5982014-10-29 15:43:02 +00003880 // Define only if AVX512VL feature is present.
Craig Topperdb290662016-05-01 05:57:06 +00003881 let Predicates = [prd, HasVLX] in {
Robert Khasanov595e5982014-10-29 15:43:02 +00003882 defm PSZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, v4f32x_info,
3883 IsCommutable>, EVEX_V128, PS,
3884 EVEX_CD8<32, CD8VF>;
3885 defm PSZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, v8f32x_info,
3886 IsCommutable>, EVEX_V256, PS,
3887 EVEX_CD8<32, CD8VF>;
3888 defm PDZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, v2f64x_info,
3889 IsCommutable>, EVEX_V128, PD, VEX_W,
3890 EVEX_CD8<64, CD8VF>;
3891 defm PDZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, v4f64x_info,
3892 IsCommutable>, EVEX_V256, PD, VEX_W,
3893 EVEX_CD8<64, CD8VF>;
Elena Demikhovskyf7c1b162014-03-06 08:45:30 +00003894 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003895}
3896
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00003897multiclass avx512_fp_binop_p_round<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd> {
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00003898 defm PSZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, v16f32_info>,
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00003899 EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00003900 defm PDZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, v8f64_info>,
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00003901 EVEX_V512, PD, VEX_W,EVEX_CD8<64, CD8VF>;
3902}
3903
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003904multiclass avx512_fp_binop_p_sae<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd> {
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00003905 defm PSZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, v16f32_info>,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003906 EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00003907 defm PDZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, v8f64_info>,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003908 EVEX_V512, PD, VEX_W,EVEX_CD8<64, CD8VF>;
3909}
3910
Craig Topperdb290662016-05-01 05:57:06 +00003911defm VADD : avx512_fp_binop_p<0x58, "vadd", fadd, HasAVX512, 1>,
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00003912 avx512_fp_binop_p_round<0x58, "vadd", X86faddRnd>;
Craig Topperdb290662016-05-01 05:57:06 +00003913defm VMUL : avx512_fp_binop_p<0x59, "vmul", fmul, HasAVX512, 1>,
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00003914 avx512_fp_binop_p_round<0x59, "vmul", X86fmulRnd>;
Craig Topperdb290662016-05-01 05:57:06 +00003915defm VSUB : avx512_fp_binop_p<0x5C, "vsub", fsub, HasAVX512>,
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00003916 avx512_fp_binop_p_round<0x5C, "vsub", X86fsubRnd>;
Craig Topperdb290662016-05-01 05:57:06 +00003917defm VDIV : avx512_fp_binop_p<0x5E, "vdiv", fdiv, HasAVX512>,
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00003918 avx512_fp_binop_p_round<0x5E, "vdiv", X86fdivRnd>;
Igor Breger58c07802016-05-03 11:51:45 +00003919defm VMIN : avx512_fp_binop_p<0x5D, "vmin", X86fmin, HasAVX512, 0>,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003920 avx512_fp_binop_p_sae<0x5D, "vmin", X86fminRnd>;
Igor Breger58c07802016-05-03 11:51:45 +00003921defm VMAX : avx512_fp_binop_p<0x5F, "vmax", X86fmax, HasAVX512, 0>,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00003922 avx512_fp_binop_p_sae<0x5F, "vmax", X86fmaxRnd>;
Igor Breger58c07802016-05-03 11:51:45 +00003923let isCodeGenOnly = 1 in {
3924 defm VMINC : avx512_fp_binop_p<0x5D, "vmin", X86fminc, HasAVX512, 1>;
3925 defm VMAXC : avx512_fp_binop_p<0x5F, "vmax", X86fmaxc, HasAVX512, 1>;
3926}
Craig Topperdb290662016-05-01 05:57:06 +00003927defm VAND : avx512_fp_binop_p<0x54, "vand", X86fand, HasDQI, 1>;
3928defm VANDN : avx512_fp_binop_p<0x55, "vandn", X86fandn, HasDQI, 0>;
3929defm VOR : avx512_fp_binop_p<0x56, "vor", X86for, HasDQI, 1>;
3930defm VXOR : avx512_fp_binop_p<0x57, "vxor", X86fxor, HasDQI, 1>;
Elena Demikhovsky52e4a0e2014-01-05 10:46:09 +00003931
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00003932multiclass avx512_fp_scalef_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
3933 X86VectorVTInfo _> {
3934 defm rr: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
3935 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
3936 "$src2, $src1", "$src1, $src2",
3937 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (i32 FROUND_CURRENT)))>, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00003938 defm rm: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
3939 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix,
3940 "$src2, $src1", "$src1, $src2",
3941 (OpNode _.RC:$src1, (_.LdFrag addr:$src2), (i32 FROUND_CURRENT))>, EVEX_4V;
3942 defm rmb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
3943 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix,
3944 "${src2}"##_.BroadcastStr##", $src1",
3945 "$src1, ${src2}"##_.BroadcastStr,
3946 (OpNode _.RC:$src1, (_.VT (X86VBroadcast
3947 (_.ScalarLdFrag addr:$src2))), (i32 FROUND_CURRENT))>,
3948 EVEX_4V, EVEX_B;
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00003949}
3950
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00003951multiclass avx512_fp_scalef_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode,
3952 X86VectorVTInfo _> {
3953 defm rr: AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
3954 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
3955 "$src2, $src1", "$src1, $src2",
3956 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (i32 FROUND_CURRENT)))>;
Craig Toppere1cac152016-06-07 07:27:54 +00003957 defm rm: AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
3958 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix,
3959 "$src2, $src1", "$src1, $src2",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00003960 (OpNode _.RC:$src1,
Craig Toppere1cac152016-06-07 07:27:54 +00003961 (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))),
3962 (i32 FROUND_CURRENT))>;
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00003963}
3964
Michael Zuckerman11b55b22016-05-21 11:09:53 +00003965multiclass avx512_fp_scalef_all<bits<8> opc, bits<8> opcScaler, string OpcodeStr, SDNode OpNode, SDNode OpNodeScal> {
Michael Liao66233b72015-08-06 09:06:20 +00003966 defm PSZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v16f32_info>,
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00003967 avx512_fp_round_packed<opc, OpcodeStr, OpNode, v16f32_info>,
3968 EVEX_V512, EVEX_CD8<32, CD8VF>;
Michael Liao66233b72015-08-06 09:06:20 +00003969 defm PDZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v8f64_info>,
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00003970 avx512_fp_round_packed<opc, OpcodeStr, OpNode, v8f64_info>,
3971 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
Michael Zuckerman11b55b22016-05-21 11:09:53 +00003972 defm SSZ128 : avx512_fp_scalef_scalar<opcScaler, OpcodeStr, OpNodeScal, f32x_info>,
3973 avx512_fp_scalar_round<opcScaler, OpcodeStr##"ss", f32x_info, OpNodeScal, SSE_ALU_ITINS_S.s>,
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00003974 EVEX_4V,EVEX_CD8<32, CD8VT1>;
Michael Zuckerman11b55b22016-05-21 11:09:53 +00003975 defm SDZ128 : avx512_fp_scalef_scalar<opcScaler, OpcodeStr, OpNodeScal, f64x_info>,
3976 avx512_fp_scalar_round<opcScaler, OpcodeStr##"sd", f64x_info, OpNodeScal, SSE_ALU_ITINS_S.d>,
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00003977 EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
3978
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00003979 // Define only if AVX512VL feature is present.
3980 let Predicates = [HasVLX] in {
3981 defm PSZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v4f32x_info>,
3982 EVEX_V128, EVEX_CD8<32, CD8VF>;
3983 defm PSZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v8f32x_info>,
3984 EVEX_V256, EVEX_CD8<32, CD8VF>;
3985 defm PDZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v2f64x_info>,
3986 EVEX_V128, VEX_W, EVEX_CD8<64, CD8VF>;
3987 defm PDZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v4f64x_info>,
3988 EVEX_V256, VEX_W, EVEX_CD8<64, CD8VF>;
3989 }
3990}
Michael Zuckerman11b55b22016-05-21 11:09:53 +00003991defm VSCALEF : avx512_fp_scalef_all<0x2C, 0x2D, "vscalef", X86scalef, X86scalefs>, T8PD;
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00003992
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003993//===----------------------------------------------------------------------===//
3994// AVX-512 VPTESTM instructions
3995//===----------------------------------------------------------------------===//
3996
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00003997multiclass avx512_vptest<bits<8> opc, string OpcodeStr, SDNode OpNode,
3998 X86VectorVTInfo _> {
Igor Breger639fde72016-03-03 14:18:38 +00003999 let isCommutable = 1 in
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00004000 defm rr : AVX512_maskable_cmp<opc, MRMSrcReg, _, (outs _.KRC:$dst),
4001 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
4002 "$src2, $src1", "$src1, $src2",
4003 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>,
4004 EVEX_4V;
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00004005 defm rm : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst),
4006 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
4007 "$src2, $src1", "$src1, $src2",
Michael Liao66233b72015-08-06 09:06:20 +00004008 (OpNode (_.VT _.RC:$src1),
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00004009 (_.VT (bitconvert (_.LdFrag addr:$src2))))>,
4010 EVEX_4V,
4011 EVEX_CD8<_.EltSize, CD8VF>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004012}
4013
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00004014multiclass avx512_vptest_mb<bits<8> opc, string OpcodeStr, SDNode OpNode,
4015 X86VectorVTInfo _> {
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00004016 defm rmb : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst),
4017 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
4018 "${src2}"##_.BroadcastStr##", $src1",
4019 "$src1, ${src2}"##_.BroadcastStr,
4020 (OpNode (_.VT _.RC:$src1), (_.VT (X86VBroadcast
4021 (_.ScalarLdFrag addr:$src2))))>,
4022 EVEX_B, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>;
Elena Demikhovskya30e4372014-02-05 07:05:03 +00004023}
Igor Bregerfca0a342016-01-28 13:19:25 +00004024
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004025// Use 512bit version to implement 128/256 bit in case NoVLX.
Igor Bregerfca0a342016-01-28 13:19:25 +00004026multiclass avx512_vptest_lowering<SDNode OpNode, X86VectorVTInfo ExtendInfo,
4027 X86VectorVTInfo _, string Suffix> {
4028 def : Pat<(_.KVT (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))),
4029 (_.KVT (COPY_TO_REGCLASS
4030 (!cast<Instruction>(NAME # Suffix # "Zrr")
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004031 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
Igor Bregerfca0a342016-01-28 13:19:25 +00004032 _.RC:$src1, _.SubRegIdx),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004033 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
Igor Bregerfca0a342016-01-28 13:19:25 +00004034 _.RC:$src2, _.SubRegIdx)),
4035 _.KRC))>;
4036}
4037
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00004038multiclass avx512_vptest_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
Igor Bregerfca0a342016-01-28 13:19:25 +00004039 AVX512VLVectorVTInfo _, string Suffix> {
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00004040 let Predicates = [HasAVX512] in
4041 defm Z : avx512_vptest<opc, OpcodeStr, OpNode, _.info512>,
4042 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
4043
4044 let Predicates = [HasAVX512, HasVLX] in {
4045 defm Z256 : avx512_vptest<opc, OpcodeStr, OpNode, _.info256>,
4046 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
4047 defm Z128 : avx512_vptest<opc, OpcodeStr, OpNode, _.info128>,
4048 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128;
4049 }
Igor Bregerfca0a342016-01-28 13:19:25 +00004050 let Predicates = [HasAVX512, NoVLX] in {
4051 defm Z256_Alt : avx512_vptest_lowering< OpNode, _.info512, _.info256, Suffix>;
4052 defm Z128_Alt : avx512_vptest_lowering< OpNode, _.info512, _.info128, Suffix>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004053 }
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00004054}
4055
4056multiclass avx512_vptest_dq<bits<8> opc, string OpcodeStr, SDNode OpNode> {
4057 defm D : avx512_vptest_dq_sizes<opc, OpcodeStr#"d", OpNode,
Igor Bregerfca0a342016-01-28 13:19:25 +00004058 avx512vl_i32_info, "D">;
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00004059 defm Q : avx512_vptest_dq_sizes<opc, OpcodeStr#"q", OpNode,
Igor Bregerfca0a342016-01-28 13:19:25 +00004060 avx512vl_i64_info, "Q">, VEX_W;
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00004061}
4062
4063multiclass avx512_vptest_wb<bits<8> opc, string OpcodeStr,
4064 SDNode OpNode> {
4065 let Predicates = [HasBWI] in {
4066 defm WZ: avx512_vptest<opc, OpcodeStr#"w", OpNode, v32i16_info>,
4067 EVEX_V512, VEX_W;
4068 defm BZ: avx512_vptest<opc, OpcodeStr#"b", OpNode, v64i8_info>,
4069 EVEX_V512;
4070 }
4071 let Predicates = [HasVLX, HasBWI] in {
4072
4073 defm WZ256: avx512_vptest<opc, OpcodeStr#"w", OpNode, v16i16x_info>,
4074 EVEX_V256, VEX_W;
4075 defm WZ128: avx512_vptest<opc, OpcodeStr#"w", OpNode, v8i16x_info>,
4076 EVEX_V128, VEX_W;
4077 defm BZ256: avx512_vptest<opc, OpcodeStr#"b", OpNode, v32i8x_info>,
4078 EVEX_V256;
4079 defm BZ128: avx512_vptest<opc, OpcodeStr#"b", OpNode, v16i8x_info>,
4080 EVEX_V128;
4081 }
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004082
Igor Bregerfca0a342016-01-28 13:19:25 +00004083 let Predicates = [HasAVX512, NoVLX] in {
4084 defm BZ256_Alt : avx512_vptest_lowering< OpNode, v64i8_info, v32i8x_info, "B">;
4085 defm BZ128_Alt : avx512_vptest_lowering< OpNode, v64i8_info, v16i8x_info, "B">;
4086 defm WZ256_Alt : avx512_vptest_lowering< OpNode, v32i16_info, v16i16x_info, "W">;
4087 defm WZ128_Alt : avx512_vptest_lowering< OpNode, v32i16_info, v8i16x_info, "W">;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004088 }
Igor Bregerfca0a342016-01-28 13:19:25 +00004089
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00004090}
4091
4092multiclass avx512_vptest_all_forms<bits<8> opc_wb, bits<8> opc_dq, string OpcodeStr,
4093 SDNode OpNode> :
4094 avx512_vptest_wb <opc_wb, OpcodeStr, OpNode>,
4095 avx512_vptest_dq<opc_dq, OpcodeStr, OpNode>;
4096
4097defm VPTESTM : avx512_vptest_all_forms<0x26, 0x27, "vptestm", X86testm>, T8PD;
4098defm VPTESTNM : avx512_vptest_all_forms<0x26, 0x27, "vptestnm", X86testnm>, T8XS;
Elena Demikhovskya30e4372014-02-05 07:05:03 +00004099
Cameron McInally9b7c15a2014-11-25 20:41:51 +00004100
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004101//===----------------------------------------------------------------------===//
4102// AVX-512 Shift instructions
4103//===----------------------------------------------------------------------===//
4104multiclass avx512_shift_rmi<bits<8> opc, Format ImmFormR, Format ImmFormM,
Michael Liao5bf95782014-12-04 05:20:33 +00004105 string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> {
Cameron McInally04400442014-11-14 15:43:00 +00004106 defm ri : AVX512_maskable<opc, ImmFormR, _, (outs _.RC:$dst),
Craig Topper7ff6ab32015-01-21 08:43:49 +00004107 (ins _.RC:$src1, u8imm:$src2), OpcodeStr,
Cameron McInally04400442014-11-14 15:43:00 +00004108 "$src2, $src1", "$src1, $src2",
4109 (_.VT (OpNode _.RC:$src1, (i8 imm:$src2))),
Elena Demikhovsky75ede682015-06-01 07:17:23 +00004110 SSE_INTSHIFT_ITINS_P.rr>;
Cameron McInally04400442014-11-14 15:43:00 +00004111 defm mi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst),
Craig Topper7ff6ab32015-01-21 08:43:49 +00004112 (ins _.MemOp:$src1, u8imm:$src2), OpcodeStr,
Cameron McInally04400442014-11-14 15:43:00 +00004113 "$src2, $src1", "$src1, $src2",
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004114 (_.VT (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
4115 (i8 imm:$src2))),
Elena Demikhovsky75ede682015-06-01 07:17:23 +00004116 SSE_INTSHIFT_ITINS_P.rm>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004117}
4118
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004119multiclass avx512_shift_rmbi<bits<8> opc, Format ImmFormM,
4120 string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> {
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004121 defm mbi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst),
4122 (ins _.ScalarMemOp:$src1, u8imm:$src2), OpcodeStr,
4123 "$src2, ${src1}"##_.BroadcastStr, "${src1}"##_.BroadcastStr##", $src2",
4124 (_.VT (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src1)), (i8 imm:$src2))),
Elena Demikhovsky75ede682015-06-01 07:17:23 +00004125 SSE_INTSHIFT_ITINS_P.rm>, EVEX_B;
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004126}
4127
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004128multiclass avx512_shift_rrm<bits<8> opc, string OpcodeStr, SDNode OpNode,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004129 ValueType SrcVT, PatFrag bc_frag, X86VectorVTInfo _> {
Cameron McInally9b7c15a2014-11-25 20:41:51 +00004130 // src2 is always 128-bit
4131 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
4132 (ins _.RC:$src1, VR128X:$src2), OpcodeStr,
4133 "$src2, $src1", "$src1, $src2",
4134 (_.VT (OpNode _.RC:$src1, (SrcVT VR128X:$src2))),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00004135 SSE_INTSHIFT_ITINS_P.rr>, AVX512BIBase, EVEX_4V;
Cameron McInally9b7c15a2014-11-25 20:41:51 +00004136 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
4137 (ins _.RC:$src1, i128mem:$src2), OpcodeStr,
4138 "$src2, $src1", "$src1, $src2",
Craig Topper820d4922015-02-09 04:04:50 +00004139 (_.VT (OpNode _.RC:$src1, (bc_frag (loadv2i64 addr:$src2)))),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00004140 SSE_INTSHIFT_ITINS_P.rm>, AVX512BIBase,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004141 EVEX_4V;
Cameron McInally9b7c15a2014-11-25 20:41:51 +00004142}
4143
Cameron McInally5fb084e2014-12-11 17:13:05 +00004144multiclass avx512_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004145 ValueType SrcVT, PatFrag bc_frag,
4146 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
4147 let Predicates = [prd] in
4148 defm Z : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag,
4149 VTInfo.info512>, EVEX_V512,
4150 EVEX_CD8<VTInfo.info512.EltSize, CD8VQ> ;
4151 let Predicates = [prd, HasVLX] in {
4152 defm Z256 : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag,
4153 VTInfo.info256>, EVEX_V256,
4154 EVEX_CD8<VTInfo.info256.EltSize, CD8VH>;
4155 defm Z128 : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag,
4156 VTInfo.info128>, EVEX_V128,
4157 EVEX_CD8<VTInfo.info128.EltSize, CD8VF>;
4158 }
Cameron McInally9b7c15a2014-11-25 20:41:51 +00004159}
4160
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004161multiclass avx512_shift_types<bits<8> opcd, bits<8> opcq, bits<8> opcw,
4162 string OpcodeStr, SDNode OpNode> {
Cameron McInally5fb084e2014-12-11 17:13:05 +00004163 defm D : avx512_shift_sizes<opcd, OpcodeStr#"d", OpNode, v4i32, bc_v4i32,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004164 avx512vl_i32_info, HasAVX512>;
Cameron McInally5fb084e2014-12-11 17:13:05 +00004165 defm Q : avx512_shift_sizes<opcq, OpcodeStr#"q", OpNode, v2i64, bc_v2i64,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004166 avx512vl_i64_info, HasAVX512>, VEX_W;
4167 defm W : avx512_shift_sizes<opcw, OpcodeStr#"w", OpNode, v8i16, bc_v8i16,
4168 avx512vl_i16_info, HasBWI>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004169}
4170
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004171multiclass avx512_shift_rmi_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM,
4172 string OpcodeStr, SDNode OpNode,
4173 AVX512VLVectorVTInfo VTInfo> {
4174 let Predicates = [HasAVX512] in
4175 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4176 VTInfo.info512>,
4177 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
4178 VTInfo.info512>, EVEX_V512;
4179 let Predicates = [HasAVX512, HasVLX] in {
4180 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4181 VTInfo.info256>,
4182 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
4183 VTInfo.info256>, EVEX_V256;
4184 defm Z128: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4185 VTInfo.info128>,
Michael Liao66233b72015-08-06 09:06:20 +00004186 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004187 VTInfo.info128>, EVEX_V128;
4188 }
4189}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004190
Michael Liao66233b72015-08-06 09:06:20 +00004191multiclass avx512_shift_rmi_w<bits<8> opcw,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004192 Format ImmFormR, Format ImmFormM,
4193 string OpcodeStr, SDNode OpNode> {
4194 let Predicates = [HasBWI] in
4195 defm WZ: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4196 v32i16_info>, EVEX_V512;
4197 let Predicates = [HasVLX, HasBWI] in {
4198 defm WZ256: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4199 v16i16x_info>, EVEX_V256;
4200 defm WZ128: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4201 v8i16x_info>, EVEX_V128;
4202 }
4203}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004204
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004205multiclass avx512_shift_rmi_dq<bits<8> opcd, bits<8> opcq,
4206 Format ImmFormR, Format ImmFormM,
4207 string OpcodeStr, SDNode OpNode> {
4208 defm D: avx512_shift_rmi_sizes<opcd, ImmFormR, ImmFormM, OpcodeStr#"d", OpNode,
4209 avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
4210 defm Q: avx512_shift_rmi_sizes<opcq, ImmFormR, ImmFormM, OpcodeStr#"q", OpNode,
4211 avx512vl_i64_info>, EVEX_CD8<64, CD8VF>, VEX_W;
4212}
Cameron McInally9b7c15a2014-11-25 20:41:51 +00004213
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004214defm VPSRL : avx512_shift_rmi_dq<0x72, 0x73, MRM2r, MRM2m, "vpsrl", X86vsrli>,
Elena Demikhovsky75ede682015-06-01 07:17:23 +00004215 avx512_shift_rmi_w<0x71, MRM2r, MRM2m, "vpsrlw", X86vsrli>, AVX512BIi8Base, EVEX_4V;
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004216
4217defm VPSLL : avx512_shift_rmi_dq<0x72, 0x73, MRM6r, MRM6m, "vpsll", X86vshli>,
Elena Demikhovsky75ede682015-06-01 07:17:23 +00004218 avx512_shift_rmi_w<0x71, MRM6r, MRM6m, "vpsllw", X86vshli>, AVX512BIi8Base, EVEX_4V;
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004219
Elena Demikhovsky1b2f2f12015-05-13 07:35:05 +00004220defm VPSRA : avx512_shift_rmi_dq<0x72, 0x72, MRM4r, MRM4m, "vpsra", X86vsrai>,
Elena Demikhovsky75ede682015-06-01 07:17:23 +00004221 avx512_shift_rmi_w<0x71, MRM4r, MRM4m, "vpsraw", X86vsrai>, AVX512BIi8Base, EVEX_4V;
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004222
Michael Zuckerman298a6802016-01-13 12:39:33 +00004223defm VPROR : avx512_shift_rmi_dq<0x72, 0x72, MRM0r, MRM0m, "vpror", X86vrotri>, AVX512BIi8Base, EVEX_4V;
Michael Zuckerman2ddcbcf2016-01-12 21:19:17 +00004224defm VPROL : avx512_shift_rmi_dq<0x72, 0x72, MRM1r, MRM1m, "vprol", X86vrotli>, AVX512BIi8Base, EVEX_4V;
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004225
4226defm VPSLL : avx512_shift_types<0xF2, 0xF3, 0xF1, "vpsll", X86vshl>;
4227defm VPSRA : avx512_shift_types<0xE2, 0xE2, 0xE1, "vpsra", X86vsra>;
4228defm VPSRL : avx512_shift_types<0xD2, 0xD3, 0xD1, "vpsrl", X86vsrl>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004229
4230//===-------------------------------------------------------------------===//
4231// Variable Bit Shifts
4232//===-------------------------------------------------------------------===//
4233multiclass avx512_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode,
Cameron McInally5fb084e2014-12-11 17:13:05 +00004234 X86VectorVTInfo _> {
4235 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
4236 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
4237 "$src2, $src1", "$src1, $src2",
4238 (_.VT (OpNode _.RC:$src1, (_.VT _.RC:$src2))),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00004239 SSE_INTSHIFT_ITINS_P.rr>, AVX5128IBase, EVEX_4V;
Cameron McInally5fb084e2014-12-11 17:13:05 +00004240 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
4241 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
4242 "$src2, $src1", "$src1, $src2",
Elena Demikhovsky4078c752015-06-04 07:07:13 +00004243 (_.VT (OpNode _.RC:$src1,
4244 (_.VT (bitconvert (_.LdFrag addr:$src2))))),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00004245 SSE_INTSHIFT_ITINS_P.rm>, AVX5128IBase, EVEX_4V,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004246 EVEX_CD8<_.EltSize, CD8VF>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004247}
4248
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004249multiclass avx512_var_shift_mb<bits<8> opc, string OpcodeStr, SDNode OpNode,
4250 X86VectorVTInfo _> {
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004251 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
4252 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
4253 "${src2}"##_.BroadcastStr##", $src1",
4254 "$src1, ${src2}"##_.BroadcastStr,
4255 (_.VT (OpNode _.RC:$src1, (_.VT (X86VBroadcast
4256 (_.ScalarLdFrag addr:$src2))))),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00004257 SSE_INTSHIFT_ITINS_P.rm>, AVX5128IBase, EVEX_B,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004258 EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>;
4259}
Cameron McInally5fb084e2014-12-11 17:13:05 +00004260multiclass avx512_var_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
4261 AVX512VLVectorVTInfo _> {
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004262 let Predicates = [HasAVX512] in
4263 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>,
4264 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
4265
4266 let Predicates = [HasAVX512, HasVLX] in {
4267 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>,
4268 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
4269 defm Z128 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info128>,
4270 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128;
4271 }
Cameron McInally5fb084e2014-12-11 17:13:05 +00004272}
4273
4274multiclass avx512_var_shift_types<bits<8> opc, string OpcodeStr,
4275 SDNode OpNode> {
4276 defm D : avx512_var_shift_sizes<opc, OpcodeStr#"d", OpNode,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004277 avx512vl_i32_info>;
Cameron McInally5fb084e2014-12-11 17:13:05 +00004278 defm Q : avx512_var_shift_sizes<opc, OpcodeStr#"q", OpNode,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004279 avx512vl_i64_info>, VEX_W;
Cameron McInally5fb084e2014-12-11 17:13:05 +00004280}
4281
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004282// Use 512bit version to implement 128/256 bit in case NoVLX.
Igor Breger7b46b4e2015-12-23 08:06:50 +00004283multiclass avx512_var_shift_w_lowering<AVX512VLVectorVTInfo _, SDNode OpNode> {
4284 let Predicates = [HasBWI, NoVLX] in {
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004285 def : Pat<(_.info256.VT (OpNode (_.info256.VT _.info256.RC:$src1),
Igor Breger7b46b4e2015-12-23 08:06:50 +00004286 (_.info256.VT _.info256.RC:$src2))),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004287 (EXTRACT_SUBREG
Igor Breger7b46b4e2015-12-23 08:06:50 +00004288 (!cast<Instruction>(NAME#"WZrr")
4289 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR256X:$src1, sub_ymm),
4290 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR256X:$src2, sub_ymm)),
4291 sub_ymm)>;
4292
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004293 def : Pat<(_.info128.VT (OpNode (_.info128.VT _.info128.RC:$src1),
Igor Breger7b46b4e2015-12-23 08:06:50 +00004294 (_.info128.VT _.info128.RC:$src2))),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004295 (EXTRACT_SUBREG
Igor Breger7b46b4e2015-12-23 08:06:50 +00004296 (!cast<Instruction>(NAME#"WZrr")
4297 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR128X:$src1, sub_xmm),
4298 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR128X:$src2, sub_xmm)),
4299 sub_xmm)>;
4300 }
4301}
4302
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004303multiclass avx512_var_shift_w<bits<8> opc, string OpcodeStr,
4304 SDNode OpNode> {
4305 let Predicates = [HasBWI] in
4306 defm WZ: avx512_var_shift<opc, OpcodeStr, OpNode, v32i16_info>,
4307 EVEX_V512, VEX_W;
4308 let Predicates = [HasVLX, HasBWI] in {
4309
4310 defm WZ256: avx512_var_shift<opc, OpcodeStr, OpNode, v16i16x_info>,
4311 EVEX_V256, VEX_W;
4312 defm WZ128: avx512_var_shift<opc, OpcodeStr, OpNode, v8i16x_info>,
4313 EVEX_V128, VEX_W;
4314 }
4315}
4316
4317defm VPSLLV : avx512_var_shift_types<0x47, "vpsllv", shl>,
Igor Breger7b46b4e2015-12-23 08:06:50 +00004318 avx512_var_shift_w<0x12, "vpsllvw", shl>,
4319 avx512_var_shift_w_lowering<avx512vl_i16_info, shl>;
Igor Bregere59165c2016-06-20 07:05:43 +00004320
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004321defm VPSRAV : avx512_var_shift_types<0x46, "vpsrav", sra>,
Igor Breger7b46b4e2015-12-23 08:06:50 +00004322 avx512_var_shift_w<0x11, "vpsravw", sra>,
4323 avx512_var_shift_w_lowering<avx512vl_i16_info, sra>;
Igor Bregere59165c2016-06-20 07:05:43 +00004324let isCodeGenOnly = 1 in
4325 defm VPSRAV_Int : avx512_var_shift_types<0x46, "vpsrav", X86vsrav>,
4326 avx512_var_shift_w<0x11, "vpsravw", X86vsrav>;
4327
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004328defm VPSRLV : avx512_var_shift_types<0x45, "vpsrlv", srl>,
Igor Breger7b46b4e2015-12-23 08:06:50 +00004329 avx512_var_shift_w<0x10, "vpsrlvw", srl>,
4330 avx512_var_shift_w_lowering<avx512vl_i16_info, srl>;
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00004331defm VPRORV : avx512_var_shift_types<0x14, "vprorv", rotr>;
4332defm VPROLV : avx512_var_shift_types<0x15, "vprolv", rotl>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004333
Elena Demikhovsky4078c752015-06-04 07:07:13 +00004334//===-------------------------------------------------------------------===//
4335// 1-src variable permutation VPERMW/D/Q
4336//===-------------------------------------------------------------------===//
4337multiclass avx512_vperm_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
4338 AVX512VLVectorVTInfo _> {
4339 let Predicates = [HasAVX512] in
4340 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>,
4341 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
4342
4343 let Predicates = [HasAVX512, HasVLX] in
4344 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>,
4345 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
4346}
4347
4348multiclass avx512_vpermi_dq_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM,
4349 string OpcodeStr, SDNode OpNode,
4350 AVX512VLVectorVTInfo VTInfo> {
4351 let Predicates = [HasAVX512] in
4352 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4353 VTInfo.info512>,
4354 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
4355 VTInfo.info512>, EVEX_V512;
4356 let Predicates = [HasAVX512, HasVLX] in
4357 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4358 VTInfo.info256>,
4359 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
4360 VTInfo.info256>, EVEX_V256;
4361}
4362
Michael Zuckermand9cac592016-01-19 17:07:43 +00004363multiclass avx512_vperm_bw<bits<8> opc, string OpcodeStr,
4364 Predicate prd, SDNode OpNode,
4365 AVX512VLVectorVTInfo _> {
4366 let Predicates = [prd] in
4367 defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>,
4368 EVEX_V512 ;
4369 let Predicates = [HasVLX, prd] in {
4370 defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>,
4371 EVEX_V256 ;
4372 defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, _.info128>,
4373 EVEX_V128 ;
4374 }
4375}
Elena Demikhovsky4078c752015-06-04 07:07:13 +00004376
Michael Zuckermand9cac592016-01-19 17:07:43 +00004377defm VPERMW : avx512_vperm_bw<0x8D, "vpermw", HasBWI, X86VPermv,
4378 avx512vl_i16_info>, VEX_W;
4379defm VPERMB : avx512_vperm_bw<0x8D, "vpermb", HasVBMI, X86VPermv,
4380 avx512vl_i8_info>;
Elena Demikhovsky4078c752015-06-04 07:07:13 +00004381
4382defm VPERMD : avx512_vperm_dq_sizes<0x36, "vpermd", X86VPermv,
4383 avx512vl_i32_info>;
4384defm VPERMQ : avx512_vperm_dq_sizes<0x36, "vpermq", X86VPermv,
4385 avx512vl_i64_info>, VEX_W;
4386defm VPERMPS : avx512_vperm_dq_sizes<0x16, "vpermps", X86VPermv,
4387 avx512vl_f32_info>;
4388defm VPERMPD : avx512_vperm_dq_sizes<0x16, "vpermpd", X86VPermv,
4389 avx512vl_f64_info>, VEX_W;
4390
4391defm VPERMQ : avx512_vpermi_dq_sizes<0x00, MRMSrcReg, MRMSrcMem, "vpermq",
4392 X86VPermi, avx512vl_i64_info>,
4393 EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, VEX_W;
4394defm VPERMPD : avx512_vpermi_dq_sizes<0x01, MRMSrcReg, MRMSrcMem, "vpermpd",
4395 X86VPermi, avx512vl_f64_info>,
4396 EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, VEX_W;
Igor Breger78741a12015-10-04 07:20:41 +00004397//===----------------------------------------------------------------------===//
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004398// AVX-512 - VPERMIL
Igor Breger78741a12015-10-04 07:20:41 +00004399//===----------------------------------------------------------------------===//
Elena Demikhovsky4078c752015-06-04 07:07:13 +00004400
Igor Breger78741a12015-10-04 07:20:41 +00004401multiclass avx512_permil_vec<bits<8> OpcVar, string OpcodeStr, SDNode OpNode,
4402 X86VectorVTInfo _, X86VectorVTInfo Ctrl> {
4403 defm rr: AVX512_maskable<OpcVar, MRMSrcReg, _, (outs _.RC:$dst),
4404 (ins _.RC:$src1, Ctrl.RC:$src2), OpcodeStr,
4405 "$src2, $src1", "$src1, $src2",
4406 (_.VT (OpNode _.RC:$src1,
4407 (Ctrl.VT Ctrl.RC:$src2)))>,
4408 T8PD, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00004409 defm rm: AVX512_maskable<OpcVar, MRMSrcMem, _, (outs _.RC:$dst),
4410 (ins _.RC:$src1, Ctrl.MemOp:$src2), OpcodeStr,
4411 "$src2, $src1", "$src1, $src2",
4412 (_.VT (OpNode
4413 _.RC:$src1,
4414 (Ctrl.VT (bitconvert(Ctrl.LdFrag addr:$src2)))))>,
4415 T8PD, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>;
4416 defm rmb: AVX512_maskable<OpcVar, MRMSrcMem, _, (outs _.RC:$dst),
4417 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
4418 "${src2}"##_.BroadcastStr##", $src1",
4419 "$src1, ${src2}"##_.BroadcastStr,
4420 (_.VT (OpNode
4421 _.RC:$src1,
4422 (Ctrl.VT (X86VBroadcast
4423 (Ctrl.ScalarLdFrag addr:$src2)))))>,
4424 T8PD, EVEX_4V, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>;
Igor Breger78741a12015-10-04 07:20:41 +00004425}
4426
4427multiclass avx512_permil_vec_common<string OpcodeStr, bits<8> OpcVar,
4428 AVX512VLVectorVTInfo _, AVX512VLVectorVTInfo Ctrl>{
4429 let Predicates = [HasAVX512] in {
4430 defm Z : avx512_permil_vec<OpcVar, OpcodeStr, X86VPermilpv, _.info512,
4431 Ctrl.info512>, EVEX_V512;
4432 }
4433 let Predicates = [HasAVX512, HasVLX] in {
4434 defm Z128 : avx512_permil_vec<OpcVar, OpcodeStr, X86VPermilpv, _.info128,
4435 Ctrl.info128>, EVEX_V128;
4436 defm Z256 : avx512_permil_vec<OpcVar, OpcodeStr, X86VPermilpv, _.info256,
4437 Ctrl.info256>, EVEX_V256;
4438 }
4439}
4440
4441multiclass avx512_permil<string OpcodeStr, bits<8> OpcImm, bits<8> OpcVar,
4442 AVX512VLVectorVTInfo _, AVX512VLVectorVTInfo Ctrl>{
4443
4444 defm NAME: avx512_permil_vec_common<OpcodeStr, OpcVar, _, Ctrl>;
4445 defm NAME: avx512_shift_rmi_sizes<OpcImm, MRMSrcReg, MRMSrcMem, OpcodeStr,
4446 X86VPermilpi, _>,
4447 EVEX, AVX512AIi8Base, EVEX_CD8<_.info128.EltSize, CD8VF>;
Igor Breger78741a12015-10-04 07:20:41 +00004448}
4449
4450defm VPERMILPS : avx512_permil<"vpermilps", 0x04, 0x0C, avx512vl_f32_info,
4451 avx512vl_i32_info>;
4452defm VPERMILPD : avx512_permil<"vpermilpd", 0x05, 0x0D, avx512vl_f64_info,
4453 avx512vl_i64_info>, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004454//===----------------------------------------------------------------------===//
Elena Demikhovsky75ede682015-06-01 07:17:23 +00004455// AVX-512 - VPSHUFD, VPSHUFLW, VPSHUFHW
4456//===----------------------------------------------------------------------===//
4457
4458defm VPSHUFD : avx512_shift_rmi_sizes<0x70, MRMSrcReg, MRMSrcMem, "vpshufd",
Michael Liao66233b72015-08-06 09:06:20 +00004459 X86PShufd, avx512vl_i32_info>,
Elena Demikhovsky75ede682015-06-01 07:17:23 +00004460 EVEX, AVX512BIi8Base, EVEX_CD8<32, CD8VF>;
4461defm VPSHUFH : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshufhw",
Igor Breger1a6fd1c2015-10-07 06:31:18 +00004462 X86PShufhw>, EVEX, AVX512XSIi8Base;
Elena Demikhovsky75ede682015-06-01 07:17:23 +00004463defm VPSHUFL : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshuflw",
Igor Breger1a6fd1c2015-10-07 06:31:18 +00004464 X86PShuflw>, EVEX, AVX512XDIi8Base;
Michael Liao66233b72015-08-06 09:06:20 +00004465
Elena Demikhovsky55a99742015-06-22 13:00:42 +00004466multiclass avx512_pshufb_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode> {
4467 let Predicates = [HasBWI] in
4468 defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, v64i8_info>, EVEX_V512;
4469
4470 let Predicates = [HasVLX, HasBWI] in {
4471 defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, v32i8x_info>, EVEX_V256;
4472 defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, v16i8x_info>, EVEX_V128;
4473 }
4474}
4475
4476defm VPSHUFB: avx512_pshufb_sizes<0x00, "vpshufb", X86pshufb>;
4477
Elena Demikhovsky75ede682015-06-01 07:17:23 +00004478//===----------------------------------------------------------------------===//
Elena Demikhovsky0a74b7d2013-11-14 11:29:27 +00004479// Move Low to High and High to Low packed FP Instructions
4480//===----------------------------------------------------------------------===//
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004481def VMOVLHPSZrr : AVX512PSI<0x16, MRMSrcReg, (outs VR128X:$dst),
4482 (ins VR128X:$src1, VR128X:$src2),
Elena Demikhovskycf088092013-12-11 14:31:04 +00004483 "vmovlhps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004484 [(set VR128X:$dst, (v4f32 (X86Movlhps VR128X:$src1, VR128X:$src2)))],
4485 IIC_SSE_MOV_LH>, EVEX_4V;
4486def VMOVHLPSZrr : AVX512PSI<0x12, MRMSrcReg, (outs VR128X:$dst),
4487 (ins VR128X:$src1, VR128X:$src2),
Elena Demikhovskycf088092013-12-11 14:31:04 +00004488 "vmovhlps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004489 [(set VR128X:$dst, (v4f32 (X86Movhlps VR128X:$src1, VR128X:$src2)))],
4490 IIC_SSE_MOV_LH>, EVEX_4V;
4491
Craig Topperdbe8b7d2013-09-27 07:20:47 +00004492let Predicates = [HasAVX512] in {
4493 // MOVLHPS patterns
4494 def : Pat<(v4i32 (X86Movlhps VR128X:$src1, VR128X:$src2)),
4495 (VMOVLHPSZrr VR128X:$src1, VR128X:$src2)>;
4496 def : Pat<(v2i64 (X86Movlhps VR128X:$src1, VR128X:$src2)),
4497 (VMOVLHPSZrr (v2i64 VR128X:$src1), VR128X:$src2)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004498
Craig Topperdbe8b7d2013-09-27 07:20:47 +00004499 // MOVHLPS patterns
4500 def : Pat<(v4i32 (X86Movhlps VR128X:$src1, VR128X:$src2)),
4501 (VMOVHLPSZrr VR128X:$src1, VR128X:$src2)>;
4502}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004503
4504//===----------------------------------------------------------------------===//
Igor Bregerb6b27af2015-11-10 07:09:07 +00004505// VMOVHPS/PD VMOVLPS Instructions
4506// All patterns was taken from SSS implementation.
4507//===----------------------------------------------------------------------===//
4508multiclass avx512_mov_hilo_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
4509 X86VectorVTInfo _> {
Craig Toppere1cac152016-06-07 07:27:54 +00004510 def rm : AVX512<opc, MRMSrcMem, (outs _.RC:$dst),
4511 (ins _.RC:$src1, f64mem:$src2),
4512 !strconcat(OpcodeStr,
4513 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
4514 [(set _.RC:$dst,
4515 (OpNode _.RC:$src1,
4516 (_.VT (bitconvert
4517 (v2f64 (scalar_to_vector (loadf64 addr:$src2)))))))],
4518 IIC_SSE_MOV_LH>, EVEX_4V;
Igor Bregerb6b27af2015-11-10 07:09:07 +00004519}
4520
4521defm VMOVHPSZ128 : avx512_mov_hilo_packed<0x16, "vmovhps", X86Movlhps,
4522 v4f32x_info>, EVEX_CD8<32, CD8VT2>, PS;
4523defm VMOVHPDZ128 : avx512_mov_hilo_packed<0x16, "vmovhpd", X86Movlhpd,
4524 v2f64x_info>, EVEX_CD8<64, CD8VT1>, PD, VEX_W;
4525defm VMOVLPSZ128 : avx512_mov_hilo_packed<0x12, "vmovlps", X86Movlps,
4526 v4f32x_info>, EVEX_CD8<32, CD8VT2>, PS;
4527defm VMOVLPDZ128 : avx512_mov_hilo_packed<0x12, "vmovlpd", X86Movlpd,
4528 v2f64x_info>, EVEX_CD8<64, CD8VT1>, PD, VEX_W;
4529
4530let Predicates = [HasAVX512] in {
4531 // VMOVHPS patterns
4532 def : Pat<(X86Movlhps VR128X:$src1,
4533 (bc_v4f32 (v2i64 (scalar_to_vector (loadi64 addr:$src2))))),
4534 (VMOVHPSZ128rm VR128X:$src1, addr:$src2)>;
4535 def : Pat<(X86Movlhps VR128X:$src1,
4536 (bc_v4i32 (v2i64 (X86vzload addr:$src2)))),
4537 (VMOVHPSZ128rm VR128X:$src1, addr:$src2)>;
4538 // VMOVHPD patterns
4539 def : Pat<(v2f64 (X86Unpckl VR128X:$src1,
4540 (scalar_to_vector (loadf64 addr:$src2)))),
4541 (VMOVHPDZ128rm VR128X:$src1, addr:$src2)>;
4542 def : Pat<(v2f64 (X86Unpckl VR128X:$src1,
4543 (bc_v2f64 (v2i64 (scalar_to_vector (loadi64 addr:$src2)))))),
4544 (VMOVHPDZ128rm VR128X:$src1, addr:$src2)>;
4545 // VMOVLPS patterns
4546 def : Pat<(v4f32 (X86Movlps VR128X:$src1, (load addr:$src2))),
4547 (VMOVLPSZ128rm VR128X:$src1, addr:$src2)>;
4548 def : Pat<(v4i32 (X86Movlps VR128X:$src1, (load addr:$src2))),
4549 (VMOVLPSZ128rm VR128X:$src1, addr:$src2)>;
4550 // VMOVLPD patterns
4551 def : Pat<(v2f64 (X86Movlpd VR128X:$src1, (load addr:$src2))),
4552 (VMOVLPDZ128rm VR128X:$src1, addr:$src2)>;
4553 def : Pat<(v2i64 (X86Movlpd VR128X:$src1, (load addr:$src2))),
4554 (VMOVLPDZ128rm VR128X:$src1, addr:$src2)>;
4555 def : Pat<(v2f64 (X86Movsd VR128X:$src1,
4556 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))),
4557 (VMOVLPDZ128rm VR128X:$src1, addr:$src2)>;
4558}
4559
Igor Bregerb6b27af2015-11-10 07:09:07 +00004560def VMOVHPSZ128mr : AVX512PSI<0x17, MRMDestMem, (outs),
4561 (ins f64mem:$dst, VR128X:$src),
4562 "vmovhps\t{$src, $dst|$dst, $src}",
Craig Topperc9b19232016-05-01 04:59:44 +00004563 [(store (f64 (extractelt
Igor Bregerb6b27af2015-11-10 07:09:07 +00004564 (X86Unpckh (bc_v2f64 (v4f32 VR128X:$src)),
4565 (bc_v2f64 (v4f32 VR128X:$src))),
4566 (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>,
4567 EVEX, EVEX_CD8<32, CD8VT2>;
4568def VMOVHPDZ128mr : AVX512PDI<0x17, MRMDestMem, (outs),
4569 (ins f64mem:$dst, VR128X:$src),
4570 "vmovhpd\t{$src, $dst|$dst, $src}",
Craig Topperc9b19232016-05-01 04:59:44 +00004571 [(store (f64 (extractelt
Igor Bregerb6b27af2015-11-10 07:09:07 +00004572 (v2f64 (X86Unpckh VR128X:$src, VR128X:$src)),
4573 (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>,
4574 EVEX, EVEX_CD8<64, CD8VT1>, VEX_W;
4575def VMOVLPSZ128mr : AVX512PSI<0x13, MRMDestMem, (outs),
4576 (ins f64mem:$dst, VR128X:$src),
4577 "vmovlps\t{$src, $dst|$dst, $src}",
Craig Topperc9b19232016-05-01 04:59:44 +00004578 [(store (f64 (extractelt (bc_v2f64 (v4f32 VR128X:$src)),
Igor Bregerb6b27af2015-11-10 07:09:07 +00004579 (iPTR 0))), addr:$dst)],
4580 IIC_SSE_MOV_LH>,
4581 EVEX, EVEX_CD8<32, CD8VT2>;
4582def VMOVLPDZ128mr : AVX512PDI<0x13, MRMDestMem, (outs),
4583 (ins f64mem:$dst, VR128X:$src),
4584 "vmovlpd\t{$src, $dst|$dst, $src}",
Craig Topperc9b19232016-05-01 04:59:44 +00004585 [(store (f64 (extractelt (v2f64 VR128X:$src),
Igor Bregerb6b27af2015-11-10 07:09:07 +00004586 (iPTR 0))), addr:$dst)],
4587 IIC_SSE_MOV_LH>,
4588 EVEX, EVEX_CD8<64, CD8VT1>, VEX_W;
Craig Toppere1cac152016-06-07 07:27:54 +00004589
Igor Bregerb6b27af2015-11-10 07:09:07 +00004590let Predicates = [HasAVX512] in {
4591 // VMOVHPD patterns
Craig Topperc9b19232016-05-01 04:59:44 +00004592 def : Pat<(store (f64 (extractelt
Igor Bregerb6b27af2015-11-10 07:09:07 +00004593 (v2f64 (X86VPermilpi VR128X:$src, (i8 1))),
4594 (iPTR 0))), addr:$dst),
4595 (VMOVHPDZ128mr addr:$dst, VR128X:$src)>;
4596 // VMOVLPS patterns
4597 def : Pat<(store (v4f32 (X86Movlps (load addr:$src1), VR128X:$src2)),
4598 addr:$src1),
4599 (VMOVLPSZ128mr addr:$src1, VR128X:$src2)>;
4600 def : Pat<(store (v4i32 (X86Movlps
4601 (bc_v4i32 (loadv2i64 addr:$src1)), VR128X:$src2)), addr:$src1),
4602 (VMOVLPSZ128mr addr:$src1, VR128X:$src2)>;
4603 // VMOVLPD patterns
4604 def : Pat<(store (v2f64 (X86Movlpd (load addr:$src1), VR128X:$src2)),
4605 addr:$src1),
4606 (VMOVLPDZ128mr addr:$src1, VR128X:$src2)>;
4607 def : Pat<(store (v2i64 (X86Movlpd (load addr:$src1), VR128X:$src2)),
4608 addr:$src1),
4609 (VMOVLPDZ128mr addr:$src1, VR128X:$src2)>;
4610}
4611//===----------------------------------------------------------------------===//
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004612// FMA - Fused Multiply Operations
4613//
Adam Nemet26371ce2014-10-24 00:02:55 +00004614
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004615let Constraints = "$src1 = $dst" in {
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004616multiclass avx512_fma3p_213_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4617 X86VectorVTInfo _> {
Adam Nemet34801422014-10-08 23:25:39 +00004618 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
Adam Nemet6bddb8c2014-09-29 22:54:41 +00004619 (ins _.RC:$src2, _.RC:$src3),
Adam Nemet2e91ee52014-08-14 17:13:19 +00004620 OpcodeStr, "$src3, $src2", "$src2, $src3",
Adam Nemet6bddb8c2014-09-29 22:54:41 +00004621 (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>,
Adam Nemet2e91ee52014-08-14 17:13:19 +00004622 AVX512FMA3Base;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004623
Craig Toppere1cac152016-06-07 07:27:54 +00004624 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
4625 (ins _.RC:$src2, _.MemOp:$src3),
4626 OpcodeStr, "$src3, $src2", "$src2, $src3",
4627 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (_.LdFrag addr:$src3)))>,
4628 AVX512FMA3Base;
Elena Demikhovskyfcea06a2014-12-23 10:30:39 +00004629
Craig Toppere1cac152016-06-07 07:27:54 +00004630 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
4631 (ins _.RC:$src2, _.ScalarMemOp:$src3),
4632 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),
4633 !strconcat("$src2, ${src3}", _.BroadcastStr ),
4634 (OpNode _.RC:$src1,
4635 _.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))))>,
4636 AVX512FMA3Base, EVEX_B;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004637}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004638
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004639multiclass avx512_fma3_213_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
4640 X86VectorVTInfo _> {
4641 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
Elena Demikhovsky7b0dd392015-01-28 10:21:27 +00004642 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
4643 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",
4644 (_.VT ( OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3, (i32 imm:$rc)))>,
4645 AVX512FMA3Base, EVEX_B, EVEX_RC;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004646}
Elena Demikhovsky7b0dd392015-01-28 10:21:27 +00004647} // Constraints = "$src1 = $dst"
4648
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004649multiclass avx512_fma3p_213_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
4650 SDNode OpNodeRnd, AVX512VLVectorVTInfo _> {
4651 let Predicates = [HasAVX512] in {
4652 defm Z : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info512>,
4653 avx512_fma3_213_round<opc, OpcodeStr, OpNodeRnd, _.info512>,
4654 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
Elena Demikhovskyfcea06a2014-12-23 10:30:39 +00004655 }
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004656 let Predicates = [HasVLX, HasAVX512] in {
4657 defm Z256 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info256>,
4658 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
4659 defm Z128 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info128>,
4660 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
Elena Demikhovskyfcea06a2014-12-23 10:30:39 +00004661 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004662}
4663
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004664multiclass avx512_fma3p_213_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
4665 SDNode OpNodeRnd > {
4666 defm PS : avx512_fma3p_213_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
4667 avx512vl_f32_info>;
4668 defm PD : avx512_fma3p_213_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
4669 avx512vl_f64_info>, VEX_W;
4670}
4671
4672defm VFMADD213 : avx512_fma3p_213_f<0xA8, "vfmadd213", X86Fmadd, X86FmaddRnd>;
4673defm VFMSUB213 : avx512_fma3p_213_f<0xAA, "vfmsub213", X86Fmsub, X86FmsubRnd>;
4674defm VFMADDSUB213 : avx512_fma3p_213_f<0xA6, "vfmaddsub213", X86Fmaddsub, X86FmaddsubRnd>;
4675defm VFMSUBADD213 : avx512_fma3p_213_f<0xA7, "vfmsubadd213", X86Fmsubadd, X86FmsubaddRnd>;
4676defm VFNMADD213 : avx512_fma3p_213_f<0xAC, "vfnmadd213", X86Fnmadd, X86FnmaddRnd>;
4677defm VFNMSUB213 : avx512_fma3p_213_f<0xAE, "vfnmsub213", X86Fnmsub, X86FnmsubRnd>;
4678
Elena Demikhovskyfcea06a2014-12-23 10:30:39 +00004679
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004680let Constraints = "$src1 = $dst" in {
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004681multiclass avx512_fma3p_231_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4682 X86VectorVTInfo _> {
4683 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
4684 (ins _.RC:$src2, _.RC:$src3),
4685 OpcodeStr, "$src3, $src2", "$src2, $src3",
4686 (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1))>,
4687 AVX512FMA3Base;
4688
Craig Toppere1cac152016-06-07 07:27:54 +00004689 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
4690 (ins _.RC:$src2, _.MemOp:$src3),
4691 OpcodeStr, "$src3, $src2", "$src2, $src3",
4692 (_.VT (OpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1))>,
4693 AVX512FMA3Base;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004694
Craig Toppere1cac152016-06-07 07:27:54 +00004695 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
4696 (ins _.RC:$src2, _.ScalarMemOp:$src3),
4697 OpcodeStr, "${src3}"##_.BroadcastStr##", $src2",
4698 "$src2, ${src3}"##_.BroadcastStr,
4699 (_.VT (OpNode _.RC:$src2,
4700 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
4701 _.RC:$src1))>, AVX512FMA3Base, EVEX_B;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004702}
4703
4704multiclass avx512_fma3_231_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
4705 X86VectorVTInfo _> {
4706 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
4707 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
4708 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",
4709 (_.VT ( OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1, (i32 imm:$rc)))>,
4710 AVX512FMA3Base, EVEX_B, EVEX_RC;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004711}
4712} // Constraints = "$src1 = $dst"
4713
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004714multiclass avx512_fma3p_231_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
4715 SDNode OpNodeRnd, AVX512VLVectorVTInfo _> {
4716 let Predicates = [HasAVX512] in {
4717 defm Z : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info512>,
4718 avx512_fma3_231_round<opc, OpcodeStr, OpNodeRnd, _.info512>,
4719 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
Elena Demikhovskyfcea06a2014-12-23 10:30:39 +00004720 }
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004721 let Predicates = [HasVLX, HasAVX512] in {
4722 defm Z256 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info256>,
4723 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
4724 defm Z128 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info128>,
4725 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
Elena Demikhovskyfcea06a2014-12-23 10:30:39 +00004726 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004727}
4728
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004729multiclass avx512_fma3p_231_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
4730 SDNode OpNodeRnd > {
4731 defm PS : avx512_fma3p_231_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
4732 avx512vl_f32_info>;
4733 defm PD : avx512_fma3p_231_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
4734 avx512vl_f64_info>, VEX_W;
4735}
4736
4737defm VFMADD231 : avx512_fma3p_231_f<0xB8, "vfmadd231", X86Fmadd, X86FmaddRnd>;
4738defm VFMSUB231 : avx512_fma3p_231_f<0xBA, "vfmsub231", X86Fmsub, X86FmsubRnd>;
4739defm VFMADDSUB231 : avx512_fma3p_231_f<0xB6, "vfmaddsub231", X86Fmaddsub, X86FmaddsubRnd>;
4740defm VFMSUBADD231 : avx512_fma3p_231_f<0xB7, "vfmsubadd231", X86Fmsubadd, X86FmsubaddRnd>;
4741defm VFNMADD231 : avx512_fma3p_231_f<0xBC, "vfnmadd231", X86Fnmadd, X86FnmaddRnd>;
4742defm VFNMSUB231 : avx512_fma3p_231_f<0xBE, "vfnmsub231", X86Fnmsub, X86FnmsubRnd>;
4743
4744let Constraints = "$src1 = $dst" in {
4745multiclass avx512_fma3p_132_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4746 X86VectorVTInfo _> {
4747 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
4748 (ins _.RC:$src3, _.RC:$src2),
4749 OpcodeStr, "$src2, $src3", "$src3, $src2",
4750 (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>,
4751 AVX512FMA3Base;
4752
Craig Toppere1cac152016-06-07 07:27:54 +00004753 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
4754 (ins _.RC:$src3, _.MemOp:$src2),
4755 OpcodeStr, "$src2, $src3", "$src3, $src2",
4756 (_.VT (OpNode _.RC:$src1, (_.LdFrag addr:$src2), _.RC:$src3))>,
4757 AVX512FMA3Base;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004758
Craig Toppere1cac152016-06-07 07:27:54 +00004759 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
4760 (ins _.RC:$src3, _.ScalarMemOp:$src2),
4761 OpcodeStr, "${src2}"##_.BroadcastStr##", $src3",
4762 "$src3, ${src2}"##_.BroadcastStr,
4763 (_.VT (OpNode _.RC:$src1,
4764 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
4765 _.RC:$src3))>, AVX512FMA3Base, EVEX_B;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00004766}
4767
4768multiclass avx512_fma3_132_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
4769 X86VectorVTInfo _> {
4770 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
4771 (ins _.RC:$src3, _.RC:$src2, AVX512RC:$rc),
4772 OpcodeStr, "$rc, $src2, $src3", "$src3, $src2, $rc",
4773 (_.VT ( OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3, (i32 imm:$rc)))>,
4774 AVX512FMA3Base, EVEX_B, EVEX_RC;
4775}
4776} // Constraints = "$src1 = $dst"
4777
4778multiclass avx512_fma3p_132_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
4779 SDNode OpNodeRnd, AVX512VLVectorVTInfo _> {
4780 let Predicates = [HasAVX512] in {
4781 defm Z : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info512>,
4782 avx512_fma3_132_round<opc, OpcodeStr, OpNodeRnd, _.info512>,
4783 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
4784 }
4785 let Predicates = [HasVLX, HasAVX512] in {
4786 defm Z256 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info256>,
4787 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
4788 defm Z128 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info128>,
4789 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
4790 }
4791}
4792
4793multiclass avx512_fma3p_132_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
4794 SDNode OpNodeRnd > {
4795 defm PS : avx512_fma3p_132_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
4796 avx512vl_f32_info>;
4797 defm PD : avx512_fma3p_132_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
4798 avx512vl_f64_info>, VEX_W;
4799}
4800
4801defm VFMADD132 : avx512_fma3p_132_f<0x98, "vfmadd132", X86Fmadd, X86FmaddRnd>;
4802defm VFMSUB132 : avx512_fma3p_132_f<0x9A, "vfmsub132", X86Fmsub, X86FmsubRnd>;
4803defm VFMADDSUB132 : avx512_fma3p_132_f<0x96, "vfmaddsub132", X86Fmaddsub, X86FmaddsubRnd>;
4804defm VFMSUBADD132 : avx512_fma3p_132_f<0x97, "vfmsubadd132", X86Fmsubadd, X86FmsubaddRnd>;
4805defm VFNMADD132 : avx512_fma3p_132_f<0x9C, "vfnmadd132", X86Fnmadd, X86FnmaddRnd>;
4806defm VFNMSUB132 : avx512_fma3p_132_f<0x9E, "vfnmsub132", X86Fnmsub, X86FnmsubRnd>;
Elena Demikhovskyfcea06a2014-12-23 10:30:39 +00004807
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004808// Scalar FMA
4809let Constraints = "$src1 = $dst" in {
Igor Breger15820b02015-07-01 13:24:28 +00004810multiclass avx512_fma3s_common<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
4811 dag RHS_VEC_r, dag RHS_VEC_m, dag RHS_VEC_rb,
4812 dag RHS_r, dag RHS_m > {
4813 defm r_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
4814 (ins _.RC:$src2, _.RC:$src3), OpcodeStr,
4815 "$src3, $src2", "$src2, $src3", RHS_VEC_r>, AVX512FMA3Base;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004816
Craig Toppere1cac152016-06-07 07:27:54 +00004817 defm m_Int: AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
4818 (ins _.RC:$src2, _.ScalarMemOp:$src3), OpcodeStr,
4819 "$src3, $src2", "$src2, $src3", RHS_VEC_m>, AVX512FMA3Base;
Igor Breger15820b02015-07-01 13:24:28 +00004820
4821 defm rb_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
4822 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
4823 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc", RHS_VEC_rb>,
4824 AVX512FMA3Base, EVEX_B, EVEX_RC;
4825
4826 let isCodeGenOnly = 1 in {
4827 def r : AVX512FMA3<opc, MRMSrcReg, (outs _.FRC:$dst),
4828 (ins _.FRC:$src1, _.FRC:$src2, _.FRC:$src3),
4829 !strconcat(OpcodeStr,
4830 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
4831 [RHS_r]>;
Craig Toppere1cac152016-06-07 07:27:54 +00004832 def m : AVX512FMA3<opc, MRMSrcMem, (outs _.FRC:$dst),
4833 (ins _.FRC:$src1, _.FRC:$src2, _.ScalarMemOp:$src3),
4834 !strconcat(OpcodeStr,
4835 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
4836 [RHS_m]>;
Igor Breger15820b02015-07-01 13:24:28 +00004837 }// isCodeGenOnly = 1
4838}
4839}// Constraints = "$src1 = $dst"
4840
4841multiclass avx512_fma3s_all<bits<8> opc213, bits<8> opc231, bits<8> opc132,
4842 string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd, X86VectorVTInfo _ ,
4843 string SUFF> {
4844
4845 defm NAME#213#SUFF: avx512_fma3s_common<opc213, OpcodeStr#"213"#_.Suffix , _ ,
Michael Zuckerman7d733602016-02-04 14:41:08 +00004846 (_.VT (OpNodeRnd _.RC:$src2, _.RC:$src1, _.RC:$src3, (i32 FROUND_CURRENT))),
4847 (_.VT (OpNodeRnd _.RC:$src2, _.RC:$src1,
4848 (_.VT (scalar_to_vector(_.ScalarLdFrag addr:$src3))), (i32 FROUND_CURRENT))),
Igor Breger15820b02015-07-01 13:24:28 +00004849 (_.VT ( OpNodeRnd _.RC:$src2, _.RC:$src1, _.RC:$src3,
4850 (i32 imm:$rc))),
4851 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1,
4852 _.FRC:$src3))),
4853 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1,
4854 (_.ScalarLdFrag addr:$src3))))>;
4855
4856 defm NAME#231#SUFF: avx512_fma3s_common<opc231, OpcodeStr#"231"#_.Suffix , _ ,
Michael Zuckerman7d733602016-02-04 14:41:08 +00004857 (_.VT (OpNodeRnd _.RC:$src2, _.RC:$src3, _.RC:$src1, (i32 FROUND_CURRENT))),
4858 (_.VT (OpNodeRnd _.RC:$src2,
Igor Breger15820b02015-07-01 13:24:28 +00004859 (_.VT (scalar_to_vector(_.ScalarLdFrag addr:$src3))),
Michael Zuckerman7d733602016-02-04 14:41:08 +00004860 _.RC:$src1, (i32 FROUND_CURRENT))),
Igor Breger15820b02015-07-01 13:24:28 +00004861 (_.VT ( OpNodeRnd _.RC:$src2, _.RC:$src3, _.RC:$src1,
4862 (i32 imm:$rc))),
4863 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src3,
4864 _.FRC:$src1))),
4865 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2,
4866 (_.ScalarLdFrag addr:$src3), _.FRC:$src1)))>;
4867
4868 defm NAME#132#SUFF: avx512_fma3s_common<opc132, OpcodeStr#"132"#_.Suffix , _ ,
Michael Zuckerman7d733602016-02-04 14:41:08 +00004869 (_.VT (OpNodeRnd _.RC:$src1, _.RC:$src3, _.RC:$src2, (i32 FROUND_CURRENT))),
4870 (_.VT (OpNodeRnd _.RC:$src1,
Igor Breger15820b02015-07-01 13:24:28 +00004871 (_.VT (scalar_to_vector(_.ScalarLdFrag addr:$src3))),
Michael Zuckerman7d733602016-02-04 14:41:08 +00004872 _.RC:$src2, (i32 FROUND_CURRENT))),
Igor Breger15820b02015-07-01 13:24:28 +00004873 (_.VT ( OpNodeRnd _.RC:$src1, _.RC:$src3, _.RC:$src2,
4874 (i32 imm:$rc))),
4875 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1, _.FRC:$src3,
4876 _.FRC:$src2))),
4877 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1,
4878 (_.ScalarLdFrag addr:$src3), _.FRC:$src2)))>;
4879}
4880
4881multiclass avx512_fma3s<bits<8> opc213, bits<8> opc231, bits<8> opc132,
4882 string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd>{
4883 let Predicates = [HasAVX512] in {
4884 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,
4885 OpNodeRnd, f32x_info, "SS">,
4886 EVEX_CD8<32, CD8VT1>, VEX_LIG;
4887 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,
4888 OpNodeRnd, f64x_info, "SD">,
4889 EVEX_CD8<64, CD8VT1>, VEX_LIG, VEX_W;
4890 }
4891}
4892
4893defm VFMADD : avx512_fma3s<0xA9, 0xB9, 0x99, "vfmadd", X86Fmadd, X86FmaddRnd>;
4894defm VFMSUB : avx512_fma3s<0xAB, 0xBB, 0x9B, "vfmsub", X86Fmsub, X86FmsubRnd>;
4895defm VFNMADD : avx512_fma3s<0xAD, 0xBD, 0x9D, "vfnmadd", X86Fnmadd, X86FnmaddRnd>;
4896defm VFNMSUB : avx512_fma3s<0xAF, 0xBF, 0x9F, "vfnmsub", X86Fnmsub, X86FnmsubRnd>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004897
4898//===----------------------------------------------------------------------===//
Asaf Badouh655822a2016-01-25 11:14:24 +00004899// AVX-512 Packed Multiply of Unsigned 52-bit Integers and Add the Low 52-bit IFMA
4900//===----------------------------------------------------------------------===//
4901let Constraints = "$src1 = $dst" in {
4902multiclass avx512_pmadd52_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4903 X86VectorVTInfo _> {
4904 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
4905 (ins _.RC:$src2, _.RC:$src3),
4906 OpcodeStr, "$src3, $src2", "$src2, $src3",
4907 (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>,
4908 AVX512FMA3Base;
4909
Craig Toppere1cac152016-06-07 07:27:54 +00004910 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
4911 (ins _.RC:$src2, _.MemOp:$src3),
4912 OpcodeStr, "$src3, $src2", "$src2, $src3",
4913 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (_.LdFrag addr:$src3)))>,
4914 AVX512FMA3Base;
Asaf Badouh655822a2016-01-25 11:14:24 +00004915
Craig Toppere1cac152016-06-07 07:27:54 +00004916 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
4917 (ins _.RC:$src2, _.ScalarMemOp:$src3),
4918 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),
4919 !strconcat("$src2, ${src3}", _.BroadcastStr ),
4920 (OpNode _.RC:$src1,
4921 _.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))))>,
4922 AVX512FMA3Base, EVEX_B;
Asaf Badouh655822a2016-01-25 11:14:24 +00004923}
4924} // Constraints = "$src1 = $dst"
4925
4926multiclass avx512_pmadd52_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
4927 AVX512VLVectorVTInfo _> {
4928 let Predicates = [HasIFMA] in {
4929 defm Z : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, _.info512>,
4930 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
4931 }
4932 let Predicates = [HasVLX, HasIFMA] in {
4933 defm Z256 : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, _.info256>,
4934 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
4935 defm Z128 : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, _.info128>,
4936 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
4937 }
4938}
4939
4940defm VPMADD52LUQ : avx512_pmadd52_common<0xb4, "vpmadd52luq", x86vpmadd52l,
4941 avx512vl_i64_info>, VEX_W;
4942defm VPMADD52HUQ : avx512_pmadd52_common<0xb5, "vpmadd52huq", x86vpmadd52h,
4943 avx512vl_i64_info>, VEX_W;
4944
4945//===----------------------------------------------------------------------===//
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004946// AVX-512 Scalar convert from sign integer to float/double
4947//===----------------------------------------------------------------------===//
4948
Igor Bregerdfcc3d32015-06-17 07:23:57 +00004949multiclass avx512_vcvtsi<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
4950 X86VectorVTInfo DstVT, X86MemOperand x86memop,
4951 PatFrag ld_frag, string asm> {
4952 let hasSideEffects = 0 in {
4953 def rr : SI<opc, MRMSrcReg, (outs DstVT.FRC:$dst),
4954 (ins DstVT.FRC:$src1, SrcRC:$src),
Craig Topperedb09112014-11-25 20:11:23 +00004955 !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00004956 EVEX_4V;
Igor Bregerdfcc3d32015-06-17 07:23:57 +00004957 let mayLoad = 1 in
4958 def rm : SI<opc, MRMSrcMem, (outs DstVT.FRC:$dst),
4959 (ins DstVT.FRC:$src1, x86memop:$src),
Craig Topperedb09112014-11-25 20:11:23 +00004960 !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00004961 EVEX_4V;
Igor Bregerdfcc3d32015-06-17 07:23:57 +00004962 } // hasSideEffects = 0
4963 let isCodeGenOnly = 1 in {
4964 def rr_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst),
4965 (ins DstVT.RC:$src1, SrcRC:$src2),
4966 !strconcat(asm,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
4967 [(set DstVT.RC:$dst,
4968 (OpNode (DstVT.VT DstVT.RC:$src1),
4969 SrcRC:$src2,
4970 (i32 FROUND_CURRENT)))]>, EVEX_4V;
4971
4972 def rm_Int : SI<opc, MRMSrcMem, (outs DstVT.RC:$dst),
4973 (ins DstVT.RC:$src1, x86memop:$src2),
4974 !strconcat(asm,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
4975 [(set DstVT.RC:$dst,
4976 (OpNode (DstVT.VT DstVT.RC:$src1),
4977 (ld_frag addr:$src2),
4978 (i32 FROUND_CURRENT)))]>, EVEX_4V;
4979 }//isCodeGenOnly = 1
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004980}
Elena Demikhovskyd8fda622015-03-30 09:29:28 +00004981
Igor Bregerabe4a792015-06-14 12:44:55 +00004982multiclass avx512_vcvtsi_round<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00004983 X86VectorVTInfo DstVT, string asm> {
Igor Bregerabe4a792015-06-14 12:44:55 +00004984 def rrb_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst),
4985 (ins DstVT.RC:$src1, SrcRC:$src2, AVX512RC:$rc),
Igor Bregerdfcc3d32015-06-17 07:23:57 +00004986 !strconcat(asm,
4987 "\t{$src2, $rc, $src1, $dst|$dst, $src1, $rc, $src2}"),
Igor Bregerabe4a792015-06-14 12:44:55 +00004988 [(set DstVT.RC:$dst,
4989 (OpNode (DstVT.VT DstVT.RC:$src1),
4990 SrcRC:$src2,
4991 (i32 imm:$rc)))]>, EVEX_4V, EVEX_B, EVEX_RC;
4992}
4993
4994multiclass avx512_vcvtsi_common<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00004995 X86VectorVTInfo DstVT, X86MemOperand x86memop,
4996 PatFrag ld_frag, string asm> {
4997 defm NAME : avx512_vcvtsi_round<opc, OpNode, SrcRC, DstVT, asm>,
4998 avx512_vcvtsi<opc, OpNode, SrcRC, DstVT, x86memop, ld_frag, asm>,
4999 VEX_LIG;
Igor Bregerabe4a792015-06-14 12:44:55 +00005000}
5001
Andrew Trick15a47742013-10-09 05:11:10 +00005002let Predicates = [HasAVX512] in {
Igor Bregerabe4a792015-06-14 12:44:55 +00005003defm VCVTSI2SSZ : avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR32,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00005004 v4f32x_info, i32mem, loadi32, "cvtsi2ss{l}">,
5005 XS, EVEX_CD8<32, CD8VT1>;
Igor Bregerabe4a792015-06-14 12:44:55 +00005006defm VCVTSI642SSZ: avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR64,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00005007 v4f32x_info, i64mem, loadi64, "cvtsi2ss{q}">,
5008 XS, VEX_W, EVEX_CD8<64, CD8VT1>;
Igor Bregerabe4a792015-06-14 12:44:55 +00005009defm VCVTSI2SDZ : avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR32,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00005010 v2f64x_info, i32mem, loadi32, "cvtsi2sd{l}">,
5011 XD, EVEX_CD8<32, CD8VT1>;
Igor Bregerabe4a792015-06-14 12:44:55 +00005012defm VCVTSI642SDZ: avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR64,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00005013 v2f64x_info, i64mem, loadi64, "cvtsi2sd{q}">,
5014 XD, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005015
5016def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))),
5017 (VCVTSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
5018def : Pat<(f32 (sint_to_fp (loadi64 addr:$src))),
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005019 (VCVTSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005020def : Pat<(f64 (sint_to_fp (loadi32 addr:$src))),
5021 (VCVTSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
5022def : Pat<(f64 (sint_to_fp (loadi64 addr:$src))),
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005023 (VCVTSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005024
5025def : Pat<(f32 (sint_to_fp GR32:$src)),
5026 (VCVTSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>;
5027def : Pat<(f32 (sint_to_fp GR64:$src)),
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005028 (VCVTSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005029def : Pat<(f64 (sint_to_fp GR32:$src)),
5030 (VCVTSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>;
5031def : Pat<(f64 (sint_to_fp GR64:$src)),
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005032 (VCVTSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>;
5033
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005034defm VCVTUSI2SSZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR32,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00005035 v4f32x_info, i32mem, loadi32,
5036 "cvtusi2ss{l}">, XS, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005037defm VCVTUSI642SSZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR64,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00005038 v4f32x_info, i64mem, loadi64, "cvtusi2ss{q}">,
5039 XS, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005040defm VCVTUSI2SDZ : avx512_vcvtsi<0x7B, X86UintToFpRnd, GR32, v2f64x_info,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00005041 i32mem, loadi32, "cvtusi2sd{l}">,
5042 XD, VEX_LIG, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005043defm VCVTUSI642SDZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR64,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00005044 v2f64x_info, i64mem, loadi64, "cvtusi2sd{q}">,
5045 XD, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005046
5047def : Pat<(f32 (uint_to_fp (loadi32 addr:$src))),
5048 (VCVTUSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
5049def : Pat<(f32 (uint_to_fp (loadi64 addr:$src))),
5050 (VCVTUSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
5051def : Pat<(f64 (uint_to_fp (loadi32 addr:$src))),
5052 (VCVTUSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
5053def : Pat<(f64 (uint_to_fp (loadi64 addr:$src))),
5054 (VCVTUSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
5055
5056def : Pat<(f32 (uint_to_fp GR32:$src)),
5057 (VCVTUSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>;
5058def : Pat<(f32 (uint_to_fp GR64:$src)),
5059 (VCVTUSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>;
5060def : Pat<(f64 (uint_to_fp GR32:$src)),
5061 (VCVTUSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>;
5062def : Pat<(f64 (uint_to_fp GR64:$src)),
5063 (VCVTUSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>;
Andrew Trick15a47742013-10-09 05:11:10 +00005064}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005065
5066//===----------------------------------------------------------------------===//
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005067// AVX-512 Scalar convert from float/double to integer
5068//===----------------------------------------------------------------------===//
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00005069multiclass avx512_cvt_s_int_round<bits<8> opc, X86VectorVTInfo SrcVT ,
5070 X86VectorVTInfo DstVT, SDNode OpNode, string asm> {
Craig Toppere1cac152016-06-07 07:27:54 +00005071 let Predicates = [HasAVX512] in {
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00005072 def rr : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst), (ins SrcVT.RC:$src),
Asaf Badouh2744d212015-09-20 14:31:19 +00005073 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00005074 [(set DstVT.RC:$dst, (OpNode (SrcVT.VT SrcVT.RC:$src),(i32 FROUND_CURRENT)))]>,
5075 EVEX, VEX_LIG;
5076 def rb : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst), (ins SrcVT.RC:$src, AVX512RC:$rc),
5077 !strconcat(asm,"\t{$rc, $src, $dst|$dst, $src, $rc}"),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00005078 [(set DstVT.RC:$dst, (OpNode (SrcVT.VT SrcVT.RC:$src),(i32 imm:$rc)))]>,
Asaf Badouh2744d212015-09-20 14:31:19 +00005079 EVEX, VEX_LIG, EVEX_B, EVEX_RC;
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00005080 def rm : SI<opc, MRMSrcMem, (outs DstVT.RC:$dst), (ins SrcVT.ScalarMemOp:$src),
5081 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00005082 [(set DstVT.RC:$dst, (OpNode
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00005083 (SrcVT.VT (scalar_to_vector (SrcVT.ScalarLdFrag addr:$src))),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00005084 (i32 FROUND_CURRENT)))]>,
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00005085 EVEX, VEX_LIG;
Simon Pilgrimb13961d2016-06-11 14:34:10 +00005086 } // Predicates = [HasAVX512]
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005087}
Asaf Badouh2744d212015-09-20 14:31:19 +00005088
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005089// Convert float/double to signed/unsigned int 32/64
Simon Pilgrimb13961d2016-06-11 14:34:10 +00005090defm VCVTSS2SIZ: avx512_cvt_s_int_round<0x2D, f32x_info, i32x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005091 X86cvts2si, "cvtss2si">,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005092 XS, EVEX_CD8<32, CD8VT1>;
Simon Pilgrimb13961d2016-06-11 14:34:10 +00005093defm VCVTSS2SI64Z: avx512_cvt_s_int_round<0x2D, f32x_info, i64x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005094 X86cvts2si, "cvtss2si">,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005095 XS, VEX_W, EVEX_CD8<32, CD8VT1>;
Simon Pilgrimb13961d2016-06-11 14:34:10 +00005096defm VCVTSS2USIZ: avx512_cvt_s_int_round<0x79, f32x_info, i32x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005097 X86cvts2usi, "cvtss2usi">,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005098 XS, EVEX_CD8<32, CD8VT1>;
Simon Pilgrimb13961d2016-06-11 14:34:10 +00005099defm VCVTSS2USI64Z: avx512_cvt_s_int_round<0x79, f32x_info, i64x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005100 X86cvts2usi, "cvtss2usi">, XS, VEX_W,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005101 EVEX_CD8<32, CD8VT1>;
Simon Pilgrimb13961d2016-06-11 14:34:10 +00005102defm VCVTSD2SIZ: avx512_cvt_s_int_round<0x2D, f64x_info, i32x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005103 X86cvts2si, "cvtsd2si">,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005104 XD, EVEX_CD8<64, CD8VT1>;
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00005105defm VCVTSD2SI64Z: avx512_cvt_s_int_round<0x2D, f64x_info, i64x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005106 X86cvts2si, "cvtsd2si">,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005107 XD, VEX_W, EVEX_CD8<64, CD8VT1>;
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00005108defm VCVTSD2USIZ: avx512_cvt_s_int_round<0x79, f64x_info, i32x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005109 X86cvts2usi, "cvtsd2usi">,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005110 XD, EVEX_CD8<64, CD8VT1>;
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00005111defm VCVTSD2USI64Z: avx512_cvt_s_int_round<0x79, f64x_info, i64x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005112 X86cvts2usi, "cvtsd2usi">, XD, VEX_W,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005113 EVEX_CD8<64, CD8VT1>;
5114
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00005115// The SSE version of these instructions are disabled for AVX512.
5116// Therefore, the SSE intrinsics are mapped to the AVX512 instructions.
5117let Predicates = [HasAVX512] in {
5118 def : Pat<(i32 (int_x86_sse_cvtss2si (v4f32 VR128X:$src))),
5119 (VCVTSS2SIZrr (COPY_TO_REGCLASS VR128X:$src, FR32X))>;
5120 def : Pat<(i64 (int_x86_sse_cvtss2si64 (v4f32 VR128X:$src))),
5121 (VCVTSS2SI64Zrr (COPY_TO_REGCLASS VR128X:$src, FR32X))>;
5122 def : Pat<(i32 (int_x86_sse2_cvtsd2si (v2f64 VR128X:$src))),
5123 (VCVTSD2SIZrr (COPY_TO_REGCLASS VR128X:$src, FR64X))>;
5124 def : Pat<(i64 (int_x86_sse2_cvtsd2si64 (v2f64 VR128X:$src))),
5125 (VCVTSD2SI64Zrr (COPY_TO_REGCLASS VR128X:$src, FR64X))>;
5126} // HasAVX512
5127
Asaf Badouh2744d212015-09-20 14:31:19 +00005128let isCodeGenOnly = 1 , Predicates = [HasAVX512] in {
Craig Topper9dd48c82014-01-02 17:28:14 +00005129 defm Int_VCVTSI2SSZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X,
5130 int_x86_sse_cvtsi2ss, i32mem, loadi32, "cvtsi2ss{l}",
5131 SSE_CVT_Scalar, 0>, XS, EVEX_4V;
5132 defm Int_VCVTSI2SS64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X,
5133 int_x86_sse_cvtsi642ss, i64mem, loadi64, "cvtsi2ss{q}",
5134 SSE_CVT_Scalar, 0>, XS, EVEX_4V, VEX_W;
5135 defm Int_VCVTSI2SDZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X,
5136 int_x86_sse2_cvtsi2sd, i32mem, loadi32, "cvtsi2sd{l}",
5137 SSE_CVT_Scalar, 0>, XD, EVEX_4V;
5138 defm Int_VCVTSI2SD64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X,
5139 int_x86_sse2_cvtsi642sd, i64mem, loadi64, "cvtsi2sd{q}",
5140 SSE_CVT_Scalar, 0>, XD, EVEX_4V, VEX_W;
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005141
Igor Breger982e4002016-06-08 07:48:23 +00005142 defm Int_VCVTUSI2SDZ : sse12_cvt_sint_3addr<0x7B, GR32, VR128X,
Craig Topper9dd48c82014-01-02 17:28:14 +00005143 int_x86_avx512_cvtusi2sd, i32mem, loadi32, "cvtusi2sd{l}",
5144 SSE_CVT_Scalar, 0>, XD, EVEX_4V;
Asaf Badouh2744d212015-09-20 14:31:19 +00005145} // isCodeGenOnly = 1, Predicates = [HasAVX512]
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005146
5147// Convert float/double to signed/unsigned int 32/64 with truncation
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005148multiclass avx512_cvt_s_all<bits<8> opc, string asm, X86VectorVTInfo _SrcRC,
5149 X86VectorVTInfo _DstRC, SDNode OpNode,
Asaf Badouh2744d212015-09-20 14:31:19 +00005150 SDNode OpNodeRnd>{
5151let Predicates = [HasAVX512] in {
5152 def rr : SI<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.FRC:$src),
5153 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
5154 [(set _DstRC.RC:$dst, (OpNode _SrcRC.FRC:$src))]>, EVEX;
5155 def rb : SI<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.FRC:$src),
5156 !strconcat(asm,"\t{{sae}, $src, $dst|$dst, $src, {sae}}"),
5157 []>, EVEX, EVEX_B;
Igor Breger4511e762016-02-22 11:48:27 +00005158 def rm : SI<opc, MRMSrcMem, (outs _DstRC.RC:$dst), (ins _SrcRC.ScalarMemOp:$src),
Asaf Badouh2744d212015-09-20 14:31:19 +00005159 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005160 [(set _DstRC.RC:$dst, (OpNode (_SrcRC.ScalarLdFrag addr:$src)))]>,
Asaf Badouh2744d212015-09-20 14:31:19 +00005161 EVEX;
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005162
Craig Toppere1cac152016-06-07 07:27:54 +00005163 let isCodeGenOnly = 1 in {
Asaf Badouh2744d212015-09-20 14:31:19 +00005164 def rr_Int : SI<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.RC:$src),
5165 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
Craig Topper19e04b62016-05-19 06:13:58 +00005166 [(set _DstRC.RC:$dst, (OpNodeRnd (_SrcRC.VT _SrcRC.RC:$src),
Asaf Badouh2744d212015-09-20 14:31:19 +00005167 (i32 FROUND_CURRENT)))]>, EVEX, VEX_LIG;
5168 def rb_Int : SI<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.RC:$src),
5169 !strconcat(asm,"\t{{sae}, $src, $dst|$dst, $src, {sae}}"),
Craig Topper19e04b62016-05-19 06:13:58 +00005170 [(set _DstRC.RC:$dst, (OpNodeRnd (_SrcRC.VT _SrcRC.RC:$src),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005171 (i32 FROUND_NO_EXC)))]>,
Asaf Badouh2744d212015-09-20 14:31:19 +00005172 EVEX,VEX_LIG , EVEX_B;
Craig Toppere1cac152016-06-07 07:27:54 +00005173 let mayLoad = 1, hasSideEffects = 0 in
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005174 def rm_Int : SI<opc, MRMSrcMem, (outs _DstRC.RC:$dst),
Asaf Badouh2744d212015-09-20 14:31:19 +00005175 (ins _SrcRC.MemOp:$src),
5176 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
5177 []>, EVEX, VEX_LIG;
5178
Craig Toppere1cac152016-06-07 07:27:54 +00005179 } // isCodeGenOnly = 1
Asaf Badouh2744d212015-09-20 14:31:19 +00005180} //HasAVX512
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005181}
5182
Asaf Badouh2744d212015-09-20 14:31:19 +00005183
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005184defm VCVTTSS2SIZ: avx512_cvt_s_all<0x2C, "cvttss2si", f32x_info, i32x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005185 fp_to_sint,X86cvtts2IntRnd>,
Asaf Badouh2744d212015-09-20 14:31:19 +00005186 XS, EVEX_CD8<32, CD8VT1>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005187defm VCVTTSS2SI64Z: avx512_cvt_s_all<0x2C, "cvttss2si", f32x_info, i64x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005188 fp_to_sint,X86cvtts2IntRnd>,
Asaf Badouh2744d212015-09-20 14:31:19 +00005189 VEX_W, XS, EVEX_CD8<32, CD8VT1>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005190defm VCVTTSD2SIZ: avx512_cvt_s_all<0x2C, "cvttsd2si", f64x_info, i32x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005191 fp_to_sint,X86cvtts2IntRnd>,
Asaf Badouh2744d212015-09-20 14:31:19 +00005192 XD, EVEX_CD8<64, CD8VT1>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005193defm VCVTTSD2SI64Z: avx512_cvt_s_all<0x2C, "cvttsd2si", f64x_info, i64x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005194 fp_to_sint,X86cvtts2IntRnd>,
Asaf Badouh2744d212015-09-20 14:31:19 +00005195 VEX_W, XD, EVEX_CD8<64, CD8VT1>;
5196
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005197defm VCVTTSS2USIZ: avx512_cvt_s_all<0x78, "cvttss2usi", f32x_info, i32x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005198 fp_to_uint,X86cvtts2UIntRnd>,
Asaf Badouh2744d212015-09-20 14:31:19 +00005199 XS, EVEX_CD8<32, CD8VT1>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005200defm VCVTTSS2USI64Z: avx512_cvt_s_all<0x78, "cvttss2usi", f32x_info, i64x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005201 fp_to_uint,X86cvtts2UIntRnd>,
Asaf Badouh2744d212015-09-20 14:31:19 +00005202 XS,VEX_W, EVEX_CD8<32, CD8VT1>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005203defm VCVTTSD2USIZ: avx512_cvt_s_all<0x78, "cvttsd2usi", f64x_info, i32x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005204 fp_to_uint,X86cvtts2UIntRnd>,
Asaf Badouh2744d212015-09-20 14:31:19 +00005205 XD, EVEX_CD8<64, CD8VT1>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005206defm VCVTTSD2USI64Z: avx512_cvt_s_all<0x78, "cvttsd2usi", f64x_info, i64x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00005207 fp_to_uint,X86cvtts2UIntRnd>,
Asaf Badouh2744d212015-09-20 14:31:19 +00005208 XD, VEX_W, EVEX_CD8<64, CD8VT1>;
5209let Predicates = [HasAVX512] in {
5210 def : Pat<(i32 (int_x86_sse_cvttss2si (v4f32 VR128X:$src))),
5211 (VCVTTSS2SIZrr_Int (COPY_TO_REGCLASS VR128X:$src, FR32X))>;
5212 def : Pat<(i64 (int_x86_sse_cvttss2si64 (v4f32 VR128X:$src))),
5213 (VCVTTSS2SI64Zrr_Int (COPY_TO_REGCLASS VR128X:$src, FR32X))>;
5214 def : Pat<(i32 (int_x86_sse2_cvttsd2si (v2f64 VR128X:$src))),
5215 (VCVTTSD2SIZrr_Int (COPY_TO_REGCLASS VR128X:$src, FR64X))>;
5216 def : Pat<(i64 (int_x86_sse2_cvttsd2si64 (v2f64 VR128X:$src))),
5217 (VCVTTSD2SI64Zrr_Int (COPY_TO_REGCLASS VR128X:$src, FR64X))>;
5218
Elena Demikhovskycf088092013-12-11 14:31:04 +00005219} // HasAVX512
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00005220//===----------------------------------------------------------------------===//
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005221// AVX-512 Convert form float to double and back
5222//===----------------------------------------------------------------------===//
Asaf Badouh2744d212015-09-20 14:31:19 +00005223multiclass avx512_cvt_fp_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5224 X86VectorVTInfo _Src, SDNode OpNode> {
5225 defm rr : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
Craig Toppera58abd12016-05-09 05:34:12 +00005226 (ins _.RC:$src1, _Src.RC:$src2), OpcodeStr,
Asaf Badouh2744d212015-09-20 14:31:19 +00005227 "$src2, $src1", "$src1, $src2",
Craig Toppera58abd12016-05-09 05:34:12 +00005228 (_.VT (OpNode (_.VT _.RC:$src1),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005229 (_Src.VT _Src.RC:$src2)))>,
Asaf Badouh2744d212015-09-20 14:31:19 +00005230 EVEX_4V, VEX_LIG, Sched<[WriteCvtF2F]>;
5231 defm rm : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
Igor Breger4511e762016-02-22 11:48:27 +00005232 (ins _Src.RC:$src1, _Src.ScalarMemOp:$src2), OpcodeStr,
Asaf Badouh2744d212015-09-20 14:31:19 +00005233 "$src2, $src1", "$src1, $src2",
Craig Toppera58abd12016-05-09 05:34:12 +00005234 (_.VT (OpNode (_.VT _.RC:$src1),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005235 (_Src.VT (scalar_to_vector
5236 (_Src.ScalarLdFrag addr:$src2)))))>,
Asaf Badouh2744d212015-09-20 14:31:19 +00005237 EVEX_4V, VEX_LIG, Sched<[WriteCvtF2FLd, ReadAfterLd]>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005238}
5239
Asaf Badouh2744d212015-09-20 14:31:19 +00005240// Scalar Coversion with SAE - suppress all exceptions
5241multiclass avx512_cvt_fp_sae_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5242 X86VectorVTInfo _Src, SDNode OpNodeRnd> {
5243 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
Craig Toppera58abd12016-05-09 05:34:12 +00005244 (ins _.RC:$src1, _Src.RC:$src2), OpcodeStr,
Asaf Badouh2744d212015-09-20 14:31:19 +00005245 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
Craig Toppera58abd12016-05-09 05:34:12 +00005246 (_.VT (OpNodeRnd (_.VT _.RC:$src1),
Asaf Badouh2744d212015-09-20 14:31:19 +00005247 (_Src.VT _Src.RC:$src2),
5248 (i32 FROUND_NO_EXC)))>,
5249 EVEX_4V, VEX_LIG, EVEX_B;
5250}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005251
Asaf Badouh2744d212015-09-20 14:31:19 +00005252// Scalar Conversion with rounding control (RC)
5253multiclass avx512_cvt_fp_rc_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5254 X86VectorVTInfo _Src, SDNode OpNodeRnd> {
5255 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
Craig Toppera58abd12016-05-09 05:34:12 +00005256 (ins _.RC:$src1, _Src.RC:$src2, AVX512RC:$rc), OpcodeStr,
Asaf Badouh2744d212015-09-20 14:31:19 +00005257 "$rc, $src2, $src1", "$src1, $src2, $rc",
Craig Toppera58abd12016-05-09 05:34:12 +00005258 (_.VT (OpNodeRnd (_.VT _.RC:$src1),
Asaf Badouh2744d212015-09-20 14:31:19 +00005259 (_Src.VT _Src.RC:$src2), (i32 imm:$rc)))>,
5260 EVEX_4V, VEX_LIG, Sched<[WriteCvtF2FLd, ReadAfterLd]>,
5261 EVEX_B, EVEX_RC;
5262}
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005263multiclass avx512_cvt_fp_scalar_sd2ss<bits<8> opc, string OpcodeStr, SDNode OpNode,
5264 SDNode OpNodeRnd, X86VectorVTInfo _src,
Asaf Badouh2744d212015-09-20 14:31:19 +00005265 X86VectorVTInfo _dst> {
5266 let Predicates = [HasAVX512] in {
5267 defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNode>,
5268 avx512_cvt_fp_rc_scalar<opc, OpcodeStr, _dst, _src,
5269 OpNodeRnd>, VEX_W, EVEX_CD8<64, CD8VT1>,
5270 EVEX_V512, XD;
5271 }
5272}
5273
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005274multiclass avx512_cvt_fp_scalar_ss2sd<bits<8> opc, string OpcodeStr, SDNode OpNode,
5275 SDNode OpNodeRnd, X86VectorVTInfo _src,
Asaf Badouh2744d212015-09-20 14:31:19 +00005276 X86VectorVTInfo _dst> {
5277 let Predicates = [HasAVX512] in {
5278 defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNode>,
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005279 avx512_cvt_fp_sae_scalar<opc, OpcodeStr, _dst, _src, OpNodeRnd>,
Asaf Badouh2744d212015-09-20 14:31:19 +00005280 EVEX_CD8<32, CD8VT1>, XS, EVEX_V512;
5281 }
5282}
5283defm VCVTSD2SS : avx512_cvt_fp_scalar_sd2ss<0x5A, "vcvtsd2ss", X86fround,
5284 X86froundRnd, f64x_info, f32x_info>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005285defm VCVTSS2SD : avx512_cvt_fp_scalar_ss2sd<0x5A, "vcvtss2sd", X86fpext,
Asaf Badouh2744d212015-09-20 14:31:19 +00005286 X86fpextRnd,f32x_info, f64x_info >;
5287
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005288def : Pat<(f64 (fextend FR32X:$src)),
5289 (COPY_TO_REGCLASS (VCVTSS2SDZrr (COPY_TO_REGCLASS FR32X:$src, VR128X),
Asaf Badouh2744d212015-09-20 14:31:19 +00005290 (COPY_TO_REGCLASS FR32X:$src, VR128X)), VR128X)>,
5291 Requires<[HasAVX512]>;
5292def : Pat<(f64 (fextend (loadf32 addr:$src))),
5293 (COPY_TO_REGCLASS (VCVTSS2SDZrm (v4f32 (IMPLICIT_DEF)), addr:$src), VR128X)>,
5294 Requires<[HasAVX512]>;
5295
5296def : Pat<(f64 (extloadf32 addr:$src)),
5297 (COPY_TO_REGCLASS (VCVTSS2SDZrm (v4f32 (IMPLICIT_DEF)), addr:$src), VR128X)>,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005298 Requires<[HasAVX512, OptForSize]>;
5299
Asaf Badouh2744d212015-09-20 14:31:19 +00005300def : Pat<(f64 (extloadf32 addr:$src)),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005301 (COPY_TO_REGCLASS (VCVTSS2SDZrr (v4f32 (IMPLICIT_DEF)),
Asaf Badouh2744d212015-09-20 14:31:19 +00005302 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)), VR128X)>,
5303 Requires<[HasAVX512, OptForSpeed]>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005304
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005305def : Pat<(f32 (fround FR64X:$src)),
5306 (COPY_TO_REGCLASS (VCVTSD2SSZrr (COPY_TO_REGCLASS FR64X:$src, VR128X),
Asaf Badouh2744d212015-09-20 14:31:19 +00005307 (COPY_TO_REGCLASS FR64X:$src, VR128X)), VR128X)>,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005308 Requires<[HasAVX512]>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005309//===----------------------------------------------------------------------===//
5310// AVX-512 Vector convert from signed/unsigned integer to float/double
5311// and from float/double to signed/unsigned integer
5312//===----------------------------------------------------------------------===//
5313
5314multiclass avx512_vcvt_fp<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5315 X86VectorVTInfo _Src, SDNode OpNode,
5316 string Broadcast = _.BroadcastStr,
5317 string Alias = ""> {
5318
5319 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5320 (ins _Src.RC:$src), OpcodeStr, "$src", "$src",
5321 (_.VT (OpNode (_Src.VT _Src.RC:$src)))>, EVEX;
5322
5323 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5324 (ins _Src.MemOp:$src), OpcodeStr#Alias, "$src", "$src",
5325 (_.VT (OpNode (_Src.VT
5326 (bitconvert (_Src.LdFrag addr:$src)))))>, EVEX;
5327
5328 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
Igor Breger4511e762016-02-22 11:48:27 +00005329 (ins _Src.ScalarMemOp:$src), OpcodeStr,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005330 "${src}"##Broadcast, "${src}"##Broadcast,
5331 (_.VT (OpNode (_Src.VT
5332 (X86VBroadcast (_Src.ScalarLdFrag addr:$src)))
5333 ))>, EVEX, EVEX_B;
5334}
5335// Coversion with SAE - suppress all exceptions
5336multiclass avx512_vcvt_fp_sae<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5337 X86VectorVTInfo _Src, SDNode OpNodeRnd> {
5338 defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5339 (ins _Src.RC:$src), OpcodeStr,
5340 "{sae}, $src", "$src, {sae}",
5341 (_.VT (OpNodeRnd (_Src.VT _Src.RC:$src),
5342 (i32 FROUND_NO_EXC)))>,
5343 EVEX, EVEX_B;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005344}
5345
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005346// Conversion with rounding control (RC)
5347multiclass avx512_vcvt_fp_rc<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5348 X86VectorVTInfo _Src, SDNode OpNodeRnd> {
5349 defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5350 (ins _Src.RC:$src, AVX512RC:$rc), OpcodeStr,
5351 "$rc, $src", "$src, $rc",
5352 (_.VT (OpNodeRnd (_Src.VT _Src.RC:$src), (i32 imm:$rc)))>,
5353 EVEX, EVEX_B, EVEX_RC;
Elena Demikhovskyde3f7512014-01-01 15:12:34 +00005354}
5355
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005356// Extend Float to Double
5357multiclass avx512_cvtps2pd<bits<8> opc, string OpcodeStr> {
5358 let Predicates = [HasAVX512] in {
5359 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8f32x_info, fextend>,
5360 avx512_vcvt_fp_sae<opc, OpcodeStr, v8f64_info, v8f32x_info,
5361 X86vfpextRnd>, EVEX_V512;
5362 }
5363 let Predicates = [HasVLX] in {
5364 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4f32x_info,
5365 X86vfpext, "{1to2}">, EVEX_V128;
5366 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4f32x_info, fextend>,
5367 EVEX_V256;
5368 }
5369}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005370
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005371// Truncate Double to Float
5372multiclass avx512_cvtpd2ps<bits<8> opc, string OpcodeStr> {
5373 let Predicates = [HasAVX512] in {
5374 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8f64_info, fround>,
5375 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8f64_info,
5376 X86vfproundRnd>, EVEX_V512;
5377 }
5378 let Predicates = [HasVLX] in {
5379 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2f64x_info,
5380 X86vfpround, "{1to2}", "{x}">, EVEX_V128;
5381 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4f64x_info, fround,
5382 "{1to4}", "{y}">, EVEX_V256;
5383 }
5384}
5385
5386defm VCVTPD2PS : avx512_cvtpd2ps<0x5A, "vcvtpd2ps">,
5387 VEX_W, PD, EVEX_CD8<64, CD8VF>;
5388defm VCVTPS2PD : avx512_cvtps2pd<0x5A, "vcvtps2pd">,
5389 PS, EVEX_CD8<32, CD8VH>;
5390
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005391def : Pat<(v8f64 (extloadv8f32 addr:$src)),
5392 (VCVTPS2PDZrm addr:$src)>;
Michael Liao5bf95782014-12-04 05:20:33 +00005393
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005394let Predicates = [HasVLX] in {
5395 def : Pat<(v4f64 (extloadv4f32 addr:$src)),
5396 (VCVTPS2PDZ256rm addr:$src)>;
5397}
Elena Demikhovsky3629b4a2014-01-06 08:45:54 +00005398
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005399// Convert Signed/Unsigned Doubleword to Double
5400multiclass avx512_cvtdq2pd<bits<8> opc, string OpcodeStr, SDNode OpNode,
5401 SDNode OpNode128> {
5402 // No rounding in this op
5403 let Predicates = [HasAVX512] in
5404 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i32x_info, OpNode>,
5405 EVEX_V512;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005406
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005407 let Predicates = [HasVLX] in {
5408 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4i32x_info,
5409 OpNode128, "{1to2}">, EVEX_V128;
5410 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i32x_info, OpNode>,
5411 EVEX_V256;
5412 }
5413}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005414
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005415// Convert Signed/Unsigned Doubleword to Float
5416multiclass avx512_cvtdq2ps<bits<8> opc, string OpcodeStr, SDNode OpNode,
5417 SDNode OpNodeRnd> {
5418 let Predicates = [HasAVX512] in
5419 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16f32_info, v16i32_info, OpNode>,
5420 avx512_vcvt_fp_rc<opc, OpcodeStr, v16f32_info, v16i32_info,
5421 OpNodeRnd>, EVEX_V512;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005422
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005423 let Predicates = [HasVLX] in {
5424 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i32x_info, OpNode>,
5425 EVEX_V128;
5426 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i32x_info, OpNode>,
5427 EVEX_V256;
5428 }
5429}
5430
5431// Convert Float to Signed/Unsigned Doubleword with truncation
5432multiclass avx512_cvttps2dq<bits<8> opc, string OpcodeStr,
5433 SDNode OpNode, SDNode OpNodeRnd> {
5434 let Predicates = [HasAVX512] in {
5435 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode>,
5436 avx512_vcvt_fp_sae<opc, OpcodeStr, v16i32_info, v16f32_info,
5437 OpNodeRnd>, EVEX_V512;
5438 }
5439 let Predicates = [HasVLX] in {
5440 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode>,
5441 EVEX_V128;
5442 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode>,
5443 EVEX_V256;
5444 }
5445}
5446
5447// Convert Float to Signed/Unsigned Doubleword
5448multiclass avx512_cvtps2dq<bits<8> opc, string OpcodeStr,
5449 SDNode OpNode, SDNode OpNodeRnd> {
5450 let Predicates = [HasAVX512] in {
5451 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode>,
5452 avx512_vcvt_fp_rc<opc, OpcodeStr, v16i32_info, v16f32_info,
5453 OpNodeRnd>, EVEX_V512;
5454 }
5455 let Predicates = [HasVLX] in {
5456 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode>,
5457 EVEX_V128;
5458 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode>,
5459 EVEX_V256;
5460 }
5461}
5462
5463// Convert Double to Signed/Unsigned Doubleword with truncation
5464multiclass avx512_cvttpd2dq<bits<8> opc, string OpcodeStr,
5465 SDNode OpNode, SDNode OpNodeRnd> {
5466 let Predicates = [HasAVX512] in {
5467 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode>,
5468 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i32x_info, v8f64_info,
5469 OpNodeRnd>, EVEX_V512;
5470 }
5471 let Predicates = [HasVLX] in {
5472 // we need "x"/"y" suffixes in order to distinguish between 128 and 256
5473 // memory forms of these instructions in Asm Parcer. They have the same
5474 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
5475 // due to the same reason.
5476 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode,
5477 "{1to2}", "{x}">, EVEX_V128;
5478 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
5479 "{1to4}", "{y}">, EVEX_V256;
5480 }
5481}
5482
5483// Convert Double to Signed/Unsigned Doubleword
5484multiclass avx512_cvtpd2dq<bits<8> opc, string OpcodeStr,
5485 SDNode OpNode, SDNode OpNodeRnd> {
5486 let Predicates = [HasAVX512] in {
5487 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode>,
5488 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i32x_info, v8f64_info,
5489 OpNodeRnd>, EVEX_V512;
5490 }
5491 let Predicates = [HasVLX] in {
5492 // we need "x"/"y" suffixes in order to distinguish between 128 and 256
5493 // memory forms of these instructions in Asm Parcer. They have the same
5494 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
5495 // due to the same reason.
5496 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode,
5497 "{1to2}", "{x}">, EVEX_V128;
5498 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
5499 "{1to4}", "{y}">, EVEX_V256;
5500 }
5501}
5502
5503// Convert Double to Signed/Unsigned Quardword
5504multiclass avx512_cvtpd2qq<bits<8> opc, string OpcodeStr,
5505 SDNode OpNode, SDNode OpNodeRnd> {
5506 let Predicates = [HasDQI] in {
5507 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode>,
5508 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f64_info,
5509 OpNodeRnd>, EVEX_V512;
5510 }
5511 let Predicates = [HasDQI, HasVLX] in {
5512 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode>,
5513 EVEX_V128;
5514 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode>,
5515 EVEX_V256;
5516 }
5517}
5518
5519// Convert Double to Signed/Unsigned Quardword with truncation
5520multiclass avx512_cvttpd2qq<bits<8> opc, string OpcodeStr,
5521 SDNode OpNode, SDNode OpNodeRnd> {
5522 let Predicates = [HasDQI] in {
5523 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode>,
5524 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f64_info,
5525 OpNodeRnd>, EVEX_V512;
5526 }
5527 let Predicates = [HasDQI, HasVLX] in {
5528 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode>,
5529 EVEX_V128;
5530 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode>,
5531 EVEX_V256;
5532 }
5533}
5534
5535// Convert Signed/Unsigned Quardword to Double
5536multiclass avx512_cvtqq2pd<bits<8> opc, string OpcodeStr,
5537 SDNode OpNode, SDNode OpNodeRnd> {
5538 let Predicates = [HasDQI] in {
5539 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i64_info, OpNode>,
5540 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f64_info, v8i64_info,
5541 OpNodeRnd>, EVEX_V512;
5542 }
5543 let Predicates = [HasDQI, HasVLX] in {
5544 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v2i64x_info, OpNode>,
5545 EVEX_V128;
5546 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i64x_info, OpNode>,
5547 EVEX_V256;
5548 }
5549}
5550
5551// Convert Float to Signed/Unsigned Quardword
5552multiclass avx512_cvtps2qq<bits<8> opc, string OpcodeStr,
5553 SDNode OpNode, SDNode OpNodeRnd> {
5554 let Predicates = [HasDQI] in {
5555 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode>,
5556 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f32x_info,
5557 OpNodeRnd>, EVEX_V512;
5558 }
5559 let Predicates = [HasDQI, HasVLX] in {
5560 // Explicitly specified broadcast string, since we take only 2 elements
5561 // from v4f32x_info source
5562 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,
5563 "{1to2}">, EVEX_V128;
5564 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode>,
5565 EVEX_V256;
5566 }
5567}
5568
5569// Convert Float to Signed/Unsigned Quardword with truncation
5570multiclass avx512_cvttps2qq<bits<8> opc, string OpcodeStr,
5571 SDNode OpNode, SDNode OpNodeRnd> {
5572 let Predicates = [HasDQI] in {
5573 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode>,
5574 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f32x_info,
5575 OpNodeRnd>, EVEX_V512;
5576 }
5577 let Predicates = [HasDQI, HasVLX] in {
5578 // Explicitly specified broadcast string, since we take only 2 elements
5579 // from v4f32x_info source
5580 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,
5581 "{1to2}">, EVEX_V128;
5582 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode>,
5583 EVEX_V256;
5584 }
5585}
5586
5587// Convert Signed/Unsigned Quardword to Float
5588multiclass avx512_cvtqq2ps<bits<8> opc, string OpcodeStr,
5589 SDNode OpNode, SDNode OpNodeRnd> {
5590 let Predicates = [HasDQI] in {
5591 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i64_info, OpNode>,
5592 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8i64_info,
5593 OpNodeRnd>, EVEX_V512;
5594 }
5595 let Predicates = [HasDQI, HasVLX] in {
5596 // we need "x"/"y" suffixes in order to distinguish between 128 and 256
5597 // memory forms of these instructions in Asm Parcer. They have the same
5598 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
5599 // due to the same reason.
5600 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2i64x_info, OpNode,
5601 "{1to2}", "{x}">, EVEX_V128;
5602 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i64x_info, OpNode,
5603 "{1to4}", "{y}">, EVEX_V256;
5604 }
5605}
5606
5607defm VCVTDQ2PD : avx512_cvtdq2pd<0xE6, "vcvtdq2pd", sint_to_fp, X86cvtdq2pd>, XS,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005608 EVEX_CD8<32, CD8VH>;
5609
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005610defm VCVTDQ2PS : avx512_cvtdq2ps<0x5B, "vcvtdq2ps", sint_to_fp,
5611 X86VSintToFpRnd>,
5612 PS, EVEX_CD8<32, CD8VF>;
5613
5614defm VCVTTPS2DQ : avx512_cvttps2dq<0x5B, "vcvttps2dq", fp_to_sint,
5615 X86VFpToSintRnd>,
5616 XS, EVEX_CD8<32, CD8VF>;
5617
5618defm VCVTTPD2DQ : avx512_cvttpd2dq<0xE6, "vcvttpd2dq", fp_to_sint,
5619 X86VFpToSintRnd>,
5620 PD, VEX_W, EVEX_CD8<64, CD8VF>;
5621
5622defm VCVTTPS2UDQ : avx512_cvttps2dq<0x78, "vcvttps2udq", fp_to_uint,
5623 X86VFpToUintRnd>, PS,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005624 EVEX_CD8<32, CD8VF>;
5625
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005626defm VCVTTPD2UDQ : avx512_cvttpd2dq<0x78, "vcvttpd2udq", fp_to_uint,
5627 X86VFpToUintRnd>, PS, VEX_W,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005628 EVEX_CD8<64, CD8VF>;
5629
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005630defm VCVTUDQ2PD : avx512_cvtdq2pd<0x7A, "vcvtudq2pd", uint_to_fp, X86cvtudq2pd>,
5631 XS, EVEX_CD8<32, CD8VH>;
5632
5633defm VCVTUDQ2PS : avx512_cvtdq2ps<0x7A, "vcvtudq2ps", uint_to_fp,
5634 X86VUintToFpRnd>, XD,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005635 EVEX_CD8<32, CD8VF>;
5636
Craig Topper19e04b62016-05-19 06:13:58 +00005637defm VCVTPS2DQ : avx512_cvtps2dq<0x5B, "vcvtps2dq", X86cvtp2Int,
5638 X86cvtp2IntRnd>, PD, EVEX_CD8<32, CD8VF>;
Elena Demikhovskyde3f7512014-01-01 15:12:34 +00005639
Craig Topper19e04b62016-05-19 06:13:58 +00005640defm VCVTPD2DQ : avx512_cvtpd2dq<0xE6, "vcvtpd2dq", X86cvtp2Int,
5641 X86cvtp2IntRnd>, XD, VEX_W,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005642 EVEX_CD8<64, CD8VF>;
Michael Liao5bf95782014-12-04 05:20:33 +00005643
Craig Topper19e04b62016-05-19 06:13:58 +00005644defm VCVTPS2UDQ : avx512_cvtps2dq<0x79, "vcvtps2udq", X86cvtp2UInt,
5645 X86cvtp2UIntRnd>,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005646 PS, EVEX_CD8<32, CD8VF>;
Craig Topper19e04b62016-05-19 06:13:58 +00005647defm VCVTPD2UDQ : avx512_cvtpd2dq<0x79, "vcvtpd2udq", X86cvtp2UInt,
5648 X86cvtp2UIntRnd>, VEX_W,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005649 PS, EVEX_CD8<64, CD8VF>;
Elena Demikhovskyde3f7512014-01-01 15:12:34 +00005650
Craig Topper19e04b62016-05-19 06:13:58 +00005651defm VCVTPD2QQ : avx512_cvtpd2qq<0x7B, "vcvtpd2qq", X86cvtp2Int,
5652 X86cvtp2IntRnd>, VEX_W,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005653 PD, EVEX_CD8<64, CD8VF>;
Michael Liao5bf95782014-12-04 05:20:33 +00005654
Craig Topper19e04b62016-05-19 06:13:58 +00005655defm VCVTPS2QQ : avx512_cvtps2qq<0x7B, "vcvtps2qq", X86cvtp2Int,
5656 X86cvtp2IntRnd>, PD, EVEX_CD8<32, CD8VH>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005657
Craig Topper19e04b62016-05-19 06:13:58 +00005658defm VCVTPD2UQQ : avx512_cvtpd2qq<0x79, "vcvtpd2uqq", X86cvtp2UInt,
5659 X86cvtp2UIntRnd>, VEX_W,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005660 PD, EVEX_CD8<64, CD8VF>;
5661
Craig Topper19e04b62016-05-19 06:13:58 +00005662defm VCVTPS2UQQ : avx512_cvtps2qq<0x79, "vcvtps2uqq", X86cvtp2UInt,
5663 X86cvtp2UIntRnd>, PD, EVEX_CD8<32, CD8VH>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005664
5665defm VCVTTPD2QQ : avx512_cvttpd2qq<0x7A, "vcvttpd2qq", fp_to_sint,
Craig Topper19e04b62016-05-19 06:13:58 +00005666 X86VFpToSintRnd>, VEX_W,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005667 PD, EVEX_CD8<64, CD8VF>;
5668
5669defm VCVTTPS2QQ : avx512_cvttps2qq<0x7A, "vcvttps2qq", fp_to_sint,
Craig Topper19e04b62016-05-19 06:13:58 +00005670 X86VFpToSintRnd>, PD, EVEX_CD8<32, CD8VH>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005671
5672defm VCVTTPD2UQQ : avx512_cvttpd2qq<0x78, "vcvttpd2uqq", fp_to_uint,
Craig Topper19e04b62016-05-19 06:13:58 +00005673 X86VFpToUintRnd>, VEX_W,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005674 PD, EVEX_CD8<64, CD8VF>;
5675
5676defm VCVTTPS2UQQ : avx512_cvttps2qq<0x78, "vcvttps2uqq", fp_to_uint,
Craig Topper19e04b62016-05-19 06:13:58 +00005677 X86VFpToUintRnd>, PD, EVEX_CD8<32, CD8VH>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005678
5679defm VCVTQQ2PD : avx512_cvtqq2pd<0xE6, "vcvtqq2pd", sint_to_fp,
Craig Topper19e04b62016-05-19 06:13:58 +00005680 X86VSintToFpRnd>, VEX_W, XS, EVEX_CD8<64, CD8VF>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005681
5682defm VCVTUQQ2PD : avx512_cvtqq2pd<0x7A, "vcvtuqq2pd", uint_to_fp,
Craig Topper19e04b62016-05-19 06:13:58 +00005683 X86VUintToFpRnd>, VEX_W, XS, EVEX_CD8<64, CD8VF>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005684
5685defm VCVTQQ2PS : avx512_cvtqq2ps<0x5B, "vcvtqq2ps", sint_to_fp,
Craig Topper19e04b62016-05-19 06:13:58 +00005686 X86VSintToFpRnd>, VEX_W, PS, EVEX_CD8<64, CD8VF>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005687
5688defm VCVTUQQ2PS : avx512_cvtqq2ps<0x7A, "vcvtuqq2ps", uint_to_fp,
Craig Topper19e04b62016-05-19 06:13:58 +00005689 X86VUintToFpRnd>, VEX_W, XD, EVEX_CD8<64, CD8VF>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00005690
Craig Toppere38c57a2015-11-27 05:44:02 +00005691let Predicates = [HasAVX512, NoVLX] in {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005692def : Pat<(v8i32 (fp_to_uint (v8f32 VR256X:$src1))),
Michael Liao5bf95782014-12-04 05:20:33 +00005693 (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005694 (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>;
Michael Liao5bf95782014-12-04 05:20:33 +00005695
Elena Demikhovsky3dcfbdf2014-04-08 07:24:02 +00005696def : Pat<(v4i32 (fp_to_uint (v4f32 VR128X:$src1))),
5697 (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr
5698 (v16f32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_xmm)>;
5699
Elena Demikhovsky95629ca2016-03-29 06:33:41 +00005700def : Pat<(v4i32 (fp_to_uint (v4f64 VR256X:$src1))),
5701 (EXTRACT_SUBREG (v8i32 (VCVTTPD2UDQZrr
5702 (v8f64 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_xmm)>;
5703
Elena Demikhovsky3dcfbdf2014-04-08 07:24:02 +00005704def : Pat<(v8f32 (uint_to_fp (v8i32 VR256X:$src1))),
5705 (EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr
5706 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>;
Michael Liao5bf95782014-12-04 05:20:33 +00005707
Elena Demikhovsky3dcfbdf2014-04-08 07:24:02 +00005708def : Pat<(v4f32 (uint_to_fp (v4i32 VR128X:$src1))),
5709 (EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr
5710 (v16i32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005711
Cameron McInallyf10a7c92014-06-18 14:04:37 +00005712def : Pat<(v4f64 (uint_to_fp (v4i32 VR128X:$src1))),
5713 (EXTRACT_SUBREG (v8f64 (VCVTUDQ2PDZrr
5714 (v8i32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_ymm)>;
Elena Demikhovskyde3f7512014-01-01 15:12:34 +00005715}
5716
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005717let Predicates = [HasAVX512] in {
5718 def : Pat<(v8f32 (fround (loadv8f64 addr:$src))),
5719 (VCVTPD2PSZrm addr:$src)>;
5720 def : Pat<(v8f64 (extloadv8f32 addr:$src)),
5721 (VCVTPS2PDZrm addr:$src)>;
5722}
5723
Elena Demikhovskydd0794e2013-10-24 07:16:35 +00005724//===----------------------------------------------------------------------===//
5725// Half precision conversion instructions
5726//===----------------------------------------------------------------------===//
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005727multiclass avx512_cvtph2ps<X86VectorVTInfo _dest, X86VectorVTInfo _src,
Asaf Badouh7c522452015-10-22 14:01:16 +00005728 X86MemOperand x86memop, PatFrag ld_frag> {
5729 defm rr : AVX512_maskable<0x13, MRMSrcReg, _dest ,(outs _dest.RC:$dst), (ins _src.RC:$src),
5730 "vcvtph2ps", "$src", "$src",
5731 (X86cvtph2ps (_src.VT _src.RC:$src),
5732 (i32 FROUND_CURRENT))>, T8PD;
Craig Toppere1cac152016-06-07 07:27:54 +00005733 defm rm : AVX512_maskable<0x13, MRMSrcMem, _dest, (outs _dest.RC:$dst), (ins x86memop:$src),
5734 "vcvtph2ps", "$src", "$src",
5735 (X86cvtph2ps (_src.VT (bitconvert (ld_frag addr:$src))),
5736 (i32 FROUND_CURRENT))>, T8PD;
Asaf Badouh7c522452015-10-22 14:01:16 +00005737}
5738
Asaf Badouhc7cb8802015-10-27 15:37:17 +00005739multiclass avx512_cvtph2ps_sae<X86VectorVTInfo _dest, X86VectorVTInfo _src> {
Asaf Badouh7c522452015-10-22 14:01:16 +00005740 defm rb : AVX512_maskable<0x13, MRMSrcReg, _dest ,(outs _dest.RC:$dst), (ins _src.RC:$src),
5741 "vcvtph2ps", "{sae}, $src", "$src, {sae}",
5742 (X86cvtph2ps (_src.VT _src.RC:$src),
5743 (i32 FROUND_NO_EXC))>, T8PD, EVEX_B;
5744
5745}
5746
5747let Predicates = [HasAVX512] in {
5748 defm VCVTPH2PSZ : avx512_cvtph2ps<v16f32_info, v16i16x_info, f256mem, loadv4i64>,
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005749 avx512_cvtph2ps_sae<v16f32_info, v16i16x_info>,
Asaf Badouh7c522452015-10-22 14:01:16 +00005750 EVEX, EVEX_V512, EVEX_CD8<32, CD8VH>;
5751 let Predicates = [HasVLX] in {
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005752 defm VCVTPH2PSZ256 : avx512_cvtph2ps<v8f32x_info, v8i16x_info, f128mem,
Asaf Badouh7c522452015-10-22 14:01:16 +00005753 loadv2i64>,EVEX, EVEX_V256, EVEX_CD8<32, CD8VH>;
5754 defm VCVTPH2PSZ128 : avx512_cvtph2ps<v4f32x_info, v8i16x_info, f64mem,
5755 loadv2i64>, EVEX, EVEX_V128, EVEX_CD8<32, CD8VH>;
5756 }
Elena Demikhovskydd0794e2013-10-24 07:16:35 +00005757}
5758
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005759multiclass avx512_cvtps2ph<X86VectorVTInfo _dest, X86VectorVTInfo _src,
Asaf Badouhc7cb8802015-10-27 15:37:17 +00005760 X86MemOperand x86memop> {
5761 defm rr : AVX512_maskable<0x1D, MRMDestReg, _dest ,(outs _dest.RC:$dst),
Igor Breger73ee8ba2016-05-31 08:04:21 +00005762 (ins _src.RC:$src1, i32u8imm:$src2),
5763 "vcvtps2ph", "$src2, $src1", "$src1, $src2",
Asaf Badouhc7cb8802015-10-27 15:37:17 +00005764 (X86cvtps2ph (_src.VT _src.RC:$src1),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005765 (i32 imm:$src2),
Igor Breger73ee8ba2016-05-31 08:04:21 +00005766 (i32 FROUND_CURRENT)),
5767 NoItinerary, 0, X86select>, AVX512AIi8Base;
Craig Toppere1cac152016-06-07 07:27:54 +00005768 def mr : AVX512AIi8<0x1D, MRMDestMem, (outs),
5769 (ins x86memop:$dst, _src.RC:$src1, i32u8imm:$src2),
5770 "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}",
5771 [(store (_dest.VT (X86cvtps2ph (_src.VT _src.RC:$src1),
5772 (i32 imm:$src2), (i32 FROUND_CURRENT) )),
5773 addr:$dst)]>;
5774 let hasSideEffects = 0, mayStore = 1 in
5775 def mrk : AVX512AIi8<0x1D, MRMDestMem, (outs),
5776 (ins x86memop:$dst, _dest.KRCWM:$mask, _src.RC:$src1, i32u8imm:$src2),
5777 "vcvtps2ph\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
5778 []>, EVEX_K;
Elena Demikhovskydd0794e2013-10-24 07:16:35 +00005779}
Asaf Badouhc7cb8802015-10-27 15:37:17 +00005780multiclass avx512_cvtps2ph_sae<X86VectorVTInfo _dest, X86VectorVTInfo _src> {
5781 defm rb : AVX512_maskable<0x1D, MRMDestReg, _dest ,(outs _dest.RC:$dst),
Igor Breger73ee8ba2016-05-31 08:04:21 +00005782 (ins _src.RC:$src1, i32u8imm:$src2),
5783 "vcvtps2ph", "$src2, {sae}, $src1", "$src1, {sae}, $src2",
Asaf Badouhc7cb8802015-10-27 15:37:17 +00005784 (X86cvtps2ph (_src.VT _src.RC:$src1),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005785 (i32 imm:$src2),
Igor Breger73ee8ba2016-05-31 08:04:21 +00005786 (i32 FROUND_NO_EXC)),
5787 NoItinerary, 0, X86select>, EVEX_B, AVX512AIi8Base;
Asaf Badouhc7cb8802015-10-27 15:37:17 +00005788}
5789let Predicates = [HasAVX512] in {
5790 defm VCVTPS2PHZ : avx512_cvtps2ph<v16i16x_info, v16f32_info, f256mem>,
5791 avx512_cvtps2ph_sae<v16i16x_info, v16f32_info>,
5792 EVEX, EVEX_V512, EVEX_CD8<32, CD8VH>;
5793 let Predicates = [HasVLX] in {
5794 defm VCVTPS2PHZ256 : avx512_cvtps2ph<v8i16x_info, v8f32x_info, f128mem>,
5795 EVEX, EVEX_V256, EVEX_CD8<32, CD8VH>;
5796 defm VCVTPS2PHZ128 : avx512_cvtps2ph<v8i16x_info, v4f32x_info, f128mem>,
5797 EVEX, EVEX_V128, EVEX_CD8<32, CD8VH>;
5798 }
5799}
Asaf Badouh2489f352015-12-02 08:17:51 +00005800
5801// Unordered/Ordered scalar fp compare with Sea and set EFLAGS
5802multiclass avx512_ord_cmp_sae<bits<8> opc, X86VectorVTInfo _, SDNode OpNode,
5803 string OpcodeStr> {
5804 def rb: AVX512<opc, MRMSrcReg, (outs), (ins _.RC:$src1, _.RC:$src2),
5805 !strconcat(OpcodeStr, "\t{{sae}, $src2, $src1|$src1, $src2, {sae}}"),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005806 [(set EFLAGS, (OpNode (_.VT _.RC:$src1), _.RC:$src2,
Asaf Badouh2489f352015-12-02 08:17:51 +00005807 (i32 FROUND_NO_EXC)))],
5808 IIC_SSE_COMIS_RR>, EVEX, EVEX_B, VEX_LIG, EVEX_V128,
5809 Sched<[WriteFAdd]>;
5810}
5811
5812let Defs = [EFLAGS], Predicates = [HasAVX512] in {
5813 defm VUCOMISSZ : avx512_ord_cmp_sae<0x2E, v4f32x_info, X86ucomiSae, "vucomiss">,
5814 AVX512PSIi8Base, EVEX_CD8<32, CD8VT1>;
5815 defm VUCOMISDZ : avx512_ord_cmp_sae<0x2E, v2f64x_info, X86ucomiSae, "vucomisd">,
5816 AVX512PDIi8Base, VEX_W, EVEX_CD8<64, CD8VT1>;
5817 defm VCOMISSZ : avx512_ord_cmp_sae<0x2F, v4f32x_info, X86comiSae, "vcomiss">,
5818 AVX512PSIi8Base, EVEX_CD8<32, CD8VT1>;
5819 defm VCOMISDZ : avx512_ord_cmp_sae<0x2F, v2f64x_info, X86comiSae, "vcomisd">,
5820 AVX512PDIi8Base, VEX_W, EVEX_CD8<64, CD8VT1>;
5821}
5822
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005823let Defs = [EFLAGS], Predicates = [HasAVX512] in {
5824 defm VUCOMISSZ : sse12_ord_cmp<0x2E, FR32X, X86cmp, f32, f32mem, loadf32,
Craig Topper5ccb6172014-02-18 00:21:49 +00005825 "ucomiss">, PS, EVEX, VEX_LIG,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005826 EVEX_CD8<32, CD8VT1>;
5827 defm VUCOMISDZ : sse12_ord_cmp<0x2E, FR64X, X86cmp, f64, f64mem, loadf64,
Craig Topperae11aed2014-01-14 07:41:20 +00005828 "ucomisd">, PD, EVEX,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005829 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
5830 let Pattern = []<dag> in {
Marina Yatsina7a4e1ba2015-08-20 11:21:36 +00005831 defm VCOMISSZ : sse12_ord_cmp<0x2F, FR32X, undef, f32, f32mem, loadf32,
Craig Topper5ccb6172014-02-18 00:21:49 +00005832 "comiss">, PS, EVEX, VEX_LIG,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005833 EVEX_CD8<32, CD8VT1>;
Marina Yatsina7a4e1ba2015-08-20 11:21:36 +00005834 defm VCOMISDZ : sse12_ord_cmp<0x2F, FR64X, undef, f64, f64mem, loadf64,
Craig Topperae11aed2014-01-14 07:41:20 +00005835 "comisd">, PD, EVEX,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005836 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
5837 }
Craig Topper9dd48c82014-01-02 17:28:14 +00005838 let isCodeGenOnly = 1 in {
5839 defm Int_VUCOMISSZ : sse12_ord_cmp<0x2E, VR128X, X86ucomi, v4f32, f128mem,
Craig Topper5ccb6172014-02-18 00:21:49 +00005840 load, "ucomiss">, PS, EVEX, VEX_LIG,
Craig Topper9dd48c82014-01-02 17:28:14 +00005841 EVEX_CD8<32, CD8VT1>;
5842 defm Int_VUCOMISDZ : sse12_ord_cmp<0x2E, VR128X, X86ucomi, v2f64, f128mem,
Craig Topperae11aed2014-01-14 07:41:20 +00005843 load, "ucomisd">, PD, EVEX,
Craig Topper9dd48c82014-01-02 17:28:14 +00005844 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005845
Craig Topper9dd48c82014-01-02 17:28:14 +00005846 defm Int_VCOMISSZ : sse12_ord_cmp<0x2F, VR128X, X86comi, v4f32, f128mem,
Craig Topper5ccb6172014-02-18 00:21:49 +00005847 load, "comiss">, PS, EVEX, VEX_LIG,
Craig Topper9dd48c82014-01-02 17:28:14 +00005848 EVEX_CD8<32, CD8VT1>;
5849 defm Int_VCOMISDZ : sse12_ord_cmp<0x2F, VR128X, X86comi, v2f64, f128mem,
Craig Topperae11aed2014-01-14 07:41:20 +00005850 load, "comisd">, PD, EVEX,
Craig Topper9dd48c82014-01-02 17:28:14 +00005851 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
5852 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005853}
Michael Liao5bf95782014-12-04 05:20:33 +00005854
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00005855/// avx512_fp14_s rcp14ss, rcp14sd, rsqrt14ss, rsqrt14sd
Asaf Badouheaf2da12015-09-21 10:23:53 +00005856multiclass avx512_fp14_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
5857 X86VectorVTInfo _> {
Craig Toppere1cac152016-06-07 07:27:54 +00005858 let AddedComplexity = 20 , Predicates = [HasAVX512] in {
Asaf Badouheaf2da12015-09-21 10:23:53 +00005859 defm rr : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5860 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
5861 "$src2, $src1", "$src1, $src2",
5862 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>, EVEX_4V;
Asaf Badouheaf2da12015-09-21 10:23:53 +00005863 defm rm : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
Igor Breger4511e762016-02-22 11:48:27 +00005864 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
Asaf Badouheaf2da12015-09-21 10:23:53 +00005865 "$src2, $src1", "$src1, $src2",
5866 (OpNode (_.VT _.RC:$src1),
5867 (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))))>, EVEX_4V;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005868}
5869}
5870
Asaf Badouheaf2da12015-09-21 10:23:53 +00005871defm VRCP14SS : avx512_fp14_s<0x4D, "vrcp14ss", X86frcp14s, f32x_info>,
5872 EVEX_CD8<32, CD8VT1>, T8PD;
5873defm VRCP14SD : avx512_fp14_s<0x4D, "vrcp14sd", X86frcp14s, f64x_info>,
5874 VEX_W, EVEX_CD8<64, CD8VT1>, T8PD;
5875defm VRSQRT14SS : avx512_fp14_s<0x4F, "vrsqrt14ss", X86frsqrt14s, f32x_info>,
5876 EVEX_CD8<32, CD8VT1>, T8PD;
5877defm VRSQRT14SD : avx512_fp14_s<0x4F, "vrsqrt14sd", X86frsqrt14s, f64x_info>,
5878 VEX_W, EVEX_CD8<64, CD8VT1>, T8PD;
Elena Demikhovskya3a71402013-10-09 08:16:14 +00005879
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00005880/// avx512_fp14_p rcp14ps, rcp14pd, rsqrt14ps, rsqrt14pd
5881multiclass avx512_fp14_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
Robert Khasanov3e534c92014-10-28 16:37:13 +00005882 X86VectorVTInfo _> {
5883 defm r: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5884 (ins _.RC:$src), OpcodeStr, "$src", "$src",
5885 (_.FloatVT (OpNode _.RC:$src))>, EVEX, T8PD;
Craig Toppere1cac152016-06-07 07:27:54 +00005886 defm m: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5887 (ins _.MemOp:$src), OpcodeStr, "$src", "$src",
5888 (OpNode (_.FloatVT
5889 (bitconvert (_.LdFrag addr:$src))))>, EVEX, T8PD;
5890 defm mb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5891 (ins _.ScalarMemOp:$src), OpcodeStr,
5892 "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr,
5893 (OpNode (_.FloatVT
5894 (X86VBroadcast (_.ScalarLdFrag addr:$src))))>,
5895 EVEX, T8PD, EVEX_B;
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00005896}
Robert Khasanov3e534c92014-10-28 16:37:13 +00005897
5898multiclass avx512_fp14_p_vl_all<bits<8> opc, string OpcodeStr, SDNode OpNode> {
5899 defm PSZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"), OpNode, v16f32_info>,
5900 EVEX_V512, EVEX_CD8<32, CD8VF>;
5901 defm PDZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"), OpNode, v8f64_info>,
5902 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
5903
5904 // Define only if AVX512VL feature is present.
5905 let Predicates = [HasVLX] in {
5906 defm PSZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"),
5907 OpNode, v4f32x_info>,
5908 EVEX_V128, EVEX_CD8<32, CD8VF>;
5909 defm PSZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"),
5910 OpNode, v8f32x_info>,
5911 EVEX_V256, EVEX_CD8<32, CD8VF>;
5912 defm PDZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"),
5913 OpNode, v2f64x_info>,
5914 EVEX_V128, VEX_W, EVEX_CD8<64, CD8VF>;
5915 defm PDZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"),
5916 OpNode, v4f64x_info>,
5917 EVEX_V256, VEX_W, EVEX_CD8<64, CD8VF>;
5918 }
5919}
5920
5921defm VRSQRT14 : avx512_fp14_p_vl_all<0x4E, "vrsqrt14", X86frsqrt>;
5922defm VRCP14 : avx512_fp14_p_vl_all<0x4C, "vrcp14", X86frcp>;
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00005923
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00005924/// avx512_fp28_s rcp28ss, rcp28sd, rsqrt28ss, rsqrt28sd
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00005925multiclass avx512_fp28_s<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
5926 SDNode OpNode> {
5927
5928 defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5929 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
5930 "$src2, $src1", "$src1, $src2",
5931 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
5932 (i32 FROUND_CURRENT))>;
5933
5934 defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5935 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00005936 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00005937 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00005938 (i32 FROUND_NO_EXC))>, EVEX_B;
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00005939
5940 defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
Igor Breger4511e762016-02-22 11:48:27 +00005941 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00005942 "$src2, $src1", "$src1, $src2",
5943 (OpNode (_.VT _.RC:$src1),
5944 (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))),
5945 (i32 FROUND_CURRENT))>;
Elena Demikhovskya3a71402013-10-09 08:16:14 +00005946}
5947
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00005948multiclass avx512_eri_s<bits<8> opc, string OpcodeStr, SDNode OpNode> {
5949 defm SS : avx512_fp28_s<opc, OpcodeStr#"ss", f32x_info, OpNode>,
5950 EVEX_CD8<32, CD8VT1>;
5951 defm SD : avx512_fp28_s<opc, OpcodeStr#"sd", f64x_info, OpNode>,
5952 EVEX_CD8<64, CD8VT1>, VEX_W;
5953}
Elena Demikhovskya3a71402013-10-09 08:16:14 +00005954
Craig Toppere1cac152016-06-07 07:27:54 +00005955let Predicates = [HasERI] in {
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00005956 defm VRCP28 : avx512_eri_s<0xCB, "vrcp28", X86rcp28s>, T8PD, EVEX_4V;
5957 defm VRSQRT28 : avx512_eri_s<0xCD, "vrsqrt28", X86rsqrt28s>, T8PD, EVEX_4V;
5958}
Igor Breger8352a0d2015-07-28 06:53:28 +00005959
5960defm VGETEXP : avx512_eri_s<0x43, "vgetexp", X86fgetexpRnds>, T8PD, EVEX_4V;
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00005961/// avx512_fp28_p rcp28ps, rcp28pd, rsqrt28ps, rsqrt28pd
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00005962
5963multiclass avx512_fp28_p<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5964 SDNode OpNode> {
5965
5966 defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5967 (ins _.RC:$src), OpcodeStr, "$src", "$src",
5968 (OpNode (_.VT _.RC:$src), (i32 FROUND_CURRENT))>;
5969
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00005970 defm m : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5971 (ins _.MemOp:$src), OpcodeStr, "$src", "$src",
5972 (OpNode (_.FloatVT
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00005973 (bitconvert (_.LdFrag addr:$src))),
5974 (i32 FROUND_CURRENT))>;
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00005975
5976 defm mb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
Igor Breger4511e762016-02-22 11:48:27 +00005977 (ins _.ScalarMemOp:$src), OpcodeStr,
Asaf Badouh402ebb32015-06-03 13:41:48 +00005978 "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr,
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00005979 (OpNode (_.FloatVT
5980 (X86VBroadcast (_.ScalarLdFrag addr:$src))),
5981 (i32 FROUND_CURRENT))>, EVEX_B;
Elena Demikhovskya3a71402013-10-09 08:16:14 +00005982}
Asaf Badouh402ebb32015-06-03 13:41:48 +00005983multiclass avx512_fp28_p_round<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5984 SDNode OpNode> {
5985 defm rb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5986 (ins _.RC:$src), OpcodeStr,
5987 "{sae}, $src", "$src, {sae}",
5988 (OpNode (_.VT _.RC:$src), (i32 FROUND_NO_EXC))>, EVEX_B;
5989}
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00005990
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00005991multiclass avx512_eri<bits<8> opc, string OpcodeStr, SDNode OpNode> {
5992 defm PS : avx512_fp28_p<opc, OpcodeStr#"ps", v16f32_info, OpNode>,
Asaf Badouh402ebb32015-06-03 13:41:48 +00005993 avx512_fp28_p_round<opc, OpcodeStr#"ps", v16f32_info, OpNode>,
5994 T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>;
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00005995 defm PD : avx512_fp28_p<opc, OpcodeStr#"pd", v8f64_info, OpNode>,
Asaf Badouh402ebb32015-06-03 13:41:48 +00005996 avx512_fp28_p_round<opc, OpcodeStr#"pd", v8f64_info, OpNode>,
5997 T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00005998}
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00005999
Asaf Badouh402ebb32015-06-03 13:41:48 +00006000multiclass avx512_fp_unaryop_packed<bits<8> opc, string OpcodeStr,
6001 SDNode OpNode> {
6002 // Define only if AVX512VL feature is present.
6003 let Predicates = [HasVLX] in {
6004 defm PSZ128 : avx512_fp28_p<opc, OpcodeStr#"ps", v4f32x_info, OpNode>,
6005 EVEX_V128, T8PD, EVEX_CD8<32, CD8VF>;
6006 defm PSZ256 : avx512_fp28_p<opc, OpcodeStr#"ps", v8f32x_info, OpNode>,
6007 EVEX_V256, T8PD, EVEX_CD8<32, CD8VF>;
6008 defm PDZ128 : avx512_fp28_p<opc, OpcodeStr#"pd", v2f64x_info, OpNode>,
6009 EVEX_V128, VEX_W, T8PD, EVEX_CD8<64, CD8VF>;
6010 defm PDZ256 : avx512_fp28_p<opc, OpcodeStr#"pd", v4f64x_info, OpNode>,
6011 EVEX_V256, VEX_W, T8PD, EVEX_CD8<64, CD8VF>;
6012 }
6013}
Craig Toppere1cac152016-06-07 07:27:54 +00006014let Predicates = [HasERI] in {
Michael Liao5bf95782014-12-04 05:20:33 +00006015
Asaf Badouh402ebb32015-06-03 13:41:48 +00006016 defm VRSQRT28 : avx512_eri<0xCC, "vrsqrt28", X86rsqrt28>, EVEX;
6017 defm VRCP28 : avx512_eri<0xCA, "vrcp28", X86rcp28>, EVEX;
6018 defm VEXP2 : avx512_eri<0xC8, "vexp2", X86exp2>, EVEX;
6019}
6020defm VGETEXP : avx512_eri<0x42, "vgetexp", X86fgetexpRnd>,
6021 avx512_fp_unaryop_packed<0x42, "vgetexp", X86fgetexpRnd> , EVEX;
6022
6023multiclass avx512_sqrt_packed_round<bits<8> opc, string OpcodeStr,
6024 SDNode OpNodeRnd, X86VectorVTInfo _>{
6025 defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
6026 (ins _.RC:$src, AVX512RC:$rc), OpcodeStr, "$rc, $src", "$src, $rc",
6027 (_.VT (OpNodeRnd _.RC:$src, (i32 imm:$rc)))>,
6028 EVEX, EVEX_B, EVEX_RC;
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00006029}
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00006030
Robert Khasanoveb126392014-10-28 18:15:20 +00006031multiclass avx512_sqrt_packed<bits<8> opc, string OpcodeStr,
6032 SDNode OpNode, X86VectorVTInfo _>{
Robert Khasanov1cf354c2014-10-28 18:22:41 +00006033 defm r: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
Robert Khasanoveb126392014-10-28 18:15:20 +00006034 (ins _.RC:$src), OpcodeStr, "$src", "$src",
6035 (_.FloatVT (OpNode _.RC:$src))>, EVEX;
Craig Toppere1cac152016-06-07 07:27:54 +00006036 defm m: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
6037 (ins _.MemOp:$src), OpcodeStr, "$src", "$src",
6038 (OpNode (_.FloatVT
6039 (bitconvert (_.LdFrag addr:$src))))>, EVEX;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006040
Craig Toppere1cac152016-06-07 07:27:54 +00006041 defm mb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
6042 (ins _.ScalarMemOp:$src), OpcodeStr,
6043 "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr,
6044 (OpNode (_.FloatVT
6045 (X86VBroadcast (_.ScalarLdFrag addr:$src))))>,
6046 EVEX, EVEX_B;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006047}
6048
Robert Khasanoveb126392014-10-28 18:15:20 +00006049multiclass avx512_sqrt_packed_all<bits<8> opc, string OpcodeStr,
6050 SDNode OpNode> {
6051 defm PSZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode,
6052 v16f32_info>,
6053 EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
6054 defm PDZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode,
6055 v8f64_info>,
6056 EVEX_V512, VEX_W, PD, EVEX_CD8<64, CD8VF>;
6057 // Define only if AVX512VL feature is present.
6058 let Predicates = [HasVLX] in {
6059 defm PSZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"),
6060 OpNode, v4f32x_info>,
6061 EVEX_V128, PS, EVEX_CD8<32, CD8VF>;
6062 defm PSZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"),
6063 OpNode, v8f32x_info>,
6064 EVEX_V256, PS, EVEX_CD8<32, CD8VF>;
6065 defm PDZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"),
6066 OpNode, v2f64x_info>,
6067 EVEX_V128, VEX_W, PD, EVEX_CD8<64, CD8VF>;
6068 defm PDZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"),
6069 OpNode, v4f64x_info>,
6070 EVEX_V256, VEX_W, PD, EVEX_CD8<64, CD8VF>;
6071 }
6072}
6073
Asaf Badouh402ebb32015-06-03 13:41:48 +00006074multiclass avx512_sqrt_packed_all_round<bits<8> opc, string OpcodeStr,
6075 SDNode OpNodeRnd> {
6076 defm PSZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "ps"), OpNodeRnd,
6077 v16f32_info>, EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
6078 defm PDZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "pd"), OpNodeRnd,
6079 v8f64_info>, EVEX_V512, VEX_W, PD, EVEX_CD8<64, CD8VF>;
6080}
6081
Igor Breger4c4cd782015-09-20 09:13:41 +00006082multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
6083 string SUFF, SDNode OpNode, SDNode OpNodeRnd> {
6084
6085 defm r_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
6086 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
6087 "$src2, $src1", "$src1, $src2",
6088 (OpNodeRnd (_.VT _.RC:$src1),
6089 (_.VT _.RC:$src2),
6090 (i32 FROUND_CURRENT))>;
Craig Toppere1cac152016-06-07 07:27:54 +00006091 defm m_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
6092 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
6093 "$src2, $src1", "$src1, $src2",
6094 (OpNodeRnd (_.VT _.RC:$src1),
6095 (_.VT (scalar_to_vector
6096 (_.ScalarLdFrag addr:$src2))),
6097 (i32 FROUND_CURRENT))>;
Igor Breger4c4cd782015-09-20 09:13:41 +00006098
6099 defm rb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
6100 (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr,
6101 "$rc, $src2, $src1", "$src1, $src2, $rc",
6102 (OpNodeRnd (_.VT _.RC:$src1),
6103 (_.VT _.RC:$src2),
6104 (i32 imm:$rc))>,
6105 EVEX_B, EVEX_RC;
6106
Craig Toppere1cac152016-06-07 07:27:54 +00006107 let isCodeGenOnly = 1, hasSideEffects = 0 in {
Elena Demikhovsky0d0692d2015-12-01 12:43:46 +00006108 def r : I<opc, MRMSrcReg, (outs _.FRC:$dst),
Igor Breger4c4cd782015-09-20 09:13:41 +00006109 (ins _.FRC:$src1, _.FRC:$src2),
6110 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>;
6111
6112 let mayLoad = 1 in
Elena Demikhovsky0d0692d2015-12-01 12:43:46 +00006113 def m : I<opc, MRMSrcMem, (outs _.FRC:$dst),
Igor Breger4c4cd782015-09-20 09:13:41 +00006114 (ins _.FRC:$src1, _.ScalarMemOp:$src2),
6115 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>;
6116 }
6117
6118 def : Pat<(_.EltVT (OpNode _.FRC:$src)),
6119 (!cast<Instruction>(NAME#SUFF#Zr)
6120 (_.EltVT (IMPLICIT_DEF)), _.FRC:$src)>;
6121
6122 def : Pat<(_.EltVT (OpNode (load addr:$src))),
6123 (!cast<Instruction>(NAME#SUFF#Zm)
Dimitry Andricdb417b62016-02-19 20:14:11 +00006124 (_.EltVT (IMPLICIT_DEF)), addr:$src)>, Requires<[HasAVX512, OptForSize]>;
Igor Breger4c4cd782015-09-20 09:13:41 +00006125}
6126
6127multiclass avx512_sqrt_scalar_all<bits<8> opc, string OpcodeStr> {
6128 defm SSZ : avx512_sqrt_scalar<opc, OpcodeStr#"ss", f32x_info, "SS", fsqrt,
6129 X86fsqrtRnds>, EVEX_CD8<32, CD8VT1>, EVEX_4V, XS;
6130 defm SDZ : avx512_sqrt_scalar<opc, OpcodeStr#"sd", f64x_info, "SD", fsqrt,
6131 X86fsqrtRnds>, EVEX_CD8<64, CD8VT1>, EVEX_4V, XD, VEX_W;
6132}
6133
Asaf Badouh402ebb32015-06-03 13:41:48 +00006134defm VSQRT : avx512_sqrt_packed_all<0x51, "vsqrt", fsqrt>,
6135 avx512_sqrt_packed_all_round<0x51, "vsqrt", X86fsqrtRnd>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006136
Igor Breger4c4cd782015-09-20 09:13:41 +00006137defm VSQRT : avx512_sqrt_scalar_all<0x51, "vsqrt">, VEX_LIG;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006138
Elena Demikhovskya3a71402013-10-09 08:16:14 +00006139let Predicates = [HasAVX512] in {
Elena Demikhovskya3a71402013-10-09 08:16:14 +00006140 def : Pat<(f32 (X86frsqrt FR32X:$src)),
Asaf Badouheaf2da12015-09-21 10:23:53 +00006141 (COPY_TO_REGCLASS (VRSQRT14SSrr (v4f32 (IMPLICIT_DEF)), (COPY_TO_REGCLASS FR32X:$src, VR128X)), VR128X)>;
Elena Demikhovskya3a71402013-10-09 08:16:14 +00006142 def : Pat<(f32 (X86frsqrt (load addr:$src))),
Asaf Badouheaf2da12015-09-21 10:23:53 +00006143 (COPY_TO_REGCLASS (VRSQRT14SSrm (v4f32 (IMPLICIT_DEF)), addr:$src), VR128X)>,
Elena Demikhovskya3a71402013-10-09 08:16:14 +00006144 Requires<[OptForSize]>;
Elena Demikhovskya3a71402013-10-09 08:16:14 +00006145 def : Pat<(f32 (X86frcp FR32X:$src)),
Asaf Badouheaf2da12015-09-21 10:23:53 +00006146 (COPY_TO_REGCLASS (VRCP14SSrr (v4f32 (IMPLICIT_DEF)), (COPY_TO_REGCLASS FR32X:$src, VR128X)), VR128X )>;
Elena Demikhovskya3a71402013-10-09 08:16:14 +00006147 def : Pat<(f32 (X86frcp (load addr:$src))),
Asaf Badouheaf2da12015-09-21 10:23:53 +00006148 (COPY_TO_REGCLASS (VRCP14SSrm (v4f32 (IMPLICIT_DEF)), addr:$src), VR128X)>,
Elena Demikhovskya3a71402013-10-09 08:16:14 +00006149 Requires<[OptForSize]>;
Elena Demikhovskya3a71402013-10-09 08:16:14 +00006150}
6151
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00006152multiclass
6153avx512_rndscale_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> {
Elena Demikhovskyde3f7512014-01-01 15:12:34 +00006154
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00006155 let ExeDomain = _.ExeDomain in {
6156 defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
6157 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr,
6158 "$src3, $src2, $src1", "$src1, $src2, $src3",
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00006159 (_.VT (X86RndScales (_.VT _.RC:$src1), (_.VT _.RC:$src2),
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00006160 (i32 imm:$src3), (i32 FROUND_CURRENT)))>;
6161
6162 defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
6163 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr,
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00006164 "$src3, {sae}, $src2, $src1", "$src1, $src2, {sae}, $src3",
6165 (_.VT (X86RndScales (_.VT _.RC:$src1), (_.VT _.RC:$src2),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00006166 (i32 imm:$src3), (i32 FROUND_NO_EXC)))>, EVEX_B;
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00006167
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00006168 defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006169 (ins _.RC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3),
6170 OpcodeStr,
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00006171 "$src3, $src2, $src1", "$src1, $src2, $src3",
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00006172 (_.VT (X86RndScales (_.VT _.RC:$src1),
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00006173 (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))),
6174 (i32 imm:$src3), (i32 FROUND_CURRENT)))>;
6175 }
6176 let Predicates = [HasAVX512] in {
6177 def : Pat<(ffloor _.FRC:$src), (COPY_TO_REGCLASS
6178 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)),
6179 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x1))), _.FRC)>;
6180 def : Pat<(fceil _.FRC:$src), (COPY_TO_REGCLASS
6181 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)),
6182 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x2))), _.FRC)>;
6183 def : Pat<(ftrunc _.FRC:$src), (COPY_TO_REGCLASS
6184 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)),
6185 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x3))), _.FRC)>;
6186 def : Pat<(frint _.FRC:$src), (COPY_TO_REGCLASS
6187 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)),
6188 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x4))), _.FRC)>;
6189 def : Pat<(fnearbyint _.FRC:$src), (COPY_TO_REGCLASS
6190 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)),
6191 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0xc))), _.FRC)>;
6192
6193 def : Pat<(ffloor (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
6194 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)),
6195 addr:$src, (i32 0x1))), _.FRC)>;
6196 def : Pat<(fceil (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
6197 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)),
6198 addr:$src, (i32 0x2))), _.FRC)>;
6199 def : Pat<(ftrunc (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
6200 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)),
6201 addr:$src, (i32 0x3))), _.FRC)>;
6202 def : Pat<(frint (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
6203 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)),
6204 addr:$src, (i32 0x4))), _.FRC)>;
6205 def : Pat<(fnearbyint (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
6206 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)),
6207 addr:$src, (i32 0xc))), _.FRC)>;
6208 }
Elena Demikhovskyde3f7512014-01-01 15:12:34 +00006209}
6210
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00006211defm VRNDSCALESS : avx512_rndscale_scalar<0x0A, "vrndscaless", f32x_info>,
6212 AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VT1>;
Michael Liao5bf95782014-12-04 05:20:33 +00006213
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00006214defm VRNDSCALESD : avx512_rndscale_scalar<0x0B, "vrndscalesd", f64x_info>, VEX_W,
6215 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VT1>;
Eric Christopher0d94fa92015-02-20 00:45:28 +00006216
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006217//-------------------------------------------------
6218// Integer truncate and extend operations
6219//-------------------------------------------------
6220
Igor Breger074a64e2015-07-24 17:24:15 +00006221multiclass avx512_trunc_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
6222 X86VectorVTInfo SrcInfo, X86VectorVTInfo DestInfo,
6223 X86MemOperand x86memop> {
6224
6225 defm rr : AVX512_maskable<opc, MRMDestReg, DestInfo, (outs DestInfo.RC:$dst),
6226 (ins SrcInfo.RC:$src1), OpcodeStr ,"$src1", "$src1",
6227 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1)))>,
6228 EVEX, T8XS;
6229
6230 // for intrinsic patter match
6231 def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask,
6232 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))),
6233 undef)),
6234 (!cast<Instruction>(NAME#SrcInfo.ZSuffix##rrkz) DestInfo.KRCWM:$mask ,
6235 SrcInfo.RC:$src1)>;
6236
6237 def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask,
6238 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))),
6239 DestInfo.ImmAllZerosV)),
6240 (!cast<Instruction>(NAME#SrcInfo.ZSuffix##rrkz) DestInfo.KRCWM:$mask ,
6241 SrcInfo.RC:$src1)>;
6242
6243 def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask,
6244 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))),
6245 DestInfo.RC:$src0)),
6246 (!cast<Instruction>(NAME#SrcInfo.ZSuffix##rrk) DestInfo.RC:$src0,
6247 DestInfo.KRCWM:$mask ,
6248 SrcInfo.RC:$src1)>;
6249
Craig Topper99f6b622016-05-01 01:03:56 +00006250 let mayStore = 1, mayLoad = 1, hasSideEffects = 0 in {
Igor Breger074a64e2015-07-24 17:24:15 +00006251 def mr : AVX512XS8I<opc, MRMDestMem, (outs),
6252 (ins x86memop:$dst, SrcInfo.RC:$src),
Craig Topper9feea572016-01-11 00:44:58 +00006253 OpcodeStr # "\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006254 []>, EVEX;
6255
Igor Breger074a64e2015-07-24 17:24:15 +00006256 def mrk : AVX512XS8I<opc, MRMDestMem, (outs),
6257 (ins x86memop:$dst, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src),
Craig Topper9feea572016-01-11 00:44:58 +00006258 OpcodeStr # "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",
Robert Khasanov189e7fd2014-04-22 11:36:19 +00006259 []>, EVEX, EVEX_K;
Craig Topper99f6b622016-05-01 01:03:56 +00006260 }//mayStore = 1, mayLoad = 1, hasSideEffects = 0
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006261}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006262
Igor Breger074a64e2015-07-24 17:24:15 +00006263multiclass avx512_trunc_mr_lowering<X86VectorVTInfo SrcInfo,
6264 X86VectorVTInfo DestInfo,
6265 PatFrag truncFrag, PatFrag mtruncFrag > {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006266
Igor Breger074a64e2015-07-24 17:24:15 +00006267 def : Pat<(truncFrag (SrcInfo.VT SrcInfo.RC:$src), addr:$dst),
6268 (!cast<Instruction>(NAME#SrcInfo.ZSuffix##mr)
6269 addr:$dst, SrcInfo.RC:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006270
Igor Breger074a64e2015-07-24 17:24:15 +00006271 def : Pat<(mtruncFrag addr:$dst, SrcInfo.KRCWM:$mask,
6272 (SrcInfo.VT SrcInfo.RC:$src)),
6273 (!cast<Instruction>(NAME#SrcInfo.ZSuffix##mrk)
6274 addr:$dst, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src)>;
6275}
6276
6277multiclass avx512_trunc_sat_mr_lowering<X86VectorVTInfo SrcInfo,
6278 X86VectorVTInfo DestInfo, string sat > {
6279
6280 def: Pat<(!cast<Intrinsic>("int_x86_avx512_mask_pmov"#sat#"_"#SrcInfo.Suffix#
6281 DestInfo.Suffix#"_mem_"#SrcInfo.Size)
6282 addr:$ptr, (SrcInfo.VT SrcInfo.RC:$src), SrcInfo.MRC:$mask),
6283 (!cast<Instruction>(NAME#SrcInfo.ZSuffix##mrk) addr:$ptr,
6284 (COPY_TO_REGCLASS SrcInfo.MRC:$mask, SrcInfo.KRCWM),
6285 (SrcInfo.VT SrcInfo.RC:$src))>;
6286
6287 def: Pat<(!cast<Intrinsic>("int_x86_avx512_mask_pmov"#sat#"_"#SrcInfo.Suffix#
6288 DestInfo.Suffix#"_mem_"#SrcInfo.Size)
6289 addr:$ptr, (SrcInfo.VT SrcInfo.RC:$src), -1),
6290 (!cast<Instruction>(NAME#SrcInfo.ZSuffix##mr) addr:$ptr,
6291 (SrcInfo.VT SrcInfo.RC:$src))>;
6292}
6293
6294multiclass avx512_trunc<bits<8> opc, string OpcodeStr, SDNode OpNode,
6295 AVX512VLVectorVTInfo VTSrcInfo, X86VectorVTInfo DestInfoZ128,
6296 X86VectorVTInfo DestInfoZ256, X86VectorVTInfo DestInfoZ,
6297 X86MemOperand x86memopZ128, X86MemOperand x86memopZ256,
6298 X86MemOperand x86memopZ, PatFrag truncFrag, PatFrag mtruncFrag,
6299 Predicate prd = HasAVX512>{
6300
6301 let Predicates = [HasVLX, prd] in {
6302 defm Z128: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info128,
6303 DestInfoZ128, x86memopZ128>,
6304 avx512_trunc_mr_lowering<VTSrcInfo.info128, DestInfoZ128,
6305 truncFrag, mtruncFrag>, EVEX_V128;
6306
6307 defm Z256: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info256,
6308 DestInfoZ256, x86memopZ256>,
6309 avx512_trunc_mr_lowering<VTSrcInfo.info256, DestInfoZ256,
6310 truncFrag, mtruncFrag>, EVEX_V256;
6311 }
6312 let Predicates = [prd] in
6313 defm Z: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info512,
6314 DestInfoZ, x86memopZ>,
6315 avx512_trunc_mr_lowering<VTSrcInfo.info512, DestInfoZ,
6316 truncFrag, mtruncFrag>, EVEX_V512;
6317}
6318
6319multiclass avx512_trunc_sat<bits<8> opc, string OpcodeStr, SDNode OpNode,
6320 AVX512VLVectorVTInfo VTSrcInfo, X86VectorVTInfo DestInfoZ128,
6321 X86VectorVTInfo DestInfoZ256, X86VectorVTInfo DestInfoZ,
6322 X86MemOperand x86memopZ128, X86MemOperand x86memopZ256,
6323 X86MemOperand x86memopZ, string sat, Predicate prd = HasAVX512>{
6324
6325 let Predicates = [HasVLX, prd] in {
6326 defm Z128: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info128,
6327 DestInfoZ128, x86memopZ128>,
6328 avx512_trunc_sat_mr_lowering<VTSrcInfo.info128, DestInfoZ128,
6329 sat>, EVEX_V128;
6330
6331 defm Z256: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info256,
6332 DestInfoZ256, x86memopZ256>,
6333 avx512_trunc_sat_mr_lowering<VTSrcInfo.info256, DestInfoZ256,
6334 sat>, EVEX_V256;
6335 }
6336 let Predicates = [prd] in
6337 defm Z: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info512,
6338 DestInfoZ, x86memopZ>,
6339 avx512_trunc_sat_mr_lowering<VTSrcInfo.info512, DestInfoZ,
6340 sat>, EVEX_V512;
6341}
6342
6343multiclass avx512_trunc_qb<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6344 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
6345 v16i8x_info, v16i8x_info, v16i8x_info, i16mem, i32mem, i64mem,
6346 truncstorevi8, masked_truncstorevi8>, EVEX_CD8<8, CD8VO>;
6347}
6348multiclass avx512_trunc_sat_qb<bits<8> opc, string sat, SDNode OpNode> {
6349 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qb", OpNode, avx512vl_i64_info,
6350 v16i8x_info, v16i8x_info, v16i8x_info, i16mem, i32mem, i64mem,
6351 sat>, EVEX_CD8<8, CD8VO>;
6352}
6353
6354multiclass avx512_trunc_qw<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6355 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
6356 v8i16x_info, v8i16x_info, v8i16x_info, i32mem, i64mem, i128mem,
6357 truncstorevi16, masked_truncstorevi16>, EVEX_CD8<16, CD8VQ>;
6358}
6359multiclass avx512_trunc_sat_qw<bits<8> opc, string sat, SDNode OpNode> {
6360 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qw", OpNode, avx512vl_i64_info,
6361 v8i16x_info, v8i16x_info, v8i16x_info, i32mem, i64mem, i128mem,
6362 sat>, EVEX_CD8<16, CD8VQ>;
6363}
6364
6365multiclass avx512_trunc_qd<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6366 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
6367 v4i32x_info, v4i32x_info, v8i32x_info, i64mem, i128mem, i256mem,
6368 truncstorevi32, masked_truncstorevi32>, EVEX_CD8<32, CD8VH>;
6369}
6370multiclass avx512_trunc_sat_qd<bits<8> opc, string sat, SDNode OpNode> {
6371 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qd", OpNode, avx512vl_i64_info,
6372 v4i32x_info, v4i32x_info, v8i32x_info, i64mem, i128mem, i256mem,
6373 sat>, EVEX_CD8<32, CD8VH>;
6374}
6375
6376multiclass avx512_trunc_db<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6377 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i32_info,
6378 v16i8x_info, v16i8x_info, v16i8x_info, i32mem, i64mem, i128mem,
6379 truncstorevi8, masked_truncstorevi8>, EVEX_CD8<8, CD8VQ>;
6380}
6381multiclass avx512_trunc_sat_db<bits<8> opc, string sat, SDNode OpNode> {
6382 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"db", OpNode, avx512vl_i32_info,
6383 v16i8x_info, v16i8x_info, v16i8x_info, i32mem, i64mem, i128mem,
6384 sat>, EVEX_CD8<8, CD8VQ>;
6385}
6386
6387multiclass avx512_trunc_dw<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6388 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i32_info,
6389 v8i16x_info, v8i16x_info, v16i16x_info, i64mem, i128mem, i256mem,
6390 truncstorevi16, masked_truncstorevi16>, EVEX_CD8<16, CD8VH>;
6391}
6392multiclass avx512_trunc_sat_dw<bits<8> opc, string sat, SDNode OpNode> {
6393 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"dw", OpNode, avx512vl_i32_info,
6394 v8i16x_info, v8i16x_info, v16i16x_info, i64mem, i128mem, i256mem,
6395 sat>, EVEX_CD8<16, CD8VH>;
6396}
6397
6398multiclass avx512_trunc_wb<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6399 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i16_info,
6400 v16i8x_info, v16i8x_info, v32i8x_info, i64mem, i128mem, i256mem,
6401 truncstorevi8, masked_truncstorevi8,HasBWI>, EVEX_CD8<16, CD8VH>;
6402}
6403multiclass avx512_trunc_sat_wb<bits<8> opc, string sat, SDNode OpNode> {
6404 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"wb", OpNode, avx512vl_i16_info,
6405 v16i8x_info, v16i8x_info, v32i8x_info, i64mem, i128mem, i256mem,
6406 sat, HasBWI>, EVEX_CD8<16, CD8VH>;
6407}
6408
6409defm VPMOVQB : avx512_trunc_qb<0x32, "vpmovqb", X86vtrunc>;
6410defm VPMOVSQB : avx512_trunc_sat_qb<0x22, "s", X86vtruncs>;
6411defm VPMOVUSQB : avx512_trunc_sat_qb<0x12, "us", X86vtruncus>;
6412
6413defm VPMOVQW : avx512_trunc_qw<0x34, "vpmovqw", X86vtrunc>;
6414defm VPMOVSQW : avx512_trunc_sat_qw<0x24, "s", X86vtruncs>;
6415defm VPMOVUSQW : avx512_trunc_sat_qw<0x14, "us", X86vtruncus>;
6416
6417defm VPMOVQD : avx512_trunc_qd<0x35, "vpmovqd", X86vtrunc>;
6418defm VPMOVSQD : avx512_trunc_sat_qd<0x25, "s", X86vtruncs>;
6419defm VPMOVUSQD : avx512_trunc_sat_qd<0x15, "us", X86vtruncus>;
6420
6421defm VPMOVDB : avx512_trunc_db<0x31, "vpmovdb", X86vtrunc>;
6422defm VPMOVSDB : avx512_trunc_sat_db<0x21, "s", X86vtruncs>;
6423defm VPMOVUSDB : avx512_trunc_sat_db<0x11, "us", X86vtruncus>;
6424
6425defm VPMOVDW : avx512_trunc_dw<0x33, "vpmovdw", X86vtrunc>;
6426defm VPMOVSDW : avx512_trunc_sat_dw<0x23, "s", X86vtruncs>;
6427defm VPMOVUSDW : avx512_trunc_sat_dw<0x13, "us", X86vtruncus>;
6428
6429defm VPMOVWB : avx512_trunc_wb<0x30, "vpmovwb", X86vtrunc>;
6430defm VPMOVSWB : avx512_trunc_sat_wb<0x20, "s", X86vtruncs>;
6431defm VPMOVUSWB : avx512_trunc_sat_wb<0x10, "us", X86vtruncus>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006432
Elena Demikhovskydb738d92015-11-01 11:45:47 +00006433let Predicates = [HasAVX512, NoVLX] in {
6434def: Pat<(v8i16 (X86vtrunc (v8i32 VR256X:$src))),
6435 (v8i16 (EXTRACT_SUBREG
6436 (v16i16 (VPMOVDWZrr (v16i32 (SUBREG_TO_REG (i32 0),
6437 VR256X:$src, sub_ymm)))), sub_xmm))>;
6438def: Pat<(v4i32 (X86vtrunc (v4i64 VR256X:$src))),
6439 (v4i32 (EXTRACT_SUBREG
6440 (v8i32 (VPMOVQDZrr (v8i64 (SUBREG_TO_REG (i32 0),
6441 VR256X:$src, sub_ymm)))), sub_xmm))>;
6442}
6443
6444let Predicates = [HasBWI, NoVLX] in {
6445def: Pat<(v16i8 (X86vtrunc (v16i16 VR256X:$src))),
6446 (v16i8 (EXTRACT_SUBREG (VPMOVWBZrr (v32i16 (SUBREG_TO_REG (i32 0),
6447 VR256X:$src, sub_ymm))), sub_xmm))>;
6448}
6449
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006450multiclass avx512_extend_common<bits<8> opc, string OpcodeStr,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006451 X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo,
6452 X86MemOperand x86memop, PatFrag LdFrag, SDPatternOperator OpNode,
6453 bit IsCodeGenOnly>{
6454 let isCodeGenOnly = IsCodeGenOnly in {
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006455 defm rr : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),
6456 (ins SrcInfo.RC:$src), OpcodeStr ,"$src", "$src",
6457 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src)))>,
6458 EVEX;
Robert Khasanov189e7fd2014-04-22 11:36:19 +00006459
Craig Toppere1cac152016-06-07 07:27:54 +00006460 defm rm : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
6461 (ins x86memop:$src), OpcodeStr ,"$src", "$src",
6462 (DestInfo.VT (LdFrag addr:$src))>,
6463 EVEX;
Igor Breger2ba64ab2016-05-22 10:21:04 +00006464 }//isCodeGenOnly
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006465}
6466
Igor Bregerc7ba5692016-02-24 08:15:20 +00006467// support full register inputs (like SSE paterns)
Igor Breger2ba64ab2016-05-22 10:21:04 +00006468multiclass avx512_extend_lowering<SDPatternOperator OpNode, X86VectorVTInfo To,
Igor Bregerc7ba5692016-02-24 08:15:20 +00006469 X86VectorVTInfo From, SubRegIndex SubRegIdx> {
6470 def : Pat<(To.VT (OpNode (From.VT From.RC:$src))),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006471 (!cast<Instruction>(NAME#To.ZSuffix#"rr")
Igor Bregerc7ba5692016-02-24 08:15:20 +00006472 (EXTRACT_SUBREG From.RC:$src, SubRegIdx))>;
6473}
6474
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006475multiclass avx512_extend_BW<bits<8> opc, string OpcodeStr,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006476 SDPatternOperator OpNode, bit IsCodeGenOnly,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006477 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
6478 let Predicates = [HasVLX, HasBWI] in {
6479 defm Z128: avx512_extend_common<opc, OpcodeStr, v8i16x_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006480 v16i8x_info, i64mem, LdFrag, OpNode, IsCodeGenOnly>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006481 EVEX_CD8<8, CD8VH>, T8PD, EVEX_V128;
Robert Khasanov189e7fd2014-04-22 11:36:19 +00006482
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006483 defm Z256: avx512_extend_common<opc, OpcodeStr, v16i16x_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006484 v16i8x_info, i128mem, LdFrag, OpNode, IsCodeGenOnly>,
Igor Bregerc7ba5692016-02-24 08:15:20 +00006485 avx512_extend_lowering<OpNode, v16i16x_info, v32i8x_info, sub_xmm>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006486 EVEX_CD8<8, CD8VH>, T8PD, EVEX_V256;
6487 }
6488 let Predicates = [HasBWI] in {
6489 defm Z : avx512_extend_common<opc, OpcodeStr, v32i16_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006490 v32i8x_info, i256mem, LdFrag, OpNode, IsCodeGenOnly>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006491 EVEX_CD8<8, CD8VH>, T8PD, EVEX_V512;
6492 }
6493}
6494
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006495multiclass avx512_extend_BD<bits<8> opc, string OpcodeStr,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006496 SDPatternOperator OpNode, bit IsCodeGenOnly,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006497 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
6498 let Predicates = [HasVLX, HasAVX512] in {
6499 defm Z128: avx512_extend_common<opc, OpcodeStr, v4i32x_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006500 v16i8x_info, i32mem, LdFrag, OpNode, IsCodeGenOnly>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006501 EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V128;
6502
6503 defm Z256: avx512_extend_common<opc, OpcodeStr, v8i32x_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006504 v16i8x_info, i64mem, LdFrag, OpNode, IsCodeGenOnly>,
Igor Bregerc7ba5692016-02-24 08:15:20 +00006505 avx512_extend_lowering<OpNode, v8i32x_info, v32i8x_info, sub_xmm>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006506 EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V256;
6507 }
6508 let Predicates = [HasAVX512] in {
6509 defm Z : avx512_extend_common<opc, OpcodeStr, v16i32_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006510 v16i8x_info, i128mem, LdFrag, OpNode, IsCodeGenOnly>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006511 EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V512;
6512 }
6513}
6514
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006515multiclass avx512_extend_BQ<bits<8> opc, string OpcodeStr,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006516 SDPatternOperator OpNode, bit IsCodeGenOnly,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006517 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
6518 let Predicates = [HasVLX, HasAVX512] in {
6519 defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006520 v16i8x_info, i16mem, LdFrag, OpNode, IsCodeGenOnly>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006521 EVEX_CD8<8, CD8VO>, T8PD, EVEX_V128;
6522
6523 defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006524 v16i8x_info, i32mem, LdFrag, OpNode, IsCodeGenOnly>,
Igor Bregerc7ba5692016-02-24 08:15:20 +00006525 avx512_extend_lowering<OpNode, v4i64x_info, v32i8x_info, sub_xmm>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006526 EVEX_CD8<8, CD8VO>, T8PD, EVEX_V256;
6527 }
6528 let Predicates = [HasAVX512] in {
6529 defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006530 v16i8x_info, i64mem, LdFrag, OpNode, IsCodeGenOnly>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006531 EVEX_CD8<8, CD8VO>, T8PD, EVEX_V512;
6532 }
6533}
6534
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006535multiclass avx512_extend_WD<bits<8> opc, string OpcodeStr,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006536 SDPatternOperator OpNode, bit IsCodeGenOnly,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006537 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> {
6538 let Predicates = [HasVLX, HasAVX512] in {
6539 defm Z128: avx512_extend_common<opc, OpcodeStr, v4i32x_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006540 v8i16x_info, i64mem, LdFrag, OpNode, IsCodeGenOnly>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006541 EVEX_CD8<16, CD8VH>, T8PD, EVEX_V128;
6542
6543 defm Z256: avx512_extend_common<opc, OpcodeStr, v8i32x_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006544 v8i16x_info, i128mem, LdFrag, OpNode, IsCodeGenOnly>,
Igor Bregerc7ba5692016-02-24 08:15:20 +00006545 avx512_extend_lowering<OpNode, v8i32x_info, v16i16x_info, sub_xmm>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006546 EVEX_CD8<16, CD8VH>, T8PD, EVEX_V256;
6547 }
6548 let Predicates = [HasAVX512] in {
6549 defm Z : avx512_extend_common<opc, OpcodeStr, v16i32_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006550 v16i16x_info, i256mem, LdFrag, OpNode, IsCodeGenOnly>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006551 EVEX_CD8<16, CD8VH>, T8PD, EVEX_V512;
6552 }
6553}
6554
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006555multiclass avx512_extend_WQ<bits<8> opc, string OpcodeStr,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006556 SDPatternOperator OpNode, bit IsCodeGenOnly,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006557 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> {
6558 let Predicates = [HasVLX, HasAVX512] in {
6559 defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006560 v8i16x_info, i32mem, LdFrag, OpNode, IsCodeGenOnly>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006561 EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V128;
6562
6563 defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006564 v8i16x_info, i64mem, LdFrag, OpNode, IsCodeGenOnly>,
Igor Bregerc7ba5692016-02-24 08:15:20 +00006565 avx512_extend_lowering<OpNode, v4i64x_info, v16i16x_info, sub_xmm>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006566 EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V256;
6567 }
6568 let Predicates = [HasAVX512] in {
6569 defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006570 v8i16x_info, i128mem, LdFrag, OpNode, IsCodeGenOnly>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006571 EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V512;
6572 }
6573}
6574
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006575multiclass avx512_extend_DQ<bits<8> opc, string OpcodeStr,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006576 SDPatternOperator OpNode, bit IsCodeGenOnly,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006577 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi32")> {
6578
6579 let Predicates = [HasVLX, HasAVX512] in {
6580 defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006581 v4i32x_info, i64mem, LdFrag, OpNode, IsCodeGenOnly>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006582 EVEX_CD8<32, CD8VH>, T8PD, EVEX_V128;
6583
6584 defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006585 v4i32x_info, i128mem, LdFrag, OpNode, IsCodeGenOnly>,
Igor Bregerc7ba5692016-02-24 08:15:20 +00006586 avx512_extend_lowering<OpNode, v4i64x_info, v8i32x_info, sub_xmm>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006587 EVEX_CD8<32, CD8VH>, T8PD, EVEX_V256;
6588 }
6589 let Predicates = [HasAVX512] in {
6590 defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info,
Igor Breger2ba64ab2016-05-22 10:21:04 +00006591 v8i32x_info, i256mem, LdFrag, OpNode, IsCodeGenOnly>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006592 EVEX_CD8<32, CD8VH>, T8PD, EVEX_V512;
6593 }
6594}
6595
Igor Breger2ba64ab2016-05-22 10:21:04 +00006596defm VPMOVZXBW : avx512_extend_BW<0x30, "vpmovzxbw", X86vzext, 0, "z">;
6597defm VPMOVZXBD : avx512_extend_BD<0x31, "vpmovzxbd", X86vzext, 0, "z">;
6598defm VPMOVZXBQ : avx512_extend_BQ<0x32, "vpmovzxbq", X86vzext, 0, "z">;
6599defm VPMOVZXWD : avx512_extend_WD<0x33, "vpmovzxwd", X86vzext, 0, "z">;
6600defm VPMOVZXWQ : avx512_extend_WQ<0x34, "vpmovzxwq", X86vzext, 0, "z">;
6601defm VPMOVZXDQ : avx512_extend_DQ<0x35, "vpmovzxdq", X86vzext, 0, "z">;
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006602
Igor Breger2ba64ab2016-05-22 10:21:04 +00006603defm VPMOVSXBW: avx512_extend_BW<0x20, "vpmovsxbw", X86vsext, 0, "s">;
6604defm VPMOVSXBD: avx512_extend_BD<0x21, "vpmovsxbd", X86vsext, 0, "s">;
6605defm VPMOVSXBQ: avx512_extend_BQ<0x22, "vpmovsxbq", X86vsext, 0, "s">;
6606defm VPMOVSXWD: avx512_extend_WD<0x23, "vpmovsxwd", X86vsext, 0, "s">;
6607defm VPMOVSXWQ: avx512_extend_WQ<0x24, "vpmovsxwq", X86vsext, 0, "s">;
6608defm VPMOVSXDQ: avx512_extend_DQ<0x25, "vpmovsxdq", X86vsext, 0, "s">;
Elena Demikhovsky3948c592015-05-27 08:15:19 +00006609
Igor Breger2ba64ab2016-05-22 10:21:04 +00006610// EXTLOAD patterns, implemented using vpmovz
6611defm VPMOVAXBW : avx512_extend_BW<0x30, "vpmovzxbw", null_frag, 1, "">;
6612defm VPMOVAXBD : avx512_extend_BD<0x31, "vpmovzxbd", null_frag, 1, "">;
6613defm VPMOVAXBQ : avx512_extend_BQ<0x32, "vpmovzxbq", null_frag, 1, "">;
6614defm VPMOVAXWD : avx512_extend_WD<0x33, "vpmovzxwd", null_frag, 1, "">;
6615defm VPMOVAXWQ : avx512_extend_WQ<0x34, "vpmovzxwq", null_frag, 1, "">;
6616defm VPMOVAXDQ : avx512_extend_DQ<0x35, "vpmovzxdq", null_frag, 1, "">;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006617
6618//===----------------------------------------------------------------------===//
6619// GATHER - SCATTER Operations
6620
Elena Demikhovskye1eda8a2015-04-30 08:38:48 +00006621multiclass avx512_gather<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
6622 X86MemOperand memop, PatFrag GatherNode> {
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006623 let Constraints = "@earlyclobber $dst, $src1 = $dst, $mask = $mask_wb",
6624 ExeDomain = _.ExeDomain in
Elena Demikhovskye1eda8a2015-04-30 08:38:48 +00006625 def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst, _.KRCWM:$mask_wb),
6626 (ins _.RC:$src1, _.KRCWM:$mask, memop:$src2),
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006627 !strconcat(OpcodeStr#_.Suffix,
Craig Topperedb09112014-11-25 20:11:23 +00006628 "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"),
Elena Demikhovskye1eda8a2015-04-30 08:38:48 +00006629 [(set _.RC:$dst, _.KRCWM:$mask_wb,
6630 (GatherNode (_.VT _.RC:$src1), _.KRCWM:$mask,
6631 vectoraddr:$src2))]>, EVEX, EVEX_K,
6632 EVEX_CD8<_.EltSize, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006633}
Cameron McInally45325962014-03-26 13:50:50 +00006634
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006635multiclass avx512_gather_q_pd<bits<8> dopc, bits<8> qopc,
6636 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
6637 defm NAME##D##SUFF##Z: avx512_gather<dopc, OpcodeStr##"d", _.info512,
Igor Breger45ef10f2016-02-25 13:30:17 +00006638 vy512mem, mgatherv8i32>, EVEX_V512, VEX_W;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006639 defm NAME##Q##SUFF##Z: avx512_gather<qopc, OpcodeStr##"q", _.info512,
Igor Breger45ef10f2016-02-25 13:30:17 +00006640 vz512mem, mgatherv8i64>, EVEX_V512, VEX_W;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006641let Predicates = [HasVLX] in {
6642 defm NAME##D##SUFF##Z256: avx512_gather<dopc, OpcodeStr##"d", _.info256,
Igor Breger45ef10f2016-02-25 13:30:17 +00006643 vx256xmem, mgatherv4i32>, EVEX_V256, VEX_W;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006644 defm NAME##Q##SUFF##Z256: avx512_gather<qopc, OpcodeStr##"q", _.info256,
Igor Breger45ef10f2016-02-25 13:30:17 +00006645 vy256xmem, mgatherv4i64>, EVEX_V256, VEX_W;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006646 defm NAME##D##SUFF##Z128: avx512_gather<dopc, OpcodeStr##"d", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00006647 vx128xmem, mgatherv4i32>, EVEX_V128, VEX_W;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006648 defm NAME##Q##SUFF##Z128: avx512_gather<qopc, OpcodeStr##"q", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00006649 vx128xmem, mgatherv2i64>, EVEX_V128, VEX_W;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006650}
Cameron McInally45325962014-03-26 13:50:50 +00006651}
6652
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006653multiclass avx512_gather_d_ps<bits<8> dopc, bits<8> qopc,
6654 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
Igor Breger45ef10f2016-02-25 13:30:17 +00006655 defm NAME##D##SUFF##Z: avx512_gather<dopc, OpcodeStr##"d", _.info512, vz512mem,
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006656 mgatherv16i32>, EVEX_V512;
Igor Breger45ef10f2016-02-25 13:30:17 +00006657 defm NAME##Q##SUFF##Z: avx512_gather<qopc, OpcodeStr##"q", _.info256, vz512mem,
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006658 mgatherv8i64>, EVEX_V512;
6659let Predicates = [HasVLX] in {
6660 defm NAME##D##SUFF##Z256: avx512_gather<dopc, OpcodeStr##"d", _.info256,
Igor Breger45ef10f2016-02-25 13:30:17 +00006661 vy256xmem, mgatherv8i32>, EVEX_V256;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006662 defm NAME##Q##SUFF##Z256: avx512_gather<qopc, OpcodeStr##"q", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00006663 vy128xmem, mgatherv4i64>, EVEX_V256;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006664 defm NAME##D##SUFF##Z128: avx512_gather<dopc, OpcodeStr##"d", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00006665 vx128xmem, mgatherv4i32>, EVEX_V128;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006666 defm NAME##Q##SUFF##Z128: avx512_gather<qopc, OpcodeStr##"q", _.info128,
6667 vx64xmem, mgatherv2i64>, EVEX_V128;
6668}
Cameron McInally45325962014-03-26 13:50:50 +00006669}
Michael Liao5bf95782014-12-04 05:20:33 +00006670
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006671
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00006672defm VGATHER : avx512_gather_q_pd<0x92, 0x93, avx512vl_f64_info, "vgather", "PD">,
6673 avx512_gather_d_ps<0x92, 0x93, avx512vl_f32_info, "vgather", "PS">;
6674
6675defm VPGATHER : avx512_gather_q_pd<0x90, 0x91, avx512vl_i64_info, "vpgather", "Q">,
6676 avx512_gather_d_ps<0x90, 0x91, avx512vl_i32_info, "vpgather", "D">;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006677
Elena Demikhovskye1eda8a2015-04-30 08:38:48 +00006678multiclass avx512_scatter<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
6679 X86MemOperand memop, PatFrag ScatterNode> {
6680
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006681let mayStore = 1, Constraints = "$mask = $mask_wb", ExeDomain = _.ExeDomain in
Elena Demikhovskye1eda8a2015-04-30 08:38:48 +00006682
6683 def mr : AVX5128I<opc, MRMDestMem, (outs _.KRCWM:$mask_wb),
6684 (ins memop:$dst, _.KRCWM:$mask, _.RC:$src),
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006685 !strconcat(OpcodeStr#_.Suffix,
Elena Demikhovskye1eda8a2015-04-30 08:38:48 +00006686 "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"),
6687 [(set _.KRCWM:$mask_wb, (ScatterNode (_.VT _.RC:$src),
6688 _.KRCWM:$mask, vectoraddr:$dst))]>,
6689 EVEX, EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006690}
6691
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006692multiclass avx512_scatter_q_pd<bits<8> dopc, bits<8> qopc,
6693 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
6694 defm NAME##D##SUFF##Z: avx512_scatter<dopc, OpcodeStr##"d", _.info512,
Igor Breger45ef10f2016-02-25 13:30:17 +00006695 vy512mem, mscatterv8i32>, EVEX_V512, VEX_W;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006696 defm NAME##Q##SUFF##Z: avx512_scatter<qopc, OpcodeStr##"q", _.info512,
Igor Breger45ef10f2016-02-25 13:30:17 +00006697 vz512mem, mscatterv8i64>, EVEX_V512, VEX_W;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006698let Predicates = [HasVLX] in {
6699 defm NAME##D##SUFF##Z256: avx512_scatter<dopc, OpcodeStr##"d", _.info256,
Igor Breger45ef10f2016-02-25 13:30:17 +00006700 vx256xmem, mscatterv4i32>, EVEX_V256, VEX_W;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006701 defm NAME##Q##SUFF##Z256: avx512_scatter<qopc, OpcodeStr##"q", _.info256,
Igor Breger45ef10f2016-02-25 13:30:17 +00006702 vy256xmem, mscatterv4i64>, EVEX_V256, VEX_W;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006703 defm NAME##D##SUFF##Z128: avx512_scatter<dopc, OpcodeStr##"d", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00006704 vx128xmem, mscatterv4i32>, EVEX_V128, VEX_W;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006705 defm NAME##Q##SUFF##Z128: avx512_scatter<qopc, OpcodeStr##"q", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00006706 vx128xmem, mscatterv2i64>, EVEX_V128, VEX_W;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006707}
Cameron McInally45325962014-03-26 13:50:50 +00006708}
6709
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006710multiclass avx512_scatter_d_ps<bits<8> dopc, bits<8> qopc,
6711 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
Igor Breger45ef10f2016-02-25 13:30:17 +00006712 defm NAME##D##SUFF##Z: avx512_scatter<dopc, OpcodeStr##"d", _.info512, vz512mem,
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006713 mscatterv16i32>, EVEX_V512;
Igor Breger45ef10f2016-02-25 13:30:17 +00006714 defm NAME##Q##SUFF##Z: avx512_scatter<qopc, OpcodeStr##"q", _.info256, vz512mem,
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006715 mscatterv8i64>, EVEX_V512;
6716let Predicates = [HasVLX] in {
6717 defm NAME##D##SUFF##Z256: avx512_scatter<dopc, OpcodeStr##"d", _.info256,
Igor Breger45ef10f2016-02-25 13:30:17 +00006718 vy256xmem, mscatterv8i32>, EVEX_V256;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006719 defm NAME##Q##SUFF##Z256: avx512_scatter<qopc, OpcodeStr##"q", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00006720 vy128xmem, mscatterv4i64>, EVEX_V256;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006721 defm NAME##D##SUFF##Z128: avx512_scatter<dopc, OpcodeStr##"d", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00006722 vx128xmem, mscatterv4i32>, EVEX_V128;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006723 defm NAME##Q##SUFF##Z128: avx512_scatter<qopc, OpcodeStr##"q", _.info128,
6724 vx64xmem, mscatterv2i64>, EVEX_V128;
6725}
Cameron McInally45325962014-03-26 13:50:50 +00006726}
6727
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006728defm VSCATTER : avx512_scatter_q_pd<0xA2, 0xA3, avx512vl_f64_info, "vscatter", "PD">,
6729 avx512_scatter_d_ps<0xA2, 0xA3, avx512vl_f32_info, "vscatter", "PS">;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006730
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00006731defm VPSCATTER : avx512_scatter_q_pd<0xA0, 0xA1, avx512vl_i64_info, "vpscatter", "Q">,
6732 avx512_scatter_d_ps<0xA0, 0xA1, avx512vl_i32_info, "vpscatter", "D">;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006733
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006734// prefetch
6735multiclass avx512_gather_scatter_prefetch<bits<8> opc, Format F, string OpcodeStr,
6736 RegisterClass KRC, X86MemOperand memop> {
6737 let Predicates = [HasPFI], hasSideEffects = 1 in
6738 def m : AVX5128I<opc, F, (outs), (ins KRC:$mask, memop:$src),
Craig Topperedb09112014-11-25 20:11:23 +00006739 !strconcat(OpcodeStr, "\t{$src {${mask}}|{${mask}}, $src}"),
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006740 []>, EVEX, EVEX_K;
6741}
6742
6743defm VGATHERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dps",
Igor Breger45ef10f2016-02-25 13:30:17 +00006744 VK16WM, vz512mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006745
6746defm VGATHERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qps",
Igor Breger45ef10f2016-02-25 13:30:17 +00006747 VK8WM, vz512mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006748
6749defm VGATHERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00006750 VK8WM, vy512mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006751
6752defm VGATHERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00006753 VK8WM, vz512mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
Michael Liao5bf95782014-12-04 05:20:33 +00006754
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006755defm VGATHERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dps",
Igor Breger45ef10f2016-02-25 13:30:17 +00006756 VK16WM, vz512mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006757
6758defm VGATHERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qps",
Igor Breger45ef10f2016-02-25 13:30:17 +00006759 VK8WM, vz512mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006760
6761defm VGATHERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00006762 VK8WM, vy512mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006763
6764defm VGATHERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00006765 VK8WM, vz512mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006766
6767defm VSCATTERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dps",
Igor Breger45ef10f2016-02-25 13:30:17 +00006768 VK16WM, vz512mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006769
6770defm VSCATTERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qps",
Igor Breger45ef10f2016-02-25 13:30:17 +00006771 VK8WM, vz512mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006772
6773defm VSCATTERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00006774 VK8WM, vy512mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006775
6776defm VSCATTERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00006777 VK8WM, vz512mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006778
6779defm VSCATTERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dps",
Igor Breger45ef10f2016-02-25 13:30:17 +00006780 VK16WM, vz512mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006781
6782defm VSCATTERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qps",
Igor Breger45ef10f2016-02-25 13:30:17 +00006783 VK8WM, vz512mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006784
6785defm VSCATTERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00006786 VK8WM, vy512mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00006787
6788defm VSCATTERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00006789 VK8WM, vz512mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006790
Elena Demikhovskybb2f6b72014-03-27 09:45:08 +00006791// Helper fragments to match sext vXi1 to vXiY.
6792def v16i1sextv16i32 : PatLeaf<(v16i32 (X86vsrai VR512:$src, (i8 31)))>;
6793def v8i1sextv8i64 : PatLeaf<(v8i64 (X86vsrai VR512:$src, (i8 63)))>;
6794
Elena Demikhovsky44bf0632014-10-05 14:11:08 +00006795multiclass cvt_by_vec_width<bits<8> opc, X86VectorVTInfo Vec, string OpcodeStr > {
Elena Demikhovsky0e6d6d52015-04-21 14:38:31 +00006796def rr : AVX512XS8I<opc, MRMSrcReg, (outs Vec.RC:$dst), (ins Vec.KRC:$src),
Craig Topperedb09112014-11-25 20:11:23 +00006797 !strconcat(OpcodeStr##Vec.Suffix, "\t{$src, $dst|$dst, $src}"),
Elena Demikhovsky44bf0632014-10-05 14:11:08 +00006798 [(set Vec.RC:$dst, (Vec.VT (X86vsext Vec.KRC:$src)))]>, EVEX;
6799}
Michael Liao5bf95782014-12-04 05:20:33 +00006800
Elena Demikhovsky44bf0632014-10-05 14:11:08 +00006801multiclass cvt_mask_by_elt_width<bits<8> opc, AVX512VLVectorVTInfo VTInfo,
6802 string OpcodeStr, Predicate prd> {
6803let Predicates = [prd] in
6804 defm Z : cvt_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512;
6805
6806 let Predicates = [prd, HasVLX] in {
6807 defm Z256 : cvt_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256;
6808 defm Z128 : cvt_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128;
6809 }
6810}
6811
6812multiclass avx512_convert_mask_to_vector<string OpcodeStr> {
6813 defm NAME##B : cvt_mask_by_elt_width<0x28, avx512vl_i8_info, OpcodeStr,
6814 HasBWI>;
6815 defm NAME##W : cvt_mask_by_elt_width<0x28, avx512vl_i16_info, OpcodeStr,
6816 HasBWI>, VEX_W;
6817 defm NAME##D : cvt_mask_by_elt_width<0x38, avx512vl_i32_info, OpcodeStr,
6818 HasDQI>;
6819 defm NAME##Q : cvt_mask_by_elt_width<0x38, avx512vl_i64_info, OpcodeStr,
6820 HasDQI>, VEX_W;
6821}
Michael Liao5bf95782014-12-04 05:20:33 +00006822
Elena Demikhovsky44bf0632014-10-05 14:11:08 +00006823defm VPMOVM2 : avx512_convert_mask_to_vector<"vpmovm2">;
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00006824
Elena Demikhovsky0e6d6d52015-04-21 14:38:31 +00006825multiclass convert_vector_to_mask_common<bits<8> opc, X86VectorVTInfo _, string OpcodeStr > {
Igor Bregerfca0a342016-01-28 13:19:25 +00006826 def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.KRC:$dst), (ins _.RC:$src),
6827 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
6828 [(set _.KRC:$dst, (X86cvt2mask (_.VT _.RC:$src)))]>, EVEX;
6829}
6830
Simon Pilgrim18bcf932016-02-03 09:41:59 +00006831// Use 512bit version to implement 128/256 bit in case NoVLX.
6832multiclass convert_vector_to_mask_lowering<X86VectorVTInfo ExtendInfo,
Igor Bregerfca0a342016-01-28 13:19:25 +00006833 X86VectorVTInfo _> {
6834
6835 def : Pat<(_.KVT (X86cvt2mask (_.VT _.RC:$src))),
6836 (_.KVT (COPY_TO_REGCLASS
6837 (!cast<Instruction>(NAME#"Zrr")
Simon Pilgrim18bcf932016-02-03 09:41:59 +00006838 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
Igor Bregerfca0a342016-01-28 13:19:25 +00006839 _.RC:$src, _.SubRegIdx)),
6840 _.KRC))>;
Elena Demikhovsky0e6d6d52015-04-21 14:38:31 +00006841}
6842
6843multiclass avx512_convert_vector_to_mask<bits<8> opc, string OpcodeStr,
Igor Bregerfca0a342016-01-28 13:19:25 +00006844 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
6845 let Predicates = [prd] in
6846 defm Z : convert_vector_to_mask_common <opc, VTInfo.info512, OpcodeStr>,
6847 EVEX_V512;
Elena Demikhovsky0e6d6d52015-04-21 14:38:31 +00006848
6849 let Predicates = [prd, HasVLX] in {
6850 defm Z256 : convert_vector_to_mask_common<opc, VTInfo.info256, OpcodeStr>,
Igor Bregerfca0a342016-01-28 13:19:25 +00006851 EVEX_V256;
Elena Demikhovsky0e6d6d52015-04-21 14:38:31 +00006852 defm Z128 : convert_vector_to_mask_common<opc, VTInfo.info128, OpcodeStr>,
Igor Bregerfca0a342016-01-28 13:19:25 +00006853 EVEX_V128;
6854 }
6855 let Predicates = [prd, NoVLX] in {
6856 defm Z256_Alt : convert_vector_to_mask_lowering<VTInfo.info512, VTInfo.info256>;
6857 defm Z128_Alt : convert_vector_to_mask_lowering<VTInfo.info512, VTInfo.info128>;
Elena Demikhovsky0e6d6d52015-04-21 14:38:31 +00006858 }
6859}
6860
6861defm VPMOVB2M : avx512_convert_vector_to_mask<0x29, "vpmovb2m",
6862 avx512vl_i8_info, HasBWI>;
6863defm VPMOVW2M : avx512_convert_vector_to_mask<0x29, "vpmovw2m",
6864 avx512vl_i16_info, HasBWI>, VEX_W;
6865defm VPMOVD2M : avx512_convert_vector_to_mask<0x39, "vpmovd2m",
6866 avx512vl_i32_info, HasDQI>;
6867defm VPMOVQ2M : avx512_convert_vector_to_mask<0x39, "vpmovq2m",
6868 avx512vl_i64_info, HasDQI>, VEX_W;
6869
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00006870//===----------------------------------------------------------------------===//
6871// AVX-512 - COMPRESS and EXPAND
6872//
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00006873
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00006874multiclass compress_by_vec_width<bits<8> opc, X86VectorVTInfo _,
6875 string OpcodeStr> {
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00006876 defm rr : AVX512_maskable<opc, MRMDestReg, _, (outs _.RC:$dst),
Michael Liao66233b72015-08-06 09:06:20 +00006877 (ins _.RC:$src1), OpcodeStr, "$src1", "$src1",
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00006878 (_.VT (X86compress _.RC:$src1))>, AVX5128IBase;
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00006879
Craig Toppere1cac152016-06-07 07:27:54 +00006880 let mayStore = 1, hasSideEffects = 0 in
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00006881 def mr : AVX5128I<opc, MRMDestMem, (outs),
6882 (ins _.MemOp:$dst, _.RC:$src),
Craig Topper9feea572016-01-11 00:44:58 +00006883 OpcodeStr # "\t{$src, $dst|$dst, $src}",
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00006884 []>, EVEX_CD8<_.EltSize, CD8VT1>;
6885
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00006886 def mrk : AVX5128I<opc, MRMDestMem, (outs),
6887 (ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src),
Craig Topper9feea572016-01-11 00:44:58 +00006888 OpcodeStr # "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",
Michael Liao66233b72015-08-06 09:06:20 +00006889 [(store (_.VT (vselect _.KRCWM:$mask,
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00006890 (_.VT (X86compress _.RC:$src)), _.ImmAllZerosV)),
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00006891 addr:$dst)]>,
6892 EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>;
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00006893}
6894
6895multiclass compress_by_elt_width<bits<8> opc, string OpcodeStr,
6896 AVX512VLVectorVTInfo VTInfo> {
6897 defm Z : compress_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512;
6898
6899 let Predicates = [HasVLX] in {
6900 defm Z256 : compress_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256;
6901 defm Z128 : compress_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128;
6902 }
6903}
6904
6905defm VPCOMPRESSD : compress_by_elt_width <0x8B, "vpcompressd", avx512vl_i32_info>,
6906 EVEX;
6907defm VPCOMPRESSQ : compress_by_elt_width <0x8B, "vpcompressq", avx512vl_i64_info>,
6908 EVEX, VEX_W;
6909defm VCOMPRESSPS : compress_by_elt_width <0x8A, "vcompressps", avx512vl_f32_info>,
6910 EVEX;
6911defm VCOMPRESSPD : compress_by_elt_width <0x8A, "vcompresspd", avx512vl_f64_info>,
6912 EVEX, VEX_W;
6913
Elena Demikhovsky72860c32014-12-15 10:03:52 +00006914// expand
6915multiclass expand_by_vec_width<bits<8> opc, X86VectorVTInfo _,
6916 string OpcodeStr> {
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00006917 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
Michael Liao66233b72015-08-06 09:06:20 +00006918 (ins _.RC:$src1), OpcodeStr, "$src1", "$src1",
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00006919 (_.VT (X86expand _.RC:$src1))>, AVX5128IBase;
Elena Demikhovsky75ede682015-06-01 07:17:23 +00006920
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00006921 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
6922 (ins _.MemOp:$src1), OpcodeStr, "$src1", "$src1",
6923 (_.VT (X86expand (_.VT (bitconvert
6924 (_.LdFrag addr:$src1)))))>,
6925 AVX5128IBase, EVEX_CD8<_.EltSize, CD8VT1>;
Elena Demikhovsky72860c32014-12-15 10:03:52 +00006926}
6927
6928multiclass expand_by_elt_width<bits<8> opc, string OpcodeStr,
6929 AVX512VLVectorVTInfo VTInfo> {
6930 defm Z : expand_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512;
6931
6932 let Predicates = [HasVLX] in {
6933 defm Z256 : expand_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256;
6934 defm Z128 : expand_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128;
6935 }
6936}
6937
6938defm VPEXPANDD : expand_by_elt_width <0x89, "vpexpandd", avx512vl_i32_info>,
6939 EVEX;
6940defm VPEXPANDQ : expand_by_elt_width <0x89, "vpexpandq", avx512vl_i64_info>,
6941 EVEX, VEX_W;
6942defm VEXPANDPS : expand_by_elt_width <0x88, "vexpandps", avx512vl_f32_info>,
6943 EVEX;
6944defm VEXPANDPD : expand_by_elt_width <0x88, "vexpandpd", avx512vl_f64_info>,
6945 EVEX, VEX_W;
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00006946
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00006947//handle instruction reg_vec1 = op(reg_vec,imm)
6948// op(mem_vec,imm)
6949// op(broadcast(eltVt),imm)
6950//all instruction created with FROUND_CURRENT
6951multiclass avx512_unary_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
6952 X86VectorVTInfo _>{
6953 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
6954 (ins _.RC:$src1, i32u8imm:$src2),
Igor Breger252c2d92016-02-22 12:37:41 +00006955 OpcodeStr##_.Suffix, "$src2, $src1", "$src1, $src2",
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00006956 (OpNode (_.VT _.RC:$src1),
6957 (i32 imm:$src2),
6958 (i32 FROUND_CURRENT))>;
Craig Toppere1cac152016-06-07 07:27:54 +00006959 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
6960 (ins _.MemOp:$src1, i32u8imm:$src2),
6961 OpcodeStr##_.Suffix, "$src2, $src1", "$src1, $src2",
6962 (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
6963 (i32 imm:$src2),
6964 (i32 FROUND_CURRENT))>;
6965 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
6966 (ins _.ScalarMemOp:$src1, i32u8imm:$src2),
6967 OpcodeStr##_.Suffix, "$src2, ${src1}"##_.BroadcastStr,
6968 "${src1}"##_.BroadcastStr##", $src2",
6969 (OpNode (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src1))),
6970 (i32 imm:$src2),
6971 (i32 FROUND_CURRENT))>, EVEX_B;
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00006972}
6973
6974//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}
6975multiclass avx512_unary_fp_sae_packed_imm<bits<8> opc, string OpcodeStr,
6976 SDNode OpNode, X86VectorVTInfo _>{
6977 defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
6978 (ins _.RC:$src1, i32u8imm:$src2),
Craig Topperbfe13ff2016-01-11 00:44:52 +00006979 OpcodeStr##_.Suffix, "$src2, {sae}, $src1",
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00006980 "$src1, {sae}, $src2",
6981 (OpNode (_.VT _.RC:$src1),
6982 (i32 imm:$src2),
6983 (i32 FROUND_NO_EXC))>, EVEX_B;
6984}
6985
6986multiclass avx512_common_unary_fp_sae_packed_imm<string OpcodeStr,
6987 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
6988 let Predicates = [prd] in {
6989 defm Z : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
6990 avx512_unary_fp_sae_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
6991 EVEX_V512;
6992 }
6993 let Predicates = [prd, HasVLX] in {
6994 defm Z128 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info128>,
6995 EVEX_V128;
6996 defm Z256 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info256>,
6997 EVEX_V256;
6998 }
6999}
7000
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00007001//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)
7002// op(reg_vec2,mem_vec,imm)
7003// op(reg_vec2,broadcast(eltVt),imm)
7004//all instruction created with FROUND_CURRENT
7005multiclass avx512_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
7006 X86VectorVTInfo _>{
7007 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00007008 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00007009 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
7010 (OpNode (_.VT _.RC:$src1),
7011 (_.VT _.RC:$src2),
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00007012 (i32 imm:$src3),
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00007013 (i32 FROUND_CURRENT))>;
Craig Toppere1cac152016-06-07 07:27:54 +00007014 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7015 (ins _.RC:$src1, _.MemOp:$src2, i32u8imm:$src3),
7016 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
7017 (OpNode (_.VT _.RC:$src1),
7018 (_.VT (bitconvert (_.LdFrag addr:$src2))),
7019 (i32 imm:$src3),
7020 (i32 FROUND_CURRENT))>;
7021 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7022 (ins _.RC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3),
7023 OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1",
7024 "$src1, ${src2}"##_.BroadcastStr##", $src3",
7025 (OpNode (_.VT _.RC:$src1),
7026 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
7027 (i32 imm:$src3),
7028 (i32 FROUND_CURRENT))>, EVEX_B;
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00007029}
7030
Elena Demikhovsky9e380862015-06-03 10:56:40 +00007031//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)
7032// op(reg_vec2,mem_vec,imm)
Igor Breger2ae0fe32015-08-31 11:14:02 +00007033multiclass avx512_3Op_rm_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
7034 X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo>{
7035
7036 defm rri : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),
7037 (ins SrcInfo.RC:$src1, SrcInfo.RC:$src2, u8imm:$src3),
7038 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
7039 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1),
7040 (SrcInfo.VT SrcInfo.RC:$src2),
7041 (i8 imm:$src3)))>;
Craig Toppere1cac152016-06-07 07:27:54 +00007042 defm rmi : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
7043 (ins SrcInfo.RC:$src1, SrcInfo.MemOp:$src2, u8imm:$src3),
7044 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
7045 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1),
7046 (SrcInfo.VT (bitconvert
7047 (SrcInfo.LdFrag addr:$src2))),
7048 (i8 imm:$src3)))>;
Igor Breger2ae0fe32015-08-31 11:14:02 +00007049}
7050
7051//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)
7052// op(reg_vec2,mem_vec,imm)
Elena Demikhovsky9e380862015-06-03 10:56:40 +00007053// op(reg_vec2,broadcast(eltVt),imm)
7054multiclass avx512_3Op_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
Igor Breger2ae0fe32015-08-31 11:14:02 +00007055 X86VectorVTInfo _>:
7056 avx512_3Op_rm_imm8<opc, OpcodeStr, OpNode, _, _>{
7057
Craig Toppere1cac152016-06-07 07:27:54 +00007058 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7059 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),
7060 OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1",
7061 "$src1, ${src2}"##_.BroadcastStr##", $src3",
7062 (OpNode (_.VT _.RC:$src1),
7063 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
7064 (i8 imm:$src3))>, EVEX_B;
Elena Demikhovsky9e380862015-06-03 10:56:40 +00007065}
7066
Elena Demikhovsky3425c932015-06-02 08:28:57 +00007067//handle scalar instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)
7068// op(reg_vec2,mem_scalar,imm)
7069//all instruction created with FROUND_CURRENT
7070multiclass avx512_fp_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
7071 X86VectorVTInfo _> {
7072
7073 defm rri : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00007074 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
Elena Demikhovsky3425c932015-06-02 08:28:57 +00007075 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
7076 (OpNode (_.VT _.RC:$src1),
7077 (_.VT _.RC:$src2),
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00007078 (i32 imm:$src3),
Elena Demikhovsky3425c932015-06-02 08:28:57 +00007079 (i32 FROUND_CURRENT))>;
Craig Toppere1cac152016-06-07 07:27:54 +00007080 defm rmi : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
7081 (ins _.RC:$src1, _.MemOp:$src2, i32u8imm:$src3),
7082 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
7083 (OpNode (_.VT _.RC:$src1),
7084 (_.VT (scalar_to_vector
7085 (_.ScalarLdFrag addr:$src2))),
7086 (i32 imm:$src3),
7087 (i32 FROUND_CURRENT))>;
Elena Demikhovsky3425c932015-06-02 08:28:57 +00007088
Craig Toppere1cac152016-06-07 07:27:54 +00007089 let isAsmParserOnly = 1, mayLoad = 1, hasSideEffects = 0 in {
7090 defm rmi_alt :AVX512_maskable_in_asm<opc, MRMSrcMem, _, (outs _.FRC:$dst),
7091 (ins _.FRC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),
7092 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
7093 []>;
Elena Demikhovsky3425c932015-06-02 08:28:57 +00007094 }
7095}
7096
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00007097//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}
7098multiclass avx512_fp_sae_packed_imm<bits<8> opc, string OpcodeStr,
7099 SDNode OpNode, X86VectorVTInfo _>{
7100 defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00007101 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
Craig Topperbfe13ff2016-01-11 00:44:52 +00007102 OpcodeStr, "$src3, {sae}, $src2, $src1",
7103 "$src1, $src2, {sae}, $src3",
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00007104 (OpNode (_.VT _.RC:$src1),
7105 (_.VT _.RC:$src2),
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00007106 (i32 imm:$src3),
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00007107 (i32 FROUND_NO_EXC))>, EVEX_B;
7108}
Elena Demikhovsky3425c932015-06-02 08:28:57 +00007109//handle scalar instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}
7110multiclass avx512_fp_sae_scalar_imm<bits<8> opc, string OpcodeStr,
7111 SDNode OpNode, X86VectorVTInfo _> {
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00007112 defm NAME#rrib : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
7113 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
Craig Topperbfe13ff2016-01-11 00:44:52 +00007114 OpcodeStr, "$src3, {sae}, $src2, $src1",
7115 "$src1, $src2, {sae}, $src3",
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00007116 (OpNode (_.VT _.RC:$src1),
7117 (_.VT _.RC:$src2),
7118 (i32 imm:$src3),
7119 (i32 FROUND_NO_EXC))>, EVEX_B;
Elena Demikhovsky3425c932015-06-02 08:28:57 +00007120}
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00007121
Elena Demikhovsky3582eb32015-06-01 11:05:34 +00007122multiclass avx512_common_fp_sae_packed_imm<string OpcodeStr,
7123 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
Elena Demikhovsky9e380862015-06-03 10:56:40 +00007124 let Predicates = [prd] in {
7125 defm Z : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
Igor Breger00d9f842015-06-08 14:03:17 +00007126 avx512_fp_sae_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00007127 EVEX_V512;
7128
Elena Demikhovsky9e380862015-06-03 10:56:40 +00007129 }
7130 let Predicates = [prd, HasVLX] in {
7131 defm Z128 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info128>,
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00007132 EVEX_V128;
Elena Demikhovsky9e380862015-06-03 10:56:40 +00007133 defm Z256 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info256>,
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00007134 EVEX_V256;
Elena Demikhovsky9e380862015-06-03 10:56:40 +00007135 }
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00007136}
7137
Igor Breger2ae0fe32015-08-31 11:14:02 +00007138multiclass avx512_common_3Op_rm_imm8<bits<8> opc, SDNode OpNode, string OpStr,
7139 AVX512VLVectorVTInfo DestInfo, AVX512VLVectorVTInfo SrcInfo>{
7140 let Predicates = [HasBWI] in {
7141 defm Z : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info512,
7142 SrcInfo.info512>, EVEX_V512, AVX512AIi8Base, EVEX_4V;
7143 }
7144 let Predicates = [HasBWI, HasVLX] in {
7145 defm Z128 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info128,
7146 SrcInfo.info128>, EVEX_V128, AVX512AIi8Base, EVEX_4V;
7147 defm Z256 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info256,
7148 SrcInfo.info256>, EVEX_V256, AVX512AIi8Base, EVEX_4V;
7149 }
7150}
7151
Igor Breger00d9f842015-06-08 14:03:17 +00007152multiclass avx512_common_3Op_imm8<string OpcodeStr, AVX512VLVectorVTInfo _,
7153 bits<8> opc, SDNode OpNode>{
7154 let Predicates = [HasAVX512] in {
7155 defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
7156 }
7157 let Predicates = [HasAVX512, HasVLX] in {
7158 defm Z128 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128;
7159 defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
7160 }
7161}
7162
Elena Demikhovsky3425c932015-06-02 08:28:57 +00007163multiclass avx512_common_fp_sae_scalar_imm<string OpcodeStr,
7164 X86VectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
7165 let Predicates = [prd] in {
7166 defm Z128 : avx512_fp_scalar_imm<opc, OpcodeStr, OpNode, _>,
7167 avx512_fp_sae_scalar_imm<opc, OpcodeStr, OpNode, _>;
Elena Demikhovsky9e380862015-06-03 10:56:40 +00007168 }
Elena Demikhovsky3425c932015-06-02 08:28:57 +00007169}
7170
Igor Breger1e58e8a2015-09-02 11:18:55 +00007171multiclass avx512_common_unary_fp_sae_packed_imm_all<string OpcodeStr,
7172 bits<8> opcPs, bits<8> opcPd, SDNode OpNode, Predicate prd>{
7173 defm PS : avx512_common_unary_fp_sae_packed_imm<OpcodeStr, avx512vl_f32_info,
7174 opcPs, OpNode, prd>, EVEX_CD8<32, CD8VF>;
7175 defm PD : avx512_common_unary_fp_sae_packed_imm<OpcodeStr, avx512vl_f64_info,
7176 opcPd, OpNode, prd>, EVEX_CD8<64, CD8VF>, VEX_W;
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00007177}
7178
Elena Demikhovsky3582eb32015-06-01 11:05:34 +00007179
Igor Breger1e58e8a2015-09-02 11:18:55 +00007180defm VREDUCE : avx512_common_unary_fp_sae_packed_imm_all<"vreduce", 0x56, 0x56,
7181 X86VReduce, HasDQI>, AVX512AIi8Base, EVEX;
7182defm VRNDSCALE : avx512_common_unary_fp_sae_packed_imm_all<"vrndscale", 0x08, 0x09,
7183 X86VRndScale, HasAVX512>, AVX512AIi8Base, EVEX;
7184defm VGETMANT : avx512_common_unary_fp_sae_packed_imm_all<"vgetmant", 0x26, 0x26,
7185 X86VGetMant, HasAVX512>, AVX512AIi8Base, EVEX;
7186
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00007187
Elena Demikhovsky3582eb32015-06-01 11:05:34 +00007188defm VRANGEPD : avx512_common_fp_sae_packed_imm<"vrangepd", avx512vl_f64_info,
7189 0x50, X86VRange, HasDQI>,
7190 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
7191defm VRANGEPS : avx512_common_fp_sae_packed_imm<"vrangeps", avx512vl_f32_info,
7192 0x50, X86VRange, HasDQI>,
7193 AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
7194
Elena Demikhovsky8938f5a2015-06-02 14:12:54 +00007195defm VRANGESD: avx512_common_fp_sae_scalar_imm<"vrangesd", f64x_info,
7196 0x51, X86VRange, HasDQI>,
7197 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
7198defm VRANGESS: avx512_common_fp_sae_scalar_imm<"vrangess", f32x_info,
7199 0x51, X86VRange, HasDQI>,
7200 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
7201
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00007202defm VREDUCESD: avx512_common_fp_sae_scalar_imm<"vreducesd", f64x_info,
7203 0x57, X86Reduces, HasDQI>,
7204 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
7205defm VREDUCESS: avx512_common_fp_sae_scalar_imm<"vreducess", f32x_info,
7206 0x57, X86Reduces, HasDQI>,
7207 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky9e380862015-06-03 10:56:40 +00007208
Igor Breger1e58e8a2015-09-02 11:18:55 +00007209defm VGETMANTSD: avx512_common_fp_sae_scalar_imm<"vgetmantsd", f64x_info,
7210 0x27, X86GetMants, HasAVX512>,
7211 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
7212defm VGETMANTSS: avx512_common_fp_sae_scalar_imm<"vgetmantss", f32x_info,
7213 0x27, X86GetMants, HasAVX512>,
7214 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
7215
Elena Demikhovsky9e380862015-06-03 10:56:40 +00007216multiclass avx512_shuff_packed_128<string OpcodeStr, AVX512VLVectorVTInfo _,
7217 bits<8> opc, SDNode OpNode = X86Shuf128>{
7218 let Predicates = [HasAVX512] in {
7219 defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
7220
7221 }
7222 let Predicates = [HasAVX512, HasVLX] in {
7223 defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
7224 }
7225}
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00007226let Predicates = [HasAVX512] in {
7227def : Pat<(v16f32 (ffloor VR512:$src)),
7228 (VRNDSCALEPSZrri VR512:$src, (i32 0x1))>;
7229def : Pat<(v16f32 (fnearbyint VR512:$src)),
7230 (VRNDSCALEPSZrri VR512:$src, (i32 0xC))>;
7231def : Pat<(v16f32 (fceil VR512:$src)),
7232 (VRNDSCALEPSZrri VR512:$src, (i32 0x2))>;
7233def : Pat<(v16f32 (frint VR512:$src)),
7234 (VRNDSCALEPSZrri VR512:$src, (i32 0x4))>;
7235def : Pat<(v16f32 (ftrunc VR512:$src)),
7236 (VRNDSCALEPSZrri VR512:$src, (i32 0x3))>;
7237
7238def : Pat<(v8f64 (ffloor VR512:$src)),
7239 (VRNDSCALEPDZrri VR512:$src, (i32 0x1))>;
7240def : Pat<(v8f64 (fnearbyint VR512:$src)),
7241 (VRNDSCALEPDZrri VR512:$src, (i32 0xC))>;
7242def : Pat<(v8f64 (fceil VR512:$src)),
7243 (VRNDSCALEPDZrri VR512:$src, (i32 0x2))>;
7244def : Pat<(v8f64 (frint VR512:$src)),
7245 (VRNDSCALEPDZrri VR512:$src, (i32 0x4))>;
7246def : Pat<(v8f64 (ftrunc VR512:$src)),
7247 (VRNDSCALEPDZrri VR512:$src, (i32 0x3))>;
7248}
Elena Demikhovsky9e380862015-06-03 10:56:40 +00007249
7250defm VSHUFF32X4 : avx512_shuff_packed_128<"vshuff32x4",avx512vl_f32_info, 0x23>,
7251 AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
7252defm VSHUFF64X2 : avx512_shuff_packed_128<"vshuff64x2",avx512vl_f64_info, 0x23>,
7253 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
7254defm VSHUFI32X4 : avx512_shuff_packed_128<"vshufi32x4",avx512vl_i32_info, 0x43>,
7255 AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
7256defm VSHUFI64X2 : avx512_shuff_packed_128<"vshufi64x2",avx512vl_i64_info, 0x43>,
7257 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
Igor Breger00d9f842015-06-08 14:03:17 +00007258
Craig Topperc48fa892015-12-27 19:45:21 +00007259multiclass avx512_valign<string OpcodeStr, AVX512VLVectorVTInfo VTInfo_I> {
Igor Breger00d9f842015-06-08 14:03:17 +00007260 defm NAME: avx512_common_3Op_imm8<OpcodeStr, VTInfo_I, 0x03, X86VAlign>,
7261 AVX512AIi8Base, EVEX_4V;
Igor Breger00d9f842015-06-08 14:03:17 +00007262}
7263
Craig Topperc48fa892015-12-27 19:45:21 +00007264defm VALIGND: avx512_valign<"valignd", avx512vl_i32_info>,
Igor Breger00d9f842015-06-08 14:03:17 +00007265 EVEX_CD8<32, CD8VF>;
Craig Topperc48fa892015-12-27 19:45:21 +00007266defm VALIGNQ: avx512_valign<"valignq", avx512vl_i64_info>,
Igor Breger00d9f842015-06-08 14:03:17 +00007267 EVEX_CD8<64, CD8VF>, VEX_W;
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00007268
Craig Topper7a299302016-06-09 07:06:38 +00007269multiclass avx512_vpalignr_lowering<X86VectorVTInfo _ , list<Predicate> p>{
Igor Breger2ae0fe32015-08-31 11:14:02 +00007270 let Predicates = p in
7271 def NAME#_.VTName#rri:
7272 Pat<(_.VT (X86PAlignr _.RC:$src1, _.RC:$src2, (i8 imm:$imm))),
7273 (!cast<Instruction>(NAME#_.ZSuffix#rri)
7274 _.RC:$src1, _.RC:$src2, imm:$imm)>;
7275}
7276
Craig Topper7a299302016-06-09 07:06:38 +00007277multiclass avx512_vpalignr_lowering_common<AVX512VLVectorVTInfo _>:
7278 avx512_vpalignr_lowering<_.info512, [HasBWI]>,
7279 avx512_vpalignr_lowering<_.info128, [HasBWI, HasVLX]>,
7280 avx512_vpalignr_lowering<_.info256, [HasBWI, HasVLX]>;
Igor Breger2ae0fe32015-08-31 11:14:02 +00007281
Craig Topper7a299302016-06-09 07:06:38 +00007282defm VPALIGNR: avx512_common_3Op_rm_imm8<0x0F, X86PAlignr, "vpalignr" ,
Igor Breger2ae0fe32015-08-31 11:14:02 +00007283 avx512vl_i8_info, avx512vl_i8_info>,
Craig Topper7a299302016-06-09 07:06:38 +00007284 avx512_vpalignr_lowering_common<avx512vl_i16_info>,
7285 avx512_vpalignr_lowering_common<avx512vl_i32_info>,
7286 avx512_vpalignr_lowering_common<avx512vl_f32_info>,
7287 avx512_vpalignr_lowering_common<avx512vl_i64_info>,
7288 avx512_vpalignr_lowering_common<avx512vl_f64_info>,
Igor Breger2ae0fe32015-08-31 11:14:02 +00007289 EVEX_CD8<8, CD8VF>;
7290
Igor Bregerf3ded812015-08-31 13:09:30 +00007291defm VDBPSADBW: avx512_common_3Op_rm_imm8<0x42, X86dbpsadbw, "vdbpsadbw" ,
7292 avx512vl_i16_info, avx512vl_i8_info>, EVEX_CD8<8, CD8VF>;
7293
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00007294multiclass avx512_unary_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
7295 X86VectorVTInfo _> {
7296 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
Igor Breger24cab0f2015-11-16 07:22:00 +00007297 (ins _.RC:$src1), OpcodeStr,
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00007298 "$src1", "$src1",
7299 (_.VT (OpNode _.RC:$src1))>, EVEX, AVX5128IBase;
7300
Craig Toppere1cac152016-06-07 07:27:54 +00007301 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7302 (ins _.MemOp:$src1), OpcodeStr,
7303 "$src1", "$src1",
7304 (_.VT (OpNode (bitconvert (_.LdFrag addr:$src1))))>,
7305 EVEX, AVX5128IBase, EVEX_CD8<_.EltSize, CD8VF>;
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00007306}
7307
7308multiclass avx512_unary_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
7309 X86VectorVTInfo _> :
7310 avx512_unary_rm<opc, OpcodeStr, OpNode, _> {
Craig Toppere1cac152016-06-07 07:27:54 +00007311 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7312 (ins _.ScalarMemOp:$src1), OpcodeStr,
7313 "${src1}"##_.BroadcastStr,
7314 "${src1}"##_.BroadcastStr,
7315 (_.VT (OpNode (X86VBroadcast
7316 (_.ScalarLdFrag addr:$src1))))>,
7317 EVEX, AVX5128IBase, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>;
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00007318}
7319
7320multiclass avx512_unary_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
7321 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
7322 let Predicates = [prd] in
7323 defm Z : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info512>, EVEX_V512;
7324
7325 let Predicates = [prd, HasVLX] in {
7326 defm Z256 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info256>,
7327 EVEX_V256;
7328 defm Z128 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info128>,
7329 EVEX_V128;
7330 }
7331}
7332
7333multiclass avx512_unary_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
7334 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
7335 let Predicates = [prd] in
7336 defm Z : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info512>,
7337 EVEX_V512;
7338
7339 let Predicates = [prd, HasVLX] in {
7340 defm Z256 : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info256>,
7341 EVEX_V256;
7342 defm Z128 : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info128>,
7343 EVEX_V128;
7344 }
7345}
7346
7347multiclass avx512_unary_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr,
7348 SDNode OpNode, Predicate prd> {
Igor Breger24cab0f2015-11-16 07:22:00 +00007349 defm Q : avx512_unary_rmb_vl<opc_q, OpcodeStr#"q", OpNode, avx512vl_i64_info,
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00007350 prd>, VEX_W;
Igor Breger24cab0f2015-11-16 07:22:00 +00007351 defm D : avx512_unary_rmb_vl<opc_d, OpcodeStr#"d", OpNode, avx512vl_i32_info,
7352 prd>;
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00007353}
7354
7355multiclass avx512_unary_rm_vl_bw<bits<8> opc_b, bits<8> opc_w, string OpcodeStr,
7356 SDNode OpNode, Predicate prd> {
Igor Breger24cab0f2015-11-16 07:22:00 +00007357 defm W : avx512_unary_rm_vl<opc_w, OpcodeStr#"w", OpNode, avx512vl_i16_info, prd>;
7358 defm B : avx512_unary_rm_vl<opc_b, OpcodeStr#"b", OpNode, avx512vl_i8_info, prd>;
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00007359}
7360
7361multiclass avx512_unary_rm_vl_all<bits<8> opc_b, bits<8> opc_w,
7362 bits<8> opc_d, bits<8> opc_q,
7363 string OpcodeStr, SDNode OpNode> {
7364 defm NAME : avx512_unary_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode,
7365 HasAVX512>,
7366 avx512_unary_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode,
7367 HasBWI>;
7368}
7369
7370defm VPABS : avx512_unary_rm_vl_all<0x1C, 0x1D, 0x1E, 0x1F, "vpabs", X86Abs>;
7371
7372def : Pat<(xor
7373 (bc_v16i32 (v16i1sextv16i32)),
7374 (bc_v16i32 (add (v16i32 VR512:$src), (v16i1sextv16i32)))),
7375 (VPABSDZrr VR512:$src)>;
7376def : Pat<(xor
7377 (bc_v8i64 (v8i1sextv8i64)),
7378 (bc_v8i64 (add (v8i64 VR512:$src), (v8i1sextv8i64)))),
7379 (VPABSQZrr VR512:$src)>;
Igor Bregerf2460112015-07-26 14:41:44 +00007380
Igor Breger0dcd8bc2015-09-03 09:05:31 +00007381multiclass avx512_ctlz<bits<8> opc, string OpcodeStr, Predicate prd>{
7382
7383 defm NAME : avx512_unary_rm_vl_dq<opc, opc, OpcodeStr, ctlz, prd>;
Igor Breger0dcd8bc2015-09-03 09:05:31 +00007384}
7385
7386defm VPLZCNT : avx512_ctlz<0x44, "vplzcnt", HasCDI>;
7387defm VPCONFLICT : avx512_unary_rm_vl_dq<0xC4, 0xC4, "vpconflict", X86Conflict, HasCDI>;
7388
Igor Breger24cab0f2015-11-16 07:22:00 +00007389//===---------------------------------------------------------------------===//
7390// Replicate Single FP - MOVSHDUP and MOVSLDUP
7391//===---------------------------------------------------------------------===//
7392multiclass avx512_replicate<bits<8> opc, string OpcodeStr, SDNode OpNode>{
7393 defm NAME: avx512_unary_rm_vl<opc, OpcodeStr, OpNode, avx512vl_f32_info,
7394 HasAVX512>, XS;
Igor Breger24cab0f2015-11-16 07:22:00 +00007395}
7396
7397defm VMOVSHDUP : avx512_replicate<0x16, "vmovshdup", X86Movshdup>;
7398defm VMOVSLDUP : avx512_replicate<0x12, "vmovsldup", X86Movsldup>;
Igor Breger1f782962015-11-19 08:26:56 +00007399
7400//===----------------------------------------------------------------------===//
7401// AVX-512 - MOVDDUP
7402//===----------------------------------------------------------------------===//
7403
7404multiclass avx512_movddup_128<bits<8> opc, string OpcodeStr, SDNode OpNode,
7405 X86VectorVTInfo _> {
7406 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
7407 (ins _.RC:$src), OpcodeStr, "$src", "$src",
7408 (_.VT (OpNode (_.VT _.RC:$src)))>, EVEX;
Craig Toppere1cac152016-06-07 07:27:54 +00007409 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7410 (ins _.ScalarMemOp:$src), OpcodeStr, "$src", "$src",
7411 (_.VT (OpNode (_.VT (scalar_to_vector
7412 (_.ScalarLdFrag addr:$src)))))>,
7413 EVEX, EVEX_CD8<_.EltSize, CD8VH>;
Igor Breger1f782962015-11-19 08:26:56 +00007414}
7415
7416multiclass avx512_movddup_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
7417 AVX512VLVectorVTInfo VTInfo> {
7418
7419 defm Z : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info512>, EVEX_V512;
7420
7421 let Predicates = [HasAVX512, HasVLX] in {
7422 defm Z256 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info256>,
7423 EVEX_V256;
7424 defm Z128 : avx512_movddup_128<opc, OpcodeStr, OpNode, VTInfo.info128>,
7425 EVEX_V128;
7426 }
7427}
7428
7429multiclass avx512_movddup<bits<8> opc, string OpcodeStr, SDNode OpNode>{
7430 defm NAME: avx512_movddup_common<opc, OpcodeStr, OpNode,
7431 avx512vl_f64_info>, XD, VEX_W;
Igor Breger1f782962015-11-19 08:26:56 +00007432}
7433
7434defm VMOVDDUP : avx512_movddup<0x12, "vmovddup", X86Movddup>;
7435
7436def : Pat<(X86Movddup (loadv2f64 addr:$src)),
7437 (VMOVDDUPZ128rm addr:$src)>, Requires<[HasAVX512, HasVLX]>;
7438def : Pat<(v2f64 (X86VBroadcast (loadf64 addr:$src))),
7439 (VMOVDDUPZ128rm addr:$src)>, Requires<[HasAVX512, HasVLX]>;
7440
Igor Bregerf2460112015-07-26 14:41:44 +00007441//===----------------------------------------------------------------------===//
7442// AVX-512 - Unpack Instructions
7443//===----------------------------------------------------------------------===//
Craig Topperdb290662016-05-01 05:57:06 +00007444defm VUNPCKH : avx512_fp_binop_p<0x15, "vunpckh", X86Unpckh, HasAVX512>;
7445defm VUNPCKL : avx512_fp_binop_p<0x14, "vunpckl", X86Unpckl, HasAVX512>;
Igor Bregerf2460112015-07-26 14:41:44 +00007446
7447defm VPUNPCKLBW : avx512_binop_rm_vl_b<0x60, "vpunpcklbw", X86Unpckl,
7448 SSE_INTALU_ITINS_P, HasBWI>;
7449defm VPUNPCKHBW : avx512_binop_rm_vl_b<0x68, "vpunpckhbw", X86Unpckh,
7450 SSE_INTALU_ITINS_P, HasBWI>;
7451defm VPUNPCKLWD : avx512_binop_rm_vl_w<0x61, "vpunpcklwd", X86Unpckl,
7452 SSE_INTALU_ITINS_P, HasBWI>;
7453defm VPUNPCKHWD : avx512_binop_rm_vl_w<0x69, "vpunpckhwd", X86Unpckh,
7454 SSE_INTALU_ITINS_P, HasBWI>;
7455
7456defm VPUNPCKLDQ : avx512_binop_rm_vl_d<0x62, "vpunpckldq", X86Unpckl,
7457 SSE_INTALU_ITINS_P, HasAVX512>;
7458defm VPUNPCKHDQ : avx512_binop_rm_vl_d<0x6A, "vpunpckhdq", X86Unpckh,
7459 SSE_INTALU_ITINS_P, HasAVX512>;
7460defm VPUNPCKLQDQ : avx512_binop_rm_vl_q<0x6C, "vpunpcklqdq", X86Unpckl,
7461 SSE_INTALU_ITINS_P, HasAVX512>;
7462defm VPUNPCKHQDQ : avx512_binop_rm_vl_q<0x6D, "vpunpckhqdq", X86Unpckh,
7463 SSE_INTALU_ITINS_P, HasAVX512>;
Igor Bregerdefab3c2015-10-08 12:55:01 +00007464
7465//===----------------------------------------------------------------------===//
7466// AVX-512 - Extract & Insert Integer Instructions
7467//===----------------------------------------------------------------------===//
7468
7469multiclass avx512_extract_elt_bw_m<bits<8> opc, string OpcodeStr, SDNode OpNode,
7470 X86VectorVTInfo _> {
Craig Toppere1cac152016-06-07 07:27:54 +00007471 def mr : AVX512Ii8<opc, MRMDestMem, (outs),
7472 (ins _.ScalarMemOp:$dst, _.RC:$src1, u8imm:$src2),
7473 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
7474 [(store (_.EltVT (trunc (assertzext (OpNode (_.VT _.RC:$src1),
7475 imm:$src2)))),
7476 addr:$dst)]>,
7477 EVEX, EVEX_CD8<_.EltSize, CD8VT1>;
Igor Bregerdefab3c2015-10-08 12:55:01 +00007478}
7479
7480multiclass avx512_extract_elt_b<string OpcodeStr, X86VectorVTInfo _> {
7481 let Predicates = [HasBWI] in {
7482 def rr : AVX512Ii8<0x14, MRMDestReg, (outs GR32orGR64:$dst),
7483 (ins _.RC:$src1, u8imm:$src2),
7484 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
7485 [(set GR32orGR64:$dst,
7486 (X86pextrb (_.VT _.RC:$src1), imm:$src2))]>,
7487 EVEX, TAPD;
7488
7489 defm NAME : avx512_extract_elt_bw_m<0x14, OpcodeStr, X86pextrb, _>, TAPD;
7490 }
7491}
7492
7493multiclass avx512_extract_elt_w<string OpcodeStr, X86VectorVTInfo _> {
7494 let Predicates = [HasBWI] in {
7495 def rr : AVX512Ii8<0xC5, MRMSrcReg, (outs GR32orGR64:$dst),
7496 (ins _.RC:$src1, u8imm:$src2),
7497 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
7498 [(set GR32orGR64:$dst,
7499 (X86pextrw (_.VT _.RC:$src1), imm:$src2))]>,
7500 EVEX, PD;
7501
Craig Topper99f6b622016-05-01 01:03:56 +00007502 let hasSideEffects = 0 in
Igor Breger55747302015-11-18 08:46:16 +00007503 def rr_REV : AVX512Ii8<0x15, MRMDestReg, (outs GR32orGR64:$dst),
7504 (ins _.RC:$src1, u8imm:$src2),
7505 OpcodeStr#".s\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
7506 EVEX, TAPD;
7507
Igor Bregerdefab3c2015-10-08 12:55:01 +00007508 defm NAME : avx512_extract_elt_bw_m<0x15, OpcodeStr, X86pextrw, _>, TAPD;
7509 }
7510}
7511
7512multiclass avx512_extract_elt_dq<string OpcodeStr, X86VectorVTInfo _,
7513 RegisterClass GRC> {
7514 let Predicates = [HasDQI] in {
7515 def rr : AVX512Ii8<0x16, MRMDestReg, (outs GRC:$dst),
7516 (ins _.RC:$src1, u8imm:$src2),
7517 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
7518 [(set GRC:$dst,
7519 (extractelt (_.VT _.RC:$src1), imm:$src2))]>,
7520 EVEX, TAPD;
7521
Craig Toppere1cac152016-06-07 07:27:54 +00007522 def mr : AVX512Ii8<0x16, MRMDestMem, (outs),
7523 (ins _.ScalarMemOp:$dst, _.RC:$src1, u8imm:$src2),
7524 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
7525 [(store (extractelt (_.VT _.RC:$src1),
7526 imm:$src2),addr:$dst)]>,
7527 EVEX, EVEX_CD8<_.EltSize, CD8VT1>, TAPD;
Igor Bregerdefab3c2015-10-08 12:55:01 +00007528 }
7529}
7530
7531defm VPEXTRBZ : avx512_extract_elt_b<"vpextrb", v16i8x_info>;
7532defm VPEXTRWZ : avx512_extract_elt_w<"vpextrw", v8i16x_info>;
7533defm VPEXTRDZ : avx512_extract_elt_dq<"vpextrd", v4i32x_info, GR32>;
7534defm VPEXTRQZ : avx512_extract_elt_dq<"vpextrq", v2i64x_info, GR64>, VEX_W;
7535
7536multiclass avx512_insert_elt_m<bits<8> opc, string OpcodeStr, SDNode OpNode,
7537 X86VectorVTInfo _, PatFrag LdFrag> {
7538 def rm : AVX512Ii8<opc, MRMSrcMem, (outs _.RC:$dst),
7539 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),
7540 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
7541 [(set _.RC:$dst,
7542 (_.VT (OpNode _.RC:$src1, (LdFrag addr:$src2), imm:$src3)))]>,
7543 EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>;
7544}
7545
7546multiclass avx512_insert_elt_bw<bits<8> opc, string OpcodeStr, SDNode OpNode,
7547 X86VectorVTInfo _, PatFrag LdFrag> {
7548 let Predicates = [HasBWI] in {
7549 def rr : AVX512Ii8<opc, MRMSrcReg, (outs _.RC:$dst),
7550 (ins _.RC:$src1, GR32orGR64:$src2, u8imm:$src3),
7551 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
7552 [(set _.RC:$dst,
7553 (OpNode _.RC:$src1, GR32orGR64:$src2, imm:$src3))]>, EVEX_4V;
7554
7555 defm NAME : avx512_insert_elt_m<opc, OpcodeStr, OpNode, _, LdFrag>;
7556 }
7557}
7558
7559multiclass avx512_insert_elt_dq<bits<8> opc, string OpcodeStr,
7560 X86VectorVTInfo _, RegisterClass GRC> {
7561 let Predicates = [HasDQI] in {
7562 def rr : AVX512Ii8<opc, MRMSrcReg, (outs _.RC:$dst),
7563 (ins _.RC:$src1, GRC:$src2, u8imm:$src3),
7564 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
7565 [(set _.RC:$dst,
7566 (_.VT (insertelt _.RC:$src1, GRC:$src2, imm:$src3)))]>,
7567 EVEX_4V, TAPD;
7568
7569 defm NAME : avx512_insert_elt_m<opc, OpcodeStr, insertelt, _,
7570 _.ScalarLdFrag>, TAPD;
7571 }
7572}
7573
7574defm VPINSRBZ : avx512_insert_elt_bw<0x20, "vpinsrb", X86pinsrb, v16i8x_info,
7575 extloadi8>, TAPD;
7576defm VPINSRWZ : avx512_insert_elt_bw<0xC4, "vpinsrw", X86pinsrw, v8i16x_info,
7577 extloadi16>, PD;
7578defm VPINSRDZ : avx512_insert_elt_dq<0x22, "vpinsrd", v4i32x_info, GR32>;
7579defm VPINSRQZ : avx512_insert_elt_dq<0x22, "vpinsrq", v2i64x_info, GR64>, VEX_W;
Igor Bregera6297c72015-09-02 10:50:58 +00007580//===----------------------------------------------------------------------===//
7581// VSHUFPS - VSHUFPD Operations
7582//===----------------------------------------------------------------------===//
7583multiclass avx512_shufp<string OpcodeStr, AVX512VLVectorVTInfo VTInfo_I,
7584 AVX512VLVectorVTInfo VTInfo_FP>{
7585 defm NAME: avx512_common_3Op_imm8<OpcodeStr, VTInfo_FP, 0xC6, X86Shufp>,
7586 EVEX_CD8<VTInfo_FP.info512.EltSize, CD8VF>,
7587 AVX512AIi8Base, EVEX_4V;
Igor Bregera6297c72015-09-02 10:50:58 +00007588}
7589
7590defm VSHUFPS: avx512_shufp<"vshufps", avx512vl_i32_info, avx512vl_f32_info>, PS;
7591defm VSHUFPD: avx512_shufp<"vshufpd", avx512vl_i64_info, avx512vl_f64_info>, PD, VEX_W;
Asaf Badouhd2c35992015-09-02 14:21:54 +00007592//===----------------------------------------------------------------------===//
7593// AVX-512 - Byte shift Left/Right
7594//===----------------------------------------------------------------------===//
7595
7596multiclass avx512_shift_packed<bits<8> opc, SDNode OpNode, Format MRMr,
7597 Format MRMm, string OpcodeStr, X86VectorVTInfo _>{
7598 def rr : AVX512<opc, MRMr,
7599 (outs _.RC:$dst), (ins _.RC:$src1, u8imm:$src2),
7600 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
7601 [(set _.RC:$dst,(_.VT (OpNode _.RC:$src1, (i8 imm:$src2))))]>;
Craig Toppere1cac152016-06-07 07:27:54 +00007602 def rm : AVX512<opc, MRMm,
7603 (outs _.RC:$dst), (ins _.MemOp:$src1, u8imm:$src2),
7604 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
7605 [(set _.RC:$dst,(_.VT (OpNode
Simon Pilgrim255fdd02016-06-11 12:54:37 +00007606 (_.VT (bitconvert (_.LdFrag addr:$src1))),
7607 (i8 imm:$src2))))]>;
Asaf Badouhd2c35992015-09-02 14:21:54 +00007608}
7609
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007610multiclass avx512_shift_packed_all<bits<8> opc, SDNode OpNode, Format MRMr,
Asaf Badouhd2c35992015-09-02 14:21:54 +00007611 Format MRMm, string OpcodeStr, Predicate prd>{
7612 let Predicates = [prd] in
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007613 defm Z512 : avx512_shift_packed<opc, OpNode, MRMr, MRMm,
Simon Pilgrim255fdd02016-06-11 12:54:37 +00007614 OpcodeStr, v64i8_info>, EVEX_V512;
Asaf Badouhd2c35992015-09-02 14:21:54 +00007615 let Predicates = [prd, HasVLX] in {
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007616 defm Z256 : avx512_shift_packed<opc, OpNode, MRMr, MRMm,
Simon Pilgrim255fdd02016-06-11 12:54:37 +00007617 OpcodeStr, v32i8x_info>, EVEX_V256;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007618 defm Z128 : avx512_shift_packed<opc, OpNode, MRMr, MRMm,
Simon Pilgrim255fdd02016-06-11 12:54:37 +00007619 OpcodeStr, v16i8x_info>, EVEX_V128;
Asaf Badouhd2c35992015-09-02 14:21:54 +00007620 }
7621}
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007622defm VPSLLDQ : avx512_shift_packed_all<0x73, X86vshldq, MRM7r, MRM7m, "vpslldq",
Asaf Badouhd2c35992015-09-02 14:21:54 +00007623 HasBWI>, AVX512PDIi8Base, EVEX_4V;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007624defm VPSRLDQ : avx512_shift_packed_all<0x73, X86vshrdq, MRM3r, MRM3m, "vpsrldq",
Asaf Badouhd2c35992015-09-02 14:21:54 +00007625 HasBWI>, AVX512PDIi8Base, EVEX_4V;
7626
7627
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007628multiclass avx512_psadbw_packed<bits<8> opc, SDNode OpNode,
Cong Houdb6220f2015-11-24 19:51:26 +00007629 string OpcodeStr, X86VectorVTInfo _dst,
7630 X86VectorVTInfo _src>{
Asaf Badouhd2c35992015-09-02 14:21:54 +00007631 def rr : AVX512BI<opc, MRMSrcReg,
Cong Houdb6220f2015-11-24 19:51:26 +00007632 (outs _dst.RC:$dst), (ins _src.RC:$src1, _src.RC:$src2),
Asaf Badouhd2c35992015-09-02 14:21:54 +00007633 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
Cong Houdb6220f2015-11-24 19:51:26 +00007634 [(set _dst.RC:$dst,(_dst.VT
7635 (OpNode (_src.VT _src.RC:$src1),
7636 (_src.VT _src.RC:$src2))))]>;
Craig Toppere1cac152016-06-07 07:27:54 +00007637 def rm : AVX512BI<opc, MRMSrcMem,
7638 (outs _dst.RC:$dst), (ins _src.RC:$src1, _src.MemOp:$src2),
7639 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
7640 [(set _dst.RC:$dst,(_dst.VT
7641 (OpNode (_src.VT _src.RC:$src1),
7642 (_src.VT (bitconvert
7643 (_src.LdFrag addr:$src2))))))]>;
Asaf Badouhd2c35992015-09-02 14:21:54 +00007644}
7645
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007646multiclass avx512_psadbw_packed_all<bits<8> opc, SDNode OpNode,
Asaf Badouhd2c35992015-09-02 14:21:54 +00007647 string OpcodeStr, Predicate prd> {
7648 let Predicates = [prd] in
Cong Houdb6220f2015-11-24 19:51:26 +00007649 defm Z512 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v8i64_info,
7650 v64i8_info>, EVEX_V512;
Asaf Badouhd2c35992015-09-02 14:21:54 +00007651 let Predicates = [prd, HasVLX] in {
Cong Houdb6220f2015-11-24 19:51:26 +00007652 defm Z256 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v4i64x_info,
7653 v32i8x_info>, EVEX_V256;
7654 defm Z128 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v2i64x_info,
7655 v16i8x_info>, EVEX_V128;
Asaf Badouhd2c35992015-09-02 14:21:54 +00007656 }
7657}
7658
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007659defm VPSADBW : avx512_psadbw_packed_all<0xf6, X86psadbw, "vpsadbw",
Asaf Badouhd2c35992015-09-02 14:21:54 +00007660 HasBWI>, EVEX_4V;
Igor Bregerb4bb1902015-10-15 12:33:24 +00007661
7662multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,
7663 X86VectorVTInfo _>{
7664 let Constraints = "$src1 = $dst" in {
7665 defm rri : AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
7666 (ins _.RC:$src2, _.RC:$src3, u8imm:$src4),
Igor Breger252c2d92016-02-22 12:37:41 +00007667 OpcodeStr, "$src4, $src3, $src2", "$src2, $src3, $src4",
Igor Bregerb4bb1902015-10-15 12:33:24 +00007668 (OpNode (_.VT _.RC:$src1),
7669 (_.VT _.RC:$src2),
7670 (_.VT _.RC:$src3),
7671 (i8 imm:$src4))>, AVX512AIi8Base, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00007672 defm rmi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
7673 (ins _.RC:$src2, _.MemOp:$src3, u8imm:$src4),
7674 OpcodeStr, "$src4, $src3, $src2", "$src2, $src3, $src4",
7675 (OpNode (_.VT _.RC:$src1),
7676 (_.VT _.RC:$src2),
7677 (_.VT (bitconvert (_.LdFrag addr:$src3))),
7678 (i8 imm:$src4))>,
7679 AVX512AIi8Base, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>;
7680 defm rmbi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
7681 (ins _.RC:$src2, _.ScalarMemOp:$src3, u8imm:$src4),
7682 OpcodeStr, "$src4, ${src3}"##_.BroadcastStr##", $src2",
7683 "$src2, ${src3}"##_.BroadcastStr##", $src4",
7684 (OpNode (_.VT _.RC:$src1),
7685 (_.VT _.RC:$src2),
7686 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
7687 (i8 imm:$src4))>, EVEX_B,
7688 AVX512AIi8Base, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>;
Igor Bregerb4bb1902015-10-15 12:33:24 +00007689 }// Constraints = "$src1 = $dst"
7690}
7691
7692multiclass avx512_common_ternlog<string OpcodeStr, AVX512VLVectorVTInfo _>{
7693 let Predicates = [HasAVX512] in
7694 defm Z : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, _.info512>, EVEX_V512;
7695 let Predicates = [HasAVX512, HasVLX] in {
7696 defm Z128 : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, _.info128>, EVEX_V128;
7697 defm Z256 : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, _.info256>, EVEX_V256;
7698 }
7699}
7700
7701defm VPTERNLOGD : avx512_common_ternlog<"vpternlogd", avx512vl_i32_info>;
7702defm VPTERNLOGQ : avx512_common_ternlog<"vpternlogq", avx512vl_i64_info>, VEX_W;
7703
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +00007704//===----------------------------------------------------------------------===//
7705// AVX-512 - FixupImm
7706//===----------------------------------------------------------------------===//
7707
7708multiclass avx512_fixupimm_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
7709 X86VectorVTInfo _>{
7710 let Constraints = "$src1 = $dst" in {
7711 defm rri : AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
7712 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),
7713 OpcodeStr##_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",
7714 (OpNode (_.VT _.RC:$src1),
7715 (_.VT _.RC:$src2),
7716 (_.IntVT _.RC:$src3),
7717 (i32 imm:$src4),
7718 (i32 FROUND_CURRENT))>;
Craig Toppere1cac152016-06-07 07:27:54 +00007719 defm rmi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
7720 (ins _.RC:$src2, _.MemOp:$src3, i32u8imm:$src4),
7721 OpcodeStr##_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",
7722 (OpNode (_.VT _.RC:$src1),
7723 (_.VT _.RC:$src2),
7724 (_.IntVT (bitconvert (_.LdFrag addr:$src3))),
7725 (i32 imm:$src4),
7726 (i32 FROUND_CURRENT))>;
7727 defm rmbi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
7728 (ins _.RC:$src2, _.ScalarMemOp:$src3, i32u8imm:$src4),
7729 OpcodeStr##_.Suffix, "$src4, ${src3}"##_.BroadcastStr##", $src2",
7730 "$src2, ${src3}"##_.BroadcastStr##", $src4",
7731 (OpNode (_.VT _.RC:$src1),
7732 (_.VT _.RC:$src2),
7733 (_.IntVT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
7734 (i32 imm:$src4),
7735 (i32 FROUND_CURRENT))>, EVEX_B;
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +00007736 } // Constraints = "$src1 = $dst"
7737}
7738
7739multiclass avx512_fixupimm_packed_sae<bits<8> opc, string OpcodeStr,
7740 SDNode OpNode, X86VectorVTInfo _>{
7741let Constraints = "$src1 = $dst" in {
7742 defm rrib : AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
7743 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007744 OpcodeStr##_.Suffix, "$src4, {sae}, $src3, $src2",
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +00007745 "$src2, $src3, {sae}, $src4",
7746 (OpNode (_.VT _.RC:$src1),
7747 (_.VT _.RC:$src2),
7748 (_.IntVT _.RC:$src3),
7749 (i32 imm:$src4),
7750 (i32 FROUND_NO_EXC))>, EVEX_B;
7751 }
7752}
7753
7754multiclass avx512_fixupimm_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode,
7755 X86VectorVTInfo _, X86VectorVTInfo _src3VT> {
7756 let Constraints = "$src1 = $dst" , Predicates = [HasAVX512] in {
7757 defm rri : AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
7758 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),
7759 OpcodeStr##_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",
7760 (OpNode (_.VT _.RC:$src1),
7761 (_.VT _.RC:$src2),
7762 (_src3VT.VT _src3VT.RC:$src3),
7763 (i32 imm:$src4),
7764 (i32 FROUND_CURRENT))>;
7765
7766 defm rrib : AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
7767 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),
7768 OpcodeStr##_.Suffix, "$src4, {sae}, $src3, $src2",
7769 "$src2, $src3, {sae}, $src4",
7770 (OpNode (_.VT _.RC:$src1),
7771 (_.VT _.RC:$src2),
7772 (_src3VT.VT _src3VT.RC:$src3),
7773 (i32 imm:$src4),
7774 (i32 FROUND_NO_EXC))>, EVEX_B;
Craig Toppere1cac152016-06-07 07:27:54 +00007775 defm rmi : AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
7776 (ins _.RC:$src2, _.ScalarMemOp:$src3, i32u8imm:$src4),
7777 OpcodeStr##_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",
7778 (OpNode (_.VT _.RC:$src1),
7779 (_.VT _.RC:$src2),
7780 (_src3VT.VT (scalar_to_vector
7781 (_src3VT.ScalarLdFrag addr:$src3))),
7782 (i32 imm:$src4),
7783 (i32 FROUND_CURRENT))>;
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +00007784 }
7785}
7786
7787multiclass avx512_fixupimm_packed_all<AVX512VLVectorVTInfo _Vec>{
7788 let Predicates = [HasAVX512] in
7789 defm Z : avx512_fixupimm_packed<0x54, "vfixupimm", X86VFixupimm, _Vec.info512>,
7790 avx512_fixupimm_packed_sae<0x54, "vfixupimm", X86VFixupimm, _Vec.info512>,
7791 AVX512AIi8Base, EVEX_4V, EVEX_V512;
7792 let Predicates = [HasAVX512, HasVLX] in {
7793 defm Z128 : avx512_fixupimm_packed<0x54, "vfixupimm", X86VFixupimm, _Vec.info128>,
7794 AVX512AIi8Base, EVEX_4V, EVEX_V128;
7795 defm Z256 : avx512_fixupimm_packed<0x54, "vfixupimm", X86VFixupimm, _Vec.info256>,
7796 AVX512AIi8Base, EVEX_4V, EVEX_V256;
7797 }
7798}
7799
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007800defm VFIXUPIMMSS : avx512_fixupimm_scalar<0x55, "vfixupimm", X86VFixupimmScalar,
7801 f32x_info, v4i32x_info>,
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +00007802 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007803defm VFIXUPIMMSD : avx512_fixupimm_scalar<0x55, "vfixupimm", X86VFixupimmScalar,
7804 f64x_info, v2i64x_info>,
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +00007805 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007806defm VFIXUPIMMPS : avx512_fixupimm_packed_all<avx512vl_f32_info>,
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +00007807 EVEX_CD8<32, CD8VF>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007808defm VFIXUPIMMPD : avx512_fixupimm_packed_all<avx512vl_f64_info>,
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +00007809 EVEX_CD8<64, CD8VF>, VEX_W;