blob: 18713f3c87456c8657dec85762525e50ed9ff55f [file] [log] [blame]
Eric Christopher06b32cd2015-02-20 00:36:53 +00001//===-- X86InstrAVX512.td - AVX512 Instruction Set ---------*- tablegen -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file describes the X86 AVX512 instruction set, defining the
11// instructions, and properties of the instructions which are needed for code
12// generation, machine code emission, and analysis.
13//
14//===----------------------------------------------------------------------===//
15
Adam Nemet5ed17da2014-08-21 19:50:07 +000016// Group template arguments that can be derived from the vector type (EltNum x
17// EltVT). These are things like the register class for the writemask, etc.
18// The idea is to pass one of these as the template argument rather than the
19// individual arguments.
Elena Demikhovskyfa4a6c12014-12-09 07:06:32 +000020// The template is also used for scalar types, in this case numelts is 1.
Robert Khasanov4204c1a2014-12-12 14:21:30 +000021class X86VectorVTInfo<int numelts, ValueType eltvt, RegisterClass rc,
Adam Nemet5ed17da2014-08-21 19:50:07 +000022 string suffix = ""> {
23 RegisterClass RC = rc;
Robert Khasanov4204c1a2014-12-12 14:21:30 +000024 ValueType EltVT = eltvt;
Adam Nemet449b3f02014-10-15 23:42:09 +000025 int NumElts = numelts;
Adam Nemet5ed17da2014-08-21 19:50:07 +000026
27 // Corresponding mask register class.
28 RegisterClass KRC = !cast<RegisterClass>("VK" # NumElts);
29
30 // Corresponding write-mask register class.
31 RegisterClass KRCWM = !cast<RegisterClass>("VK" # NumElts # "WM");
32
Igor Bregerfca0a342016-01-28 13:19:25 +000033 // The mask VT.
Guy Blank548e22a2017-05-19 12:35:15 +000034 ValueType KVT = !cast<ValueType>("v" # NumElts # "i1");
Simon Pilgrimb13961d2016-06-11 14:34:10 +000035
Adam Nemet5ed17da2014-08-21 19:50:07 +000036 // Suffix used in the instruction mnemonic.
37 string Suffix = suffix;
38
Elena Demikhovskyfa4a6c12014-12-09 07:06:32 +000039 // VTName is a string name for vector VT. For vector types it will be
40 // v # NumElts # EltVT, so for vector of 8 elements of i32 it will be v8i32
41 // It is a little bit complex for scalar types, where NumElts = 1.
42 // In this case we build v4f32 or v2f64
43 string VTName = "v" # !if (!eq (NumElts, 1),
44 !if (!eq (EltVT.Size, 32), 4,
45 !if (!eq (EltVT.Size, 64), 2, NumElts)), NumElts) # EltVT;
Robert Khasanov2ea081d2014-08-25 14:49:34 +000046
Adam Nemet5ed17da2014-08-21 19:50:07 +000047 // The vector VT.
Robert Khasanov2ea081d2014-08-25 14:49:34 +000048 ValueType VT = !cast<ValueType>(VTName);
Adam Nemet5ed17da2014-08-21 19:50:07 +000049
50 string EltTypeName = !cast<string>(EltVT);
51 // Size of the element type in bits, e.g. 32 for v16i32.
Robert Khasanov2ea081d2014-08-25 14:49:34 +000052 string EltSizeName = !subst("i", "", !subst("f", "", EltTypeName));
53 int EltSize = EltVT.Size;
Adam Nemet5ed17da2014-08-21 19:50:07 +000054
55 // "i" for integer types and "f" for floating-point types
Robert Khasanov2ea081d2014-08-25 14:49:34 +000056 string TypeVariantName = !subst(EltSizeName, "", EltTypeName);
Adam Nemet5ed17da2014-08-21 19:50:07 +000057
58 // Size of RC in bits, e.g. 512 for VR512.
59 int Size = VT.Size;
60
61 // The corresponding memory operand, e.g. i512mem for VR512.
62 X86MemOperand MemOp = !cast<X86MemOperand>(TypeVariantName # Size # "mem");
Robert Khasanov2ea081d2014-08-25 14:49:34 +000063 X86MemOperand ScalarMemOp = !cast<X86MemOperand>(EltVT # "mem");
Ayman Musaf77219e2017-02-13 09:55:48 +000064 // FP scalar memory operand for intrinsics - ssmem/sdmem.
65 Operand IntScalarMemOp = !if (!eq (EltTypeName, "f32"), !cast<Operand>("ssmem"),
66 !if (!eq (EltTypeName, "f64"), !cast<Operand>("sdmem"), ?));
Robert Khasanov2ea081d2014-08-25 14:49:34 +000067
68 // Load patterns
69 // Note: For 128/256-bit integer VT we choose loadv2i64/loadv4i64
70 // due to load promotion during legalization
71 PatFrag LdFrag = !cast<PatFrag>("load" #
72 !if (!eq (TypeVariantName, "i"),
73 !if (!eq (Size, 128), "v2i64",
74 !if (!eq (Size, 256), "v4i64",
Craig Toppera78b7682016-08-11 06:04:07 +000075 !if (!eq (Size, 512), "v8i64",
76 VTName))), VTName));
Elena Demikhovsky2689d782015-03-02 12:46:21 +000077
78 PatFrag AlignedLdFrag = !cast<PatFrag>("alignedload" #
Craig Toppera78b7682016-08-11 06:04:07 +000079 !if (!eq (TypeVariantName, "i"),
80 !if (!eq (Size, 128), "v2i64",
81 !if (!eq (Size, 256), "v4i64",
82 !if (!eq (Size, 512), "v8i64",
83 VTName))), VTName));
Elena Demikhovsky2689d782015-03-02 12:46:21 +000084
Robert Khasanov2ea081d2014-08-25 14:49:34 +000085 PatFrag ScalarLdFrag = !cast<PatFrag>("load" # EltVT);
Adam Nemet5ed17da2014-08-21 19:50:07 +000086
Craig Topperd9fe6642017-02-21 04:26:10 +000087 ComplexPattern ScalarIntMemCPat = !if (!eq (EltTypeName, "f32"),
88 !cast<ComplexPattern>("sse_load_f32"),
89 !if (!eq (EltTypeName, "f64"),
90 !cast<ComplexPattern>("sse_load_f64"),
91 ?));
92
Adam Nemet5ed17da2014-08-21 19:50:07 +000093 // The corresponding float type, e.g. v16f32 for v16i32
Robert Khasanov2ea081d2014-08-25 14:49:34 +000094 // Note: For EltSize < 32, FloatVT is illegal and TableGen
95 // fails to compile, so we choose FloatVT = VT
96 ValueType FloatVT = !cast<ValueType>(
97 !if (!eq (!srl(EltSize,5),0),
98 VTName,
99 !if (!eq(TypeVariantName, "i"),
100 "v" # NumElts # "f" # EltSize,
101 VTName)));
Adam Nemet5ed17da2014-08-21 19:50:07 +0000102
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +0000103 ValueType IntVT = !cast<ValueType>(
104 !if (!eq (!srl(EltSize,5),0),
105 VTName,
106 !if (!eq(TypeVariantName, "f"),
107 "v" # NumElts # "i" # EltSize,
108 VTName)));
Adam Nemet5ed17da2014-08-21 19:50:07 +0000109 // The string to specify embedded broadcast in assembly.
110 string BroadcastStr = "{1to" # NumElts # "}";
Adam Nemet55536c62014-09-25 23:48:45 +0000111
Adam Nemet449b3f02014-10-15 23:42:09 +0000112 // 8-bit compressed displacement tuple/subvector format. This is only
113 // defined for NumElts <= 8.
114 CD8VForm CD8TupleForm = !if (!eq (!srl(NumElts, 4), 0),
115 !cast<CD8VForm>("CD8VT" # NumElts), ?);
116
Adam Nemet55536c62014-09-25 23:48:45 +0000117 SubRegIndex SubRegIdx = !if (!eq (Size, 128), sub_xmm,
118 !if (!eq (Size, 256), sub_ymm, ?));
119
120 Domain ExeDomain = !if (!eq (EltTypeName, "f32"), SSEPackedSingle,
121 !if (!eq (EltTypeName, "f64"), SSEPackedDouble,
122 SSEPackedInt));
Adam Nemet09377232014-10-08 23:25:31 +0000123
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +0000124 RegisterClass FRC = !if (!eq (EltTypeName, "f32"), FR32X, FR64X);
125
Craig Topperabe80cc2016-08-28 06:06:28 +0000126 // A vector tye of the same width with element type i64. This is used to
127 // create patterns for logic ops.
128 ValueType i64VT = !cast<ValueType>("v" # !srl(Size, 6) # "i64");
129
Adam Nemet09377232014-10-08 23:25:31 +0000130 // A vector type of the same width with element type i32. This is used to
131 // create the canonical constant zero node ImmAllZerosV.
132 ValueType i32VT = !cast<ValueType>("v" # !srl(Size, 5) # "i32");
133 dag ImmAllZerosV = (VT (bitconvert (i32VT immAllZerosV)));
Elena Demikhovskyd207f172015-03-03 15:03:35 +0000134
135 string ZSuffix = !if (!eq (Size, 128), "Z128",
136 !if (!eq (Size, 256), "Z256", "Z"));
Adam Nemet5ed17da2014-08-21 19:50:07 +0000137}
138
Robert Khasanov2ea081d2014-08-25 14:49:34 +0000139def v64i8_info : X86VectorVTInfo<64, i8, VR512, "b">;
140def v32i16_info : X86VectorVTInfo<32, i16, VR512, "w">;
Adam Nemet5ed17da2014-08-21 19:50:07 +0000141def v16i32_info : X86VectorVTInfo<16, i32, VR512, "d">;
142def v8i64_info : X86VectorVTInfo<8, i64, VR512, "q">;
Adam Nemet6bddb8c2014-09-29 22:54:41 +0000143def v16f32_info : X86VectorVTInfo<16, f32, VR512, "ps">;
144def v8f64_info : X86VectorVTInfo<8, f64, VR512, "pd">;
Adam Nemet5ed17da2014-08-21 19:50:07 +0000145
Robert Khasanov2ea081d2014-08-25 14:49:34 +0000146// "x" in v32i8x_info means RC = VR256X
147def v32i8x_info : X86VectorVTInfo<32, i8, VR256X, "b">;
148def v16i16x_info : X86VectorVTInfo<16, i16, VR256X, "w">;
149def v8i32x_info : X86VectorVTInfo<8, i32, VR256X, "d">;
150def v4i64x_info : X86VectorVTInfo<4, i64, VR256X, "q">;
Robert Khasanov3e534c92014-10-28 16:37:13 +0000151def v8f32x_info : X86VectorVTInfo<8, f32, VR256X, "ps">;
152def v4f64x_info : X86VectorVTInfo<4, f64, VR256X, "pd">;
Robert Khasanov2ea081d2014-08-25 14:49:34 +0000153
154def v16i8x_info : X86VectorVTInfo<16, i8, VR128X, "b">;
155def v8i16x_info : X86VectorVTInfo<8, i16, VR128X, "w">;
156def v4i32x_info : X86VectorVTInfo<4, i32, VR128X, "d">;
157def v2i64x_info : X86VectorVTInfo<2, i64, VR128X, "q">;
Robert Khasanov3e534c92014-10-28 16:37:13 +0000158def v4f32x_info : X86VectorVTInfo<4, f32, VR128X, "ps">;
159def v2f64x_info : X86VectorVTInfo<2, f64, VR128X, "pd">;
Robert Khasanov2ea081d2014-08-25 14:49:34 +0000160
Elena Demikhovskyfa4a6c12014-12-09 07:06:32 +0000161// We map scalar types to the smallest (128-bit) vector type
162// with the appropriate element type. This allows to use the same masking logic.
Asaf Badouh2744d212015-09-20 14:31:19 +0000163def i32x_info : X86VectorVTInfo<1, i32, GR32, "si">;
164def i64x_info : X86VectorVTInfo<1, i64, GR64, "sq">;
Elena Demikhovsky905a5a62014-11-26 10:46:49 +0000165def f32x_info : X86VectorVTInfo<1, f32, VR128X, "ss">;
166def f64x_info : X86VectorVTInfo<1, f64, VR128X, "sd">;
167
Robert Khasanov2ea081d2014-08-25 14:49:34 +0000168class AVX512VLVectorVTInfo<X86VectorVTInfo i512, X86VectorVTInfo i256,
169 X86VectorVTInfo i128> {
170 X86VectorVTInfo info512 = i512;
171 X86VectorVTInfo info256 = i256;
172 X86VectorVTInfo info128 = i128;
173}
174
175def avx512vl_i8_info : AVX512VLVectorVTInfo<v64i8_info, v32i8x_info,
176 v16i8x_info>;
177def avx512vl_i16_info : AVX512VLVectorVTInfo<v32i16_info, v16i16x_info,
178 v8i16x_info>;
179def avx512vl_i32_info : AVX512VLVectorVTInfo<v16i32_info, v8i32x_info,
180 v4i32x_info>;
181def avx512vl_i64_info : AVX512VLVectorVTInfo<v8i64_info, v4i64x_info,
182 v2i64x_info>;
Robert Khasanovaf318f72014-10-30 14:21:47 +0000183def avx512vl_f32_info : AVX512VLVectorVTInfo<v16f32_info, v8f32x_info,
184 v4f32x_info>;
185def avx512vl_f64_info : AVX512VLVectorVTInfo<v8f64_info, v4f64x_info,
186 v2f64x_info>;
Robert Khasanov2ea081d2014-08-25 14:49:34 +0000187
Ayman Musa721d97f2017-06-27 12:08:37 +0000188class X86KVectorVTInfo<RegisterClass _krc, RegisterClass _krcwm,
189 ValueType _vt> {
190 RegisterClass KRC = _krc;
191 RegisterClass KRCWM = _krcwm;
192 ValueType KVT = _vt;
193}
194
195def v2i1_info : X86KVectorVTInfo<VK2, VK2WM, v2i1>;
196def v4i1_info : X86KVectorVTInfo<VK4, VK4WM, v4i1>;
197def v8i1_info : X86KVectorVTInfo<VK8, VK8WM, v8i1>;
198def v16i1_info : X86KVectorVTInfo<VK16, VK16WM, v16i1>;
199def v32i1_info : X86KVectorVTInfo<VK32, VK32WM, v32i1>;
200def v64i1_info : X86KVectorVTInfo<VK64, VK64WM, v64i1>;
201
Adam Nemet52bb6cf2014-10-08 23:25:23 +0000202// This multiclass generates the masking variants from the non-masking
203// variant. It only provides the assembly pieces for the masking variants.
204// It assumes custom ISel patterns for masking which can be provided as
205// template arguments.
Adam Nemet34801422014-10-08 23:25:39 +0000206multiclass AVX512_maskable_custom<bits<8> O, Format F,
207 dag Outs,
208 dag Ins, dag MaskingIns, dag ZeroMaskingIns,
209 string OpcodeStr,
210 string AttSrcAsm, string IntelSrcAsm,
211 list<dag> Pattern,
212 list<dag> MaskingPattern,
213 list<dag> ZeroMaskingPattern,
214 string MaskingConstraint = "",
215 InstrItinClass itin = NoItinerary,
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +0000216 bit IsCommutable = 0,
217 bit IsKCommutable = 0> {
Adam Nemet52bb6cf2014-10-08 23:25:23 +0000218 let isCommutable = IsCommutable in
219 def NAME: AVX512<O, F, Outs, Ins,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000220 OpcodeStr#"\t{"#AttSrcAsm#", $dst|"#
Craig Topper9d2cab72016-01-11 01:03:40 +0000221 "$dst, "#IntelSrcAsm#"}",
Adam Nemet52bb6cf2014-10-08 23:25:23 +0000222 Pattern, itin>;
223
224 // Prefer over VMOV*rrk Pat<>
Craig Topper63801df2017-02-19 21:44:35 +0000225 let isCommutable = IsKCommutable in
Adam Nemet52bb6cf2014-10-08 23:25:23 +0000226 def NAME#k: AVX512<O, F, Outs, MaskingIns,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000227 OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"#
228 "$dst {${mask}}, "#IntelSrcAsm#"}",
Adam Nemet52bb6cf2014-10-08 23:25:23 +0000229 MaskingPattern, itin>,
230 EVEX_K {
231 // In case of the 3src subclass this is overridden with a let.
232 string Constraints = MaskingConstraint;
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +0000233 }
234
235 // Zero mask does not add any restrictions to commute operands transformation.
236 // So, it is Ok to use IsCommutable instead of IsKCommutable.
Craig Topper63801df2017-02-19 21:44:35 +0000237 let isCommutable = IsCommutable in // Prefer over VMOV*rrkz Pat<>
Adam Nemet52bb6cf2014-10-08 23:25:23 +0000238 def NAME#kz: AVX512<O, F, Outs, ZeroMaskingIns,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000239 OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}} {z}|"#
240 "$dst {${mask}} {z}, "#IntelSrcAsm#"}",
Adam Nemet52bb6cf2014-10-08 23:25:23 +0000241 ZeroMaskingPattern,
242 itin>,
243 EVEX_KZ;
244}
245
Robert Khasanov2ea081d2014-08-25 14:49:34 +0000246
Adam Nemet34801422014-10-08 23:25:39 +0000247// Common base class of AVX512_maskable and AVX512_maskable_3src.
248multiclass AVX512_maskable_common<bits<8> O, Format F, X86VectorVTInfo _,
249 dag Outs,
250 dag Ins, dag MaskingIns, dag ZeroMaskingIns,
251 string OpcodeStr,
252 string AttSrcAsm, string IntelSrcAsm,
253 dag RHS, dag MaskingRHS,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000254 SDNode Select = vselect,
Adam Nemet34801422014-10-08 23:25:39 +0000255 string MaskingConstraint = "",
256 InstrItinClass itin = NoItinerary,
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +0000257 bit IsCommutable = 0,
258 bit IsKCommutable = 0> :
Adam Nemet34801422014-10-08 23:25:39 +0000259 AVX512_maskable_custom<O, F, Outs, Ins, MaskingIns, ZeroMaskingIns, OpcodeStr,
260 AttSrcAsm, IntelSrcAsm,
261 [(set _.RC:$dst, RHS)],
262 [(set _.RC:$dst, MaskingRHS)],
263 [(set _.RC:$dst,
Elena Demikhovsky905a5a62014-11-26 10:46:49 +0000264 (Select _.KRCWM:$mask, RHS, _.ImmAllZerosV))],
Craig Topperb9e3e112017-08-14 15:28:48 +0000265 MaskingConstraint, itin, IsCommutable,
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +0000266 IsKCommutable>;
Adam Nemet2e2537f2014-08-07 17:53:55 +0000267
Ayman Musa6e670cf2017-02-23 07:24:21 +0000268// Similar to AVX512_maskable_common, but with scalar types.
269multiclass AVX512_maskable_fp_common<bits<8> O, Format F, X86VectorVTInfo _,
270 dag Outs,
271 dag Ins, dag MaskingIns, dag ZeroMaskingIns,
272 string OpcodeStr,
273 string AttSrcAsm, string IntelSrcAsm,
274 SDNode Select = vselect,
275 string MaskingConstraint = "",
276 InstrItinClass itin = NoItinerary,
277 bit IsCommutable = 0,
278 bit IsKCommutable = 0> :
279 AVX512_maskable_custom<O, F, Outs, Ins, MaskingIns, ZeroMaskingIns, OpcodeStr,
280 AttSrcAsm, IntelSrcAsm,
281 [], [], [],
Craig Topperb9e3e112017-08-14 15:28:48 +0000282 MaskingConstraint, itin, IsCommutable,
Ayman Musa6e670cf2017-02-23 07:24:21 +0000283 IsKCommutable>;
284
Adam Nemet2e91ee52014-08-14 17:13:19 +0000285// This multiclass generates the unconditional/non-masking, the masking and
Elena Demikhovsky905a5a62014-11-26 10:46:49 +0000286// the zero-masking variant of the vector instruction. In the masking case, the
Adam Nemet2e91ee52014-08-14 17:13:19 +0000287// perserved vector elements come from a new dummy input operand tied to $dst.
Adam Nemet34801422014-10-08 23:25:39 +0000288multiclass AVX512_maskable<bits<8> O, Format F, X86VectorVTInfo _,
289 dag Outs, dag Ins, string OpcodeStr,
290 string AttSrcAsm, string IntelSrcAsm,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000291 dag RHS,
Elena Demikhovskybe8808d2014-11-12 07:31:03 +0000292 InstrItinClass itin = NoItinerary,
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +0000293 bit IsCommutable = 0, bit IsKCommutable = 0,
294 SDNode Select = vselect> :
Adam Nemet34801422014-10-08 23:25:39 +0000295 AVX512_maskable_common<O, F, _, Outs, Ins,
296 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),
297 !con((ins _.KRCWM:$mask), Ins),
298 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
Igor Breger73ee8ba2016-05-31 08:04:21 +0000299 (Select _.KRCWM:$mask, RHS, _.RC:$src0), Select,
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +0000300 "$src0 = $dst", itin, IsCommutable, IsKCommutable>;
Elena Demikhovsky905a5a62014-11-26 10:46:49 +0000301
302// This multiclass generates the unconditional/non-masking, the masking and
303// the zero-masking variant of the scalar instruction.
304multiclass AVX512_maskable_scalar<bits<8> O, Format F, X86VectorVTInfo _,
305 dag Outs, dag Ins, string OpcodeStr,
306 string AttSrcAsm, string IntelSrcAsm,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000307 dag RHS,
Elena Demikhovsky905a5a62014-11-26 10:46:49 +0000308 InstrItinClass itin = NoItinerary,
309 bit IsCommutable = 0> :
310 AVX512_maskable_common<O, F, _, Outs, Ins,
311 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),
312 !con((ins _.KRCWM:$mask), Ins),
313 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
Craig Topper74ed0872016-05-18 06:55:59 +0000314 (X86selects _.KRCWM:$mask, RHS, _.RC:$src0),
315 X86selects, "$src0 = $dst", itin, IsCommutable>;
Adam Nemet2e91ee52014-08-14 17:13:19 +0000316
Adam Nemet34801422014-10-08 23:25:39 +0000317// Similar to AVX512_maskable but in this case one of the source operands
Adam Nemet2e91ee52014-08-14 17:13:19 +0000318// ($src1) is already tied to $dst so we just use that for the preserved
319// vector elements. NOTE that the NonTiedIns (the ins dag) should exclude
320// $src1.
Adam Nemet34801422014-10-08 23:25:39 +0000321multiclass AVX512_maskable_3src<bits<8> O, Format F, X86VectorVTInfo _,
322 dag Outs, dag NonTiedIns, string OpcodeStr,
323 string AttSrcAsm, string IntelSrcAsm,
Simon Pilgrim916485c2016-08-18 11:22:22 +0000324 dag RHS, bit IsCommutable = 0,
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +0000325 bit IsKCommutable = 0> :
Adam Nemet34801422014-10-08 23:25:39 +0000326 AVX512_maskable_common<O, F, _, Outs,
327 !con((ins _.RC:$src1), NonTiedIns),
328 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),
329 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),
330 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +0000331 (vselect _.KRCWM:$mask, RHS, _.RC:$src1),
332 vselect, "", NoItinerary, IsCommutable, IsKCommutable>;
Adam Nemet2e91ee52014-08-14 17:13:19 +0000333
Igor Breger15820b02015-07-01 13:24:28 +0000334multiclass AVX512_maskable_3src_scalar<bits<8> O, Format F, X86VectorVTInfo _,
335 dag Outs, dag NonTiedIns, string OpcodeStr,
336 string AttSrcAsm, string IntelSrcAsm,
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +0000337 dag RHS, bit IsCommutable = 0,
338 bit IsKCommutable = 0> :
Igor Breger15820b02015-07-01 13:24:28 +0000339 AVX512_maskable_common<O, F, _, Outs,
340 !con((ins _.RC:$src1), NonTiedIns),
341 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),
342 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),
343 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
Craig Topper74ed0872016-05-18 06:55:59 +0000344 (X86selects _.KRCWM:$mask, RHS, _.RC:$src1),
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +0000345 X86selects, "", NoItinerary, IsCommutable,
346 IsKCommutable>;
Adam Nemet2b5cdbb2014-10-08 23:25:33 +0000347
Adam Nemet34801422014-10-08 23:25:39 +0000348multiclass AVX512_maskable_in_asm<bits<8> O, Format F, X86VectorVTInfo _,
349 dag Outs, dag Ins,
350 string OpcodeStr,
351 string AttSrcAsm, string IntelSrcAsm,
352 list<dag> Pattern> :
353 AVX512_maskable_custom<O, F, Outs, Ins,
354 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),
355 !con((ins _.KRCWM:$mask), Ins),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +0000356 OpcodeStr, AttSrcAsm, IntelSrcAsm, Pattern, [], [],
Adam Nemet34801422014-10-08 23:25:39 +0000357 "$src0 = $dst">;
Adam Nemet2b5cdbb2014-10-08 23:25:33 +0000358
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000359
360// Instruction with mask that puts result in mask register,
361// like "compare" and "vptest"
362multiclass AVX512_maskable_custom_cmp<bits<8> O, Format F,
363 dag Outs,
364 dag Ins, dag MaskingIns,
365 string OpcodeStr,
366 string AttSrcAsm, string IntelSrcAsm,
367 list<dag> Pattern,
Craig Topper225da2c2016-08-27 05:22:15 +0000368 list<dag> MaskingPattern,
369 bit IsCommutable = 0> {
370 let isCommutable = IsCommutable in
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000371 def NAME: AVX512<O, F, Outs, Ins,
Craig Topper156622a2016-01-11 00:44:56 +0000372 OpcodeStr#"\t{"#AttSrcAsm#", $dst|"#
373 "$dst, "#IntelSrcAsm#"}",
374 Pattern, NoItinerary>;
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000375
376 def NAME#k: AVX512<O, F, Outs, MaskingIns,
Craig Topper156622a2016-01-11 00:44:56 +0000377 OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"#
378 "$dst {${mask}}, "#IntelSrcAsm#"}",
379 MaskingPattern, NoItinerary>, EVEX_K;
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000380}
381
382multiclass AVX512_maskable_common_cmp<bits<8> O, Format F, X86VectorVTInfo _,
383 dag Outs,
384 dag Ins, dag MaskingIns,
385 string OpcodeStr,
386 string AttSrcAsm, string IntelSrcAsm,
Craig Topper225da2c2016-08-27 05:22:15 +0000387 dag RHS, dag MaskingRHS,
388 bit IsCommutable = 0> :
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000389 AVX512_maskable_custom_cmp<O, F, Outs, Ins, MaskingIns, OpcodeStr,
390 AttSrcAsm, IntelSrcAsm,
391 [(set _.KRC:$dst, RHS)],
Craig Topper225da2c2016-08-27 05:22:15 +0000392 [(set _.KRC:$dst, MaskingRHS)], IsCommutable>;
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000393
394multiclass AVX512_maskable_cmp<bits<8> O, Format F, X86VectorVTInfo _,
395 dag Outs, dag Ins, string OpcodeStr,
396 string AttSrcAsm, string IntelSrcAsm,
Craig Topper225da2c2016-08-27 05:22:15 +0000397 dag RHS, bit IsCommutable = 0> :
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000398 AVX512_maskable_common_cmp<O, F, _, Outs, Ins,
399 !con((ins _.KRCWM:$mask), Ins),
400 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
Craig Topper225da2c2016-08-27 05:22:15 +0000401 (and _.KRCWM:$mask, RHS), IsCommutable>;
Elena Demikhovsky431b81e2015-04-21 13:13:46 +0000402
Elena Demikhovsky29792e92015-05-07 11:24:42 +0000403multiclass AVX512_maskable_cmp_alt<bits<8> O, Format F, X86VectorVTInfo _,
404 dag Outs, dag Ins, string OpcodeStr,
405 string AttSrcAsm, string IntelSrcAsm> :
406 AVX512_maskable_custom_cmp<O, F, Outs,
407 Ins, !con((ins _.KRCWM:$mask),Ins), OpcodeStr,
Craig Topper156622a2016-01-11 00:44:56 +0000408 AttSrcAsm, IntelSrcAsm, [],[]>;
Elena Demikhovsky29792e92015-05-07 11:24:42 +0000409
Craig Topperabe80cc2016-08-28 06:06:28 +0000410// This multiclass generates the unconditional/non-masking, the masking and
411// the zero-masking variant of the vector instruction. In the masking case, the
412// perserved vector elements come from a new dummy input operand tied to $dst.
413multiclass AVX512_maskable_logic<bits<8> O, Format F, X86VectorVTInfo _,
414 dag Outs, dag Ins, string OpcodeStr,
415 string AttSrcAsm, string IntelSrcAsm,
416 dag RHS, dag MaskedRHS,
417 InstrItinClass itin = NoItinerary,
418 bit IsCommutable = 0, SDNode Select = vselect> :
419 AVX512_maskable_custom<O, F, Outs, Ins,
420 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),
421 !con((ins _.KRCWM:$mask), Ins),
422 OpcodeStr, AttSrcAsm, IntelSrcAsm,
423 [(set _.RC:$dst, RHS)],
424 [(set _.RC:$dst,
425 (Select _.KRCWM:$mask, MaskedRHS, _.RC:$src0))],
426 [(set _.RC:$dst,
427 (Select _.KRCWM:$mask, MaskedRHS,
428 _.ImmAllZerosV))],
429 "$src0 = $dst", itin, IsCommutable>;
430
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000431// Bitcasts between 512-bit vector types. Return the original type since
Craig Topper2388b462016-06-03 04:15:27 +0000432// no instruction is needed for the conversion.
433def : Pat<(v8f64 (bitconvert (v8i64 VR512:$src))), (v8f64 VR512:$src)>;
434def : Pat<(v8f64 (bitconvert (v16i32 VR512:$src))), (v8f64 VR512:$src)>;
435def : Pat<(v8f64 (bitconvert (v32i16 VR512:$src))), (v8f64 VR512:$src)>;
436def : Pat<(v8f64 (bitconvert (v64i8 VR512:$src))), (v8f64 VR512:$src)>;
437def : Pat<(v8f64 (bitconvert (v16f32 VR512:$src))), (v8f64 VR512:$src)>;
438def : Pat<(v16f32 (bitconvert (v8i64 VR512:$src))), (v16f32 VR512:$src)>;
439def : Pat<(v16f32 (bitconvert (v16i32 VR512:$src))), (v16f32 VR512:$src)>;
440def : Pat<(v16f32 (bitconvert (v32i16 VR512:$src))), (v16f32 VR512:$src)>;
441def : Pat<(v16f32 (bitconvert (v64i8 VR512:$src))), (v16f32 VR512:$src)>;
442def : Pat<(v16f32 (bitconvert (v8f64 VR512:$src))), (v16f32 VR512:$src)>;
443def : Pat<(v8i64 (bitconvert (v16i32 VR512:$src))), (v8i64 VR512:$src)>;
444def : Pat<(v8i64 (bitconvert (v32i16 VR512:$src))), (v8i64 VR512:$src)>;
445def : Pat<(v8i64 (bitconvert (v64i8 VR512:$src))), (v8i64 VR512:$src)>;
446def : Pat<(v8i64 (bitconvert (v8f64 VR512:$src))), (v8i64 VR512:$src)>;
447def : Pat<(v8i64 (bitconvert (v16f32 VR512:$src))), (v8i64 VR512:$src)>;
448def : Pat<(v16i32 (bitconvert (v8i64 VR512:$src))), (v16i32 VR512:$src)>;
449def : Pat<(v16i32 (bitconvert (v16f32 VR512:$src))), (v16i32 VR512:$src)>;
450def : Pat<(v16i32 (bitconvert (v32i16 VR512:$src))), (v16i32 VR512:$src)>;
451def : Pat<(v16i32 (bitconvert (v64i8 VR512:$src))), (v16i32 VR512:$src)>;
452def : Pat<(v16i32 (bitconvert (v8f64 VR512:$src))), (v16i32 VR512:$src)>;
453def : Pat<(v32i16 (bitconvert (v8i64 VR512:$src))), (v32i16 VR512:$src)>;
454def : Pat<(v32i16 (bitconvert (v16i32 VR512:$src))), (v32i16 VR512:$src)>;
455def : Pat<(v32i16 (bitconvert (v64i8 VR512:$src))), (v32i16 VR512:$src)>;
456def : Pat<(v32i16 (bitconvert (v8f64 VR512:$src))), (v32i16 VR512:$src)>;
457def : Pat<(v32i16 (bitconvert (v16f32 VR512:$src))), (v32i16 VR512:$src)>;
458def : Pat<(v32i16 (bitconvert (v16f32 VR512:$src))), (v32i16 VR512:$src)>;
459def : Pat<(v64i8 (bitconvert (v8i64 VR512:$src))), (v64i8 VR512:$src)>;
460def : Pat<(v64i8 (bitconvert (v16i32 VR512:$src))), (v64i8 VR512:$src)>;
461def : Pat<(v64i8 (bitconvert (v32i16 VR512:$src))), (v64i8 VR512:$src)>;
462def : Pat<(v64i8 (bitconvert (v8f64 VR512:$src))), (v64i8 VR512:$src)>;
463def : Pat<(v64i8 (bitconvert (v16f32 VR512:$src))), (v64i8 VR512:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000464
Craig Topper9d9251b2016-05-08 20:10:20 +0000465// Alias instruction that maps zero vector to pxor / xorp* for AVX-512.
466// This is expanded by ExpandPostRAPseudos to an xorps / vxorps, and then
467// swizzled by ExecutionDepsFix to pxor.
468// We set canFoldAsLoad because this can be converted to a constant-pool
469// load of an all-zeros value if folding it would be beneficial.
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000470let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
Craig Topper86748492016-07-11 05:36:41 +0000471 isPseudo = 1, Predicates = [HasAVX512], SchedRW = [WriteZero] in {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000472def AVX512_512_SET0 : I<0, Pseudo, (outs VR512:$dst), (ins), "",
Craig Topper9d9251b2016-05-08 20:10:20 +0000473 [(set VR512:$dst, (v16i32 immAllZerosV))]>;
Craig Topper516e14c2016-07-11 05:36:48 +0000474def AVX512_512_SETALLONES : I<0, Pseudo, (outs VR512:$dst), (ins), "",
475 [(set VR512:$dst, (v16i32 immAllOnesV))]>;
Craig Topperfb1746b2014-01-30 06:03:19 +0000476}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000477
Craig Topper6393afc2017-01-09 02:44:34 +0000478// Alias instructions that allow VPTERNLOG to be used with a mask to create
479// a mix of all ones and all zeros elements. This is done this way to force
480// the same register to be used as input for all three sources.
481let isPseudo = 1, Predicates = [HasAVX512] in {
482def AVX512_512_SEXT_MASK_32 : I<0, Pseudo, (outs VR512:$dst),
483 (ins VK16WM:$mask), "",
484 [(set VR512:$dst, (vselect (v16i1 VK16WM:$mask),
485 (v16i32 immAllOnesV),
486 (v16i32 immAllZerosV)))]>;
487def AVX512_512_SEXT_MASK_64 : I<0, Pseudo, (outs VR512:$dst),
488 (ins VK8WM:$mask), "",
489 [(set VR512:$dst, (vselect (v8i1 VK8WM:$mask),
490 (bc_v8i64 (v16i32 immAllOnesV)),
491 (bc_v8i64 (v16i32 immAllZerosV))))]>;
492}
493
Craig Toppere5ce84a2016-05-08 21:33:53 +0000494let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
Craig Topper09b7e0f2017-01-14 07:29:24 +0000495 isPseudo = 1, Predicates = [HasAVX512], SchedRW = [WriteZero] in {
Craig Toppere5ce84a2016-05-08 21:33:53 +0000496def AVX512_128_SET0 : I<0, Pseudo, (outs VR128X:$dst), (ins), "",
497 [(set VR128X:$dst, (v4i32 immAllZerosV))]>;
498def AVX512_256_SET0 : I<0, Pseudo, (outs VR256X:$dst), (ins), "",
499 [(set VR256X:$dst, (v8i32 immAllZerosV))]>;
500}
501
Craig Topperadd9cc62016-12-18 06:23:14 +0000502// Alias instructions that map fld0 to xorps for sse or vxorps for avx.
503// This is expanded by ExpandPostRAPseudos.
504let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
Craig Topper09b7e0f2017-01-14 07:29:24 +0000505 isPseudo = 1, SchedRW = [WriteZero], Predicates = [HasAVX512] in {
Craig Topperadd9cc62016-12-18 06:23:14 +0000506 def AVX512_FsFLD0SS : I<0, Pseudo, (outs FR32X:$dst), (ins), "",
507 [(set FR32X:$dst, fp32imm0)]>;
508 def AVX512_FsFLD0SD : I<0, Pseudo, (outs FR64X:$dst), (ins), "",
509 [(set FR64X:$dst, fpimm0)]>;
510}
511
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000512//===----------------------------------------------------------------------===//
513// AVX-512 - VECTOR INSERT
514//
Igor Breger0ede3cb2015-09-20 06:52:42 +0000515multiclass vinsert_for_size<int Opcode, X86VectorVTInfo From, X86VectorVTInfo To,
516 PatFrag vinsert_insert> {
Craig Toppere1cac152016-06-07 07:27:54 +0000517 let ExeDomain = To.ExeDomain in {
Igor Breger0ede3cb2015-09-20 06:52:42 +0000518 defm rr : AVX512_maskable<Opcode, MRMSrcReg, To, (outs To.RC:$dst),
Ayman Musaf77219e2017-02-13 09:55:48 +0000519 (ins To.RC:$src1, From.RC:$src2, u8imm:$src3),
Igor Breger0ede3cb2015-09-20 06:52:42 +0000520 "vinsert" # From.EltTypeName # "x" # From.NumElts,
521 "$src3, $src2, $src1", "$src1, $src2, $src3",
522 (vinsert_insert:$src3 (To.VT To.RC:$src1),
523 (From.VT From.RC:$src2),
524 (iPTR imm))>, AVX512AIi8Base, EVEX_4V;
Adam Nemet4e2ef472014-10-02 23:18:28 +0000525
Igor Breger0ede3cb2015-09-20 06:52:42 +0000526 defm rm : AVX512_maskable<Opcode, MRMSrcMem, To, (outs To.RC:$dst),
Ayman Musaf77219e2017-02-13 09:55:48 +0000527 (ins To.RC:$src1, From.MemOp:$src2, u8imm:$src3),
Igor Breger0ede3cb2015-09-20 06:52:42 +0000528 "vinsert" # From.EltTypeName # "x" # From.NumElts,
529 "$src3, $src2, $src1", "$src1, $src2, $src3",
530 (vinsert_insert:$src3 (To.VT To.RC:$src1),
531 (From.VT (bitconvert (From.LdFrag addr:$src2))),
532 (iPTR imm))>, AVX512AIi8Base, EVEX_4V,
533 EVEX_CD8<From.EltSize, From.CD8TupleForm>;
Adam Nemet4e2ef472014-10-02 23:18:28 +0000534 }
Adam Nemet4285c1f2014-10-15 23:42:17 +0000535}
Adam Nemet4e2ef472014-10-02 23:18:28 +0000536
Igor Breger0ede3cb2015-09-20 06:52:42 +0000537multiclass vinsert_for_size_lowering<string InstrStr, X86VectorVTInfo From,
538 X86VectorVTInfo To, PatFrag vinsert_insert,
539 SDNodeXForm INSERT_get_vinsert_imm , list<Predicate> p> {
540 let Predicates = p in {
Adam Nemet4285c1f2014-10-15 23:42:17 +0000541 def : Pat<(vinsert_insert:$ins
Igor Breger0ede3cb2015-09-20 06:52:42 +0000542 (To.VT To.RC:$src1), (From.VT From.RC:$src2), (iPTR imm)),
543 (To.VT (!cast<Instruction>(InstrStr#"rr")
544 To.RC:$src1, From.RC:$src2,
545 (INSERT_get_vinsert_imm To.RC:$ins)))>;
546
547 def : Pat<(vinsert_insert:$ins
548 (To.VT To.RC:$src1),
549 (From.VT (bitconvert (From.LdFrag addr:$src2))),
550 (iPTR imm)),
551 (To.VT (!cast<Instruction>(InstrStr#"rm")
552 To.RC:$src1, addr:$src2,
553 (INSERT_get_vinsert_imm To.RC:$ins)))>;
554 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000555}
556
Adam Nemetb1c3ef42014-10-15 23:42:04 +0000557multiclass vinsert_for_type<ValueType EltVT32, int Opcode128,
558 ValueType EltVT64, int Opcode256> {
Igor Breger0ede3cb2015-09-20 06:52:42 +0000559
560 let Predicates = [HasVLX] in
561 defm NAME # "32x4Z256" : vinsert_for_size<Opcode128,
562 X86VectorVTInfo< 4, EltVT32, VR128X>,
563 X86VectorVTInfo< 8, EltVT32, VR256X>,
564 vinsert128_insert>, EVEX_V256;
565
566 defm NAME # "32x4Z" : vinsert_for_size<Opcode128,
Adam Nemet4e2ef472014-10-02 23:18:28 +0000567 X86VectorVTInfo< 4, EltVT32, VR128X>,
568 X86VectorVTInfo<16, EltVT32, VR512>,
Igor Breger0ede3cb2015-09-20 06:52:42 +0000569 vinsert128_insert>, EVEX_V512;
570
571 defm NAME # "64x4Z" : vinsert_for_size<Opcode256,
Adam Nemet4e2ef472014-10-02 23:18:28 +0000572 X86VectorVTInfo< 4, EltVT64, VR256X>,
573 X86VectorVTInfo< 8, EltVT64, VR512>,
Igor Breger0ede3cb2015-09-20 06:52:42 +0000574 vinsert256_insert>, VEX_W, EVEX_V512;
575
576 let Predicates = [HasVLX, HasDQI] in
577 defm NAME # "64x2Z256" : vinsert_for_size<Opcode128,
578 X86VectorVTInfo< 2, EltVT64, VR128X>,
579 X86VectorVTInfo< 4, EltVT64, VR256X>,
580 vinsert128_insert>, VEX_W, EVEX_V256;
581
582 let Predicates = [HasDQI] in {
583 defm NAME # "64x2Z" : vinsert_for_size<Opcode128,
584 X86VectorVTInfo< 2, EltVT64, VR128X>,
585 X86VectorVTInfo< 8, EltVT64, VR512>,
586 vinsert128_insert>, VEX_W, EVEX_V512;
587
588 defm NAME # "32x8Z" : vinsert_for_size<Opcode256,
589 X86VectorVTInfo< 8, EltVT32, VR256X>,
590 X86VectorVTInfo<16, EltVT32, VR512>,
591 vinsert256_insert>, EVEX_V512;
592 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000593}
594
Adam Nemet4e2ef472014-10-02 23:18:28 +0000595defm VINSERTF : vinsert_for_type<f32, 0x18, f64, 0x1a>;
596defm VINSERTI : vinsert_for_type<i32, 0x38, i64, 0x3a>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000597
Igor Breger0ede3cb2015-09-20 06:52:42 +0000598// Codegen pattern with the alternative types,
599// Only add this if 64x2 and its friends are not supported natively via AVX512DQ.
600defm : vinsert_for_size_lowering<"VINSERTF32x4Z256", v2f64x_info, v4f64x_info,
601 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX, NoDQI]>;
602defm : vinsert_for_size_lowering<"VINSERTI32x4Z256", v2i64x_info, v4i64x_info,
603 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX, NoDQI]>;
604
605defm : vinsert_for_size_lowering<"VINSERTF32x4Z", v2f64x_info, v8f64_info,
606 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512, NoDQI]>;
607defm : vinsert_for_size_lowering<"VINSERTI32x4Z", v2i64x_info, v8i64_info,
608 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512, NoDQI]>;
609
610defm : vinsert_for_size_lowering<"VINSERTF64x4Z", v8f32x_info, v16f32_info,
611 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512, NoDQI]>;
612defm : vinsert_for_size_lowering<"VINSERTI64x4Z", v8i32x_info, v16i32_info,
613 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512, NoDQI]>;
614
615// Codegen pattern with the alternative types insert VEC128 into VEC256
616defm : vinsert_for_size_lowering<"VINSERTI32x4Z256", v8i16x_info, v16i16x_info,
617 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>;
618defm : vinsert_for_size_lowering<"VINSERTI32x4Z256", v16i8x_info, v32i8x_info,
619 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>;
620// Codegen pattern with the alternative types insert VEC128 into VEC512
621defm : vinsert_for_size_lowering<"VINSERTI32x4Z", v8i16x_info, v32i16_info,
622 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>;
623defm : vinsert_for_size_lowering<"VINSERTI32x4Z", v16i8x_info, v64i8_info,
624 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>;
625// Codegen pattern with the alternative types insert VEC256 into VEC512
626defm : vinsert_for_size_lowering<"VINSERTI64x4Z", v16i16x_info, v32i16_info,
627 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>;
628defm : vinsert_for_size_lowering<"VINSERTI64x4Z", v32i8x_info, v64i8_info,
629 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>;
630
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000631// vinsertps - insert f32 to XMM
Craig Topper43973152016-10-09 06:41:47 +0000632let ExeDomain = SSEPackedSingle in {
Craig Topper6189d3e2016-07-19 01:26:19 +0000633def VINSERTPSZrr : AVX512AIi8<0x21, MRMSrcReg, (outs VR128X:$dst),
Craig Topper7ff6ab32015-01-21 08:43:49 +0000634 (ins VR128X:$src1, VR128X:$src2, u8imm:$src3),
Elena Demikhovskycf088092013-12-11 14:31:04 +0000635 "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
Filipe Cabecinhas20352212014-04-21 20:07:29 +0000636 [(set VR128X:$dst, (X86insertps VR128X:$src1, VR128X:$src2, imm:$src3))]>,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000637 EVEX_4V;
Craig Topper6189d3e2016-07-19 01:26:19 +0000638def VINSERTPSZrm: AVX512AIi8<0x21, MRMSrcMem, (outs VR128X:$dst),
Craig Topper7ff6ab32015-01-21 08:43:49 +0000639 (ins VR128X:$src1, f32mem:$src2, u8imm:$src3),
Elena Demikhovskycf088092013-12-11 14:31:04 +0000640 "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
Filipe Cabecinhas20352212014-04-21 20:07:29 +0000641 [(set VR128X:$dst, (X86insertps VR128X:$src1,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000642 (v4f32 (scalar_to_vector (loadf32 addr:$src2))),
643 imm:$src3))]>, EVEX_4V, EVEX_CD8<32, CD8VT1>;
Craig Topper43973152016-10-09 06:41:47 +0000644}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000645
646//===----------------------------------------------------------------------===//
647// AVX-512 VECTOR EXTRACT
648//---
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000649
Igor Breger7f69a992015-09-10 12:54:54 +0000650multiclass vextract_for_size<int Opcode,
Craig Topperd4e58072016-10-31 05:55:57 +0000651 X86VectorVTInfo From, X86VectorVTInfo To,
Craig Topperca98bb92017-08-14 05:09:33 +0000652 PatFrag vextract_extract> {
Igor Breger7f69a992015-09-10 12:54:54 +0000653
654 let hasSideEffects = 0, ExeDomain = To.ExeDomain in {
Craig Topperaadec702017-08-14 01:53:10 +0000655 defm rr : AVX512_maskable<Opcode, MRMDestReg, To, (outs To.RC:$dst),
Ayman Musaf77219e2017-02-13 09:55:48 +0000656 (ins From.RC:$src1, u8imm:$idx),
Igor Breger7f69a992015-09-10 12:54:54 +0000657 "vextract" # To.EltTypeName # "x" # To.NumElts,
658 "$idx, $src1", "$src1, $idx",
Craig Topperaadec702017-08-14 01:53:10 +0000659 (vextract_extract:$idx (From.VT From.RC:$src1), (iPTR imm))>,
Igor Breger7f69a992015-09-10 12:54:54 +0000660 AVX512AIi8Base, EVEX;
Craig Toppere1cac152016-06-07 07:27:54 +0000661 def mr : AVX512AIi8<Opcode, MRMDestMem, (outs),
Ayman Musaf77219e2017-02-13 09:55:48 +0000662 (ins To.MemOp:$dst, From.RC:$src1, u8imm:$idx),
Craig Toppere1cac152016-06-07 07:27:54 +0000663 "vextract" # To.EltTypeName # "x" # To.NumElts #
664 "\t{$idx, $src1, $dst|$dst, $src1, $idx}",
665 [(store (To.VT (vextract_extract:$idx
666 (From.VT From.RC:$src1), (iPTR imm))),
667 addr:$dst)]>, EVEX;
Igor Breger7f69a992015-09-10 12:54:54 +0000668
Craig Toppere1cac152016-06-07 07:27:54 +0000669 let mayStore = 1, hasSideEffects = 0 in
670 def mrk : AVX512AIi8<Opcode, MRMDestMem, (outs),
671 (ins To.MemOp:$dst, To.KRCWM:$mask,
Ayman Musaf77219e2017-02-13 09:55:48 +0000672 From.RC:$src1, u8imm:$idx),
Craig Toppere1cac152016-06-07 07:27:54 +0000673 "vextract" # To.EltTypeName # "x" # To.NumElts #
674 "\t{$idx, $src1, $dst {${mask}}|"
675 "$dst {${mask}}, $src1, $idx}",
676 []>, EVEX_K, EVEX;
Igor Breger7f69a992015-09-10 12:54:54 +0000677 }
Igor Bregerac29a822015-09-09 14:35:09 +0000678}
679
Igor Bregerdefab3c2015-10-08 12:55:01 +0000680// Codegen pattern for the alternative types
681multiclass vextract_for_size_lowering<string InstrStr, X86VectorVTInfo From,
682 X86VectorVTInfo To, PatFrag vextract_extract,
Craig Topper5f3fef82016-05-22 07:40:58 +0000683 SDNodeXForm EXTRACT_get_vextract_imm, list<Predicate> p> {
Craig Topperdb960ed2016-05-21 22:50:14 +0000684 let Predicates = p in {
Igor Bregerdefab3c2015-10-08 12:55:01 +0000685 def : Pat<(vextract_extract:$ext (From.VT From.RC:$src1), (iPTR imm)),
686 (To.VT (!cast<Instruction>(InstrStr#"rr")
687 From.RC:$src1,
688 (EXTRACT_get_vextract_imm To.RC:$ext)))>;
Craig Topperdb960ed2016-05-21 22:50:14 +0000689 def : Pat<(store (To.VT (vextract_extract:$ext (From.VT From.RC:$src1),
690 (iPTR imm))), addr:$dst),
691 (!cast<Instruction>(InstrStr#"mr") addr:$dst, From.RC:$src1,
692 (EXTRACT_get_vextract_imm To.RC:$ext))>;
693 }
Igor Breger7f69a992015-09-10 12:54:54 +0000694}
695
696multiclass vextract_for_type<ValueType EltVT32, int Opcode128,
Craig Topperd4e58072016-10-31 05:55:57 +0000697 ValueType EltVT64, int Opcode256> {
Craig Topperaadec702017-08-14 01:53:10 +0000698 let Predicates = [HasAVX512] in {
699 defm NAME # "32x4Z" : vextract_for_size<Opcode128,
700 X86VectorVTInfo<16, EltVT32, VR512>,
701 X86VectorVTInfo< 4, EltVT32, VR128X>,
Craig Topperca98bb92017-08-14 05:09:33 +0000702 vextract128_extract>,
Craig Topperaadec702017-08-14 01:53:10 +0000703 EVEX_V512, EVEX_CD8<32, CD8VT4>;
704 defm NAME # "64x4Z" : vextract_for_size<Opcode256,
705 X86VectorVTInfo< 8, EltVT64, VR512>,
706 X86VectorVTInfo< 4, EltVT64, VR256X>,
Craig Topperca98bb92017-08-14 05:09:33 +0000707 vextract256_extract>,
Craig Topperaadec702017-08-14 01:53:10 +0000708 VEX_W, EVEX_V512, EVEX_CD8<64, CD8VT4>;
709 }
Igor Breger7f69a992015-09-10 12:54:54 +0000710 let Predicates = [HasVLX] in
Igor Bregerdefab3c2015-10-08 12:55:01 +0000711 defm NAME # "32x4Z256" : vextract_for_size<Opcode128,
Igor Breger7f69a992015-09-10 12:54:54 +0000712 X86VectorVTInfo< 8, EltVT32, VR256X>,
713 X86VectorVTInfo< 4, EltVT32, VR128X>,
Craig Topperca98bb92017-08-14 05:09:33 +0000714 vextract128_extract>,
Igor Breger7f69a992015-09-10 12:54:54 +0000715 EVEX_V256, EVEX_CD8<32, CD8VT4>;
716 let Predicates = [HasVLX, HasDQI] in
717 defm NAME # "64x2Z256" : vextract_for_size<Opcode128,
718 X86VectorVTInfo< 4, EltVT64, VR256X>,
719 X86VectorVTInfo< 2, EltVT64, VR128X>,
Craig Topperca98bb92017-08-14 05:09:33 +0000720 vextract128_extract>,
Igor Breger7f69a992015-09-10 12:54:54 +0000721 VEX_W, EVEX_V256, EVEX_CD8<64, CD8VT2>;
722 let Predicates = [HasDQI] in {
723 defm NAME # "64x2Z" : vextract_for_size<Opcode128,
724 X86VectorVTInfo< 8, EltVT64, VR512>,
725 X86VectorVTInfo< 2, EltVT64, VR128X>,
Craig Topperca98bb92017-08-14 05:09:33 +0000726 vextract128_extract>,
Igor Breger7f69a992015-09-10 12:54:54 +0000727 VEX_W, EVEX_V512, EVEX_CD8<64, CD8VT2>;
728 defm NAME # "32x8Z" : vextract_for_size<Opcode256,
729 X86VectorVTInfo<16, EltVT32, VR512>,
730 X86VectorVTInfo< 8, EltVT32, VR256X>,
Craig Topperca98bb92017-08-14 05:09:33 +0000731 vextract256_extract>,
Igor Breger7f69a992015-09-10 12:54:54 +0000732 EVEX_V512, EVEX_CD8<32, CD8VT8>;
733 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000734}
735
Adam Nemet55536c62014-09-25 23:48:45 +0000736defm VEXTRACTF : vextract_for_type<f32, 0x19, f64, 0x1b>;
737defm VEXTRACTI : vextract_for_type<i32, 0x39, i64, 0x3b>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000738
Igor Bregerdefab3c2015-10-08 12:55:01 +0000739// extract_subvector codegen patterns with the alternative types.
740// Only add this if 64x2 and its friends are not supported natively via AVX512DQ.
741defm : vextract_for_size_lowering<"VEXTRACTF32x4Z", v8f64_info, v2f64x_info,
742 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512, NoDQI]>;
743defm : vextract_for_size_lowering<"VEXTRACTI32x4Z", v8i64_info, v2i64x_info,
744 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512, NoDQI]>;
745
746defm : vextract_for_size_lowering<"VEXTRACTF64x4Z", v16f32_info, v8f32x_info,
Igor Breger684af812015-10-26 12:26:34 +0000747 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512, NoDQI]>;
Igor Bregerdefab3c2015-10-08 12:55:01 +0000748defm : vextract_for_size_lowering<"VEXTRACTI64x4Z", v16i32_info, v8i32x_info,
749 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512, NoDQI]>;
750
751defm : vextract_for_size_lowering<"VEXTRACTF32x4Z256", v4f64x_info, v2f64x_info,
752 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX, NoDQI]>;
753defm : vextract_for_size_lowering<"VEXTRACTI32x4Z256", v4i64x_info, v2i64x_info,
754 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX, NoDQI]>;
755
Craig Topper08a68572016-05-21 22:50:04 +0000756// Codegen pattern with the alternative types extract VEC128 from VEC256
Craig Topper02626c02016-05-21 07:08:56 +0000757defm : vextract_for_size_lowering<"VEXTRACTI32x4Z256", v16i16x_info, v8i16x_info,
758 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>;
759defm : vextract_for_size_lowering<"VEXTRACTI32x4Z256", v32i8x_info, v16i8x_info,
760 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>;
761
762// Codegen pattern with the alternative types extract VEC128 from VEC512
Igor Bregerdefab3c2015-10-08 12:55:01 +0000763defm : vextract_for_size_lowering<"VEXTRACTI32x4Z", v32i16_info, v8i16x_info,
764 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>;
765defm : vextract_for_size_lowering<"VEXTRACTI32x4Z", v64i8_info, v16i8x_info,
766 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>;
767// Codegen pattern with the alternative types extract VEC256 from VEC512
768defm : vextract_for_size_lowering<"VEXTRACTI64x4Z", v32i16_info, v16i16x_info,
769 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>;
770defm : vextract_for_size_lowering<"VEXTRACTI64x4Z", v64i8_info, v32i8x_info,
771 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>;
772
Craig Topper5f3fef82016-05-22 07:40:58 +0000773// A 128-bit subvector extract from the first 256-bit vector position
774// is a subregister copy that needs no instruction.
775def : Pat<(v2i64 (extract_subvector (v8i64 VR512:$src), (iPTR 0))),
776 (v2i64 (EXTRACT_SUBREG (v8i64 VR512:$src), sub_xmm))>;
777def : Pat<(v2f64 (extract_subvector (v8f64 VR512:$src), (iPTR 0))),
778 (v2f64 (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm))>;
779def : Pat<(v4i32 (extract_subvector (v16i32 VR512:$src), (iPTR 0))),
780 (v4i32 (EXTRACT_SUBREG (v16i32 VR512:$src), sub_xmm))>;
781def : Pat<(v4f32 (extract_subvector (v16f32 VR512:$src), (iPTR 0))),
782 (v4f32 (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm))>;
783def : Pat<(v8i16 (extract_subvector (v32i16 VR512:$src), (iPTR 0))),
784 (v8i16 (EXTRACT_SUBREG (v32i16 VR512:$src), sub_xmm))>;
785def : Pat<(v16i8 (extract_subvector (v64i8 VR512:$src), (iPTR 0))),
786 (v16i8 (EXTRACT_SUBREG (v64i8 VR512:$src), sub_xmm))>;
787
788// A 256-bit subvector extract from the first 256-bit vector position
789// is a subregister copy that needs no instruction.
790def : Pat<(v4i64 (extract_subvector (v8i64 VR512:$src), (iPTR 0))),
791 (v4i64 (EXTRACT_SUBREG (v8i64 VR512:$src), sub_ymm))>;
792def : Pat<(v4f64 (extract_subvector (v8f64 VR512:$src), (iPTR 0))),
793 (v4f64 (EXTRACT_SUBREG (v8f64 VR512:$src), sub_ymm))>;
794def : Pat<(v8i32 (extract_subvector (v16i32 VR512:$src), (iPTR 0))),
795 (v8i32 (EXTRACT_SUBREG (v16i32 VR512:$src), sub_ymm))>;
796def : Pat<(v8f32 (extract_subvector (v16f32 VR512:$src), (iPTR 0))),
797 (v8f32 (EXTRACT_SUBREG (v16f32 VR512:$src), sub_ymm))>;
798def : Pat<(v16i16 (extract_subvector (v32i16 VR512:$src), (iPTR 0))),
799 (v16i16 (EXTRACT_SUBREG (v32i16 VR512:$src), sub_ymm))>;
800def : Pat<(v32i8 (extract_subvector (v64i8 VR512:$src), (iPTR 0))),
801 (v32i8 (EXTRACT_SUBREG (v64i8 VR512:$src), sub_ymm))>;
802
803let AddedComplexity = 25 in { // to give priority over vinsertf128rm
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000804// A 128-bit subvector insert to the first 512-bit vector position
805// is a subregister copy that needs no instruction.
Igor Bregerfca0a342016-01-28 13:19:25 +0000806def : Pat<(v8i64 (insert_subvector undef, (v2i64 VR128X:$src), (iPTR 0))),
807 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)>;
808def : Pat<(v8f64 (insert_subvector undef, (v2f64 VR128X:$src), (iPTR 0))),
809 (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)>;
810def : Pat<(v16i32 (insert_subvector undef, (v4i32 VR128X:$src), (iPTR 0))),
811 (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)>;
812def : Pat<(v16f32 (insert_subvector undef, (v4f32 VR128X:$src), (iPTR 0))),
813 (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)>;
814def : Pat<(v32i16 (insert_subvector undef, (v8i16 VR128X:$src), (iPTR 0))),
815 (INSERT_SUBREG (v32i16 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)>;
816def : Pat<(v64i8 (insert_subvector undef, (v16i8 VR128X:$src), (iPTR 0))),
817 (INSERT_SUBREG (v64i8 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000818
Craig Topper5f3fef82016-05-22 07:40:58 +0000819// A 256-bit subvector insert to the first 512-bit vector position
820// is a subregister copy that needs no instruction.
Igor Bregerfca0a342016-01-28 13:19:25 +0000821def : Pat<(v8i64 (insert_subvector undef, (v4i64 VR256X:$src), (iPTR 0))),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000822 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
Igor Bregerfca0a342016-01-28 13:19:25 +0000823def : Pat<(v8f64 (insert_subvector undef, (v4f64 VR256X:$src), (iPTR 0))),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000824 (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
Igor Bregerfca0a342016-01-28 13:19:25 +0000825def : Pat<(v16i32 (insert_subvector undef, (v8i32 VR256X:$src), (iPTR 0))),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000826 (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
Igor Bregerfca0a342016-01-28 13:19:25 +0000827def : Pat<(v16f32 (insert_subvector undef, (v8f32 VR256X:$src), (iPTR 0))),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000828 (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
Igor Bregerfca0a342016-01-28 13:19:25 +0000829def : Pat<(v32i16 (insert_subvector undef, (v16i16 VR256X:$src), (iPTR 0))),
Igor Bregercbb95502015-10-18 09:56:39 +0000830 (INSERT_SUBREG (v32i16 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
Igor Bregerfca0a342016-01-28 13:19:25 +0000831def : Pat<(v64i8 (insert_subvector undef, (v32i8 VR256X:$src), (iPTR 0))),
Igor Bregercbb95502015-10-18 09:56:39 +0000832 (INSERT_SUBREG (v64i8 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
Craig Toppera1041ff2016-05-22 07:40:40 +0000833}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000834
835// vextractps - extract 32 bits from XMM
Craig Topper03b849e2016-05-21 22:50:11 +0000836def VEXTRACTPSZrr : AVX512AIi8<0x17, MRMDestReg, (outs GR32:$dst),
Craig Topperfc946a02015-01-25 02:21:13 +0000837 (ins VR128X:$src1, u8imm:$src2),
Elena Demikhovskycf088092013-12-11 14:31:04 +0000838 "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000839 [(set GR32:$dst, (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2))]>,
840 EVEX;
841
Craig Topper03b849e2016-05-21 22:50:11 +0000842def VEXTRACTPSZmr : AVX512AIi8<0x17, MRMDestMem, (outs),
Craig Topperfc946a02015-01-25 02:21:13 +0000843 (ins f32mem:$dst, VR128X:$src1, u8imm:$src2),
Elena Demikhovskycf088092013-12-11 14:31:04 +0000844 "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000845 [(store (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2),
Elena Demikhovsky2aafc222014-02-11 07:25:59 +0000846 addr:$dst)]>, EVEX, EVEX_CD8<32, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000847
848//===---------------------------------------------------------------------===//
849// AVX-512 BROADCAST
850//---
Igor Breger131008f2016-05-01 08:40:00 +0000851// broadcast with a scalar argument.
852multiclass avx512_broadcast_scalar<bits<8> opc, string OpcodeStr,
853 X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo> {
Craig Topperf6df4a62017-01-30 06:59:06 +0000854 def : Pat<(DestInfo.VT (X86VBroadcast SrcInfo.FRC:$src)),
855 (!cast<Instruction>(NAME#DestInfo.ZSuffix#r)
856 (COPY_TO_REGCLASS SrcInfo.FRC:$src, SrcInfo.RC))>;
857 def : Pat<(DestInfo.VT (vselect DestInfo.KRCWM:$mask,
858 (X86VBroadcast SrcInfo.FRC:$src),
859 DestInfo.RC:$src0)),
860 (!cast<Instruction>(NAME#DestInfo.ZSuffix#rk)
861 DestInfo.RC:$src0, DestInfo.KRCWM:$mask,
862 (COPY_TO_REGCLASS SrcInfo.FRC:$src, SrcInfo.RC))>;
863 def : Pat<(DestInfo.VT (vselect DestInfo.KRCWM:$mask,
864 (X86VBroadcast SrcInfo.FRC:$src),
865 DestInfo.ImmAllZerosV)),
866 (!cast<Instruction>(NAME#DestInfo.ZSuffix#rkz)
867 DestInfo.KRCWM:$mask, (COPY_TO_REGCLASS SrcInfo.FRC:$src, SrcInfo.RC))>;
Igor Breger131008f2016-05-01 08:40:00 +0000868}
Robert Khasanovaf318f72014-10-30 14:21:47 +0000869
Igor Breger21296d22015-10-20 11:56:42 +0000870multiclass avx512_broadcast_rm<bits<8> opc, string OpcodeStr,
871 X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo> {
Craig Topper80934372016-07-16 03:42:59 +0000872 let ExeDomain = DestInfo.ExeDomain in {
Igor Breger21296d22015-10-20 11:56:42 +0000873 defm r : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),
874 (ins SrcInfo.RC:$src), OpcodeStr, "$src", "$src",
875 (DestInfo.VT (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src)))>,
876 T8PD, EVEX;
Craig Toppere1cac152016-06-07 07:27:54 +0000877 defm m : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
Igor Breger52bd1d52016-05-31 07:43:39 +0000878 (ins SrcInfo.ScalarMemOp:$src), OpcodeStr, "$src", "$src",
Craig Toppere1cac152016-06-07 07:27:54 +0000879 (DestInfo.VT (X86VBroadcast
880 (SrcInfo.ScalarLdFrag addr:$src)))>,
881 T8PD, EVEX, EVEX_CD8<SrcInfo.EltSize, CD8VT1>;
Craig Topper80934372016-07-16 03:42:59 +0000882 }
Craig Toppere1cac152016-06-07 07:27:54 +0000883
Craig Topper80934372016-07-16 03:42:59 +0000884 def : Pat<(DestInfo.VT (X86VBroadcast
885 (SrcInfo.VT (scalar_to_vector
886 (SrcInfo.ScalarLdFrag addr:$src))))),
887 (!cast<Instruction>(NAME#DestInfo.ZSuffix#m) addr:$src)>;
Craig Topper80934372016-07-16 03:42:59 +0000888 def : Pat<(DestInfo.VT (vselect DestInfo.KRCWM:$mask,
889 (X86VBroadcast
890 (SrcInfo.VT (scalar_to_vector
891 (SrcInfo.ScalarLdFrag addr:$src)))),
892 DestInfo.RC:$src0)),
893 (!cast<Instruction>(NAME#DestInfo.ZSuffix#mk)
894 DestInfo.RC:$src0, DestInfo.KRCWM:$mask, addr:$src)>;
Craig Topper80934372016-07-16 03:42:59 +0000895 def : Pat<(DestInfo.VT (vselect DestInfo.KRCWM:$mask,
896 (X86VBroadcast
897 (SrcInfo.VT (scalar_to_vector
898 (SrcInfo.ScalarLdFrag addr:$src)))),
899 DestInfo.ImmAllZerosV)),
900 (!cast<Instruction>(NAME#DestInfo.ZSuffix#mkz)
901 DestInfo.KRCWM:$mask, addr:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000902}
Robert Khasanovaf318f72014-10-30 14:21:47 +0000903
Craig Topper80934372016-07-16 03:42:59 +0000904multiclass avx512_fp_broadcast_sd<bits<8> opc, string OpcodeStr,
Igor Breger21296d22015-10-20 11:56:42 +0000905 AVX512VLVectorVTInfo _> {
Craig Topper80934372016-07-16 03:42:59 +0000906 let Predicates = [HasAVX512] in
907 defm Z : avx512_broadcast_rm<opc, OpcodeStr, _.info512, _.info128>,
908 avx512_broadcast_scalar<opc, OpcodeStr, _.info512, _.info128>,
909 EVEX_V512;
Robert Khasanovaf318f72014-10-30 14:21:47 +0000910
911 let Predicates = [HasVLX] in {
Igor Breger21296d22015-10-20 11:56:42 +0000912 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, _.info256, _.info128>,
Igor Breger131008f2016-05-01 08:40:00 +0000913 avx512_broadcast_scalar<opc, OpcodeStr, _.info256, _.info128>,
Igor Breger21296d22015-10-20 11:56:42 +0000914 EVEX_V256;
Robert Khasanovaf318f72014-10-30 14:21:47 +0000915 }
916}
917
Craig Topper80934372016-07-16 03:42:59 +0000918multiclass avx512_fp_broadcast_ss<bits<8> opc, string OpcodeStr,
919 AVX512VLVectorVTInfo _> {
920 let Predicates = [HasAVX512] in
921 defm Z : avx512_broadcast_rm<opc, OpcodeStr, _.info512, _.info128>,
922 avx512_broadcast_scalar<opc, OpcodeStr, _.info512, _.info128>,
923 EVEX_V512;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000924
Craig Topper80934372016-07-16 03:42:59 +0000925 let Predicates = [HasVLX] in {
926 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, _.info256, _.info128>,
927 avx512_broadcast_scalar<opc, OpcodeStr, _.info256, _.info128>,
928 EVEX_V256;
929 defm Z128 : avx512_broadcast_rm<opc, OpcodeStr, _.info128, _.info128>,
930 avx512_broadcast_scalar<opc, OpcodeStr, _.info128, _.info128>,
931 EVEX_V128;
932 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000933}
Craig Topper80934372016-07-16 03:42:59 +0000934defm VBROADCASTSS : avx512_fp_broadcast_ss<0x18, "vbroadcastss",
935 avx512vl_f32_info>;
936defm VBROADCASTSD : avx512_fp_broadcast_sd<0x19, "vbroadcastsd",
937 avx512vl_f64_info>, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000938
Quentin Colombet4bf1c282013-10-25 17:47:18 +0000939def : Pat<(int_x86_avx512_vbroadcast_ss_512 addr:$src),
Robert Khasanovaf318f72014-10-30 14:21:47 +0000940 (VBROADCASTSSZm addr:$src)>;
Quentin Colombet4bf1c282013-10-25 17:47:18 +0000941def : Pat<(int_x86_avx512_vbroadcast_sd_512 addr:$src),
Robert Khasanovaf318f72014-10-30 14:21:47 +0000942 (VBROADCASTSDZm addr:$src)>;
Quentin Colombet4bf1c282013-10-25 17:47:18 +0000943
Robert Khasanovcbc57032014-12-09 16:38:41 +0000944multiclass avx512_int_broadcast_reg<bits<8> opc, X86VectorVTInfo _,
Craig Topper49ba3f52017-02-26 06:45:48 +0000945 SDPatternOperator OpNode,
Robert Khasanovcbc57032014-12-09 16:38:41 +0000946 RegisterClass SrcRC> {
Craig Topperfe259882017-02-26 06:45:51 +0000947 let ExeDomain = _.ExeDomain in
Igor Breger0aeda372016-02-07 08:30:50 +0000948 defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
Simon Pilgrimb13961d2016-06-11 14:34:10 +0000949 (ins SrcRC:$src),
950 "vpbroadcast"##_.Suffix, "$src", "$src",
Craig Topper49ba3f52017-02-26 06:45:48 +0000951 (_.VT (OpNode SrcRC:$src))>, T8PD, EVEX;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +0000952}
953
Guy Blank7f60c992017-08-09 17:21:01 +0000954multiclass avx512_int_broadcastbw_reg<bits<8> opc, string Name,
955 X86VectorVTInfo _, SDPatternOperator OpNode,
956 RegisterClass SrcRC, SubRegIndex Subreg> {
Craig Topper508aa972017-08-14 05:09:34 +0000957 let hasSideEffects = 0, ExeDomain = _.ExeDomain in
Guy Blank7f60c992017-08-09 17:21:01 +0000958 defm r : AVX512_maskable_custom<opc, MRMSrcReg,
959 (outs _.RC:$dst), (ins GR32:$src),
960 !con((ins _.RC:$src0, _.KRCWM:$mask), (ins GR32:$src)),
961 !con((ins _.KRCWM:$mask), (ins GR32:$src)),
962 "vpbroadcast"##_.Suffix, "$src", "$src", [], [], [],
963 "$src0 = $dst">, T8PD, EVEX;
964
965 def : Pat <(_.VT (OpNode SrcRC:$src)),
966 (!cast<Instruction>(Name#r)
967 (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SrcRC:$src, Subreg)))>;
968
969 def : Pat <(vselect _.KRCWM:$mask, (_.VT (OpNode SrcRC:$src)), _.RC:$src0),
970 (!cast<Instruction>(Name#rk) _.RC:$src0, _.KRCWM:$mask,
971 (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SrcRC:$src, Subreg)))>;
972
973 def : Pat <(vselect _.KRCWM:$mask, (_.VT (OpNode SrcRC:$src)), _.ImmAllZerosV),
974 (!cast<Instruction>(Name#rkz) _.KRCWM:$mask,
975 (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SrcRC:$src, Subreg)))>;
976}
977
978multiclass avx512_int_broadcastbw_reg_vl<bits<8> opc, string Name,
979 AVX512VLVectorVTInfo _, SDPatternOperator OpNode,
980 RegisterClass SrcRC, SubRegIndex Subreg, Predicate prd> {
981 let Predicates = [prd] in
982 defm Z : avx512_int_broadcastbw_reg<opc, Name#Z, _.info512, OpNode, SrcRC,
983 Subreg>, EVEX_V512;
984 let Predicates = [prd, HasVLX] in {
985 defm Z256 : avx512_int_broadcastbw_reg<opc, Name#Z256, _.info256, OpNode,
986 SrcRC, Subreg>, EVEX_V256;
987 defm Z128 : avx512_int_broadcastbw_reg<opc, Name#Z128, _.info128, OpNode,
988 SrcRC, Subreg>, EVEX_V128;
989 }
990}
991
Robert Khasanovcbc57032014-12-09 16:38:41 +0000992multiclass avx512_int_broadcast_reg_vl<bits<8> opc, AVX512VLVectorVTInfo _,
Craig Topper49ba3f52017-02-26 06:45:48 +0000993 SDPatternOperator OpNode,
Robert Khasanovcbc57032014-12-09 16:38:41 +0000994 RegisterClass SrcRC, Predicate prd> {
995 let Predicates = [prd] in
Craig Topper49ba3f52017-02-26 06:45:48 +0000996 defm Z : avx512_int_broadcast_reg<opc, _.info512, OpNode, SrcRC>, EVEX_V512;
Robert Khasanovcbc57032014-12-09 16:38:41 +0000997 let Predicates = [prd, HasVLX] in {
Craig Topper49ba3f52017-02-26 06:45:48 +0000998 defm Z256 : avx512_int_broadcast_reg<opc, _.info256, OpNode, SrcRC>, EVEX_V256;
999 defm Z128 : avx512_int_broadcast_reg<opc, _.info128, OpNode, SrcRC>, EVEX_V128;
Robert Khasanovcbc57032014-12-09 16:38:41 +00001000 }
1001}
1002
Guy Blank7f60c992017-08-09 17:21:01 +00001003defm VPBROADCASTBr : avx512_int_broadcastbw_reg_vl<0x7A, "VPBROADCASTBr",
1004 avx512vl_i8_info, X86VBroadcast, GR8, sub_8bit, HasBWI>;
1005defm VPBROADCASTWr : avx512_int_broadcastbw_reg_vl<0x7B, "VPBROADCASTWr",
1006 avx512vl_i16_info, X86VBroadcast, GR16, sub_16bit,
1007 HasBWI>;
Craig Topper49ba3f52017-02-26 06:45:48 +00001008defm VPBROADCASTDr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i32_info,
1009 X86VBroadcast, GR32, HasAVX512>;
1010defm VPBROADCASTQr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i64_info,
1011 X86VBroadcast, GR64, HasAVX512>, VEX_W;
Michael Liao5bf95782014-12-04 05:20:33 +00001012
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001013def : Pat <(v16i32 (X86vzext VK16WM:$mask)),
Robert Khasanovcbc57032014-12-09 16:38:41 +00001014 (VPBROADCASTDrZrkz VK16WM:$mask, (i32 (MOV32ri 0x1)))>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001015def : Pat <(v8i64 (X86vzext VK8WM:$mask)),
Robert Khasanovcbc57032014-12-09 16:38:41 +00001016 (VPBROADCASTQrZrkz VK8WM:$mask, (i64 (MOV64ri 0x1)))>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001017
Igor Breger21296d22015-10-20 11:56:42 +00001018// Provide aliases for broadcast from the same register class that
1019// automatically does the extract.
1020multiclass avx512_int_broadcast_rm_lowering<X86VectorVTInfo DestInfo,
1021 X86VectorVTInfo SrcInfo> {
1022 def : Pat<(DestInfo.VT (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src))),
1023 (!cast<Instruction>(NAME#DestInfo.ZSuffix#"r")
1024 (EXTRACT_SUBREG (SrcInfo.VT SrcInfo.RC:$src), sub_xmm))>;
1025}
1026
1027multiclass avx512_int_broadcast_rm_vl<bits<8> opc, string OpcodeStr,
1028 AVX512VLVectorVTInfo _, Predicate prd> {
1029 let Predicates = [prd] in {
1030 defm Z : avx512_broadcast_rm<opc, OpcodeStr, _.info512, _.info128>,
1031 avx512_int_broadcast_rm_lowering<_.info512, _.info256>,
1032 EVEX_V512;
1033 // Defined separately to avoid redefinition.
1034 defm Z_Alt : avx512_int_broadcast_rm_lowering<_.info512, _.info512>;
1035 }
1036 let Predicates = [prd, HasVLX] in {
1037 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, _.info256, _.info128>,
1038 avx512_int_broadcast_rm_lowering<_.info256, _.info256>,
1039 EVEX_V256;
1040 defm Z128 : avx512_broadcast_rm<opc, OpcodeStr, _.info128, _.info128>,
1041 EVEX_V128;
Elena Demikhovskydd0794e2013-10-24 07:16:35 +00001042 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001043}
1044
Igor Breger21296d22015-10-20 11:56:42 +00001045defm VPBROADCASTB : avx512_int_broadcast_rm_vl<0x78, "vpbroadcastb",
1046 avx512vl_i8_info, HasBWI>;
1047defm VPBROADCASTW : avx512_int_broadcast_rm_vl<0x79, "vpbroadcastw",
1048 avx512vl_i16_info, HasBWI>;
1049defm VPBROADCASTD : avx512_int_broadcast_rm_vl<0x58, "vpbroadcastd",
1050 avx512vl_i32_info, HasAVX512>;
1051defm VPBROADCASTQ : avx512_int_broadcast_rm_vl<0x59, "vpbroadcastq",
1052 avx512vl_i64_info, HasAVX512>, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001053
Elena Demikhovskyad9c3962015-05-18 06:42:57 +00001054multiclass avx512_subvec_broadcast_rm<bits<8> opc, string OpcodeStr,
1055 X86VectorVTInfo _Dst, X86VectorVTInfo _Src> {
Simon Pilgrimb13961d2016-06-11 14:34:10 +00001056 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
Craig Toppere1cac152016-06-07 07:27:54 +00001057 (ins _Src.MemOp:$src), OpcodeStr, "$src", "$src",
1058 (_Dst.VT (X86SubVBroadcast
Simon Pilgrimb13961d2016-06-11 14:34:10 +00001059 (_Src.VT (bitconvert (_Src.LdFrag addr:$src)))))>,
Craig Toppere1cac152016-06-07 07:27:54 +00001060 AVX5128IBase, EVEX;
Adam Nemet73f72e12014-06-27 00:43:38 +00001061}
1062
Simon Pilgrim79195582017-02-21 16:41:44 +00001063let Predicates = [HasAVX512] in {
1064 // 32-bit targets will fail to load a i64 directly but can use ZEXT_LOAD.
1065 def : Pat<(v8i64 (X86VBroadcast (v8i64 (X86vzload addr:$src)))),
1066 (VPBROADCASTQZm addr:$src)>;
1067}
1068
Craig Topperbe351ee2016-10-01 06:01:23 +00001069let Predicates = [HasVLX, HasBWI] in {
Simon Pilgrim79195582017-02-21 16:41:44 +00001070 // 32-bit targets will fail to load a i64 directly but can use ZEXT_LOAD.
1071 def : Pat<(v2i64 (X86VBroadcast (v2i64 (X86vzload addr:$src)))),
1072 (VPBROADCASTQZ128m addr:$src)>;
1073 def : Pat<(v4i64 (X86VBroadcast (v4i64 (X86vzload addr:$src)))),
1074 (VPBROADCASTQZ256m addr:$src)>;
Craig Topperbe351ee2016-10-01 06:01:23 +00001075 // loadi16 is tricky to fold, because !isTypeDesirableForOp, justifiably.
1076 // This means we'll encounter truncated i32 loads; match that here.
1077 def : Pat<(v8i16 (X86VBroadcast (i16 (trunc (i32 (load addr:$src)))))),
1078 (VPBROADCASTWZ128m addr:$src)>;
1079 def : Pat<(v16i16 (X86VBroadcast (i16 (trunc (i32 (load addr:$src)))))),
1080 (VPBROADCASTWZ256m addr:$src)>;
1081 def : Pat<(v8i16 (X86VBroadcast
1082 (i16 (trunc (i32 (zextloadi16 addr:$src)))))),
1083 (VPBROADCASTWZ128m addr:$src)>;
1084 def : Pat<(v16i16 (X86VBroadcast
1085 (i16 (trunc (i32 (zextloadi16 addr:$src)))))),
1086 (VPBROADCASTWZ256m addr:$src)>;
1087}
1088
Simon Pilgrimea0d4f92016-07-22 13:58:44 +00001089//===----------------------------------------------------------------------===//
1090// AVX-512 BROADCAST SUBVECTORS
1091//
1092
Elena Demikhovskyad9c3962015-05-18 06:42:57 +00001093defm VBROADCASTI32X4 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4",
1094 v16i32_info, v4i32x_info>,
Adam Nemet73f72e12014-06-27 00:43:38 +00001095 EVEX_V512, EVEX_CD8<32, CD8VT4>;
Elena Demikhovskyad9c3962015-05-18 06:42:57 +00001096defm VBROADCASTF32X4 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4",
1097 v16f32_info, v4f32x_info>,
1098 EVEX_V512, EVEX_CD8<32, CD8VT4>;
1099defm VBROADCASTI64X4 : avx512_subvec_broadcast_rm<0x5b, "vbroadcasti64x4",
1100 v8i64_info, v4i64x_info>, VEX_W,
Adam Nemet73f72e12014-06-27 00:43:38 +00001101 EVEX_V512, EVEX_CD8<64, CD8VT4>;
Elena Demikhovskyad9c3962015-05-18 06:42:57 +00001102defm VBROADCASTF64X4 : avx512_subvec_broadcast_rm<0x1b, "vbroadcastf64x4",
1103 v8f64_info, v4f64x_info>, VEX_W,
1104 EVEX_V512, EVEX_CD8<64, CD8VT4>;
1105
Craig Topper715ad7f2016-10-16 23:29:51 +00001106let Predicates = [HasAVX512] in {
1107def : Pat<(v32i16 (X86SubVBroadcast (bc_v16i16 (loadv4i64 addr:$src)))),
1108 (VBROADCASTI64X4rm addr:$src)>;
1109def : Pat<(v64i8 (X86SubVBroadcast (bc_v32i8 (loadv4i64 addr:$src)))),
1110 (VBROADCASTI64X4rm addr:$src)>;
1111
1112// Provide fallback in case the load node that is used in the patterns above
1113// is used by additional users, which prevents the pattern selection.
Ayman Musa7ec4ed52016-12-11 20:11:17 +00001114def : Pat<(v8f64 (X86SubVBroadcast (v4f64 VR256X:$src))),
1115 (VINSERTF64x4Zrr (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
Simon Pilgrimb2a80952017-01-08 16:45:39 +00001116 (v4f64 VR256X:$src), 1)>;
Ayman Musa7ec4ed52016-12-11 20:11:17 +00001117def : Pat<(v8i64 (X86SubVBroadcast (v4i64 VR256X:$src))),
1118 (VINSERTI64x4Zrr (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
Simon Pilgrimb2a80952017-01-08 16:45:39 +00001119 (v4i64 VR256X:$src), 1)>;
Craig Topper715ad7f2016-10-16 23:29:51 +00001120def : Pat<(v32i16 (X86SubVBroadcast (v16i16 VR256X:$src))),
1121 (VINSERTI64x4Zrr (INSERT_SUBREG (v32i16 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
1122 (v16i16 VR256X:$src), 1)>;
1123def : Pat<(v64i8 (X86SubVBroadcast (v32i8 VR256X:$src))),
1124 (VINSERTI64x4Zrr (INSERT_SUBREG (v64i8 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
1125 (v32i8 VR256X:$src), 1)>;
Craig Toppera4dc3402016-10-19 04:44:17 +00001126
1127def : Pat<(v32i16 (X86SubVBroadcast (bc_v8i16 (loadv2i64 addr:$src)))),
1128 (VBROADCASTI32X4rm addr:$src)>;
1129def : Pat<(v64i8 (X86SubVBroadcast (bc_v16i8 (loadv2i64 addr:$src)))),
1130 (VBROADCASTI32X4rm addr:$src)>;
Craig Topper715ad7f2016-10-16 23:29:51 +00001131}
1132
Elena Demikhovskyad9c3962015-05-18 06:42:57 +00001133let Predicates = [HasVLX] in {
1134defm VBROADCASTI32X4Z256 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4",
1135 v8i32x_info, v4i32x_info>,
1136 EVEX_V256, EVEX_CD8<32, CD8VT4>;
1137defm VBROADCASTF32X4Z256 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4",
1138 v8f32x_info, v4f32x_info>,
1139 EVEX_V256, EVEX_CD8<32, CD8VT4>;
Simon Pilgrimea0d4f92016-07-22 13:58:44 +00001140
1141def : Pat<(v16i16 (X86SubVBroadcast (bc_v8i16 (loadv2i64 addr:$src)))),
1142 (VBROADCASTI32X4Z256rm addr:$src)>;
1143def : Pat<(v32i8 (X86SubVBroadcast (bc_v16i8 (loadv2i64 addr:$src)))),
1144 (VBROADCASTI32X4Z256rm addr:$src)>;
Simon Pilgrim0ad9f3e2016-08-25 12:45:16 +00001145
Simon Pilgrim6fe4a9e2016-08-25 15:45:27 +00001146// Provide fallback in case the load node that is used in the patterns above
1147// is used by additional users, which prevents the pattern selection.
Simon Pilgrim0ad9f3e2016-08-25 12:45:16 +00001148def : Pat<(v8f32 (X86SubVBroadcast (v4f32 VR128X:$src))),
Simon Pilgrim6fe4a9e2016-08-25 15:45:27 +00001149 (VINSERTF32x4Z256rr (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
Simon Pilgrim0ad9f3e2016-08-25 12:45:16 +00001150 (v4f32 VR128X:$src), 1)>;
1151def : Pat<(v8i32 (X86SubVBroadcast (v4i32 VR128X:$src))),
Simon Pilgrim6fe4a9e2016-08-25 15:45:27 +00001152 (VINSERTI32x4Z256rr (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
Simon Pilgrim0ad9f3e2016-08-25 12:45:16 +00001153 (v4i32 VR128X:$src), 1)>;
1154def : Pat<(v16i16 (X86SubVBroadcast (v8i16 VR128X:$src))),
Simon Pilgrim6fe4a9e2016-08-25 15:45:27 +00001155 (VINSERTI32x4Z256rr (INSERT_SUBREG (v16i16 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
Simon Pilgrim0ad9f3e2016-08-25 12:45:16 +00001156 (v8i16 VR128X:$src), 1)>;
1157def : Pat<(v32i8 (X86SubVBroadcast (v16i8 VR128X:$src))),
Simon Pilgrim6fe4a9e2016-08-25 15:45:27 +00001158 (VINSERTI32x4Z256rr (INSERT_SUBREG (v32i8 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
Simon Pilgrim0ad9f3e2016-08-25 12:45:16 +00001159 (v16i8 VR128X:$src), 1)>;
Elena Demikhovskyad9c3962015-05-18 06:42:57 +00001160}
Simon Pilgrimea0d4f92016-07-22 13:58:44 +00001161
Elena Demikhovskyad9c3962015-05-18 06:42:57 +00001162let Predicates = [HasVLX, HasDQI] in {
1163defm VBROADCASTI64X2Z128 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti64x2",
1164 v4i64x_info, v2i64x_info>, VEX_W,
1165 EVEX_V256, EVEX_CD8<64, CD8VT2>;
1166defm VBROADCASTF64X2Z128 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf64x2",
1167 v4f64x_info, v2f64x_info>, VEX_W,
1168 EVEX_V256, EVEX_CD8<64, CD8VT2>;
Craig Topperf18b9202016-10-16 04:54:26 +00001169
1170// Provide fallback in case the load node that is used in the patterns above
1171// is used by additional users, which prevents the pattern selection.
1172def : Pat<(v4f64 (X86SubVBroadcast (v2f64 VR128X:$src))),
1173 (VINSERTF64x2Z256rr (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
1174 (v2f64 VR128X:$src), 1)>;
1175def : Pat<(v4i64 (X86SubVBroadcast (v2i64 VR128X:$src))),
1176 (VINSERTI64x2Z256rr (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
1177 (v2i64 VR128X:$src), 1)>;
Elena Demikhovskyad9c3962015-05-18 06:42:57 +00001178}
Simon Pilgrimea0d4f92016-07-22 13:58:44 +00001179
1180let Predicates = [HasVLX, NoDQI] in {
1181def : Pat<(v4f64 (X86SubVBroadcast (loadv2f64 addr:$src))),
1182 (VBROADCASTF32X4Z256rm addr:$src)>;
1183def : Pat<(v4i64 (X86SubVBroadcast (loadv2i64 addr:$src))),
1184 (VBROADCASTI32X4Z256rm addr:$src)>;
Simon Pilgrim0ad9f3e2016-08-25 12:45:16 +00001185
Simon Pilgrim6fe4a9e2016-08-25 15:45:27 +00001186// Provide fallback in case the load node that is used in the patterns above
1187// is used by additional users, which prevents the pattern selection.
Simon Pilgrim0ad9f3e2016-08-25 12:45:16 +00001188def : Pat<(v4f64 (X86SubVBroadcast (v2f64 VR128X:$src))),
Simon Pilgrim6fe4a9e2016-08-25 15:45:27 +00001189 (VINSERTF32x4Z256rr (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
Simon Pilgrim0ad9f3e2016-08-25 12:45:16 +00001190 (v2f64 VR128X:$src), 1)>;
1191def : Pat<(v4i64 (X86SubVBroadcast (v2i64 VR128X:$src))),
Simon Pilgrim6fe4a9e2016-08-25 15:45:27 +00001192 (VINSERTI32x4Z256rr (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
1193 (v2i64 VR128X:$src), 1)>;
Simon Pilgrimea0d4f92016-07-22 13:58:44 +00001194}
1195
Craig Topper715ad7f2016-10-16 23:29:51 +00001196let Predicates = [HasAVX512, NoDQI] in {
Craig Toppera4dc3402016-10-19 04:44:17 +00001197def : Pat<(v8f64 (X86SubVBroadcast (loadv2f64 addr:$src))),
1198 (VBROADCASTF32X4rm addr:$src)>;
1199def : Pat<(v8i64 (X86SubVBroadcast (loadv2i64 addr:$src))),
1200 (VBROADCASTI32X4rm addr:$src)>;
1201
Craig Topper715ad7f2016-10-16 23:29:51 +00001202def : Pat<(v16f32 (X86SubVBroadcast (loadv8f32 addr:$src))),
1203 (VBROADCASTF64X4rm addr:$src)>;
1204def : Pat<(v16i32 (X86SubVBroadcast (bc_v8i32 (loadv4i64 addr:$src)))),
1205 (VBROADCASTI64X4rm addr:$src)>;
1206
1207// Provide fallback in case the load node that is used in the patterns above
1208// is used by additional users, which prevents the pattern selection.
1209def : Pat<(v16f32 (X86SubVBroadcast (v8f32 VR256X:$src))),
1210 (VINSERTF64x4Zrr (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
1211 (v8f32 VR256X:$src), 1)>;
1212def : Pat<(v16i32 (X86SubVBroadcast (v8i32 VR256X:$src))),
1213 (VINSERTI64x4Zrr (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
1214 (v8i32 VR256X:$src), 1)>;
1215}
1216
Elena Demikhovskyad9c3962015-05-18 06:42:57 +00001217let Predicates = [HasDQI] in {
1218defm VBROADCASTI64X2 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti64x2",
1219 v8i64_info, v2i64x_info>, VEX_W,
1220 EVEX_V512, EVEX_CD8<64, CD8VT2>;
1221defm VBROADCASTI32X8 : avx512_subvec_broadcast_rm<0x5b, "vbroadcasti32x8",
1222 v16i32_info, v8i32x_info>,
1223 EVEX_V512, EVEX_CD8<32, CD8VT8>;
1224defm VBROADCASTF64X2 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf64x2",
1225 v8f64_info, v2f64x_info>, VEX_W,
1226 EVEX_V512, EVEX_CD8<64, CD8VT2>;
1227defm VBROADCASTF32X8 : avx512_subvec_broadcast_rm<0x1b, "vbroadcastf32x8",
1228 v16f32_info, v8f32x_info>,
1229 EVEX_V512, EVEX_CD8<32, CD8VT8>;
Craig Topper715ad7f2016-10-16 23:29:51 +00001230
1231// Provide fallback in case the load node that is used in the patterns above
1232// is used by additional users, which prevents the pattern selection.
1233def : Pat<(v16f32 (X86SubVBroadcast (v8f32 VR256X:$src))),
1234 (VINSERTF32x8Zrr (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
1235 (v8f32 VR256X:$src), 1)>;
1236def : Pat<(v16i32 (X86SubVBroadcast (v8i32 VR256X:$src))),
1237 (VINSERTI32x8Zrr (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
1238 (v8i32 VR256X:$src), 1)>;
Elena Demikhovskyad9c3962015-05-18 06:42:57 +00001239}
Adam Nemet73f72e12014-06-27 00:43:38 +00001240
Igor Bregerfa798a92015-11-02 07:39:36 +00001241multiclass avx512_common_broadcast_32x2<bits<8> opc, string OpcodeStr,
Igor Breger52bd1d52016-05-31 07:43:39 +00001242 AVX512VLVectorVTInfo _Dst, AVX512VLVectorVTInfo _Src> {
Igor Bregerfa798a92015-11-02 07:39:36 +00001243 let Predicates = [HasDQI] in
Igor Breger52bd1d52016-05-31 07:43:39 +00001244 defm Z : avx512_broadcast_rm<opc, OpcodeStr, _Dst.info512, _Src.info128>,
Igor Bregerfa798a92015-11-02 07:39:36 +00001245 EVEX_V512;
1246 let Predicates = [HasDQI, HasVLX] in
Igor Breger52bd1d52016-05-31 07:43:39 +00001247 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, _Dst.info256, _Src.info128>,
Igor Bregerfa798a92015-11-02 07:39:36 +00001248 EVEX_V256;
1249}
1250
1251multiclass avx512_common_broadcast_i32x2<bits<8> opc, string OpcodeStr,
Igor Breger52bd1d52016-05-31 07:43:39 +00001252 AVX512VLVectorVTInfo _Dst, AVX512VLVectorVTInfo _Src> :
1253 avx512_common_broadcast_32x2<opc, OpcodeStr, _Dst, _Src> {
Igor Bregerfa798a92015-11-02 07:39:36 +00001254
1255 let Predicates = [HasDQI, HasVLX] in
Igor Breger52bd1d52016-05-31 07:43:39 +00001256 defm Z128 : avx512_broadcast_rm<opc, OpcodeStr, _Dst.info128, _Src.info128>,
1257 EVEX_V128;
Igor Bregerfa798a92015-11-02 07:39:36 +00001258}
1259
Craig Topper51e052f2016-10-15 16:26:02 +00001260defm VBROADCASTI32X2 : avx512_common_broadcast_i32x2<0x59, "vbroadcasti32x2",
1261 avx512vl_i32_info, avx512vl_i64_info>;
1262defm VBROADCASTF32X2 : avx512_common_broadcast_32x2<0x19, "vbroadcastf32x2",
1263 avx512vl_f32_info, avx512vl_f64_info>;
Igor Bregerfa798a92015-11-02 07:39:36 +00001264
Craig Topper52317e82017-01-15 05:47:45 +00001265let Predicates = [HasVLX] in {
1266def : Pat<(v8f32 (X86VBroadcast (v8f32 VR256X:$src))),
1267 (VBROADCASTSSZ256r (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm))>;
1268def : Pat<(v4f64 (X86VBroadcast (v4f64 VR256X:$src))),
1269 (VBROADCASTSDZ256r (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm))>;
1270}
1271
Robert Khasanovdd09a8f2014-10-28 12:28:51 +00001272def : Pat<(v16f32 (X86VBroadcast (v16f32 VR512:$src))),
Robert Khasanovaf318f72014-10-30 14:21:47 +00001273 (VBROADCASTSSZr (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm))>;
Elena Demikhovsky08ce53c2015-05-18 07:06:23 +00001274def : Pat<(v16f32 (X86VBroadcast (v8f32 VR256X:$src))),
1275 (VBROADCASTSSZr (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm))>;
1276
Robert Khasanovdd09a8f2014-10-28 12:28:51 +00001277def : Pat<(v8f64 (X86VBroadcast (v8f64 VR512:$src))),
Robert Khasanovaf318f72014-10-30 14:21:47 +00001278 (VBROADCASTSDZr (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm))>;
Elena Demikhovsky08ce53c2015-05-18 07:06:23 +00001279def : Pat<(v8f64 (X86VBroadcast (v4f64 VR256X:$src))),
1280 (VBROADCASTSDZr (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm))>;
Robert Khasanovdd09a8f2014-10-28 12:28:51 +00001281
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001282//===----------------------------------------------------------------------===//
1283// AVX-512 BROADCAST MASK TO VECTOR REGISTER
1284//---
Asaf Badouh0d957b82015-11-18 09:42:45 +00001285multiclass avx512_mask_broadcastm<bits<8> opc, string OpcodeStr,
1286 X86VectorVTInfo _, RegisterClass KRC> {
1287 def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.RC:$dst), (ins KRC:$src),
Craig Topperedb09112014-11-25 20:11:23 +00001288 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
Asaf Badouh0d957b82015-11-18 09:42:45 +00001289 [(set _.RC:$dst, (_.VT (X86VBroadcastm KRC:$src)))]>, EVEX;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001290}
1291
Simon Pilgrimb13961d2016-06-11 14:34:10 +00001292multiclass avx512_mask_broadcast<bits<8> opc, string OpcodeStr,
Asaf Badouh0d957b82015-11-18 09:42:45 +00001293 AVX512VLVectorVTInfo VTInfo, RegisterClass KRC> {
1294 let Predicates = [HasCDI] in
1295 defm Z : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info512, KRC>, EVEX_V512;
1296 let Predicates = [HasCDI, HasVLX] in {
1297 defm Z256 : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info256, KRC>, EVEX_V256;
1298 defm Z128 : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info128, KRC>, EVEX_V128;
1299 }
1300}
1301
Elena Demikhovsky4b01b732014-10-26 09:52:24 +00001302defm VPBROADCASTMW2D : avx512_mask_broadcast<0x3A, "vpbroadcastmw2d",
Asaf Badouh0d957b82015-11-18 09:42:45 +00001303 avx512vl_i32_info, VK16>;
Elena Demikhovsky4b01b732014-10-26 09:52:24 +00001304defm VPBROADCASTMB2Q : avx512_mask_broadcast<0x2A, "vpbroadcastmb2q",
Asaf Badouh0d957b82015-11-18 09:42:45 +00001305 avx512vl_i64_info, VK8>, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001306
1307//===----------------------------------------------------------------------===//
Craig Topperaad5f112015-11-30 00:13:24 +00001308// -- VPERMI2 - 3 source operands form --
Craig Topper4fa3b502016-09-06 06:56:59 +00001309multiclass avx512_perm_i<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> {
Craig Topper4729fe82016-10-16 04:54:31 +00001310let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {
Craig Topper4fa3b502016-09-06 06:56:59 +00001311 // The index operand in the pattern should really be an integer type. However,
1312 // if we do that and it happens to come from a bitcast, then it becomes
1313 // difficult to find the bitcast needed to convert the index to the
1314 // destination type for the passthru since it will be folded with the bitcast
1315 // of the index operand.
1316 defm rr: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001317 (ins _.RC:$src2, _.RC:$src3),
1318 OpcodeStr, "$src3, $src2", "$src2, $src3",
Craig Toppercada9f22016-11-22 04:57:34 +00001319 (_.VT (X86VPermi2X _.RC:$src1, _.RC:$src2, _.RC:$src3)), 1>, EVEX_4V,
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001320 AVX5128IBase;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001321
Craig Topper4fa3b502016-09-06 06:56:59 +00001322 defm rm: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001323 (ins _.RC:$src2, _.MemOp:$src3),
1324 OpcodeStr, "$src3, $src2", "$src2, $src3",
Craig Topper4fa3b502016-09-06 06:56:59 +00001325 (_.VT (X86VPermi2X _.RC:$src1, _.RC:$src2,
Craig Toppercada9f22016-11-22 04:57:34 +00001326 (_.VT (bitconvert (_.LdFrag addr:$src3))))), 1>,
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001327 EVEX_4V, AVX5128IBase;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001328 }
1329}
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001330multiclass avx512_perm_i_mb<bits<8> opc, string OpcodeStr,
Craig Topper4fa3b502016-09-06 06:56:59 +00001331 X86VectorVTInfo _> {
Craig Topper4729fe82016-10-16 04:54:31 +00001332 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in
Craig Topper4fa3b502016-09-06 06:56:59 +00001333 defm rmb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001334 (ins _.RC:$src2, _.ScalarMemOp:$src3),
1335 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),
1336 !strconcat("$src2, ${src3}", _.BroadcastStr ),
Craig Topper4fa3b502016-09-06 06:56:59 +00001337 (_.VT (X86VPermi2X _.RC:$src1,
Craig Toppercada9f22016-11-22 04:57:34 +00001338 _.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))))),
1339 1>, AVX5128IBase, EVEX_4V, EVEX_B;
Adam Nemetefe9c982014-07-02 21:25:58 +00001340}
1341
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001342multiclass avx512_perm_i_sizes<bits<8> opc, string OpcodeStr,
Craig Topper4fa3b502016-09-06 06:56:59 +00001343 AVX512VLVectorVTInfo VTInfo> {
1344 defm NAME: avx512_perm_i<opc, OpcodeStr, VTInfo.info512>,
1345 avx512_perm_i_mb<opc, OpcodeStr, VTInfo.info512>, EVEX_V512;
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001346 let Predicates = [HasVLX] in {
Craig Topper4fa3b502016-09-06 06:56:59 +00001347 defm NAME#128: avx512_perm_i<opc, OpcodeStr, VTInfo.info128>,
1348 avx512_perm_i_mb<opc, OpcodeStr, VTInfo.info128>, EVEX_V128;
1349 defm NAME#256: avx512_perm_i<opc, OpcodeStr, VTInfo.info256>,
1350 avx512_perm_i_mb<opc, OpcodeStr, VTInfo.info256>, EVEX_V256;
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001351 }
1352}
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001353
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001354multiclass avx512_perm_i_sizes_bw<bits<8> opc, string OpcodeStr,
Craig Topperaad5f112015-11-30 00:13:24 +00001355 AVX512VLVectorVTInfo VTInfo,
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001356 Predicate Prd> {
1357 let Predicates = [Prd] in
Craig Topper4fa3b502016-09-06 06:56:59 +00001358 defm NAME: avx512_perm_i<opc, OpcodeStr, VTInfo.info512>, EVEX_V512;
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001359 let Predicates = [Prd, HasVLX] in {
Craig Topper4fa3b502016-09-06 06:56:59 +00001360 defm NAME#128: avx512_perm_i<opc, OpcodeStr, VTInfo.info128>, EVEX_V128;
1361 defm NAME#256: avx512_perm_i<opc, OpcodeStr, VTInfo.info256>, EVEX_V256;
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001362 }
1363}
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001364
Craig Topperaad5f112015-11-30 00:13:24 +00001365defm VPERMI2D : avx512_perm_i_sizes<0x76, "vpermi2d",
Craig Topper4fa3b502016-09-06 06:56:59 +00001366 avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
Craig Topperaad5f112015-11-30 00:13:24 +00001367defm VPERMI2Q : avx512_perm_i_sizes<0x76, "vpermi2q",
Craig Topper4fa3b502016-09-06 06:56:59 +00001368 avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001369defm VPERMI2W : avx512_perm_i_sizes_bw<0x75, "vpermi2w",
Craig Topper4fa3b502016-09-06 06:56:59 +00001370 avx512vl_i16_info, HasBWI>,
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001371 VEX_W, EVEX_CD8<16, CD8VF>;
1372defm VPERMI2B : avx512_perm_i_sizes_bw<0x75, "vpermi2b",
Craig Topper4fa3b502016-09-06 06:56:59 +00001373 avx512vl_i8_info, HasVBMI>,
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001374 EVEX_CD8<8, CD8VF>;
Craig Topperaad5f112015-11-30 00:13:24 +00001375defm VPERMI2PS : avx512_perm_i_sizes<0x77, "vpermi2ps",
Craig Topper4fa3b502016-09-06 06:56:59 +00001376 avx512vl_f32_info>, EVEX_CD8<32, CD8VF>;
Craig Topperaad5f112015-11-30 00:13:24 +00001377defm VPERMI2PD : avx512_perm_i_sizes<0x77, "vpermi2pd",
Craig Topper4fa3b502016-09-06 06:56:59 +00001378 avx512vl_f64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
Elena Demikhovskyd3057e52015-06-18 08:56:19 +00001379
Craig Topperaad5f112015-11-30 00:13:24 +00001380// VPERMT2
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001381multiclass avx512_perm_t<bits<8> opc, string OpcodeStr,
Craig Toppera47576f2015-11-26 20:21:29 +00001382 X86VectorVTInfo _, X86VectorVTInfo IdxVT> {
Craig Topper4729fe82016-10-16 04:54:31 +00001383let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001384 defm rr: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
1385 (ins IdxVT.RC:$src2, _.RC:$src3),
1386 OpcodeStr, "$src3, $src2", "$src2, $src3",
Craig Toppercada9f22016-11-22 04:57:34 +00001387 (_.VT (X86VPermt2 _.RC:$src1, IdxVT.RC:$src2, _.RC:$src3)), 1>,
1388 EVEX_4V, AVX5128IBase;
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001389
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001390 defm rm: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
1391 (ins IdxVT.RC:$src2, _.MemOp:$src3),
1392 OpcodeStr, "$src3, $src2", "$src2, $src3",
Craig Toppera47576f2015-11-26 20:21:29 +00001393 (_.VT (X86VPermt2 _.RC:$src1, IdxVT.RC:$src2,
Craig Toppercada9f22016-11-22 04:57:34 +00001394 (bitconvert (_.LdFrag addr:$src3)))), 1>,
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001395 EVEX_4V, AVX5128IBase;
1396 }
1397}
1398multiclass avx512_perm_t_mb<bits<8> opc, string OpcodeStr,
Craig Toppera47576f2015-11-26 20:21:29 +00001399 X86VectorVTInfo _, X86VectorVTInfo IdxVT> {
Craig Topper4729fe82016-10-16 04:54:31 +00001400 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001401 defm rmb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
1402 (ins IdxVT.RC:$src2, _.ScalarMemOp:$src3),
1403 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),
1404 !strconcat("$src2, ${src3}", _.BroadcastStr ),
Craig Toppera47576f2015-11-26 20:21:29 +00001405 (_.VT (X86VPermt2 _.RC:$src1,
Craig Toppercada9f22016-11-22 04:57:34 +00001406 IdxVT.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))))),
1407 1>, AVX5128IBase, EVEX_4V, EVEX_B;
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001408}
1409
1410multiclass avx512_perm_t_sizes<bits<8> opc, string OpcodeStr,
Craig Toppera47576f2015-11-26 20:21:29 +00001411 AVX512VLVectorVTInfo VTInfo,
1412 AVX512VLVectorVTInfo ShuffleMask> {
1413 defm NAME: avx512_perm_t<opc, OpcodeStr, VTInfo.info512,
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001414 ShuffleMask.info512>,
Craig Toppera47576f2015-11-26 20:21:29 +00001415 avx512_perm_t_mb<opc, OpcodeStr, VTInfo.info512,
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001416 ShuffleMask.info512>, EVEX_V512;
1417 let Predicates = [HasVLX] in {
Craig Toppera47576f2015-11-26 20:21:29 +00001418 defm NAME#128: avx512_perm_t<opc, OpcodeStr, VTInfo.info128,
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001419 ShuffleMask.info128>,
Craig Toppera47576f2015-11-26 20:21:29 +00001420 avx512_perm_t_mb<opc, OpcodeStr, VTInfo.info128,
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001421 ShuffleMask.info128>, EVEX_V128;
Craig Toppera47576f2015-11-26 20:21:29 +00001422 defm NAME#256: avx512_perm_t<opc, OpcodeStr, VTInfo.info256,
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001423 ShuffleMask.info256>,
Craig Toppera47576f2015-11-26 20:21:29 +00001424 avx512_perm_t_mb<opc, OpcodeStr, VTInfo.info256,
1425 ShuffleMask.info256>, EVEX_V256;
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001426 }
1427}
1428
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001429multiclass avx512_perm_t_sizes_bw<bits<8> opc, string OpcodeStr,
Craig Toppera47576f2015-11-26 20:21:29 +00001430 AVX512VLVectorVTInfo VTInfo,
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001431 AVX512VLVectorVTInfo Idx,
1432 Predicate Prd> {
1433 let Predicates = [Prd] in
Craig Toppera47576f2015-11-26 20:21:29 +00001434 defm NAME: avx512_perm_t<opc, OpcodeStr, VTInfo.info512,
1435 Idx.info512>, EVEX_V512;
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001436 let Predicates = [Prd, HasVLX] in {
Craig Toppera47576f2015-11-26 20:21:29 +00001437 defm NAME#128: avx512_perm_t<opc, OpcodeStr, VTInfo.info128,
1438 Idx.info128>, EVEX_V128;
1439 defm NAME#256: avx512_perm_t<opc, OpcodeStr, VTInfo.info256,
1440 Idx.info256>, EVEX_V256;
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001441 }
1442}
1443
Craig Toppera47576f2015-11-26 20:21:29 +00001444defm VPERMT2D : avx512_perm_t_sizes<0x7E, "vpermt2d",
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001445 avx512vl_i32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
Craig Toppera47576f2015-11-26 20:21:29 +00001446defm VPERMT2Q : avx512_perm_t_sizes<0x7E, "vpermt2q",
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001447 avx512vl_i64_info, avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
Michael Zuckerman4582bda2016-01-19 18:47:02 +00001448defm VPERMT2W : avx512_perm_t_sizes_bw<0x7D, "vpermt2w",
1449 avx512vl_i16_info, avx512vl_i16_info, HasBWI>,
1450 VEX_W, EVEX_CD8<16, CD8VF>;
1451defm VPERMT2B : avx512_perm_t_sizes_bw<0x7D, "vpermt2b",
1452 avx512vl_i8_info, avx512vl_i8_info, HasVBMI>,
1453 EVEX_CD8<8, CD8VF>;
Craig Toppera47576f2015-11-26 20:21:29 +00001454defm VPERMT2PS : avx512_perm_t_sizes<0x7F, "vpermt2ps",
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001455 avx512vl_f32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
Craig Toppera47576f2015-11-26 20:21:29 +00001456defm VPERMT2PD : avx512_perm_t_sizes<0x7F, "vpermt2pd",
Elena Demikhovskyf07df9f2015-11-25 08:17:56 +00001457 avx512vl_f64_info, avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
Elena Demikhovsky299cf5112014-04-29 09:09:15 +00001458
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001459//===----------------------------------------------------------------------===//
1460// AVX-512 - BLEND using mask
1461//
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001462multiclass avx512_blendmask<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> {
Craig Toppera74e3082017-01-07 22:20:34 +00001463 let ExeDomain = _.ExeDomain, hasSideEffects = 0 in {
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001464 def rr : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),
1465 (ins _.RC:$src1, _.RC:$src2),
1466 !strconcat(OpcodeStr,
Craig Topper9feea572016-01-11 00:44:58 +00001467 "\t{$src2, $src1, ${dst}|${dst}, $src1, $src2}"),
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001468 []>, EVEX_4V;
1469 def rrk : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),
1470 (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),
Elena Demikhovsky172a27c2014-01-08 10:54:22 +00001471 !strconcat(OpcodeStr,
Craig Topperedb09112014-11-25 20:11:23 +00001472 "\t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"),
Craig Toppera74e3082017-01-07 22:20:34 +00001473 []>, EVEX_4V, EVEX_K;
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001474 def rrkz : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),
1475 (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),
1476 !strconcat(OpcodeStr,
1477 "\t{$src2, $src1, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src1, $src2}"),
1478 []>, EVEX_4V, EVEX_KZ;
Craig Toppera74e3082017-01-07 22:20:34 +00001479 let mayLoad = 1 in {
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001480 def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1481 (ins _.RC:$src1, _.MemOp:$src2),
1482 !strconcat(OpcodeStr,
Craig Topper9feea572016-01-11 00:44:58 +00001483 "\t{$src2, $src1, ${dst}|${dst}, $src1, $src2}"),
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001484 []>, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>;
1485 def rmk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1486 (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2),
Elena Demikhovsky172a27c2014-01-08 10:54:22 +00001487 !strconcat(OpcodeStr,
Craig Topperedb09112014-11-25 20:11:23 +00001488 "\t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"),
Craig Toppera74e3082017-01-07 22:20:34 +00001489 []>, EVEX_4V, EVEX_K, EVEX_CD8<_.EltSize, CD8VF>;
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001490 def rmkz : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1491 (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2),
1492 !strconcat(OpcodeStr,
1493 "\t{$src2, $src1, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src1, $src2}"),
1494 []>, EVEX_4V, EVEX_KZ, EVEX_CD8<_.EltSize, CD8VF>;
1495 }
Craig Toppera74e3082017-01-07 22:20:34 +00001496 }
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001497}
1498multiclass avx512_blendmask_rmb<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> {
1499
Craig Topper81f20aa2017-01-07 22:20:26 +00001500 let mayLoad = 1, hasSideEffects = 0 in {
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001501 def rmbk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1502 (ins _.KRCWM:$mask, _.RC:$src1, _.ScalarMemOp:$src2),
1503 !strconcat(OpcodeStr,
1504 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",
1505 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"),
Craig Topper81f20aa2017-01-07 22:20:26 +00001506 []>, EVEX_4V, EVEX_K, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>;
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001507
1508 def rmb : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1509 (ins _.RC:$src1, _.ScalarMemOp:$src2),
1510 !strconcat(OpcodeStr,
1511 "\t{${src2}", _.BroadcastStr, ", $src1, $dst|",
1512 "$dst, $src1, ${src2}", _.BroadcastStr, "}"),
Elena Demikhovsky31214492014-12-23 09:36:28 +00001513 []>, EVEX_4V, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>;
Craig Topper81f20aa2017-01-07 22:20:26 +00001514 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001515}
1516
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001517multiclass blendmask_dq <bits<8> opc, string OpcodeStr,
1518 AVX512VLVectorVTInfo VTInfo> {
1519 defm Z : avx512_blendmask <opc, OpcodeStr, VTInfo.info512>,
1520 avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info512>, EVEX_V512;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001521
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001522 let Predicates = [HasVLX] in {
1523 defm Z256 : avx512_blendmask<opc, OpcodeStr, VTInfo.info256>,
1524 avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info256>, EVEX_V256;
1525 defm Z128 : avx512_blendmask<opc, OpcodeStr, VTInfo.info128>,
1526 avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info128>, EVEX_V128;
1527 }
1528}
Elena Demikhovsky172a27c2014-01-08 10:54:22 +00001529
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001530multiclass blendmask_bw <bits<8> opc, string OpcodeStr,
1531 AVX512VLVectorVTInfo VTInfo> {
1532 let Predicates = [HasBWI] in
1533 defm Z : avx512_blendmask <opc, OpcodeStr, VTInfo.info512>, EVEX_V512;
Elena Demikhovsky172a27c2014-01-08 10:54:22 +00001534
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001535 let Predicates = [HasBWI, HasVLX] in {
1536 defm Z256 : avx512_blendmask <opc, OpcodeStr, VTInfo.info256>, EVEX_V256;
1537 defm Z128 : avx512_blendmask <opc, OpcodeStr, VTInfo.info128>, EVEX_V128;
1538 }
1539}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001540
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001541
Elena Demikhovsky949b0d42014-12-22 13:52:48 +00001542defm VBLENDMPS : blendmask_dq <0x65, "vblendmps", avx512vl_f32_info>;
1543defm VBLENDMPD : blendmask_dq <0x65, "vblendmpd", avx512vl_f64_info>, VEX_W;
1544defm VPBLENDMD : blendmask_dq <0x64, "vpblendmd", avx512vl_i32_info>;
1545defm VPBLENDMQ : blendmask_dq <0x64, "vpblendmq", avx512vl_i64_info>, VEX_W;
1546defm VPBLENDMB : blendmask_bw <0x66, "vpblendmb", avx512vl_i8_info>;
1547defm VPBLENDMW : blendmask_bw <0x66, "vpblendmw", avx512vl_i16_info>, VEX_W;
Elena Demikhovsky172a27c2014-01-08 10:54:22 +00001548
Elena Demikhovsky172a27c2014-01-08 10:54:22 +00001549
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00001550//===----------------------------------------------------------------------===//
1551// Compare Instructions
1552//===----------------------------------------------------------------------===//
1553
1554// avx512_cmp_scalar - AVX512 CMPSS and CMPSD
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001555
1556multiclass avx512_cmp_scalar<X86VectorVTInfo _, SDNode OpNode, SDNode OpNodeRnd>{
1557
1558 defm rr_Int : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
1559 (outs _.KRC:$dst),
1560 (ins _.RC:$src1, _.RC:$src2, AVXCC:$cc),
1561 "vcmp${cc}"#_.Suffix,
1562 "$src2, $src1", "$src1, $src2",
1563 (OpNode (_.VT _.RC:$src1),
1564 (_.VT _.RC:$src2),
1565 imm:$cc)>, EVEX_4V;
Ayman Musa62d1c712017-04-13 10:03:45 +00001566 let mayLoad = 1 in
Craig Toppere1cac152016-06-07 07:27:54 +00001567 defm rm_Int : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,
1568 (outs _.KRC:$dst),
Craig Topperd9fe6642017-02-21 04:26:10 +00001569 (ins _.RC:$src1, _.IntScalarMemOp:$src2, AVXCC:$cc),
Craig Toppere1cac152016-06-07 07:27:54 +00001570 "vcmp${cc}"#_.Suffix,
1571 "$src2, $src1", "$src1, $src2",
Craig Topperd9fe6642017-02-21 04:26:10 +00001572 (OpNode (_.VT _.RC:$src1), _.ScalarIntMemCPat:$src2,
Craig Toppere1cac152016-06-07 07:27:54 +00001573 imm:$cc)>, EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>;
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001574
1575 defm rrb_Int : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
1576 (outs _.KRC:$dst),
1577 (ins _.RC:$src1, _.RC:$src2, AVXCC:$cc),
1578 "vcmp${cc}"#_.Suffix,
Craig Topperbfe13ff2016-01-11 00:44:52 +00001579 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001580 (OpNodeRnd (_.VT _.RC:$src1),
1581 (_.VT _.RC:$src2),
1582 imm:$cc,
1583 (i32 FROUND_NO_EXC))>, EVEX_4V, EVEX_B;
1584 // Accept explicit immediate argument form instead of comparison code.
Craig Topper0550ce72014-01-05 04:55:55 +00001585 let isAsmParserOnly = 1, hasSideEffects = 0 in {
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001586 defm rri_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _,
1587 (outs VK1:$dst),
1588 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
1589 "vcmp"#_.Suffix,
1590 "$cc, $src2, $src1", "$src1, $src2, $cc">, EVEX_4V;
Ayman Musa62d1c712017-04-13 10:03:45 +00001591 let mayLoad = 1 in
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001592 defm rmi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _,
1593 (outs _.KRC:$dst),
Igor Breger4511e762016-02-22 11:48:27 +00001594 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$cc),
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001595 "vcmp"#_.Suffix,
1596 "$cc, $src2, $src1", "$src1, $src2, $cc">,
1597 EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>;
1598
1599 defm rrb_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _,
1600 (outs _.KRC:$dst),
1601 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
1602 "vcmp"#_.Suffix,
Craig Topperbfe13ff2016-01-11 00:44:52 +00001603 "$cc, {sae}, $src2, $src1","$src1, $src2, {sae}, $cc">,
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001604 EVEX_4V, EVEX_B;
1605 }// let isAsmParserOnly = 1, hasSideEffects = 0
1606
1607 let isCodeGenOnly = 1 in {
Craig Topper225da2c2016-08-27 05:22:15 +00001608 let isCommutable = 1 in
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001609 def rr : AVX512Ii8<0xC2, MRMSrcReg,
1610 (outs _.KRC:$dst), (ins _.FRC:$src1, _.FRC:$src2, AVXCC:$cc),
1611 !strconcat("vcmp${cc}", _.Suffix,
1612 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
1613 [(set _.KRC:$dst, (OpNode _.FRC:$src1,
1614 _.FRC:$src2,
1615 imm:$cc))],
1616 IIC_SSE_ALU_F32S_RR>, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00001617 def rm : AVX512Ii8<0xC2, MRMSrcMem,
1618 (outs _.KRC:$dst),
1619 (ins _.FRC:$src1, _.ScalarMemOp:$src2, AVXCC:$cc),
1620 !strconcat("vcmp${cc}", _.Suffix,
1621 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
1622 [(set _.KRC:$dst, (OpNode _.FRC:$src1,
1623 (_.ScalarLdFrag addr:$src2),
1624 imm:$cc))],
1625 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>;
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00001626 }
1627}
1628
1629let Predicates = [HasAVX512] in {
Craig Topperd890db62017-02-21 04:26:04 +00001630 let ExeDomain = SSEPackedSingle in
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001631 defm VCMPSSZ : avx512_cmp_scalar<f32x_info, X86cmpms, X86cmpmsRnd>,
1632 AVX512XSIi8Base;
Craig Topperd890db62017-02-21 04:26:04 +00001633 let ExeDomain = SSEPackedDouble in
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00001634 defm VCMPSDZ : avx512_cmp_scalar<f64x_info, X86cmpms, X86cmpmsRnd>,
1635 AVX512XDIi8Base, VEX_W;
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00001636}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001637
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001638multiclass avx512_icmp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper392cd032016-09-03 16:28:03 +00001639 X86VectorVTInfo _, bit IsCommutable> {
1640 let isCommutable = IsCommutable in
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001641 def rr : AVX512BI<opc, MRMSrcReg,
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001642 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2),
1643 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
1644 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2)))],
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001645 IIC_SSE_ALU_F32P_RR>, EVEX_4V;
1646 def rm : AVX512BI<opc, MRMSrcMem,
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001647 (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2),
1648 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
1649 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
1650 (_.VT (bitconvert (_.LdFrag addr:$src2)))))],
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001651 IIC_SSE_ALU_F32P_RM>, EVEX_4V;
Craig Toppere1d81032017-06-13 07:13:47 +00001652 let isCommutable = IsCommutable in
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001653 def rrk : AVX512BI<opc, MRMSrcReg,
1654 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),
1655 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst {${mask}}|",
1656 "$dst {${mask}}, $src1, $src2}"),
1657 [(set _.KRC:$dst, (and _.KRCWM:$mask,
1658 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))))],
1659 IIC_SSE_ALU_F32P_RR>, EVEX_4V, EVEX_K;
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001660 def rmk : AVX512BI<opc, MRMSrcMem,
1661 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2),
1662 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst {${mask}}|",
1663 "$dst {${mask}}, $src1, $src2}"),
1664 [(set _.KRC:$dst, (and _.KRCWM:$mask,
1665 (OpNode (_.VT _.RC:$src1),
1666 (_.VT (bitconvert
1667 (_.LdFrag addr:$src2))))))],
1668 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001669}
1670
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001671multiclass avx512_icmp_packed_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper392cd032016-09-03 16:28:03 +00001672 X86VectorVTInfo _, bit IsCommutable> :
1673 avx512_icmp_packed<opc, OpcodeStr, OpNode, _, IsCommutable> {
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001674 def rmb : AVX512BI<opc, MRMSrcMem,
1675 (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2),
1676 !strconcat(OpcodeStr, "\t{${src2}", _.BroadcastStr, ", $src1, $dst",
1677 "|$dst, $src1, ${src2}", _.BroadcastStr, "}"),
1678 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
1679 (X86VBroadcast (_.ScalarLdFrag addr:$src2))))],
1680 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_B;
1681 def rmbk : AVX512BI<opc, MRMSrcMem,
1682 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1,
1683 _.ScalarMemOp:$src2),
1684 !strconcat(OpcodeStr,
1685 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",
1686 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"),
1687 [(set _.KRC:$dst, (and _.KRCWM:$mask,
1688 (OpNode (_.VT _.RC:$src1),
1689 (X86VBroadcast
1690 (_.ScalarLdFrag addr:$src2)))))],
1691 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K, EVEX_B;
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001692}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001693
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001694multiclass avx512_icmp_packed_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper392cd032016-09-03 16:28:03 +00001695 AVX512VLVectorVTInfo VTInfo, Predicate prd,
1696 bit IsCommutable = 0> {
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001697 let Predicates = [prd] in
Craig Topper392cd032016-09-03 16:28:03 +00001698 defm Z : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info512,
1699 IsCommutable>, EVEX_V512;
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001700
1701 let Predicates = [prd, HasVLX] in {
Craig Topper392cd032016-09-03 16:28:03 +00001702 defm Z256 : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info256,
1703 IsCommutable>, EVEX_V256;
1704 defm Z128 : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info128,
1705 IsCommutable>, EVEX_V128;
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001706 }
1707}
1708
1709multiclass avx512_icmp_packed_rmb_vl<bits<8> opc, string OpcodeStr,
1710 SDNode OpNode, AVX512VLVectorVTInfo VTInfo,
Craig Topper392cd032016-09-03 16:28:03 +00001711 Predicate prd, bit IsCommutable = 0> {
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001712 let Predicates = [prd] in
Craig Topper392cd032016-09-03 16:28:03 +00001713 defm Z : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info512,
1714 IsCommutable>, EVEX_V512;
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001715
1716 let Predicates = [prd, HasVLX] in {
Craig Topper392cd032016-09-03 16:28:03 +00001717 defm Z256 : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info256,
1718 IsCommutable>, EVEX_V256;
1719 defm Z128 : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info128,
1720 IsCommutable>, EVEX_V128;
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001721 }
1722}
1723
1724defm VPCMPEQB : avx512_icmp_packed_vl<0x74, "vpcmpeqb", X86pcmpeqm,
Craig Topper392cd032016-09-03 16:28:03 +00001725 avx512vl_i8_info, HasBWI, 1>,
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001726 EVEX_CD8<8, CD8VF>;
1727
1728defm VPCMPEQW : avx512_icmp_packed_vl<0x75, "vpcmpeqw", X86pcmpeqm,
Craig Topper392cd032016-09-03 16:28:03 +00001729 avx512vl_i16_info, HasBWI, 1>,
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001730 EVEX_CD8<16, CD8VF>;
1731
Robert Khasanovf70f7982014-09-18 14:06:55 +00001732defm VPCMPEQD : avx512_icmp_packed_rmb_vl<0x76, "vpcmpeqd", X86pcmpeqm,
Craig Topper392cd032016-09-03 16:28:03 +00001733 avx512vl_i32_info, HasAVX512, 1>,
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001734 EVEX_CD8<32, CD8VF>;
1735
Robert Khasanovf70f7982014-09-18 14:06:55 +00001736defm VPCMPEQQ : avx512_icmp_packed_rmb_vl<0x29, "vpcmpeqq", X86pcmpeqm,
Craig Topper392cd032016-09-03 16:28:03 +00001737 avx512vl_i64_info, HasAVX512, 1>,
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001738 T8PD, VEX_W, EVEX_CD8<64, CD8VF>;
1739
1740defm VPCMPGTB : avx512_icmp_packed_vl<0x64, "vpcmpgtb", X86pcmpgtm,
1741 avx512vl_i8_info, HasBWI>,
1742 EVEX_CD8<8, CD8VF>;
1743
1744defm VPCMPGTW : avx512_icmp_packed_vl<0x65, "vpcmpgtw", X86pcmpgtm,
1745 avx512vl_i16_info, HasBWI>,
1746 EVEX_CD8<16, CD8VF>;
1747
Robert Khasanovf70f7982014-09-18 14:06:55 +00001748defm VPCMPGTD : avx512_icmp_packed_rmb_vl<0x66, "vpcmpgtd", X86pcmpgtm,
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001749 avx512vl_i32_info, HasAVX512>,
1750 EVEX_CD8<32, CD8VF>;
1751
Robert Khasanovf70f7982014-09-18 14:06:55 +00001752defm VPCMPGTQ : avx512_icmp_packed_rmb_vl<0x37, "vpcmpgtq", X86pcmpgtm,
Robert Khasanov2ea081d2014-08-25 14:49:34 +00001753 avx512vl_i64_info, HasAVX512>,
1754 T8PD, VEX_W, EVEX_CD8<64, CD8VF>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001755
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001756
Ayman Musa721d97f2017-06-27 12:08:37 +00001757multiclass avx512_icmp_packed_lowering<X86VectorVTInfo _, X86KVectorVTInfo NewInf,
1758 SDNode OpNode, string InstrStr,
1759 list<Predicate> Preds> {
1760let Predicates = Preds in {
1761 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
1762 (_.KVT (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))),
1763 (i64 0)),
1764 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rr) _.RC:$src1, _.RC:$src2),
1765 NewInf.KRC)>;
Simon Pilgrim64fff142017-07-16 18:37:23 +00001766
Ayman Musa721d97f2017-06-27 12:08:37 +00001767 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
Simon Pilgrim64fff142017-07-16 18:37:23 +00001768 (_.KVT (OpNode (_.VT _.RC:$src1),
Ayman Musa721d97f2017-06-27 12:08:37 +00001769 (_.VT (bitconvert (_.LdFrag addr:$src2))))),
1770 (i64 0)),
1771 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rm) _.RC:$src1, addr:$src2),
1772 NewInf.KRC)>;
Simon Pilgrim64fff142017-07-16 18:37:23 +00001773
Ayman Musa721d97f2017-06-27 12:08:37 +00001774 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
Simon Pilgrim64fff142017-07-16 18:37:23 +00001775 (_.KVT (and _.KRCWM:$mask,
Ayman Musa721d97f2017-06-27 12:08:37 +00001776 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2)))),
1777 (i64 0)),
1778 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rrk) _.KRCWM:$mask,
1779 _.RC:$src1, _.RC:$src2),
1780 NewInf.KRC)>;
Simon Pilgrim64fff142017-07-16 18:37:23 +00001781
Ayman Musa721d97f2017-06-27 12:08:37 +00001782 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
Simon Pilgrim64fff142017-07-16 18:37:23 +00001783 (_.KVT (and (_.KVT _.KRCWM:$mask),
1784 (_.KVT (OpNode (_.VT _.RC:$src1),
1785 (_.VT (bitconvert
Ayman Musa721d97f2017-06-27 12:08:37 +00001786 (_.LdFrag addr:$src2))))))),
1787 (i64 0)),
Simon Pilgrim64fff142017-07-16 18:37:23 +00001788 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rmk) _.KRCWM:$mask,
Ayman Musa721d97f2017-06-27 12:08:37 +00001789 _.RC:$src1, addr:$src2),
1790 NewInf.KRC)>;
Craig Topper8b9e6712016-09-02 04:25:30 +00001791}
Ayman Musa721d97f2017-06-27 12:08:37 +00001792}
1793
1794multiclass avx512_icmp_packed_rmb_lowering<X86VectorVTInfo _, X86KVectorVTInfo NewInf,
1795 SDNode OpNode, string InstrStr,
1796 list<Predicate> Preds>
1797 : avx512_icmp_packed_lowering<_, NewInf, OpNode, InstrStr, Preds> {
1798let Predicates = Preds in {
1799 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
1800 (_.KVT (OpNode (_.VT _.RC:$src1),
1801 (X86VBroadcast (_.ScalarLdFrag addr:$src2)))),
1802 (i64 0)),
1803 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rmb) _.RC:$src1, addr:$src2),
1804 NewInf.KRC)>;
Simon Pilgrim64fff142017-07-16 18:37:23 +00001805
Ayman Musa721d97f2017-06-27 12:08:37 +00001806 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
1807 (_.KVT (and (_.KVT _.KRCWM:$mask),
1808 (_.KVT (OpNode (_.VT _.RC:$src1),
1809 (X86VBroadcast
1810 (_.ScalarLdFrag addr:$src2)))))),
1811 (i64 0)),
1812 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rmbk) _.KRCWM:$mask,
1813 _.RC:$src1, addr:$src2),
1814 NewInf.KRC)>;
1815}
1816}
1817
1818// VPCMPEQB - i8
1819defm : avx512_icmp_packed_lowering<v16i8x_info, v32i1_info, X86pcmpeqm,
1820 "VPCMPEQBZ128", [HasBWI, HasVLX]>;
1821defm : avx512_icmp_packed_lowering<v16i8x_info, v64i1_info, X86pcmpeqm,
1822 "VPCMPEQBZ128", [HasBWI, HasVLX]>;
1823
1824defm : avx512_icmp_packed_lowering<v32i8x_info, v64i1_info, X86pcmpeqm,
1825 "VPCMPEQBZ256", [HasBWI, HasVLX]>;
1826
1827// VPCMPEQW - i16
1828defm : avx512_icmp_packed_lowering<v8i16x_info, v16i1_info, X86pcmpeqm,
1829 "VPCMPEQWZ128", [HasBWI, HasVLX]>;
1830defm : avx512_icmp_packed_lowering<v8i16x_info, v32i1_info, X86pcmpeqm,
1831 "VPCMPEQWZ128", [HasBWI, HasVLX]>;
1832defm : avx512_icmp_packed_lowering<v8i16x_info, v64i1_info, X86pcmpeqm,
1833 "VPCMPEQWZ128", [HasBWI, HasVLX]>;
1834
1835defm : avx512_icmp_packed_lowering<v16i16x_info, v32i1_info, X86pcmpeqm,
1836 "VPCMPEQWZ256", [HasBWI, HasVLX]>;
1837defm : avx512_icmp_packed_lowering<v16i16x_info, v64i1_info, X86pcmpeqm,
1838 "VPCMPEQWZ256", [HasBWI, HasVLX]>;
1839
1840defm : avx512_icmp_packed_lowering<v32i16_info, v64i1_info, X86pcmpeqm,
1841 "VPCMPEQWZ", [HasBWI]>;
1842
1843// VPCMPEQD - i32
1844defm : avx512_icmp_packed_rmb_lowering<v4i32x_info, v8i1_info, X86pcmpeqm,
1845 "VPCMPEQDZ128", [HasAVX512, HasVLX]>;
1846defm : avx512_icmp_packed_rmb_lowering<v4i32x_info, v16i1_info, X86pcmpeqm,
1847 "VPCMPEQDZ128", [HasAVX512, HasVLX]>;
1848defm : avx512_icmp_packed_rmb_lowering<v4i32x_info, v32i1_info, X86pcmpeqm,
1849 "VPCMPEQDZ128", [HasAVX512, HasVLX]>;
1850defm : avx512_icmp_packed_rmb_lowering<v4i32x_info, v64i1_info, X86pcmpeqm,
1851 "VPCMPEQDZ128", [HasAVX512, HasVLX]>;
1852
1853defm : avx512_icmp_packed_rmb_lowering<v8i32x_info, v16i1_info, X86pcmpeqm,
1854 "VPCMPEQDZ256", [HasAVX512, HasVLX]>;
1855defm : avx512_icmp_packed_rmb_lowering<v8i32x_info, v32i1_info, X86pcmpeqm,
1856 "VPCMPEQDZ256", [HasAVX512, HasVLX]>;
1857defm : avx512_icmp_packed_rmb_lowering<v8i32x_info, v64i1_info, X86pcmpeqm,
1858 "VPCMPEQDZ256", [HasAVX512, HasVLX]>;
1859
1860defm : avx512_icmp_packed_rmb_lowering<v16i32_info, v32i1_info, X86pcmpeqm,
1861 "VPCMPEQDZ", [HasAVX512]>;
1862defm : avx512_icmp_packed_rmb_lowering<v16i32_info, v64i1_info, X86pcmpeqm,
1863 "VPCMPEQDZ", [HasAVX512]>;
1864
1865// VPCMPEQQ - i64
1866defm : avx512_icmp_packed_rmb_lowering<v2i64x_info, v4i1_info, X86pcmpeqm,
1867 "VPCMPEQQZ128", [HasAVX512, HasVLX]>;
1868defm : avx512_icmp_packed_rmb_lowering<v2i64x_info, v8i1_info, X86pcmpeqm,
1869 "VPCMPEQQZ128", [HasAVX512, HasVLX]>;
1870defm : avx512_icmp_packed_rmb_lowering<v2i64x_info, v16i1_info, X86pcmpeqm,
1871 "VPCMPEQQZ128", [HasAVX512, HasVLX]>;
1872defm : avx512_icmp_packed_rmb_lowering<v2i64x_info, v32i1_info, X86pcmpeqm,
1873 "VPCMPEQQZ128", [HasAVX512, HasVLX]>;
1874defm : avx512_icmp_packed_rmb_lowering<v2i64x_info, v64i1_info, X86pcmpeqm,
1875 "VPCMPEQQZ128", [HasAVX512, HasVLX]>;
1876
1877defm : avx512_icmp_packed_rmb_lowering<v4i64x_info, v8i1_info, X86pcmpeqm,
1878 "VPCMPEQQZ256", [HasAVX512, HasVLX]>;
1879defm : avx512_icmp_packed_rmb_lowering<v4i64x_info, v16i1_info, X86pcmpeqm,
1880 "VPCMPEQQZ256", [HasAVX512, HasVLX]>;
1881defm : avx512_icmp_packed_rmb_lowering<v4i64x_info, v32i1_info, X86pcmpeqm,
1882 "VPCMPEQQZ256", [HasAVX512, HasVLX]>;
1883defm : avx512_icmp_packed_rmb_lowering<v4i64x_info, v64i1_info, X86pcmpeqm,
1884 "VPCMPEQQZ256", [HasAVX512, HasVLX]>;
1885
Simon Pilgrim64fff142017-07-16 18:37:23 +00001886defm : avx512_icmp_packed_rmb_lowering<v8i64_info, v16i1_info, X86pcmpeqm,
Ayman Musa721d97f2017-06-27 12:08:37 +00001887 "VPCMPEQQZ", [HasAVX512]>;
1888defm : avx512_icmp_packed_rmb_lowering<v8i64_info, v32i1_info, X86pcmpeqm,
1889 "VPCMPEQQZ", [HasAVX512]>;
1890defm : avx512_icmp_packed_rmb_lowering<v8i64_info, v64i1_info, X86pcmpeqm,
1891 "VPCMPEQQZ", [HasAVX512]>;
1892
1893// VPCMPGTB - i8
1894defm : avx512_icmp_packed_lowering<v16i8x_info, v32i1_info, X86pcmpgtm,
1895 "VPCMPGTBZ128", [HasBWI, HasVLX]>;
1896defm : avx512_icmp_packed_lowering<v16i8x_info, v64i1_info, X86pcmpgtm,
1897 "VPCMPGTBZ128", [HasBWI, HasVLX]>;
1898
1899defm : avx512_icmp_packed_lowering<v32i8x_info, v64i1_info, X86pcmpgtm,
1900 "VPCMPGTBZ256", [HasBWI, HasVLX]>;
1901
1902// VPCMPGTW - i16
1903defm : avx512_icmp_packed_lowering<v8i16x_info, v16i1_info, X86pcmpgtm,
1904 "VPCMPGTWZ128", [HasBWI, HasVLX]>;
1905defm : avx512_icmp_packed_lowering<v8i16x_info, v32i1_info, X86pcmpgtm,
1906 "VPCMPGTWZ128", [HasBWI, HasVLX]>;
1907defm : avx512_icmp_packed_lowering<v8i16x_info, v64i1_info, X86pcmpgtm,
1908 "VPCMPGTWZ128", [HasBWI, HasVLX]>;
1909
1910defm : avx512_icmp_packed_lowering<v16i16x_info, v32i1_info, X86pcmpgtm,
1911 "VPCMPGTWZ256", [HasBWI, HasVLX]>;
1912defm : avx512_icmp_packed_lowering<v16i16x_info, v64i1_info, X86pcmpgtm,
1913 "VPCMPGTWZ256", [HasBWI, HasVLX]>;
1914
1915defm : avx512_icmp_packed_lowering<v32i16_info, v64i1_info, X86pcmpgtm,
1916 "VPCMPGTWZ", [HasBWI]>;
1917
1918// VPCMPGTD - i32
1919defm : avx512_icmp_packed_rmb_lowering<v4i32x_info, v8i1_info, X86pcmpgtm,
1920 "VPCMPGTDZ128", [HasAVX512, HasVLX]>;
1921defm : avx512_icmp_packed_rmb_lowering<v4i32x_info, v16i1_info, X86pcmpgtm,
1922 "VPCMPGTDZ128", [HasAVX512, HasVLX]>;
1923defm : avx512_icmp_packed_rmb_lowering<v4i32x_info, v32i1_info, X86pcmpgtm,
1924 "VPCMPGTDZ128", [HasAVX512, HasVLX]>;
1925defm : avx512_icmp_packed_rmb_lowering<v4i32x_info, v64i1_info, X86pcmpgtm,
1926 "VPCMPGTDZ128", [HasAVX512, HasVLX]>;
1927
1928defm : avx512_icmp_packed_rmb_lowering<v8i32x_info, v16i1_info, X86pcmpgtm,
1929 "VPCMPGTDZ256", [HasAVX512, HasVLX]>;
1930defm : avx512_icmp_packed_rmb_lowering<v8i32x_info, v32i1_info, X86pcmpgtm,
1931 "VPCMPGTDZ256", [HasAVX512, HasVLX]>;
1932defm : avx512_icmp_packed_rmb_lowering<v8i32x_info, v64i1_info, X86pcmpgtm,
1933 "VPCMPGTDZ256", [HasAVX512, HasVLX]>;
1934
1935defm : avx512_icmp_packed_rmb_lowering<v16i32_info, v32i1_info, X86pcmpgtm,
1936 "VPCMPGTDZ", [HasAVX512]>;
1937defm : avx512_icmp_packed_rmb_lowering<v16i32_info, v64i1_info, X86pcmpgtm,
1938 "VPCMPGTDZ", [HasAVX512]>;
1939
1940// VPCMPGTQ - i64
1941defm : avx512_icmp_packed_rmb_lowering<v2i64x_info, v4i1_info, X86pcmpgtm,
1942 "VPCMPGTQZ128", [HasAVX512, HasVLX]>;
1943defm : avx512_icmp_packed_rmb_lowering<v2i64x_info, v8i1_info, X86pcmpgtm,
1944 "VPCMPGTQZ128", [HasAVX512, HasVLX]>;
1945defm : avx512_icmp_packed_rmb_lowering<v2i64x_info, v16i1_info, X86pcmpgtm,
1946 "VPCMPGTQZ128", [HasAVX512, HasVLX]>;
1947defm : avx512_icmp_packed_rmb_lowering<v2i64x_info, v32i1_info, X86pcmpgtm,
1948 "VPCMPGTQZ128", [HasAVX512, HasVLX]>;
1949defm : avx512_icmp_packed_rmb_lowering<v2i64x_info, v64i1_info, X86pcmpgtm,
1950 "VPCMPGTQZ128", [HasAVX512, HasVLX]>;
1951
1952defm : avx512_icmp_packed_rmb_lowering<v4i64x_info, v8i1_info, X86pcmpgtm,
1953 "VPCMPGTQZ256", [HasAVX512, HasVLX]>;
1954defm : avx512_icmp_packed_rmb_lowering<v4i64x_info, v16i1_info, X86pcmpgtm,
1955 "VPCMPGTQZ256", [HasAVX512, HasVLX]>;
1956defm : avx512_icmp_packed_rmb_lowering<v4i64x_info, v32i1_info, X86pcmpgtm,
1957 "VPCMPGTQZ256", [HasAVX512, HasVLX]>;
1958defm : avx512_icmp_packed_rmb_lowering<v4i64x_info, v64i1_info, X86pcmpgtm,
1959 "VPCMPGTQZ256", [HasAVX512, HasVLX]>;
1960
1961defm : avx512_icmp_packed_rmb_lowering<v8i64_info, v16i1_info, X86pcmpgtm,
1962 "VPCMPGTQZ", [HasAVX512]>;
1963defm : avx512_icmp_packed_rmb_lowering<v8i64_info, v32i1_info, X86pcmpgtm,
1964 "VPCMPGTQZ", [HasAVX512]>;
1965defm : avx512_icmp_packed_rmb_lowering<v8i64_info, v64i1_info, X86pcmpgtm,
1966 "VPCMPGTQZ", [HasAVX512]>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001967
Robert Khasanov29e3b962014-08-27 09:34:37 +00001968multiclass avx512_icmp_cc<bits<8> opc, string Suffix, SDNode OpNode,
1969 X86VectorVTInfo _> {
Craig Topper149e6bd2016-09-09 01:36:10 +00001970 let isCommutable = 1 in
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001971 def rri : AVX512AIi8<opc, MRMSrcReg,
Craig Topper7d3c6d32015-01-28 10:09:56 +00001972 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, AVX512ICC:$cc),
Adam Nemet1efcb902014-07-01 18:03:43 +00001973 !strconcat("vpcmp${cc}", Suffix,
1974 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001975 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
1976 imm:$cc))],
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00001977 IIC_SSE_ALU_F32P_RR>, EVEX_4V;
1978 def rmi : AVX512AIi8<opc, MRMSrcMem,
Craig Topper7d3c6d32015-01-28 10:09:56 +00001979 (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2, AVX512ICC:$cc),
Adam Nemet1efcb902014-07-01 18:03:43 +00001980 !strconcat("vpcmp${cc}", Suffix,
1981 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001982 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
1983 (_.VT (bitconvert (_.LdFrag addr:$src2))),
Craig Topper6e3a5822014-12-27 20:08:45 +00001984 imm:$cc))],
Robert Khasanov29e3b962014-08-27 09:34:37 +00001985 IIC_SSE_ALU_F32P_RM>, EVEX_4V;
Craig Topper8b876762017-06-13 07:13:50 +00001986 let isCommutable = 1 in
Robert Khasanov29e3b962014-08-27 09:34:37 +00001987 def rrik : AVX512AIi8<opc, MRMSrcReg,
1988 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2,
Craig Topper7d3c6d32015-01-28 10:09:56 +00001989 AVX512ICC:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00001990 !strconcat("vpcmp${cc}", Suffix,
1991 "\t{$src2, $src1, $dst {${mask}}|",
1992 "$dst {${mask}}, $src1, $src2}"),
1993 [(set _.KRC:$dst, (and _.KRCWM:$mask,
1994 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
Craig Topper6e3a5822014-12-27 20:08:45 +00001995 imm:$cc)))],
Robert Khasanov29e3b962014-08-27 09:34:37 +00001996 IIC_SSE_ALU_F32P_RR>, EVEX_4V, EVEX_K;
Robert Khasanov29e3b962014-08-27 09:34:37 +00001997 def rmik : AVX512AIi8<opc, MRMSrcMem,
1998 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2,
Craig Topper7d3c6d32015-01-28 10:09:56 +00001999 AVX512ICC:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00002000 !strconcat("vpcmp${cc}", Suffix,
2001 "\t{$src2, $src1, $dst {${mask}}|",
2002 "$dst {${mask}}, $src1, $src2}"),
2003 [(set _.KRC:$dst, (and _.KRCWM:$mask,
2004 (OpNode (_.VT _.RC:$src1),
2005 (_.VT (bitconvert (_.LdFrag addr:$src2))),
Craig Topper6e3a5822014-12-27 20:08:45 +00002006 imm:$cc)))],
Robert Khasanov29e3b962014-08-27 09:34:37 +00002007 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K;
2008
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002009 // Accept explicit immediate argument form instead of comparison code.
Craig Topper0550ce72014-01-05 04:55:55 +00002010 let isAsmParserOnly = 1, hasSideEffects = 0 in {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002011 def rri_alt : AVX512AIi8<opc, MRMSrcReg,
Craig Topper7ff6ab32015-01-21 08:43:49 +00002012 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00002013 !strconcat("vpcmp", Suffix, "\t{$cc, $src2, $src1, $dst|",
2014 "$dst, $src1, $src2, $cc}"),
Adam Nemet1efcb902014-07-01 18:03:43 +00002015 [], IIC_SSE_ALU_F32P_RR>, EVEX_4V;
Craig Topper9f4d4852015-01-20 12:15:30 +00002016 let mayLoad = 1 in
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002017 def rmi_alt : AVX512AIi8<opc, MRMSrcMem,
Craig Topper7ff6ab32015-01-21 08:43:49 +00002018 (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00002019 !strconcat("vpcmp", Suffix, "\t{$cc, $src2, $src1, $dst|",
2020 "$dst, $src1, $src2, $cc}"),
Adam Nemet1efcb902014-07-01 18:03:43 +00002021 [], IIC_SSE_ALU_F32P_RM>, EVEX_4V;
Robert Khasanov29e3b962014-08-27 09:34:37 +00002022 def rrik_alt : AVX512AIi8<opc, MRMSrcReg,
2023 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2,
Craig Topper7ff6ab32015-01-21 08:43:49 +00002024 u8imm:$cc),
Adam Nemet16de2482014-07-01 18:03:45 +00002025 !strconcat("vpcmp", Suffix,
Robert Khasanov29e3b962014-08-27 09:34:37 +00002026 "\t{$cc, $src2, $src1, $dst {${mask}}|",
2027 "$dst {${mask}}, $src1, $src2, $cc}"),
2028 [], IIC_SSE_ALU_F32P_RR>, EVEX_4V, EVEX_K;
Craig Topper9f4d4852015-01-20 12:15:30 +00002029 let mayLoad = 1 in
Robert Khasanov29e3b962014-08-27 09:34:37 +00002030 def rmik_alt : AVX512AIi8<opc, MRMSrcMem,
2031 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2,
Craig Topper7ff6ab32015-01-21 08:43:49 +00002032 u8imm:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00002033 !strconcat("vpcmp", Suffix,
2034 "\t{$cc, $src2, $src1, $dst {${mask}}|",
2035 "$dst {${mask}}, $src1, $src2, $cc}"),
Adam Nemet16de2482014-07-01 18:03:45 +00002036 [], IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002037 }
2038}
2039
Robert Khasanov29e3b962014-08-27 09:34:37 +00002040multiclass avx512_icmp_cc_rmb<bits<8> opc, string Suffix, SDNode OpNode,
Robert Khasanovf70f7982014-09-18 14:06:55 +00002041 X86VectorVTInfo _> :
2042 avx512_icmp_cc<opc, Suffix, OpNode, _> {
Robert Khasanov29e3b962014-08-27 09:34:37 +00002043 def rmib : AVX512AIi8<opc, MRMSrcMem,
2044 (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2,
Craig Topper7d3c6d32015-01-28 10:09:56 +00002045 AVX512ICC:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00002046 !strconcat("vpcmp${cc}", Suffix,
2047 "\t{${src2}", _.BroadcastStr, ", $src1, $dst|",
2048 "$dst, $src1, ${src2}", _.BroadcastStr, "}"),
2049 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
2050 (X86VBroadcast (_.ScalarLdFrag addr:$src2)),
Craig Topper6e3a5822014-12-27 20:08:45 +00002051 imm:$cc))],
Robert Khasanov29e3b962014-08-27 09:34:37 +00002052 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_B;
2053 def rmibk : AVX512AIi8<opc, MRMSrcMem,
2054 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1,
Craig Topper7d3c6d32015-01-28 10:09:56 +00002055 _.ScalarMemOp:$src2, AVX512ICC:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00002056 !strconcat("vpcmp${cc}", Suffix,
2057 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",
2058 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"),
2059 [(set _.KRC:$dst, (and _.KRCWM:$mask,
2060 (OpNode (_.VT _.RC:$src1),
2061 (X86VBroadcast (_.ScalarLdFrag addr:$src2)),
Craig Topper6e3a5822014-12-27 20:08:45 +00002062 imm:$cc)))],
Robert Khasanov29e3b962014-08-27 09:34:37 +00002063 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K, EVEX_B;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002064
Robert Khasanov29e3b962014-08-27 09:34:37 +00002065 // Accept explicit immediate argument form instead of comparison code.
Craig Topper9f4d4852015-01-20 12:15:30 +00002066 let isAsmParserOnly = 1, hasSideEffects = 0, mayLoad = 1 in {
Robert Khasanov29e3b962014-08-27 09:34:37 +00002067 def rmib_alt : AVX512AIi8<opc, MRMSrcMem,
2068 (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2,
Craig Topper7ff6ab32015-01-21 08:43:49 +00002069 u8imm:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00002070 !strconcat("vpcmp", Suffix,
2071 "\t{$cc, ${src2}", _.BroadcastStr, ", $src1, $dst|",
2072 "$dst, $src1, ${src2}", _.BroadcastStr, ", $cc}"),
2073 [], IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_B;
2074 def rmibk_alt : AVX512AIi8<opc, MRMSrcMem,
2075 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1,
Craig Topper7ff6ab32015-01-21 08:43:49 +00002076 _.ScalarMemOp:$src2, u8imm:$cc),
Robert Khasanov29e3b962014-08-27 09:34:37 +00002077 !strconcat("vpcmp", Suffix,
2078 "\t{$cc, ${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",
2079 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, ", $cc}"),
2080 [], IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K, EVEX_B;
2081 }
2082}
2083
2084multiclass avx512_icmp_cc_vl<bits<8> opc, string Suffix, SDNode OpNode,
2085 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
2086 let Predicates = [prd] in
2087 defm Z : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info512>, EVEX_V512;
2088
2089 let Predicates = [prd, HasVLX] in {
2090 defm Z256 : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info256>, EVEX_V256;
2091 defm Z128 : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info128>, EVEX_V128;
2092 }
2093}
2094
2095multiclass avx512_icmp_cc_rmb_vl<bits<8> opc, string Suffix, SDNode OpNode,
2096 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
2097 let Predicates = [prd] in
2098 defm Z : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info512>,
2099 EVEX_V512;
2100
2101 let Predicates = [prd, HasVLX] in {
2102 defm Z256 : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info256>,
2103 EVEX_V256;
2104 defm Z128 : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info128>,
2105 EVEX_V128;
2106 }
2107}
2108
2109defm VPCMPB : avx512_icmp_cc_vl<0x3F, "b", X86cmpm, avx512vl_i8_info,
2110 HasBWI>, EVEX_CD8<8, CD8VF>;
2111defm VPCMPUB : avx512_icmp_cc_vl<0x3E, "ub", X86cmpmu, avx512vl_i8_info,
2112 HasBWI>, EVEX_CD8<8, CD8VF>;
2113
2114defm VPCMPW : avx512_icmp_cc_vl<0x3F, "w", X86cmpm, avx512vl_i16_info,
2115 HasBWI>, VEX_W, EVEX_CD8<16, CD8VF>;
2116defm VPCMPUW : avx512_icmp_cc_vl<0x3E, "uw", X86cmpmu, avx512vl_i16_info,
2117 HasBWI>, VEX_W, EVEX_CD8<16, CD8VF>;
2118
Robert Khasanovf70f7982014-09-18 14:06:55 +00002119defm VPCMPD : avx512_icmp_cc_rmb_vl<0x1F, "d", X86cmpm, avx512vl_i32_info,
Robert Khasanov29e3b962014-08-27 09:34:37 +00002120 HasAVX512>, EVEX_CD8<32, CD8VF>;
Robert Khasanovf70f7982014-09-18 14:06:55 +00002121defm VPCMPUD : avx512_icmp_cc_rmb_vl<0x1E, "ud", X86cmpmu, avx512vl_i32_info,
Robert Khasanov29e3b962014-08-27 09:34:37 +00002122 HasAVX512>, EVEX_CD8<32, CD8VF>;
2123
Robert Khasanovf70f7982014-09-18 14:06:55 +00002124defm VPCMPQ : avx512_icmp_cc_rmb_vl<0x1F, "q", X86cmpm, avx512vl_i64_info,
Robert Khasanov29e3b962014-08-27 09:34:37 +00002125 HasAVX512>, VEX_W, EVEX_CD8<64, CD8VF>;
Robert Khasanovf70f7982014-09-18 14:06:55 +00002126defm VPCMPUQ : avx512_icmp_cc_rmb_vl<0x1E, "uq", X86cmpmu, avx512vl_i64_info,
Robert Khasanov29e3b962014-08-27 09:34:37 +00002127 HasAVX512>, VEX_W, EVEX_CD8<64, CD8VF>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002128
Ayman Musa721d97f2017-06-27 12:08:37 +00002129multiclass avx512_icmp_cc_packed_lowering<X86VectorVTInfo _, X86KVectorVTInfo NewInf,
2130 SDNode OpNode, string InstrStr,
2131 list<Predicate> Preds> {
2132let Predicates = Preds in {
2133 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
Simon Pilgrim64fff142017-07-16 18:37:23 +00002134 (_.KVT (OpNode (_.VT _.RC:$src1),
2135 (_.VT _.RC:$src2),
Ayman Musa721d97f2017-06-27 12:08:37 +00002136 imm:$cc)),
2137 (i64 0)),
Simon Pilgrim64fff142017-07-16 18:37:23 +00002138 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rri) _.RC:$src1,
Ayman Musa721d97f2017-06-27 12:08:37 +00002139 _.RC:$src2,
2140 imm:$cc),
2141 NewInf.KRC)>;
Simon Pilgrim64fff142017-07-16 18:37:23 +00002142
Ayman Musa721d97f2017-06-27 12:08:37 +00002143 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
Simon Pilgrim64fff142017-07-16 18:37:23 +00002144 (_.KVT (OpNode (_.VT _.RC:$src1),
Ayman Musa721d97f2017-06-27 12:08:37 +00002145 (_.VT (bitconvert (_.LdFrag addr:$src2))),
2146 imm:$cc)),
2147 (i64 0)),
2148 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rmi) _.RC:$src1,
2149 addr:$src2,
2150 imm:$cc),
2151 NewInf.KRC)>;
Simon Pilgrim64fff142017-07-16 18:37:23 +00002152
Ayman Musa721d97f2017-06-27 12:08:37 +00002153 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
Simon Pilgrim64fff142017-07-16 18:37:23 +00002154 (_.KVT (and _.KRCWM:$mask,
Ayman Musa721d97f2017-06-27 12:08:37 +00002155 (OpNode (_.VT _.RC:$src1),
2156 (_.VT _.RC:$src2),
2157 imm:$cc))),
2158 (i64 0)),
2159 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rrik) _.KRCWM:$mask,
Simon Pilgrim64fff142017-07-16 18:37:23 +00002160 _.RC:$src1,
Ayman Musa721d97f2017-06-27 12:08:37 +00002161 _.RC:$src2,
2162 imm:$cc),
2163 NewInf.KRC)>;
Simon Pilgrim64fff142017-07-16 18:37:23 +00002164
Ayman Musa721d97f2017-06-27 12:08:37 +00002165 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
Simon Pilgrim64fff142017-07-16 18:37:23 +00002166 (_.KVT (and (_.KVT _.KRCWM:$mask),
2167 (_.KVT (OpNode (_.VT _.RC:$src1),
2168 (_.VT (bitconvert
Ayman Musa721d97f2017-06-27 12:08:37 +00002169 (_.LdFrag addr:$src2))),
2170 imm:$cc)))),
2171 (i64 0)),
Simon Pilgrim64fff142017-07-16 18:37:23 +00002172 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rmik) _.KRCWM:$mask,
Ayman Musa721d97f2017-06-27 12:08:37 +00002173 _.RC:$src1,
2174 addr:$src2,
2175 imm:$cc),
2176 NewInf.KRC)>;
2177}
2178}
Simon Pilgrim64fff142017-07-16 18:37:23 +00002179
Ayman Musa721d97f2017-06-27 12:08:37 +00002180multiclass avx512_icmp_cc_packed_rmb_lowering<X86VectorVTInfo _, X86KVectorVTInfo NewInf,
2181 SDNode OpNode, string InstrStr,
Simon Pilgrim64fff142017-07-16 18:37:23 +00002182 list<Predicate> Preds>
Ayman Musa721d97f2017-06-27 12:08:37 +00002183 : avx512_icmp_cc_packed_lowering<_, NewInf, OpNode, InstrStr, Preds> {
2184let Predicates = Preds in {
2185 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
2186 (_.KVT (OpNode (_.VT _.RC:$src1),
2187 (X86VBroadcast (_.ScalarLdFrag addr:$src2)),
2188 imm:$cc)),
2189 (i64 0)),
2190 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rmib) _.RC:$src1,
2191 addr:$src2,
2192 imm:$cc),
2193 NewInf.KRC)>;
Simon Pilgrim64fff142017-07-16 18:37:23 +00002194
Ayman Musa721d97f2017-06-27 12:08:37 +00002195 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
2196 (_.KVT (and (_.KVT _.KRCWM:$mask),
2197 (_.KVT (OpNode (_.VT _.RC:$src1),
2198 (X86VBroadcast
2199 (_.ScalarLdFrag addr:$src2)),
2200 imm:$cc)))),
2201 (i64 0)),
2202 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rmibk) _.KRCWM:$mask,
2203 _.RC:$src1,
2204 addr:$src2,
2205 imm:$cc),
2206 NewInf.KRC)>;
2207}
2208}
2209
2210// VPCMPB - i8
2211defm : avx512_icmp_cc_packed_lowering<v16i8x_info, v32i1_info, X86cmpm,
2212 "VPCMPBZ128", [HasBWI, HasVLX]>;
2213defm : avx512_icmp_cc_packed_lowering<v16i8x_info, v64i1_info, X86cmpm,
2214 "VPCMPBZ128", [HasBWI, HasVLX]>;
2215
2216defm : avx512_icmp_cc_packed_lowering<v32i8x_info, v64i1_info, X86cmpm,
2217 "VPCMPBZ256", [HasBWI, HasVLX]>;
2218
2219// VPCMPW - i16
2220defm : avx512_icmp_cc_packed_lowering<v8i16x_info, v16i1_info, X86cmpm,
2221 "VPCMPWZ128", [HasBWI, HasVLX]>;
2222defm : avx512_icmp_cc_packed_lowering<v8i16x_info, v32i1_info, X86cmpm,
2223 "VPCMPWZ128", [HasBWI, HasVLX]>;
2224defm : avx512_icmp_cc_packed_lowering<v8i16x_info, v64i1_info, X86cmpm,
2225 "VPCMPWZ128", [HasBWI, HasVLX]>;
2226
2227defm : avx512_icmp_cc_packed_lowering<v16i16x_info, v32i1_info, X86cmpm,
2228 "VPCMPWZ256", [HasBWI, HasVLX]>;
2229defm : avx512_icmp_cc_packed_lowering<v16i16x_info, v64i1_info, X86cmpm,
2230 "VPCMPWZ256", [HasBWI, HasVLX]>;
2231
2232defm : avx512_icmp_cc_packed_lowering<v32i16_info, v64i1_info, X86cmpm,
2233 "VPCMPWZ", [HasBWI]>;
2234
2235// VPCMPD - i32
2236defm : avx512_icmp_cc_packed_rmb_lowering<v4i32x_info, v8i1_info, X86cmpm,
2237 "VPCMPDZ128", [HasAVX512, HasVLX]>;
2238defm : avx512_icmp_cc_packed_rmb_lowering<v4i32x_info, v16i1_info, X86cmpm,
2239 "VPCMPDZ128", [HasAVX512, HasVLX]>;
2240defm : avx512_icmp_cc_packed_rmb_lowering<v4i32x_info, v32i1_info, X86cmpm,
2241 "VPCMPDZ128", [HasAVX512, HasVLX]>;
2242defm : avx512_icmp_cc_packed_rmb_lowering<v4i32x_info, v64i1_info, X86cmpm,
2243 "VPCMPDZ128", [HasAVX512, HasVLX]>;
2244
2245defm : avx512_icmp_cc_packed_rmb_lowering<v8i32x_info, v16i1_info, X86cmpm,
2246 "VPCMPDZ256", [HasAVX512, HasVLX]>;
2247defm : avx512_icmp_cc_packed_rmb_lowering<v8i32x_info, v32i1_info, X86cmpm,
2248 "VPCMPDZ256", [HasAVX512, HasVLX]>;
2249defm : avx512_icmp_cc_packed_rmb_lowering<v8i32x_info, v64i1_info, X86cmpm,
2250 "VPCMPDZ256", [HasAVX512, HasVLX]>;
2251
2252defm : avx512_icmp_cc_packed_rmb_lowering<v16i32_info, v32i1_info, X86cmpm,
2253 "VPCMPDZ", [HasAVX512]>;
2254defm : avx512_icmp_cc_packed_rmb_lowering<v16i32_info, v64i1_info, X86cmpm,
2255 "VPCMPDZ", [HasAVX512]>;
2256
2257// VPCMPQ - i64
2258defm : avx512_icmp_cc_packed_rmb_lowering<v2i64x_info, v4i1_info, X86cmpm,
2259 "VPCMPQZ128", [HasAVX512, HasVLX]>;
2260defm : avx512_icmp_cc_packed_rmb_lowering<v2i64x_info, v8i1_info, X86cmpm,
2261 "VPCMPQZ128", [HasAVX512, HasVLX]>;
2262defm : avx512_icmp_cc_packed_rmb_lowering<v2i64x_info, v16i1_info, X86cmpm,
2263 "VPCMPQZ128", [HasAVX512, HasVLX]>;
2264defm : avx512_icmp_cc_packed_rmb_lowering<v2i64x_info, v32i1_info, X86cmpm,
2265 "VPCMPQZ128", [HasAVX512, HasVLX]>;
2266defm : avx512_icmp_cc_packed_rmb_lowering<v2i64x_info, v64i1_info, X86cmpm,
2267 "VPCMPQZ128", [HasAVX512, HasVLX]>;
2268
2269defm : avx512_icmp_cc_packed_rmb_lowering<v4i64x_info, v8i1_info, X86cmpm,
2270 "VPCMPQZ256", [HasAVX512, HasVLX]>;
2271defm : avx512_icmp_cc_packed_rmb_lowering<v4i64x_info, v16i1_info, X86cmpm,
2272 "VPCMPQZ256", [HasAVX512, HasVLX]>;
2273defm : avx512_icmp_cc_packed_rmb_lowering<v4i64x_info, v32i1_info, X86cmpm,
2274 "VPCMPQZ256", [HasAVX512, HasVLX]>;
2275defm : avx512_icmp_cc_packed_rmb_lowering<v4i64x_info, v64i1_info, X86cmpm,
2276 "VPCMPQZ256", [HasAVX512, HasVLX]>;
2277
2278defm : avx512_icmp_cc_packed_rmb_lowering<v8i64_info, v16i1_info, X86cmpm,
2279 "VPCMPQZ", [HasAVX512]>;
2280defm : avx512_icmp_cc_packed_rmb_lowering<v8i64_info, v32i1_info, X86cmpm,
2281 "VPCMPQZ", [HasAVX512]>;
2282defm : avx512_icmp_cc_packed_rmb_lowering<v8i64_info, v64i1_info, X86cmpm,
2283 "VPCMPQZ", [HasAVX512]>;
2284
2285// VPCMPUB - i8
2286defm : avx512_icmp_cc_packed_lowering<v16i8x_info, v32i1_info, X86cmpmu,
2287 "VPCMPUBZ128", [HasBWI, HasVLX]>;
2288defm : avx512_icmp_cc_packed_lowering<v16i8x_info, v64i1_info, X86cmpmu,
2289 "VPCMPUBZ128", [HasBWI, HasVLX]>;
2290
2291defm : avx512_icmp_cc_packed_lowering<v32i8x_info, v64i1_info, X86cmpmu,
2292 "VPCMPUBZ256", [HasBWI, HasVLX]>;
2293
2294// VPCMPUW - i16
2295defm : avx512_icmp_cc_packed_lowering<v8i16x_info, v16i1_info, X86cmpmu,
2296 "VPCMPUWZ128", [HasBWI, HasVLX]>;
2297defm : avx512_icmp_cc_packed_lowering<v8i16x_info, v32i1_info, X86cmpmu,
2298 "VPCMPUWZ128", [HasBWI, HasVLX]>;
2299defm : avx512_icmp_cc_packed_lowering<v8i16x_info, v64i1_info, X86cmpmu,
2300 "VPCMPUWZ128", [HasBWI, HasVLX]>;
2301
2302defm : avx512_icmp_cc_packed_lowering<v16i16x_info, v32i1_info, X86cmpmu,
2303 "VPCMPUWZ256", [HasBWI, HasVLX]>;
2304defm : avx512_icmp_cc_packed_lowering<v16i16x_info, v64i1_info, X86cmpmu,
2305 "VPCMPUWZ256", [HasBWI, HasVLX]>;
2306
2307defm : avx512_icmp_cc_packed_lowering<v32i16_info, v64i1_info, X86cmpmu,
2308 "VPCMPUWZ", [HasBWI]>;
2309
2310// VPCMPUD - i32
2311defm : avx512_icmp_cc_packed_rmb_lowering<v4i32x_info, v8i1_info, X86cmpmu,
2312 "VPCMPUDZ128", [HasAVX512, HasVLX]>;
2313defm : avx512_icmp_cc_packed_rmb_lowering<v4i32x_info, v16i1_info, X86cmpmu,
2314 "VPCMPUDZ128", [HasAVX512, HasVLX]>;
2315defm : avx512_icmp_cc_packed_rmb_lowering<v4i32x_info, v32i1_info, X86cmpmu,
2316 "VPCMPUDZ128", [HasAVX512, HasVLX]>;
2317defm : avx512_icmp_cc_packed_rmb_lowering<v4i32x_info, v64i1_info, X86cmpmu,
2318 "VPCMPUDZ128", [HasAVX512, HasVLX]>;
2319
2320defm : avx512_icmp_cc_packed_rmb_lowering<v8i32x_info, v16i1_info, X86cmpmu,
2321 "VPCMPUDZ256", [HasAVX512, HasVLX]>;
2322defm : avx512_icmp_cc_packed_rmb_lowering<v8i32x_info, v32i1_info, X86cmpmu,
2323 "VPCMPUDZ256", [HasAVX512, HasVLX]>;
2324defm : avx512_icmp_cc_packed_rmb_lowering<v8i32x_info, v64i1_info, X86cmpmu,
2325 "VPCMPUDZ256", [HasAVX512, HasVLX]>;
2326
2327defm : avx512_icmp_cc_packed_rmb_lowering<v16i32_info, v32i1_info, X86cmpmu,
2328 "VPCMPUDZ", [HasAVX512]>;
2329defm : avx512_icmp_cc_packed_rmb_lowering<v16i32_info, v64i1_info, X86cmpmu,
2330 "VPCMPUDZ", [HasAVX512]>;
2331
2332// VPCMPUQ - i64
2333defm : avx512_icmp_cc_packed_rmb_lowering<v2i64x_info, v4i1_info, X86cmpmu,
2334 "VPCMPUQZ128", [HasAVX512, HasVLX]>;
2335defm : avx512_icmp_cc_packed_rmb_lowering<v2i64x_info, v8i1_info, X86cmpmu,
2336 "VPCMPUQZ128", [HasAVX512, HasVLX]>;
2337defm : avx512_icmp_cc_packed_rmb_lowering<v2i64x_info, v16i1_info, X86cmpmu,
2338 "VPCMPUQZ128", [HasAVX512, HasVLX]>;
2339defm : avx512_icmp_cc_packed_rmb_lowering<v2i64x_info, v32i1_info, X86cmpmu,
2340 "VPCMPUQZ128", [HasAVX512, HasVLX]>;
2341defm : avx512_icmp_cc_packed_rmb_lowering<v2i64x_info, v64i1_info, X86cmpmu,
2342 "VPCMPUQZ128", [HasAVX512, HasVLX]>;
2343
2344defm : avx512_icmp_cc_packed_rmb_lowering<v4i64x_info, v8i1_info, X86cmpmu,
2345 "VPCMPUQZ256", [HasAVX512, HasVLX]>;
2346defm : avx512_icmp_cc_packed_rmb_lowering<v4i64x_info, v16i1_info, X86cmpmu,
2347 "VPCMPUQZ256", [HasAVX512, HasVLX]>;
2348defm : avx512_icmp_cc_packed_rmb_lowering<v4i64x_info, v32i1_info, X86cmpmu,
2349 "VPCMPUQZ256", [HasAVX512, HasVLX]>;
2350defm : avx512_icmp_cc_packed_rmb_lowering<v4i64x_info, v64i1_info, X86cmpmu,
2351 "VPCMPUQZ256", [HasAVX512, HasVLX]>;
2352
2353defm : avx512_icmp_cc_packed_rmb_lowering<v8i64_info, v16i1_info, X86cmpmu,
2354 "VPCMPUQZ", [HasAVX512]>;
2355defm : avx512_icmp_cc_packed_rmb_lowering<v8i64_info, v32i1_info, X86cmpmu,
2356 "VPCMPUQZ", [HasAVX512]>;
2357defm : avx512_icmp_cc_packed_rmb_lowering<v8i64_info, v64i1_info, X86cmpmu,
2358 "VPCMPUQZ", [HasAVX512]>;
2359
Elena Demikhovsky29792e92015-05-07 11:24:42 +00002360multiclass avx512_vcmp_common<X86VectorVTInfo _> {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002361
Elena Demikhovsky29792e92015-05-07 11:24:42 +00002362 defm rri : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
2363 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2,AVXCC:$cc),
2364 "vcmp${cc}"#_.Suffix,
2365 "$src2, $src1", "$src1, $src2",
2366 (X86cmpm (_.VT _.RC:$src1),
2367 (_.VT _.RC:$src2),
Craig Topper225da2c2016-08-27 05:22:15 +00002368 imm:$cc), 1>;
Elena Demikhovsky29792e92015-05-07 11:24:42 +00002369
Craig Toppere1cac152016-06-07 07:27:54 +00002370 defm rmi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,
2371 (outs _.KRC:$dst),(ins _.RC:$src1, _.MemOp:$src2, AVXCC:$cc),
2372 "vcmp${cc}"#_.Suffix,
2373 "$src2, $src1", "$src1, $src2",
2374 (X86cmpm (_.VT _.RC:$src1),
2375 (_.VT (bitconvert (_.LdFrag addr:$src2))),
2376 imm:$cc)>;
Elena Demikhovsky29792e92015-05-07 11:24:42 +00002377
Craig Toppere1cac152016-06-07 07:27:54 +00002378 defm rmbi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,
2379 (outs _.KRC:$dst),
2380 (ins _.RC:$src1, _.ScalarMemOp:$src2, AVXCC:$cc),
2381 "vcmp${cc}"#_.Suffix,
2382 "${src2}"##_.BroadcastStr##", $src1",
2383 "$src1, ${src2}"##_.BroadcastStr,
2384 (X86cmpm (_.VT _.RC:$src1),
2385 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
2386 imm:$cc)>,EVEX_B;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002387 // Accept explicit immediate argument form instead of comparison code.
Craig Topper0550ce72014-01-05 04:55:55 +00002388 let isAsmParserOnly = 1, hasSideEffects = 0 in {
Elena Demikhovsky29792e92015-05-07 11:24:42 +00002389 defm rri_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _,
2390 (outs _.KRC:$dst),
2391 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
2392 "vcmp"#_.Suffix,
2393 "$cc, $src2, $src1", "$src1, $src2, $cc">;
2394
2395 let mayLoad = 1 in {
2396 defm rmi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _,
2397 (outs _.KRC:$dst),
2398 (ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc),
2399 "vcmp"#_.Suffix,
2400 "$cc, $src2, $src1", "$src1, $src2, $cc">;
2401
2402 defm rmbi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _,
2403 (outs _.KRC:$dst),
2404 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$cc),
2405 "vcmp"#_.Suffix,
2406 "$cc, ${src2}"##_.BroadcastStr##", $src1",
2407 "$src1, ${src2}"##_.BroadcastStr##", $cc">,EVEX_B;
2408 }
2409 }
2410}
2411
2412multiclass avx512_vcmp_sae<X86VectorVTInfo _> {
2413 // comparison code form (VCMP[EQ/LT/LE/...]
2414 defm rrib : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
2415 (outs _.KRC:$dst),(ins _.RC:$src1, _.RC:$src2, AVXCC:$cc),
2416 "vcmp${cc}"#_.Suffix,
Craig Topperbfe13ff2016-01-11 00:44:52 +00002417 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
Elena Demikhovsky29792e92015-05-07 11:24:42 +00002418 (X86cmpmRnd (_.VT _.RC:$src1),
2419 (_.VT _.RC:$src2),
2420 imm:$cc,
2421 (i32 FROUND_NO_EXC))>, EVEX_B;
2422
2423 let isAsmParserOnly = 1, hasSideEffects = 0 in {
2424 defm rrib_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _,
2425 (outs _.KRC:$dst),
2426 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
2427 "vcmp"#_.Suffix,
Craig Topperbfe13ff2016-01-11 00:44:52 +00002428 "$cc, {sae}, $src2, $src1",
2429 "$src1, $src2, {sae}, $cc">, EVEX_B;
Elena Demikhovsky29792e92015-05-07 11:24:42 +00002430 }
2431}
2432
2433multiclass avx512_vcmp<AVX512VLVectorVTInfo _> {
2434 let Predicates = [HasAVX512] in {
2435 defm Z : avx512_vcmp_common<_.info512>,
2436 avx512_vcmp_sae<_.info512>, EVEX_V512;
2437
2438 }
2439 let Predicates = [HasAVX512,HasVLX] in {
2440 defm Z128 : avx512_vcmp_common<_.info128>, EVEX_V128;
2441 defm Z256 : avx512_vcmp_common<_.info256>, EVEX_V256;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002442 }
2443}
2444
Elena Demikhovsky29792e92015-05-07 11:24:42 +00002445defm VCMPPD : avx512_vcmp<avx512vl_f64_info>,
2446 AVX512PDIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
2447defm VCMPPS : avx512_vcmp<avx512vl_f32_info>,
2448 AVX512PSIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002449
Ayman Musa721d97f2017-06-27 12:08:37 +00002450multiclass avx512_fcmp_cc_packed_lowering<X86VectorVTInfo _, X86KVectorVTInfo NewInf,
2451 string InstrStr, list<Predicate> Preds> {
2452let Predicates = Preds in {
2453 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
Simon Pilgrim64fff142017-07-16 18:37:23 +00002454 (_.KVT (X86cmpm (_.VT _.RC:$src1),
2455 (_.VT _.RC:$src2),
Ayman Musa721d97f2017-06-27 12:08:37 +00002456 imm:$cc)),
2457 (i64 0)),
Simon Pilgrim64fff142017-07-16 18:37:23 +00002458 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rri) _.RC:$src1,
Ayman Musa721d97f2017-06-27 12:08:37 +00002459 _.RC:$src2,
2460 imm:$cc),
2461 NewInf.KRC)>;
Simon Pilgrim64fff142017-07-16 18:37:23 +00002462
Ayman Musa721d97f2017-06-27 12:08:37 +00002463 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
Ayman Musab16ce772017-07-24 08:10:32 +00002464 (_.KVT (and _.KRCWM:$mask,
2465 (X86cmpm (_.VT _.RC:$src1),
2466 (_.VT _.RC:$src2),
2467 imm:$cc))),
2468 (i64 0)),
2469 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rrik) _.KRCWM:$mask,
2470 _.RC:$src1,
2471 _.RC:$src2,
2472 imm:$cc),
2473 NewInf.KRC)>;
2474
2475 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
2476 (_.KVT (X86cmpm (_.VT _.RC:$src1),
Ayman Musa721d97f2017-06-27 12:08:37 +00002477 (_.VT (bitconvert (_.LdFrag addr:$src2))),
2478 imm:$cc)),
2479 (i64 0)),
2480 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rmi) _.RC:$src1,
2481 addr:$src2,
2482 imm:$cc),
2483 NewInf.KRC)>;
2484
2485 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
Ayman Musab16ce772017-07-24 08:10:32 +00002486 (_.KVT (and _.KRCWM:$mask,
2487 (X86cmpm (_.VT _.RC:$src1),
2488 (_.VT (bitconvert
2489 (_.LdFrag addr:$src2))),
2490 imm:$cc))),
2491 (i64 0)),
2492 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rmik) _.KRCWM:$mask,
2493 _.RC:$src1,
2494 addr:$src2,
2495 imm:$cc),
2496 NewInf.KRC)>;
2497
2498 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
Ayman Musa721d97f2017-06-27 12:08:37 +00002499 (_.KVT (X86cmpm (_.VT _.RC:$src1),
2500 (X86VBroadcast (_.ScalarLdFrag addr:$src2)),
2501 imm:$cc)),
2502 (i64 0)),
2503 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rmbi) _.RC:$src1,
2504 addr:$src2,
2505 imm:$cc),
2506 NewInf.KRC)>;
Ayman Musab16ce772017-07-24 08:10:32 +00002507
2508 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
2509 (_.KVT (and _.KRCWM:$mask,
2510 (X86cmpm (_.VT _.RC:$src1),
2511 (X86VBroadcast
2512 (_.ScalarLdFrag addr:$src2)),
2513 imm:$cc))),
2514 (i64 0)),
2515 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rmbik) _.KRCWM:$mask,
2516 _.RC:$src1,
2517 addr:$src2,
2518 imm:$cc),
2519 NewInf.KRC)>;
Ayman Musa721d97f2017-06-27 12:08:37 +00002520}
2521}
Simon Pilgrim64fff142017-07-16 18:37:23 +00002522
Ayman Musa721d97f2017-06-27 12:08:37 +00002523multiclass avx512_fcmp_cc_packed_sae_lowering<X86VectorVTInfo _, X86KVectorVTInfo NewInf,
Simon Pilgrim64fff142017-07-16 18:37:23 +00002524 string InstrStr, list<Predicate> Preds>
Ayman Musa721d97f2017-06-27 12:08:37 +00002525 : avx512_fcmp_cc_packed_lowering<_, NewInf, InstrStr, Preds> {
2526
2527let Predicates = Preds in
2528 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
Simon Pilgrim64fff142017-07-16 18:37:23 +00002529 (_.KVT (X86cmpmRnd (_.VT _.RC:$src1),
2530 (_.VT _.RC:$src2),
Ayman Musa721d97f2017-06-27 12:08:37 +00002531 imm:$cc,
2532 (i32 FROUND_NO_EXC))),
2533 (i64 0)),
Simon Pilgrim64fff142017-07-16 18:37:23 +00002534 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rrib) _.RC:$src1,
Ayman Musa721d97f2017-06-27 12:08:37 +00002535 _.RC:$src2,
2536 imm:$cc),
2537 NewInf.KRC)>;
Ayman Musab16ce772017-07-24 08:10:32 +00002538
2539 def : Pat<(insert_subvector (NewInf.KVT immAllZerosV),
2540 (_.KVT (and _.KRCWM:$mask,
2541 (X86cmpmRnd (_.VT _.RC:$src1),
2542 (_.VT _.RC:$src2),
2543 imm:$cc,
2544 (i32 FROUND_NO_EXC)))),
2545 (i64 0)),
2546 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr##rribk) _.KRCWM:$mask,
2547 _.RC:$src1,
2548 _.RC:$src2,
2549 imm:$cc),
2550 NewInf.KRC)>;
Ayman Musa721d97f2017-06-27 12:08:37 +00002551}
2552
2553
2554// VCMPPS - f32
2555defm : avx512_fcmp_cc_packed_lowering<v4f32x_info, v8i1_info, "VCMPPSZ128",
2556 [HasAVX512, HasVLX]>;
2557defm : avx512_fcmp_cc_packed_lowering<v4f32x_info, v16i1_info, "VCMPPSZ128",
2558 [HasAVX512, HasVLX]>;
2559defm : avx512_fcmp_cc_packed_lowering<v4f32x_info, v32i1_info, "VCMPPSZ128",
2560 [HasAVX512, HasVLX]>;
2561defm : avx512_fcmp_cc_packed_lowering<v4f32x_info, v64i1_info, "VCMPPSZ128",
2562 [HasAVX512, HasVLX]>;
2563
2564defm : avx512_fcmp_cc_packed_lowering<v8f32x_info, v16i1_info, "VCMPPSZ256",
2565 [HasAVX512, HasVLX]>;
2566defm : avx512_fcmp_cc_packed_lowering<v8f32x_info, v32i1_info, "VCMPPSZ256",
2567 [HasAVX512, HasVLX]>;
2568defm : avx512_fcmp_cc_packed_lowering<v8f32x_info, v64i1_info, "VCMPPSZ256",
2569 [HasAVX512, HasVLX]>;
2570
2571defm : avx512_fcmp_cc_packed_sae_lowering<v16f32_info, v32i1_info, "VCMPPSZ",
2572 [HasAVX512]>;
2573defm : avx512_fcmp_cc_packed_sae_lowering<v16f32_info, v64i1_info, "VCMPPSZ",
2574 [HasAVX512]>;
2575
2576// VCMPPD - f64
2577defm : avx512_fcmp_cc_packed_lowering<v2f64x_info, v4i1_info, "VCMPPDZ128",
2578 [HasAVX512, HasVLX]>;
2579defm : avx512_fcmp_cc_packed_lowering<v2f64x_info, v8i1_info, "VCMPPDZ128",
2580 [HasAVX512, HasVLX]>;
2581defm : avx512_fcmp_cc_packed_lowering<v2f64x_info, v16i1_info, "VCMPPDZ128",
2582 [HasAVX512, HasVLX]>;
2583defm : avx512_fcmp_cc_packed_lowering<v2f64x_info, v32i1_info, "VCMPPDZ128",
2584 [HasAVX512, HasVLX]>;
2585defm : avx512_fcmp_cc_packed_lowering<v2f64x_info, v64i1_info, "VCMPPDZ128",
2586 [HasAVX512, HasVLX]>;
2587
2588defm : avx512_fcmp_cc_packed_lowering<v4f64x_info, v8i1_info, "VCMPPDZ256",
2589 [HasAVX512, HasVLX]>;
2590defm : avx512_fcmp_cc_packed_lowering<v4f64x_info, v16i1_info, "VCMPPDZ256",
2591 [HasAVX512, HasVLX]>;
2592defm : avx512_fcmp_cc_packed_lowering<v4f64x_info, v32i1_info, "VCMPPDZ256",
2593 [HasAVX512, HasVLX]>;
2594defm : avx512_fcmp_cc_packed_lowering<v4f64x_info, v64i1_info, "VCMPPDZ256",
2595 [HasAVX512, HasVLX]>;
2596
2597defm : avx512_fcmp_cc_packed_sae_lowering<v8f64_info, v16i1_info, "VCMPPDZ",
2598 [HasAVX512]>;
2599defm : avx512_fcmp_cc_packed_sae_lowering<v8f64_info, v32i1_info, "VCMPPDZ",
2600 [HasAVX512]>;
2601defm : avx512_fcmp_cc_packed_sae_lowering<v8f64_info, v64i1_info, "VCMPPDZ",
2602 [HasAVX512]>;
Elena Demikhovskyde3f7512014-01-01 15:12:34 +00002603
Asaf Badouh572bbce2015-09-20 08:46:07 +00002604// ----------------------------------------------------------------
2605// FPClass
Asaf Badouh696e8e02015-10-18 11:04:38 +00002606//handle fpclass instruction mask = op(reg_scalar,imm)
2607// op(mem_scalar,imm)
2608multiclass avx512_scalar_fpclass<bits<8> opc, string OpcodeStr, SDNode OpNode,
2609 X86VectorVTInfo _, Predicate prd> {
2610 let Predicates = [prd] in {
2611 def rr : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),//_.KRC:$dst),
2612 (ins _.RC:$src1, i32u8imm:$src2),
Craig Topper048e7002016-01-08 06:09:20 +00002613 OpcodeStr##_.Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Asaf Badouh696e8e02015-10-18 11:04:38 +00002614 [(set _.KRC:$dst,(OpNode (_.VT _.RC:$src1),
2615 (i32 imm:$src2)))], NoItinerary>;
2616 def rrk : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
2617 (ins _.KRCWM:$mask, _.RC:$src1, i32u8imm:$src2),
2618 OpcodeStr##_.Suffix#
Craig Topper048e7002016-01-08 06:09:20 +00002619 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00002620 [(set _.KRC:$dst,(or _.KRCWM:$mask,
Asaf Badouh696e8e02015-10-18 11:04:38 +00002621 (OpNode (_.VT _.RC:$src1),
2622 (i32 imm:$src2))))], NoItinerary>, EVEX_K;
Craig Topper63801df2017-02-19 21:44:35 +00002623 def rm : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
2624 (ins _.MemOp:$src1, i32u8imm:$src2),
2625 OpcodeStr##_.Suffix##
2626 "\t{$src2, $src1, $dst|$dst, $src1, $src2}",
2627 [(set _.KRC:$dst,
Asaf Badouh696e8e02015-10-18 11:04:38 +00002628 (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
Craig Topper63801df2017-02-19 21:44:35 +00002629 (i32 imm:$src2)))], NoItinerary>;
2630 def rmk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
2631 (ins _.KRCWM:$mask, _.MemOp:$src1, i32u8imm:$src2),
2632 OpcodeStr##_.Suffix##
2633 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
2634 [(set _.KRC:$dst,(or _.KRCWM:$mask,
2635 (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
2636 (i32 imm:$src2))))], NoItinerary>, EVEX_K;
Asaf Badouh696e8e02015-10-18 11:04:38 +00002637 }
2638}
2639
Asaf Badouh572bbce2015-09-20 08:46:07 +00002640//handle fpclass instruction mask = fpclass(reg_vec, reg_vec, imm)
2641// fpclass(reg_vec, mem_vec, imm)
2642// fpclass(reg_vec, broadcast(eltVt), imm)
2643multiclass avx512_vector_fpclass<bits<8> opc, string OpcodeStr, SDNode OpNode,
2644 X86VectorVTInfo _, string mem, string broadcast>{
2645 def rr : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
2646 (ins _.RC:$src1, i32u8imm:$src2),
Craig Topper048e7002016-01-08 06:09:20 +00002647 OpcodeStr##_.Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Asaf Badouh572bbce2015-09-20 08:46:07 +00002648 [(set _.KRC:$dst,(OpNode (_.VT _.RC:$src1),
2649 (i32 imm:$src2)))], NoItinerary>;
2650 def rrk : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
2651 (ins _.KRCWM:$mask, _.RC:$src1, i32u8imm:$src2),
2652 OpcodeStr##_.Suffix#
Craig Topper048e7002016-01-08 06:09:20 +00002653 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00002654 [(set _.KRC:$dst,(or _.KRCWM:$mask,
Asaf Badouh572bbce2015-09-20 08:46:07 +00002655 (OpNode (_.VT _.RC:$src1),
2656 (i32 imm:$src2))))], NoItinerary>, EVEX_K;
Craig Toppere1cac152016-06-07 07:27:54 +00002657 def rm : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
2658 (ins _.MemOp:$src1, i32u8imm:$src2),
2659 OpcodeStr##_.Suffix##mem#
2660 "\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00002661 [(set _.KRC:$dst,(OpNode
Craig Toppere1cac152016-06-07 07:27:54 +00002662 (_.VT (bitconvert (_.LdFrag addr:$src1))),
2663 (i32 imm:$src2)))], NoItinerary>;
2664 def rmk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
2665 (ins _.KRCWM:$mask, _.MemOp:$src1, i32u8imm:$src2),
2666 OpcodeStr##_.Suffix##mem#
2667 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00002668 [(set _.KRC:$dst, (or _.KRCWM:$mask, (OpNode
Craig Toppere1cac152016-06-07 07:27:54 +00002669 (_.VT (bitconvert (_.LdFrag addr:$src1))),
2670 (i32 imm:$src2))))], NoItinerary>, EVEX_K;
2671 def rmb : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
2672 (ins _.ScalarMemOp:$src1, i32u8imm:$src2),
2673 OpcodeStr##_.Suffix##broadcast##"\t{$src2, ${src1}"##
2674 _.BroadcastStr##", $dst|$dst, ${src1}"
2675 ##_.BroadcastStr##", $src2}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00002676 [(set _.KRC:$dst,(OpNode
2677 (_.VT (X86VBroadcast
Craig Toppere1cac152016-06-07 07:27:54 +00002678 (_.ScalarLdFrag addr:$src1))),
2679 (i32 imm:$src2)))], NoItinerary>,EVEX_B;
2680 def rmbk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
2681 (ins _.KRCWM:$mask, _.ScalarMemOp:$src1, i32u8imm:$src2),
2682 OpcodeStr##_.Suffix##broadcast##"\t{$src2, ${src1}"##
2683 _.BroadcastStr##", $dst {${mask}}|$dst {${mask}}, ${src1}"##
2684 _.BroadcastStr##", $src2}",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00002685 [(set _.KRC:$dst,(or _.KRCWM:$mask, (OpNode
2686 (_.VT (X86VBroadcast
Craig Toppere1cac152016-06-07 07:27:54 +00002687 (_.ScalarLdFrag addr:$src1))),
2688 (i32 imm:$src2))))], NoItinerary>,
2689 EVEX_B, EVEX_K;
Asaf Badouh572bbce2015-09-20 08:46:07 +00002690}
2691
Asaf Badouh572bbce2015-09-20 08:46:07 +00002692multiclass avx512_vector_fpclass_all<string OpcodeStr,
Simon Pilgrimb13961d2016-06-11 14:34:10 +00002693 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd,
Asaf Badouh572bbce2015-09-20 08:46:07 +00002694 string broadcast>{
2695 let Predicates = [prd] in {
Simon Pilgrimb13961d2016-06-11 14:34:10 +00002696 defm Z : avx512_vector_fpclass<opc, OpcodeStr, OpNode, _.info512, "{z}",
Asaf Badouh572bbce2015-09-20 08:46:07 +00002697 broadcast>, EVEX_V512;
2698 }
2699 let Predicates = [prd, HasVLX] in {
2700 defm Z128 : avx512_vector_fpclass<opc, OpcodeStr, OpNode, _.info128, "{x}",
2701 broadcast>, EVEX_V128;
2702 defm Z256 : avx512_vector_fpclass<opc, OpcodeStr, OpNode, _.info256, "{y}",
2703 broadcast>, EVEX_V256;
2704 }
2705}
2706
2707multiclass avx512_fp_fpclass_all<string OpcodeStr, bits<8> opcVec,
Asaf Badouh696e8e02015-10-18 11:04:38 +00002708 bits<8> opcScalar, SDNode VecOpNode, SDNode ScalarOpNode, Predicate prd>{
Simon Pilgrim18bcf932016-02-03 09:41:59 +00002709 defm PS : avx512_vector_fpclass_all<OpcodeStr, avx512vl_f32_info, opcVec,
Asaf Badouh696e8e02015-10-18 11:04:38 +00002710 VecOpNode, prd, "{l}">, EVEX_CD8<32, CD8VF>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00002711 defm PD : avx512_vector_fpclass_all<OpcodeStr, avx512vl_f64_info, opcVec,
Asaf Badouh696e8e02015-10-18 11:04:38 +00002712 VecOpNode, prd, "{q}">,EVEX_CD8<64, CD8VF> , VEX_W;
2713 defm SS : avx512_scalar_fpclass<opcScalar, OpcodeStr, ScalarOpNode,
2714 f32x_info, prd>, EVEX_CD8<32, CD8VT1>;
2715 defm SD : avx512_scalar_fpclass<opcScalar, OpcodeStr, ScalarOpNode,
2716 f64x_info, prd>, EVEX_CD8<64, CD8VT1>, VEX_W;
Asaf Badouh572bbce2015-09-20 08:46:07 +00002717}
2718
Asaf Badouh696e8e02015-10-18 11:04:38 +00002719defm VFPCLASS : avx512_fp_fpclass_all<"vfpclass", 0x66, 0x67, X86Vfpclass,
2720 X86Vfpclasss, HasDQI>, AVX512AIi8Base,EVEX;
Asaf Badouh572bbce2015-09-20 08:46:07 +00002721
Elena Demikhovsky29792e92015-05-07 11:24:42 +00002722//-----------------------------------------------------------------
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002723// Mask register copy, including
2724// - copy between mask registers
2725// - load/store mask registers
2726// - copy from GPR to mask register and vice versa
2727//
2728multiclass avx512_mask_mov<bits<8> opc_kk, bits<8> opc_km, bits<8> opc_mk,
2729 string OpcodeStr, RegisterClass KRC,
Elena Demikhovskyba846722015-02-17 09:20:12 +00002730 ValueType vvt, X86MemOperand x86memop> {
Craig Toppere1cac152016-06-07 07:27:54 +00002731 let hasSideEffects = 0 in
2732 def kk : I<opc_kk, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src),
2733 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>;
2734 def km : I<opc_km, MRMSrcMem, (outs KRC:$dst), (ins x86memop:$src),
2735 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
2736 [(set KRC:$dst, (vvt (load addr:$src)))]>;
2737 def mk : I<opc_mk, MRMDestMem, (outs), (ins x86memop:$dst, KRC:$src),
2738 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
2739 [(store KRC:$src, addr:$dst)]>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002740}
2741
2742multiclass avx512_mask_mov_gpr<bits<8> opc_kr, bits<8> opc_rk,
2743 string OpcodeStr,
2744 RegisterClass KRC, RegisterClass GRC> {
Elena Demikhovskyf404e052014-01-05 14:21:07 +00002745 let hasSideEffects = 0 in {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002746 def kr : I<opc_kr, MRMSrcReg, (outs KRC:$dst), (ins GRC:$src),
Craig Topperedb09112014-11-25 20:11:23 +00002747 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002748 def rk : I<opc_rk, MRMSrcReg, (outs GRC:$dst), (ins KRC:$src),
Craig Topperedb09112014-11-25 20:11:23 +00002749 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002750 }
2751}
2752
Robert Khasanov74acbb72014-07-23 14:49:42 +00002753let Predicates = [HasDQI] in
Elena Demikhovskyba846722015-02-17 09:20:12 +00002754 defm KMOVB : avx512_mask_mov<0x90, 0x90, 0x91, "kmovb", VK8, v8i1, i8mem>,
Robert Khasanov74acbb72014-07-23 14:49:42 +00002755 avx512_mask_mov_gpr<0x92, 0x93, "kmovb", VK8, GR32>,
2756 VEX, PD;
2757
2758let Predicates = [HasAVX512] in
Elena Demikhovskyba846722015-02-17 09:20:12 +00002759 defm KMOVW : avx512_mask_mov<0x90, 0x90, 0x91, "kmovw", VK16, v16i1, i16mem>,
Robert Khasanov74acbb72014-07-23 14:49:42 +00002760 avx512_mask_mov_gpr<0x92, 0x93, "kmovw", VK16, GR32>,
Craig Topper5ccb6172014-02-18 00:21:49 +00002761 VEX, PS;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002762
2763let Predicates = [HasBWI] in {
Elena Demikhovskyba846722015-02-17 09:20:12 +00002764 defm KMOVD : avx512_mask_mov<0x90, 0x90, 0x91, "kmovd", VK32, v32i1,i32mem>,
2765 VEX, PD, VEX_W;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002766 defm KMOVD : avx512_mask_mov_gpr<0x92, 0x93, "kmovd", VK32, GR32>,
2767 VEX, XD;
Elena Demikhovskyba846722015-02-17 09:20:12 +00002768 defm KMOVQ : avx512_mask_mov<0x90, 0x90, 0x91, "kmovq", VK64, v64i1, i64mem>,
2769 VEX, PS, VEX_W;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002770 defm KMOVQ : avx512_mask_mov_gpr<0x92, 0x93, "kmovq", VK64, GR64>,
2771 VEX, XD, VEX_W;
2772}
2773
2774// GR from/to mask register
Elena Demikhovskydca03be2016-08-07 13:05:58 +00002775def : Pat<(v16i1 (bitconvert (i16 GR16:$src))),
Craig Topper058f2f62017-03-28 16:35:29 +00002776 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), GR16:$src, sub_16bit)), VK16)>;
Elena Demikhovskydca03be2016-08-07 13:05:58 +00002777def : Pat<(i16 (bitconvert (v16i1 VK16:$src))),
Craig Topper058f2f62017-03-28 16:35:29 +00002778 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS VK16:$src, GR32)), sub_16bit)>;
Elena Demikhovskydca03be2016-08-07 13:05:58 +00002779
2780def : Pat<(v8i1 (bitconvert (i8 GR8:$src))),
Craig Topper058f2f62017-03-28 16:35:29 +00002781 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), GR8:$src, sub_8bit)), VK8)>;
Elena Demikhovskydca03be2016-08-07 13:05:58 +00002782def : Pat<(i8 (bitconvert (v8i1 VK8:$src))),
Craig Topper058f2f62017-03-28 16:35:29 +00002783 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS VK8:$src, GR32)), sub_8bit)>;
Elena Demikhovskydca03be2016-08-07 13:05:58 +00002784
2785def : Pat<(i32 (zext (i16 (bitconvert (v16i1 VK16:$src))))),
Igor Bregera2f8ca92016-09-05 08:26:51 +00002786 (KMOVWrk VK16:$src)>;
Elena Demikhovskydca03be2016-08-07 13:05:58 +00002787def : Pat<(i32 (anyext (i16 (bitconvert (v16i1 VK16:$src))))),
Craig Topper058f2f62017-03-28 16:35:29 +00002788 (COPY_TO_REGCLASS VK16:$src, GR32)>;
Elena Demikhovskydca03be2016-08-07 13:05:58 +00002789
2790def : Pat<(i32 (zext (i8 (bitconvert (v8i1 VK8:$src))))),
Craig Topper058f2f62017-03-28 16:35:29 +00002791 (MOVZX32rr8 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS VK8:$src, GR32)), sub_8bit))>, Requires<[NoDQI]>;
Igor Bregera2f8ca92016-09-05 08:26:51 +00002792def : Pat<(i32 (zext (i8 (bitconvert (v8i1 VK8:$src))))),
2793 (KMOVBrk VK8:$src)>, Requires<[HasDQI]>;
Elena Demikhovskydca03be2016-08-07 13:05:58 +00002794def : Pat<(i32 (anyext (i8 (bitconvert (v8i1 VK8:$src))))),
Craig Topper058f2f62017-03-28 16:35:29 +00002795 (COPY_TO_REGCLASS VK8:$src, GR32)>;
Elena Demikhovskydca03be2016-08-07 13:05:58 +00002796
2797def : Pat<(v32i1 (bitconvert (i32 GR32:$src))),
2798 (COPY_TO_REGCLASS GR32:$src, VK32)>;
2799def : Pat<(i32 (bitconvert (v32i1 VK32:$src))),
2800 (COPY_TO_REGCLASS VK32:$src, GR32)>;
2801def : Pat<(v64i1 (bitconvert (i64 GR64:$src))),
2802 (COPY_TO_REGCLASS GR64:$src, VK64)>;
2803def : Pat<(i64 (bitconvert (v64i1 VK64:$src))),
2804 (COPY_TO_REGCLASS VK64:$src, GR64)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002805
Robert Khasanov74acbb72014-07-23 14:49:42 +00002806// Load/store kreg
2807let Predicates = [HasDQI] in {
2808 def : Pat<(store (i8 (bitconvert (v8i1 VK8:$src))), addr:$dst),
2809 (KMOVBmk addr:$dst, VK8:$src)>;
Elena Demikhovskyba846722015-02-17 09:20:12 +00002810 def : Pat<(v8i1 (bitconvert (i8 (load addr:$src)))),
2811 (KMOVBkm addr:$src)>;
Elena Demikhovsky9f83c732015-09-02 09:20:58 +00002812
2813 def : Pat<(store VK4:$src, addr:$dst),
2814 (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK4:$src, VK8))>;
2815 def : Pat<(store VK2:$src, addr:$dst),
2816 (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK2:$src, VK8))>;
Igor Bregerd6c187b2016-01-27 08:43:25 +00002817 def : Pat<(store VK1:$src, addr:$dst),
2818 (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK1:$src, VK8))>;
Elena Demikhovsky5e426f72016-04-03 08:41:12 +00002819
2820 def : Pat<(v2i1 (load addr:$src)),
2821 (COPY_TO_REGCLASS (KMOVBkm addr:$src), VK2)>;
2822 def : Pat<(v4i1 (load addr:$src)),
2823 (COPY_TO_REGCLASS (KMOVBkm addr:$src), VK4)>;
Elena Demikhovskyba846722015-02-17 09:20:12 +00002824}
2825let Predicates = [HasAVX512, NoDQI] in {
Igor Bregerd6c187b2016-01-27 08:43:25 +00002826 def : Pat<(store VK1:$src, addr:$dst),
2827 (MOV8mr addr:$dst,
Craig Topperd9f51352017-03-29 07:31:56 +00002828 (i8 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS VK1:$src, GR32)),
2829 sub_8bit)))>;
Igor Bregerd6c187b2016-01-27 08:43:25 +00002830 def : Pat<(store VK2:$src, addr:$dst),
2831 (MOV8mr addr:$dst,
Craig Topperd9f51352017-03-29 07:31:56 +00002832 (i8 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS VK2:$src, GR32)),
2833 sub_8bit)))>;
Igor Bregerd6c187b2016-01-27 08:43:25 +00002834 def : Pat<(store VK4:$src, addr:$dst),
2835 (MOV8mr addr:$dst,
Craig Topperd9f51352017-03-29 07:31:56 +00002836 (i8 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS VK4:$src, GR32)),
2837 sub_8bit)))>;
Igor Bregerd6c187b2016-01-27 08:43:25 +00002838 def : Pat<(store VK8:$src, addr:$dst),
2839 (MOV8mr addr:$dst,
Craig Topperd9f51352017-03-29 07:31:56 +00002840 (i8 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS VK8:$src, GR32)),
2841 sub_8bit)))>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00002842
Elena Demikhovsky5e426f72016-04-03 08:41:12 +00002843 def : Pat<(v8i1 (load addr:$src)),
Craig Topper99e30e62016-06-14 03:13:00 +00002844 (COPY_TO_REGCLASS (MOVZX32rm8 addr:$src), VK8)>;
Elena Demikhovsky5e426f72016-04-03 08:41:12 +00002845 def : Pat<(v2i1 (load addr:$src)),
Craig Topper99e30e62016-06-14 03:13:00 +00002846 (COPY_TO_REGCLASS (MOVZX32rm8 addr:$src), VK2)>;
Elena Demikhovsky5e426f72016-04-03 08:41:12 +00002847 def : Pat<(v4i1 (load addr:$src)),
Craig Topper99e30e62016-06-14 03:13:00 +00002848 (COPY_TO_REGCLASS (MOVZX32rm8 addr:$src), VK4)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002849}
Elena Demikhovsky5e426f72016-04-03 08:41:12 +00002850
Robert Khasanov74acbb72014-07-23 14:49:42 +00002851let Predicates = [HasAVX512] in {
2852 def : Pat<(store (i16 (bitconvert (v16i1 VK16:$src))), addr:$dst),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002853 (KMOVWmk addr:$dst, VK16:$src)>;
Guy Blank548e22a2017-05-19 12:35:15 +00002854 def : Pat<(v1i1 (load addr:$src)),
Craig Topper34d97072016-06-14 03:13:03 +00002855 (COPY_TO_REGCLASS (AND32ri8 (MOVZX32rm8 addr:$src), (i32 1)), VK1)>;
Elena Demikhovskyba846722015-02-17 09:20:12 +00002856 def : Pat<(v16i1 (bitconvert (i16 (load addr:$src)))),
2857 (KMOVWkm addr:$src)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002858}
2859let Predicates = [HasBWI] in {
2860 def : Pat<(store (i32 (bitconvert (v32i1 VK32:$src))), addr:$dst),
2861 (KMOVDmk addr:$dst, VK32:$src)>;
Elena Demikhovskyba846722015-02-17 09:20:12 +00002862 def : Pat<(v32i1 (bitconvert (i32 (load addr:$src)))),
2863 (KMOVDkm addr:$src)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002864 def : Pat<(store (i64 (bitconvert (v64i1 VK64:$src))), addr:$dst),
2865 (KMOVQmk addr:$dst, VK64:$src)>;
Elena Demikhovskyba846722015-02-17 09:20:12 +00002866 def : Pat<(v64i1 (bitconvert (i64 (load addr:$src)))),
2867 (KMOVQkm addr:$src)>;
Robert Khasanov74acbb72014-07-23 14:49:42 +00002868}
Elena Demikhovskyc5f67262013-12-17 08:33:15 +00002869
Robert Khasanov74acbb72014-07-23 14:49:42 +00002870let Predicates = [HasAVX512] in {
Guy Blank548e22a2017-05-19 12:35:15 +00002871 multiclass operation_gpr_mask_copy_lowering<RegisterClass maskRC, ValueType maskVT> {
2872 def : Pat<(maskVT (scalar_to_vector GR32:$src)),
2873 (COPY_TO_REGCLASS GR32:$src, maskRC)>;
Elena Demikhovsky6e9b1602016-07-31 06:48:01 +00002874
Simon Pilgrim64fff142017-07-16 18:37:23 +00002875 def : Pat<(i32 (X86Vextract maskRC:$src, (iPTR 0))),
Guy Blank548e22a2017-05-19 12:35:15 +00002876 (COPY_TO_REGCLASS maskRC:$src, GR32)>;
Elena Demikhovsky6e9b1602016-07-31 06:48:01 +00002877
Guy Blank548e22a2017-05-19 12:35:15 +00002878 def : Pat<(maskVT (scalar_to_vector GR8:$src)),
2879 (COPY_TO_REGCLASS (INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR8:$src, sub_8bit), maskRC)>;
Elena Demikhovskyb906df92016-09-13 07:57:00 +00002880
Simon Pilgrim64fff142017-07-16 18:37:23 +00002881 def : Pat<(i8 (X86Vextract maskRC:$src, (iPTR 0))),
Guy Blank548e22a2017-05-19 12:35:15 +00002882 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS maskRC:$src, GR32)), sub_8bit)>;
Elena Demikhovskyb906df92016-09-13 07:57:00 +00002883
Simon Pilgrim64fff142017-07-16 18:37:23 +00002884 def : Pat<(i32 (anyext (i8 (X86Vextract maskRC:$src, (iPTR 0))))),
Guy Blank548e22a2017-05-19 12:35:15 +00002885 (COPY_TO_REGCLASS maskRC:$src, GR32)>;
2886 }
Elena Demikhovsky6e9b1602016-07-31 06:48:01 +00002887
Guy Blank548e22a2017-05-19 12:35:15 +00002888 defm : operation_gpr_mask_copy_lowering<VK1, v1i1>;
2889 defm : operation_gpr_mask_copy_lowering<VK2, v2i1>;
2890 defm : operation_gpr_mask_copy_lowering<VK4, v4i1>;
2891 defm : operation_gpr_mask_copy_lowering<VK8, v8i1>;
2892 defm : operation_gpr_mask_copy_lowering<VK16, v16i1>;
2893 defm : operation_gpr_mask_copy_lowering<VK32, v32i1>;
2894 defm : operation_gpr_mask_copy_lowering<VK64, v64i1>;
Elena Demikhovskyb906df92016-09-13 07:57:00 +00002895
Guy Blank548e22a2017-05-19 12:35:15 +00002896 def : Pat<(X86kshiftr (X86kshiftl (v1i1 (scalar_to_vector GR8:$src)), (i8 15)), (i8 15)) ,
2897 (COPY_TO_REGCLASS
2898 (KMOVWkr (AND32ri8 (INSERT_SUBREG (i32 (IMPLICIT_DEF)),
2899 GR8:$src, sub_8bit), (i32 1))), VK1)>;
2900 def : Pat<(X86kshiftr (X86kshiftl (v16i1 (scalar_to_vector GR8:$src)), (i8 15)), (i8 15)) ,
2901 (COPY_TO_REGCLASS
2902 (KMOVWkr (AND32ri8 (INSERT_SUBREG (i32 (IMPLICIT_DEF)),
2903 GR8:$src, sub_8bit), (i32 1))), VK16)>;
2904 def : Pat<(X86kshiftr (X86kshiftl (v8i1 (scalar_to_vector GR8:$src)), (i8 15)), (i8 15)) ,
2905 (COPY_TO_REGCLASS
2906 (KMOVWkr (AND32ri8 (INSERT_SUBREG (i32 (IMPLICIT_DEF)),
2907 GR8:$src, sub_8bit), (i32 1))), VK8)>;
Igor Bregerb7e1f9d2015-09-20 15:15:10 +00002908
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002909}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002910
2911// Mask unary operation
2912// - KNOT
2913multiclass avx512_mask_unop<bits<8> opc, string OpcodeStr,
Robert Khasanov74acbb72014-07-23 14:49:42 +00002914 RegisterClass KRC, SDPatternOperator OpNode,
2915 Predicate prd> {
2916 let Predicates = [prd] in
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002917 def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src),
Craig Topperedb09112014-11-25 20:11:23 +00002918 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002919 [(set KRC:$dst, (OpNode KRC:$src))]>;
2920}
2921
Robert Khasanov74acbb72014-07-23 14:49:42 +00002922multiclass avx512_mask_unop_all<bits<8> opc, string OpcodeStr,
2923 SDPatternOperator OpNode> {
2924 defm B : avx512_mask_unop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,
2925 HasDQI>, VEX, PD;
2926 defm W : avx512_mask_unop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,
2927 HasAVX512>, VEX, PS;
2928 defm D : avx512_mask_unop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,
2929 HasBWI>, VEX, PD, VEX_W;
2930 defm Q : avx512_mask_unop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,
2931 HasBWI>, VEX, PS, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002932}
2933
Craig Topper7b9cc142016-11-03 06:04:28 +00002934defm KNOT : avx512_mask_unop_all<0x44, "knot", vnot>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002935
Robert Khasanov74acbb72014-07-23 14:49:42 +00002936// KNL does not support KMOVB, 8-bit mask is promoted to 16-bit
Craig Topper7b9cc142016-11-03 06:04:28 +00002937let Predicates = [HasAVX512, NoDQI] in
2938def : Pat<(vnot VK8:$src),
2939 (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$src, VK16)), VK8)>;
2940
2941def : Pat<(vnot VK4:$src),
2942 (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK4:$src, VK16)), VK4)>;
2943def : Pat<(vnot VK2:$src),
2944 (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK2:$src, VK16)), VK2)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002945
2946// Mask binary operation
Elena Demikhovskye382c3f2013-12-10 13:53:10 +00002947// - KAND, KANDN, KOR, KXNOR, KXOR
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002948multiclass avx512_mask_binop<bits<8> opc, string OpcodeStr,
Robert Khasanov595683d2014-07-28 13:46:45 +00002949 RegisterClass KRC, SDPatternOperator OpNode,
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002950 Predicate prd, bit IsCommutable> {
2951 let Predicates = [prd], isCommutable = IsCommutable in
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002952 def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2),
2953 !strconcat(OpcodeStr,
Craig Topperedb09112014-11-25 20:11:23 +00002954 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002955 [(set KRC:$dst, (OpNode KRC:$src1, KRC:$src2))]>;
2956}
2957
Robert Khasanov595683d2014-07-28 13:46:45 +00002958multiclass avx512_mask_binop_all<bits<8> opc, string OpcodeStr,
Igor Breger59ac3392015-08-31 11:50:23 +00002959 SDPatternOperator OpNode, bit IsCommutable,
2960 Predicate prdW = HasAVX512> {
Robert Khasanov595683d2014-07-28 13:46:45 +00002961 defm B : avx512_mask_binop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002962 HasDQI, IsCommutable>, VEX_4V, VEX_L, PD;
Robert Khasanov595683d2014-07-28 13:46:45 +00002963 defm W : avx512_mask_binop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,
Igor Breger59ac3392015-08-31 11:50:23 +00002964 prdW, IsCommutable>, VEX_4V, VEX_L, PS;
Robert Khasanov595683d2014-07-28 13:46:45 +00002965 defm D : avx512_mask_binop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002966 HasBWI, IsCommutable>, VEX_4V, VEX_L, VEX_W, PD;
Robert Khasanov595683d2014-07-28 13:46:45 +00002967 defm Q : avx512_mask_binop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002968 HasBWI, IsCommutable>, VEX_4V, VEX_L, VEX_W, PS;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002969}
2970
2971def andn : PatFrag<(ops node:$i0, node:$i1), (and (not node:$i0), node:$i1)>;
2972def xnor : PatFrag<(ops node:$i0, node:$i1), (not (xor node:$i0, node:$i1))>;
Craig Topper7b9cc142016-11-03 06:04:28 +00002973// These nodes use 'vnot' instead of 'not' to support vectors.
2974def vandn : PatFrag<(ops node:$i0, node:$i1), (and (vnot node:$i0), node:$i1)>;
2975def vxnor : PatFrag<(ops node:$i0, node:$i1), (vnot (xor node:$i0, node:$i1))>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00002976
Craig Topper7b9cc142016-11-03 06:04:28 +00002977defm KAND : avx512_mask_binop_all<0x41, "kand", and, 1>;
2978defm KOR : avx512_mask_binop_all<0x45, "kor", or, 1>;
2979defm KXNOR : avx512_mask_binop_all<0x46, "kxnor", vxnor, 1>;
2980defm KXOR : avx512_mask_binop_all<0x47, "kxor", xor, 1>;
2981defm KANDN : avx512_mask_binop_all<0x42, "kandn", vandn, 0>;
2982defm KADD : avx512_mask_binop_all<0x4A, "kadd", add, 1, HasDQI>;
Elena Demikhovskyb64d7e82013-12-25 10:06:40 +00002983
Craig Topper7b9cc142016-11-03 06:04:28 +00002984multiclass avx512_binop_pat<SDPatternOperator VOpNode, SDPatternOperator OpNode,
2985 Instruction Inst> {
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002986 // With AVX512F, 8-bit mask is promoted to 16-bit mask,
2987 // for the DQI set, this type is legal and KxxxB instruction is used
2988 let Predicates = [NoDQI] in
Craig Topper7b9cc142016-11-03 06:04:28 +00002989 def : Pat<(VOpNode VK8:$src1, VK8:$src2),
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00002990 (COPY_TO_REGCLASS
2991 (Inst (COPY_TO_REGCLASS VK8:$src1, VK16),
2992 (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>;
2993
2994 // All types smaller than 8 bits require conversion anyway
2995 def : Pat<(OpNode VK1:$src1, VK1:$src2),
2996 (COPY_TO_REGCLASS (Inst
2997 (COPY_TO_REGCLASS VK1:$src1, VK16),
2998 (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>;
Craig Topper7b9cc142016-11-03 06:04:28 +00002999 def : Pat<(VOpNode VK2:$src1, VK2:$src2),
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00003000 (COPY_TO_REGCLASS (Inst
3001 (COPY_TO_REGCLASS VK2:$src1, VK16),
3002 (COPY_TO_REGCLASS VK2:$src2, VK16)), VK1)>;
Craig Topper7b9cc142016-11-03 06:04:28 +00003003 def : Pat<(VOpNode VK4:$src1, VK4:$src2),
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00003004 (COPY_TO_REGCLASS (Inst
3005 (COPY_TO_REGCLASS VK4:$src1, VK16),
3006 (COPY_TO_REGCLASS VK4:$src2, VK16)), VK1)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003007}
3008
Craig Topper7b9cc142016-11-03 06:04:28 +00003009defm : avx512_binop_pat<and, and, KANDWrr>;
3010defm : avx512_binop_pat<vandn, andn, KANDNWrr>;
3011defm : avx512_binop_pat<or, or, KORWrr>;
3012defm : avx512_binop_pat<vxnor, xnor, KXNORWrr>;
3013defm : avx512_binop_pat<xor, xor, KXORWrr>;
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00003014
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003015// Mask unpacking
Igor Bregera54a1a82015-09-08 13:10:00 +00003016multiclass avx512_mask_unpck<string Suffix,RegisterClass KRC, ValueType VT,
3017 RegisterClass KRCSrc, Predicate prd> {
3018 let Predicates = [prd] in {
Craig Topperad2ce362016-01-05 07:44:08 +00003019 let hasSideEffects = 0 in
Igor Bregera54a1a82015-09-08 13:10:00 +00003020 def rr : I<0x4b, MRMSrcReg, (outs KRC:$dst),
3021 (ins KRC:$src1, KRC:$src2),
3022 "kunpck"#Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
3023 VEX_4V, VEX_L;
3024
3025 def : Pat<(VT (concat_vectors KRCSrc:$src1, KRCSrc:$src2)),
3026 (!cast<Instruction>(NAME##rr)
3027 (COPY_TO_REGCLASS KRCSrc:$src2, KRC),
3028 (COPY_TO_REGCLASS KRCSrc:$src1, KRC))>;
3029 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003030}
3031
Igor Bregera54a1a82015-09-08 13:10:00 +00003032defm KUNPCKBW : avx512_mask_unpck<"bw", VK16, v16i1, VK8, HasAVX512>, PD;
3033defm KUNPCKWD : avx512_mask_unpck<"wd", VK32, v32i1, VK16, HasBWI>, PS;
3034defm KUNPCKDQ : avx512_mask_unpck<"dq", VK64, v64i1, VK32, HasBWI>, PS, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003035
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003036// Mask bit testing
3037multiclass avx512_mask_testop<bits<8> opc, string OpcodeStr, RegisterClass KRC,
Igor Breger5ea0a6812015-08-31 13:30:19 +00003038 SDNode OpNode, Predicate prd> {
3039 let Predicates = [prd], Defs = [EFLAGS] in
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003040 def rr : I<opc, MRMSrcReg, (outs), (ins KRC:$src1, KRC:$src2),
Craig Topperedb09112014-11-25 20:11:23 +00003041 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003042 [(set EFLAGS, (OpNode KRC:$src1, KRC:$src2))]>;
3043}
3044
Igor Breger5ea0a6812015-08-31 13:30:19 +00003045multiclass avx512_mask_testop_w<bits<8> opc, string OpcodeStr, SDNode OpNode,
3046 Predicate prdW = HasAVX512> {
3047 defm B : avx512_mask_testop<opc, OpcodeStr#"b", VK8, OpNode, HasDQI>,
3048 VEX, PD;
3049 defm W : avx512_mask_testop<opc, OpcodeStr#"w", VK16, OpNode, prdW>,
3050 VEX, PS;
3051 defm Q : avx512_mask_testop<opc, OpcodeStr#"q", VK64, OpNode, HasBWI>,
3052 VEX, PS, VEX_W;
3053 defm D : avx512_mask_testop<opc, OpcodeStr#"d", VK32, OpNode, HasBWI>,
3054 VEX, PD, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003055}
3056
3057defm KORTEST : avx512_mask_testop_w<0x98, "kortest", X86kortest>;
Igor Breger5ea0a6812015-08-31 13:30:19 +00003058defm KTEST : avx512_mask_testop_w<0x99, "ktest", X86ktest, HasDQI>;
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00003059
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003060// Mask shift
3061multiclass avx512_mask_shiftop<bits<8> opc, string OpcodeStr, RegisterClass KRC,
3062 SDNode OpNode> {
3063 let Predicates = [HasAVX512] in
Craig Topper7ff6ab32015-01-21 08:43:49 +00003064 def ri : Ii8<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src, u8imm:$imm),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003065 !strconcat(OpcodeStr,
Craig Topperedb09112014-11-25 20:11:23 +00003066 "\t{$imm, $src, $dst|$dst, $src, $imm}"),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003067 [(set KRC:$dst, (OpNode KRC:$src, (i8 imm:$imm)))]>;
3068}
3069
3070multiclass avx512_mask_shiftop_w<bits<8> opc1, bits<8> opc2, string OpcodeStr,
3071 SDNode OpNode> {
3072 defm W : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "w"), VK16, OpNode>,
Elena Demikhovsky1a603b32015-01-25 12:47:15 +00003073 VEX, TAPD, VEX_W;
3074 let Predicates = [HasDQI] in
3075 defm B : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "b"), VK8, OpNode>,
3076 VEX, TAPD;
3077 let Predicates = [HasBWI] in {
3078 defm Q : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "q"), VK64, OpNode>,
3079 VEX, TAPD, VEX_W;
Elena Demikhovsky1a603b32015-01-25 12:47:15 +00003080 defm D : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "d"), VK32, OpNode>,
3081 VEX, TAPD;
Michael Liao66233b72015-08-06 09:06:20 +00003082 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003083}
3084
Craig Topper3b7e8232017-01-30 00:06:01 +00003085defm KSHIFTL : avx512_mask_shiftop_w<0x32, 0x33, "kshiftl", X86kshiftl>;
3086defm KSHIFTR : avx512_mask_shiftop_w<0x30, 0x31, "kshiftr", X86kshiftr>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003087
Ayman Musa721d97f2017-06-27 12:08:37 +00003088multiclass axv512_icmp_packed_no_vlx_lowering<SDNode OpNode, string InstStr> {
3089def : Pat<(v8i1 (OpNode (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))),
3090 (COPY_TO_REGCLASS (!cast<Instruction>(InstStr##Zrr)
3091 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
3092 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm))), VK8)>;
3093
Simon Pilgrim64fff142017-07-16 18:37:23 +00003094def : Pat<(insert_subvector (v16i1 immAllZerosV),
Ayman Musa721d97f2017-06-27 12:08:37 +00003095 (v8i1 (OpNode (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))),
3096 (i64 0)),
3097 (KSHIFTRWri (KSHIFTLWri (!cast<Instruction>(InstStr##Zrr)
3098 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
3099 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm))),
3100 (i8 8)), (i8 8))>;
3101
Simon Pilgrim64fff142017-07-16 18:37:23 +00003102def : Pat<(insert_subvector (v16i1 immAllZerosV),
3103 (v8i1 (and VK8:$mask,
Ayman Musa721d97f2017-06-27 12:08:37 +00003104 (OpNode (v8i32 VR256X:$src1), (v8i32 VR256X:$src2)))),
3105 (i64 0)),
3106 (KSHIFTRWri (KSHIFTLWri (!cast<Instruction>(InstStr##Zrrk)
3107 (COPY_TO_REGCLASS VK8:$mask, VK16),
3108 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
3109 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm))),
3110 (i8 8)), (i8 8))>;
3111}
3112
3113multiclass axv512_icmp_packed_cc_no_vlx_lowering<SDNode OpNode, string InstStr,
3114 AVX512VLVectorVTInfo _> {
3115def : Pat<(v8i1 (OpNode (_.info256.VT VR256X:$src1), (_.info256.VT VR256X:$src2), imm:$cc)),
3116 (COPY_TO_REGCLASS (!cast<Instruction>(InstStr##Zrri)
3117 (_.info512.VT (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
3118 (_.info512.VT (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm)),
3119 imm:$cc), VK8)>;
3120
Simon Pilgrim64fff142017-07-16 18:37:23 +00003121def : Pat<(insert_subvector (v16i1 immAllZerosV),
Ayman Musa721d97f2017-06-27 12:08:37 +00003122 (v8i1 (OpNode (_.info256.VT VR256X:$src1), (_.info256.VT VR256X:$src2), imm:$cc)),
3123 (i64 0)),
3124 (KSHIFTRWri (KSHIFTLWri (!cast<Instruction>(InstStr##Zrri)
3125 (_.info512.VT (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
3126 (_.info512.VT (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm)),
3127 imm:$cc),
3128 (i8 8)), (i8 8))>;
3129
Simon Pilgrim64fff142017-07-16 18:37:23 +00003130def : Pat<(insert_subvector (v16i1 immAllZerosV),
3131 (v8i1 (and VK8:$mask,
Ayman Musa721d97f2017-06-27 12:08:37 +00003132 (OpNode (_.info256.VT VR256X:$src1), (_.info256.VT VR256X:$src2), imm:$cc))),
3133 (i64 0)),
3134 (KSHIFTRWri (KSHIFTLWri (!cast<Instruction>(InstStr##Zrrik)
3135 (COPY_TO_REGCLASS VK8:$mask, VK16),
3136 (_.info512.VT (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
3137 (_.info512.VT (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm)),
3138 imm:$cc),
3139 (i8 8)), (i8 8))>;
3140}
3141
3142let Predicates = [HasAVX512, NoVLX] in {
3143 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpgtm, "VPCMPGTD">;
3144 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpeqm, "VPCMPEQD">;
3145
3146 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86cmpm, "VCMPPS", avx512vl_f32_info>;
3147 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86cmpm, "VPCMPD", avx512vl_i32_info>;
3148 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86cmpmu, "VPCMPUD", avx512vl_i32_info>;
3149}
3150
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003151// Mask setting all 0s or 1s
3152multiclass avx512_mask_setop<RegisterClass KRC, ValueType VT, PatFrag Val> {
3153 let Predicates = [HasAVX512] in
3154 let isReMaterializable = 1, isAsCheapAsAMove = 1, isPseudo = 1 in
3155 def #NAME# : I<0, Pseudo, (outs KRC:$dst), (ins), "",
3156 [(set KRC:$dst, (VT Val))]>;
3157}
3158
3159multiclass avx512_mask_setop_w<PatFrag Val> {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003160 defm W : avx512_mask_setop<VK16, v16i1, Val>;
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00003161 defm D : avx512_mask_setop<VK32, v32i1, Val>;
3162 defm Q : avx512_mask_setop<VK64, v64i1, Val>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003163}
3164
3165defm KSET0 : avx512_mask_setop_w<immAllZerosV>;
3166defm KSET1 : avx512_mask_setop_w<immAllOnesV>;
3167
3168// With AVX-512 only, 8-bit mask is promoted to 16-bit mask.
3169let Predicates = [HasAVX512] in {
3170 def : Pat<(v8i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK8)>;
Igor Breger86724082016-08-14 05:25:07 +00003171 def : Pat<(v4i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK4)>;
3172 def : Pat<(v2i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK2)>;
Guy Blank548e22a2017-05-19 12:35:15 +00003173 def : Pat<(v1i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK1)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003174 def : Pat<(v8i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK8)>;
Elena Demikhovskyd1084c52015-04-27 12:57:59 +00003175 def : Pat<(v4i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK4)>;
3176 def : Pat<(v2i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK2)>;
Guy Blank548e22a2017-05-19 12:35:15 +00003177 def : Pat<(v1i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK1)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003178}
Igor Bregerf1bd7612016-03-06 07:46:03 +00003179
3180// Patterns for kmask insert_subvector/extract_subvector to/from index=0
3181multiclass operation_subvector_mask_lowering<RegisterClass subRC, ValueType subVT,
3182 RegisterClass RC, ValueType VT> {
3183 def : Pat<(subVT (extract_subvector (VT RC:$src), (iPTR 0))),
3184 (subVT (COPY_TO_REGCLASS RC:$src, subRC))>;
Simon Pilgrimb13961d2016-06-11 14:34:10 +00003185
Igor Bregerf1bd7612016-03-06 07:46:03 +00003186 def : Pat<(VT (insert_subvector undef, subRC:$src, (iPTR 0))),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00003187 (VT (COPY_TO_REGCLASS subRC:$src, RC))>;
Igor Bregerf1bd7612016-03-06 07:46:03 +00003188}
Guy Blank548e22a2017-05-19 12:35:15 +00003189defm : operation_subvector_mask_lowering<VK1, v1i1, VK2, v2i1>;
3190defm : operation_subvector_mask_lowering<VK1, v1i1, VK4, v4i1>;
3191defm : operation_subvector_mask_lowering<VK1, v1i1, VK8, v8i1>;
3192defm : operation_subvector_mask_lowering<VK1, v1i1, VK16, v16i1>;
3193defm : operation_subvector_mask_lowering<VK1, v1i1, VK32, v32i1>;
3194defm : operation_subvector_mask_lowering<VK1, v1i1, VK64, v64i1>;
Igor Bregerf1bd7612016-03-06 07:46:03 +00003195
3196defm : operation_subvector_mask_lowering<VK2, v2i1, VK4, v4i1>;
3197defm : operation_subvector_mask_lowering<VK2, v2i1, VK8, v8i1>;
3198defm : operation_subvector_mask_lowering<VK2, v2i1, VK16, v16i1>;
3199defm : operation_subvector_mask_lowering<VK2, v2i1, VK32, v32i1>;
3200defm : operation_subvector_mask_lowering<VK2, v2i1, VK64, v64i1>;
3201
3202defm : operation_subvector_mask_lowering<VK4, v4i1, VK8, v8i1>;
3203defm : operation_subvector_mask_lowering<VK4, v4i1, VK16, v16i1>;
3204defm : operation_subvector_mask_lowering<VK4, v4i1, VK32, v32i1>;
3205defm : operation_subvector_mask_lowering<VK4, v4i1, VK64, v64i1>;
3206
3207defm : operation_subvector_mask_lowering<VK8, v8i1, VK16, v16i1>;
3208defm : operation_subvector_mask_lowering<VK8, v8i1, VK32, v32i1>;
3209defm : operation_subvector_mask_lowering<VK8, v8i1, VK64, v64i1>;
3210
3211defm : operation_subvector_mask_lowering<VK16, v16i1, VK32, v32i1>;
3212defm : operation_subvector_mask_lowering<VK16, v16i1, VK64, v64i1>;
3213
3214defm : operation_subvector_mask_lowering<VK32, v32i1, VK64, v64i1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003215
Igor Breger999ac752016-03-08 15:21:25 +00003216def : Pat<(v2i1 (extract_subvector (v4i1 VK4:$src), (iPTR 2))),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00003217 (v2i1 (COPY_TO_REGCLASS
Igor Breger999ac752016-03-08 15:21:25 +00003218 (KSHIFTRWri (COPY_TO_REGCLASS VK4:$src, VK16), (i8 2)),
3219 VK2))>;
3220def : Pat<(v4i1 (extract_subvector (v8i1 VK8:$src), (iPTR 4))),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00003221 (v4i1 (COPY_TO_REGCLASS
Igor Breger999ac752016-03-08 15:21:25 +00003222 (KSHIFTRWri (COPY_TO_REGCLASS VK8:$src, VK16), (i8 4)),
3223 VK4))>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003224def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 8))),
3225 (v8i1 (COPY_TO_REGCLASS (KSHIFTRWri VK16:$src, (i8 8)), VK8))>;
Elena Demikhovsky6015f5c2015-12-15 08:40:41 +00003226def : Pat<(v16i1 (extract_subvector (v32i1 VK32:$src), (iPTR 16))),
3227 (v16i1 (COPY_TO_REGCLASS (KSHIFTRDri VK32:$src, (i8 16)), VK16))>;
Elena Demikhovsky86c7b462015-05-27 14:09:33 +00003228def : Pat<(v32i1 (extract_subvector (v64i1 VK64:$src), (iPTR 32))),
3229 (v32i1 (COPY_TO_REGCLASS (KSHIFTRQri VK64:$src, (i8 32)), VK32))>;
3230
Elena Demikhovsky9737e382014-03-02 09:19:44 +00003231
Igor Breger86724082016-08-14 05:25:07 +00003232// Patterns for kmask shift
3233multiclass mask_shift_lowering<RegisterClass RC, ValueType VT> {
Craig Topper3b7e8232017-01-30 00:06:01 +00003234 def : Pat<(VT (X86kshiftl RC:$src, (i8 imm:$imm))),
Simon Pilgrim916485c2016-08-18 11:22:22 +00003235 (VT (COPY_TO_REGCLASS
Igor Breger86724082016-08-14 05:25:07 +00003236 (KSHIFTLWri (COPY_TO_REGCLASS RC:$src, VK16),
Simon Pilgrim916485c2016-08-18 11:22:22 +00003237 (I8Imm $imm)),
Igor Breger86724082016-08-14 05:25:07 +00003238 RC))>;
Craig Topper3b7e8232017-01-30 00:06:01 +00003239 def : Pat<(VT (X86kshiftr RC:$src, (i8 imm:$imm))),
Simon Pilgrim916485c2016-08-18 11:22:22 +00003240 (VT (COPY_TO_REGCLASS
Igor Breger86724082016-08-14 05:25:07 +00003241 (KSHIFTRWri (COPY_TO_REGCLASS RC:$src, VK16),
Simon Pilgrim916485c2016-08-18 11:22:22 +00003242 (I8Imm $imm)),
Igor Breger86724082016-08-14 05:25:07 +00003243 RC))>;
3244}
3245
3246defm : mask_shift_lowering<VK8, v8i1>, Requires<[HasAVX512, NoDQI]>;
3247defm : mask_shift_lowering<VK4, v4i1>, Requires<[HasAVX512]>;
3248defm : mask_shift_lowering<VK2, v2i1>, Requires<[HasAVX512]>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003249//===----------------------------------------------------------------------===//
3250// AVX-512 - Aligned and unaligned load and store
3251//
3252
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003253
3254multiclass avx512_load<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
Elena Demikhovskyd207f172015-03-03 15:03:35 +00003255 PatFrag ld_frag, PatFrag mload,
Craig Toppercb0e7492017-07-31 17:35:44 +00003256 bit NoRMPattern = 0,
Craig Topperc9293492016-02-26 06:50:29 +00003257 SDPatternOperator SelectOprr = vselect> {
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003258 let hasSideEffects = 0 in {
3259 def rr : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst), (ins _.RC:$src),
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003260 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [],
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003261 _.ExeDomain>, EVEX;
3262 def rrkz : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst),
3263 (ins _.KRCWM:$mask, _.RC:$src),
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003264 !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}} {z}|",
Simon Pilgrim18bcf932016-02-03 09:41:59 +00003265 "${dst} {${mask}} {z}, $src}"),
Craig Topper5c46c752017-01-08 05:46:21 +00003266 [(set _.RC:$dst, (_.VT (SelectOprr _.KRCWM:$mask,
Igor Breger7a000f52016-01-21 14:18:11 +00003267 (_.VT _.RC:$src),
3268 _.ImmAllZerosV)))], _.ExeDomain>,
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003269 EVEX, EVEX_KZ;
3270
Craig Toppercb0e7492017-07-31 17:35:44 +00003271 let mayLoad = 1, canFoldAsLoad = 1, isReMaterializable = 1,
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003272 SchedRW = [WriteLoad] in
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003273 def rm : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst), (ins _.MemOp:$src),
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003274 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
Craig Toppercb0e7492017-07-31 17:35:44 +00003275 !if(NoRMPattern, [],
3276 [(set _.RC:$dst,
3277 (_.VT (bitconvert (ld_frag addr:$src))))]),
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003278 _.ExeDomain>, EVEX;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003279
Craig Topper63e2cd62017-01-14 07:50:52 +00003280 let Constraints = "$src0 = $dst", isConvertibleToThreeAddress = 1 in {
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003281 def rrk : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst),
3282 (ins _.RC:$src0, _.KRCWM:$mask, _.RC:$src1),
3283 !strconcat(OpcodeStr, "\t{$src1, ${dst} {${mask}}|",
3284 "${dst} {${mask}}, $src1}"),
Craig Topperc9293492016-02-26 06:50:29 +00003285 [(set _.RC:$dst, (_.VT (SelectOprr _.KRCWM:$mask,
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003286 (_.VT _.RC:$src1),
3287 (_.VT _.RC:$src0))))], _.ExeDomain>,
3288 EVEX, EVEX_K;
Craig Toppere1cac152016-06-07 07:27:54 +00003289 let SchedRW = [WriteLoad] in
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003290 def rmk : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst),
3291 (ins _.RC:$src0, _.KRCWM:$mask, _.MemOp:$src1),
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003292 !strconcat(OpcodeStr, "\t{$src1, ${dst} {${mask}}|",
3293 "${dst} {${mask}}, $src1}"),
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003294 [(set _.RC:$dst, (_.VT
3295 (vselect _.KRCWM:$mask,
3296 (_.VT (bitconvert (ld_frag addr:$src1))),
3297 (_.VT _.RC:$src0))))], _.ExeDomain>, EVEX, EVEX_K;
Elena Demikhovskyfd056672014-03-13 12:05:52 +00003298 }
Craig Toppere1cac152016-06-07 07:27:54 +00003299 let SchedRW = [WriteLoad] in
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003300 def rmkz : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst),
3301 (ins _.KRCWM:$mask, _.MemOp:$src),
3302 OpcodeStr #"\t{$src, ${dst} {${mask}} {z}|"#
3303 "${dst} {${mask}} {z}, $src}",
3304 [(set _.RC:$dst, (_.VT (vselect _.KRCWM:$mask,
3305 (_.VT (bitconvert (ld_frag addr:$src))), _.ImmAllZerosV)))],
3306 _.ExeDomain>, EVEX, EVEX_KZ;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003307 }
Elena Demikhovskyd207f172015-03-03 15:03:35 +00003308 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, undef)),
3309 (!cast<Instruction>(NAME#_.ZSuffix##rmkz) _.KRCWM:$mask, addr:$ptr)>;
3310
3311 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, _.ImmAllZerosV)),
3312 (!cast<Instruction>(NAME#_.ZSuffix##rmkz) _.KRCWM:$mask, addr:$ptr)>;
3313
3314 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src0))),
3315 (!cast<Instruction>(NAME#_.ZSuffix##rmk) _.RC:$src0,
3316 _.KRCWM:$mask, addr:$ptr)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003317}
3318
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003319multiclass avx512_alignedload_vl<bits<8> opc, string OpcodeStr,
3320 AVX512VLVectorVTInfo _,
Craig Topper4e7b8882016-10-03 02:00:29 +00003321 Predicate prd> {
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003322 let Predicates = [prd] in
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003323 defm Z : avx512_load<opc, OpcodeStr, _.info512, _.info512.AlignedLdFrag,
Craig Topper4e7b8882016-10-03 02:00:29 +00003324 masked_load_aligned512>, EVEX_V512;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003325
3326 let Predicates = [prd, HasVLX] in {
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003327 defm Z256 : avx512_load<opc, OpcodeStr, _.info256, _.info256.AlignedLdFrag,
Craig Topper4e7b8882016-10-03 02:00:29 +00003328 masked_load_aligned256>, EVEX_V256;
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003329 defm Z128 : avx512_load<opc, OpcodeStr, _.info128, _.info128.AlignedLdFrag,
Craig Topper4e7b8882016-10-03 02:00:29 +00003330 masked_load_aligned128>, EVEX_V128;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003331 }
3332}
3333
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003334multiclass avx512_load_vl<bits<8> opc, string OpcodeStr,
3335 AVX512VLVectorVTInfo _,
3336 Predicate prd,
Craig Toppercb0e7492017-07-31 17:35:44 +00003337 bit NoRMPattern = 0,
Craig Topperc9293492016-02-26 06:50:29 +00003338 SDPatternOperator SelectOprr = vselect> {
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003339 let Predicates = [prd] in
3340 defm Z : avx512_load<opc, OpcodeStr, _.info512, _.info512.LdFrag,
Craig Toppercb0e7492017-07-31 17:35:44 +00003341 masked_load_unaligned, NoRMPattern,
3342 SelectOprr>, EVEX_V512;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003343
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003344 let Predicates = [prd, HasVLX] in {
3345 defm Z256 : avx512_load<opc, OpcodeStr, _.info256, _.info256.LdFrag,
Craig Toppercb0e7492017-07-31 17:35:44 +00003346 masked_load_unaligned, NoRMPattern,
3347 SelectOprr>, EVEX_V256;
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003348 defm Z128 : avx512_load<opc, OpcodeStr, _.info128, _.info128.LdFrag,
Craig Toppercb0e7492017-07-31 17:35:44 +00003349 masked_load_unaligned, NoRMPattern,
3350 SelectOprr>, EVEX_V128;
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003351 }
3352}
3353
3354multiclass avx512_store<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
Craig Topper2462a712017-08-01 15:31:24 +00003355 PatFrag st_frag, PatFrag mstore, string Name,
3356 bit NoMRPattern = 0> {
Igor Breger81b79de2015-11-19 07:43:43 +00003357
Craig Topper99f6b622016-05-01 01:03:56 +00003358 let hasSideEffects = 0 in {
Igor Breger81b79de2015-11-19 07:43:43 +00003359 def rr_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst), (ins _.RC:$src),
3360 OpcodeStr # ".s\t{$src, $dst|$dst, $src}",
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003361 [], _.ExeDomain>, EVEX, FoldGenData<Name#rr>;
Igor Breger81b79de2015-11-19 07:43:43 +00003362 def rrk_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst),
3363 (ins _.KRCWM:$mask, _.RC:$src),
3364 OpcodeStr # ".s\t{$src, ${dst} {${mask}}|"#
3365 "${dst} {${mask}}, $src}",
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003366 [], _.ExeDomain>, EVEX, EVEX_K, FoldGenData<Name#rrk>;
Igor Breger81b79de2015-11-19 07:43:43 +00003367 def rrkz_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst),
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003368 (ins _.KRCWM:$mask, _.RC:$src),
Igor Breger81b79de2015-11-19 07:43:43 +00003369 OpcodeStr # ".s\t{$src, ${dst} {${mask}} {z}|" #
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003370 "${dst} {${mask}} {z}, $src}",
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003371 [], _.ExeDomain>, EVEX, EVEX_KZ, FoldGenData<Name#rrkz>;
Craig Topper99f6b622016-05-01 01:03:56 +00003372 }
Igor Breger81b79de2015-11-19 07:43:43 +00003373
Craig Topper2462a712017-08-01 15:31:24 +00003374 let hasSideEffects = 0, mayStore = 1 in
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003375 def mr : AVX512PI<opc, MRMDestMem, (outs), (ins _.MemOp:$dst, _.RC:$src),
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003376 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
Craig Topper2462a712017-08-01 15:31:24 +00003377 !if(NoMRPattern, [],
3378 [(st_frag (_.VT _.RC:$src), addr:$dst)]),
3379 _.ExeDomain>, EVEX;
Elena Demikhovskyfd056672014-03-13 12:05:52 +00003380 def mrk : AVX512PI<opc, MRMDestMem, (outs),
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003381 (ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src),
3382 OpcodeStr # "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}",
3383 [], _.ExeDomain>, EVEX, EVEX_K;
Elena Demikhovskyd207f172015-03-03 15:03:35 +00003384
3385 def: Pat<(mstore addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src)),
3386 (!cast<Instruction>(NAME#_.ZSuffix##mrk) addr:$ptr,
3387 _.KRCWM:$mask, _.RC:$src)>;
Elena Demikhovskyfd056672014-03-13 12:05:52 +00003388}
3389
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003390
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003391multiclass avx512_store_vl< bits<8> opc, string OpcodeStr,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003392 AVX512VLVectorVTInfo _, Predicate prd,
Craig Topper2462a712017-08-01 15:31:24 +00003393 string Name, bit NoMRPattern = 0> {
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003394 let Predicates = [prd] in
Elena Demikhovskyd207f172015-03-03 15:03:35 +00003395 defm Z : avx512_store<opc, OpcodeStr, _.info512, store,
Craig Topper2462a712017-08-01 15:31:24 +00003396 masked_store_unaligned, Name#Z, NoMRPattern>, EVEX_V512;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003397
3398 let Predicates = [prd, HasVLX] in {
Elena Demikhovskyd207f172015-03-03 15:03:35 +00003399 defm Z256 : avx512_store<opc, OpcodeStr, _.info256, store,
Craig Topper2462a712017-08-01 15:31:24 +00003400 masked_store_unaligned, Name#Z256,
3401 NoMRPattern>, EVEX_V256;
Elena Demikhovskyd207f172015-03-03 15:03:35 +00003402 defm Z128 : avx512_store<opc, OpcodeStr, _.info128, store,
Craig Topper2462a712017-08-01 15:31:24 +00003403 masked_store_unaligned, Name#Z128,
3404 NoMRPattern>, EVEX_V128;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003405 }
3406}
3407
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003408multiclass avx512_alignedstore_vl<bits<8> opc, string OpcodeStr,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003409 AVX512VLVectorVTInfo _, Predicate prd,
3410 string Name> {
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003411 let Predicates = [prd] in
Elena Demikhovskyd207f172015-03-03 15:03:35 +00003412 defm Z : avx512_store<opc, OpcodeStr, _.info512, alignedstore512,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003413 masked_store_aligned512, Name#Z>, EVEX_V512;
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003414
3415 let Predicates = [prd, HasVLX] in {
Elena Demikhovskyd207f172015-03-03 15:03:35 +00003416 defm Z256 : avx512_store<opc, OpcodeStr, _.info256, alignedstore256,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003417 masked_store_aligned256, Name#Z256>, EVEX_V256;
Elena Demikhovskyd207f172015-03-03 15:03:35 +00003418 defm Z128 : avx512_store<opc, OpcodeStr, _.info128, alignedstore,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003419 masked_store_aligned128, Name#Z128>, EVEX_V128;
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003420 }
3421}
3422
3423defm VMOVAPS : avx512_alignedload_vl<0x28, "vmovaps", avx512vl_f32_info,
3424 HasAVX512>,
3425 avx512_alignedstore_vl<0x29, "vmovaps", avx512vl_f32_info,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003426 HasAVX512, "VMOVAPS">,
3427 PS, EVEX_CD8<32, CD8VF>;
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003428
3429defm VMOVAPD : avx512_alignedload_vl<0x28, "vmovapd", avx512vl_f64_info,
3430 HasAVX512>,
3431 avx512_alignedstore_vl<0x29, "vmovapd", avx512vl_f64_info,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003432 HasAVX512, "VMOVAPD">,
3433 PD, VEX_W, EVEX_CD8<64, CD8VF>;
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003434
Craig Topperc9293492016-02-26 06:50:29 +00003435defm VMOVUPS : avx512_load_vl<0x10, "vmovups", avx512vl_f32_info, HasAVX512,
Craig Toppercb0e7492017-07-31 17:35:44 +00003436 0, null_frag>,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003437 avx512_store_vl<0x11, "vmovups", avx512vl_f32_info, HasAVX512,
3438 "VMOVUPS">,
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003439 PS, EVEX_CD8<32, CD8VF>;
3440
Craig Topper4e7b8882016-10-03 02:00:29 +00003441defm VMOVUPD : avx512_load_vl<0x10, "vmovupd", avx512vl_f64_info, HasAVX512,
Craig Toppercb0e7492017-07-31 17:35:44 +00003442 0, null_frag>,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003443 avx512_store_vl<0x11, "vmovupd", avx512vl_f64_info, HasAVX512,
3444 "VMOVUPD">,
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003445 PD, VEX_W, EVEX_CD8<64, CD8VF>;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003446
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003447defm VMOVDQA32 : avx512_alignedload_vl<0x6F, "vmovdqa32", avx512vl_i32_info,
3448 HasAVX512>,
3449 avx512_alignedstore_vl<0x7F, "vmovdqa32", avx512vl_i32_info,
Simon Pilgrim64fff142017-07-16 18:37:23 +00003450 HasAVX512, "VMOVDQA32">,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003451 PD, EVEX_CD8<32, CD8VF>;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003452
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003453defm VMOVDQA64 : avx512_alignedload_vl<0x6F, "vmovdqa64", avx512vl_i64_info,
3454 HasAVX512>,
3455 avx512_alignedstore_vl<0x7F, "vmovdqa64", avx512vl_i64_info,
Simon Pilgrim64fff142017-07-16 18:37:23 +00003456 HasAVX512, "VMOVDQA64">,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003457 PD, VEX_W, EVEX_CD8<64, CD8VF>;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003458
Craig Toppercb0e7492017-07-31 17:35:44 +00003459defm VMOVDQU8 : avx512_load_vl<0x6F, "vmovdqu8", avx512vl_i8_info, HasBWI, 1>,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003460 avx512_store_vl<0x7F, "vmovdqu8", avx512vl_i8_info,
Craig Topper2462a712017-08-01 15:31:24 +00003461 HasBWI, "VMOVDQU8", 1>,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003462 XD, EVEX_CD8<8, CD8VF>;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003463
Craig Toppercb0e7492017-07-31 17:35:44 +00003464defm VMOVDQU16 : avx512_load_vl<0x6F, "vmovdqu16", avx512vl_i16_info, HasBWI, 1>,
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003465 avx512_store_vl<0x7F, "vmovdqu16", avx512vl_i16_info,
Craig Topper2462a712017-08-01 15:31:24 +00003466 HasBWI, "VMOVDQU16", 1>,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003467 XD, VEX_W, EVEX_CD8<16, CD8VF>;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003468
Craig Topperc9293492016-02-26 06:50:29 +00003469defm VMOVDQU32 : avx512_load_vl<0x6F, "vmovdqu32", avx512vl_i32_info, HasAVX512,
Craig Toppercb0e7492017-07-31 17:35:44 +00003470 0, null_frag>,
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003471 avx512_store_vl<0x7F, "vmovdqu32", avx512vl_i32_info,
Simon Pilgrim64fff142017-07-16 18:37:23 +00003472 HasAVX512, "VMOVDQU32">,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003473 XS, EVEX_CD8<32, CD8VF>;
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003474
Craig Topperc9293492016-02-26 06:50:29 +00003475defm VMOVDQU64 : avx512_load_vl<0x6F, "vmovdqu64", avx512vl_i64_info, HasAVX512,
Craig Toppercb0e7492017-07-31 17:35:44 +00003476 0, null_frag>,
Elena Demikhovsky2689d782015-03-02 12:46:21 +00003477 avx512_store_vl<0x7F, "vmovdqu64", avx512vl_i64_info,
Simon Pilgrim64fff142017-07-16 18:37:23 +00003478 HasAVX512, "VMOVDQU64">,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00003479 XS, VEX_W, EVEX_CD8<64, CD8VF>;
Elena Demikhovsky1f3ed412013-10-22 09:19:28 +00003480
Craig Topperd875d6b2016-09-29 06:07:09 +00003481// Special instructions to help with spilling when we don't have VLX. We need
3482// to load or store from a ZMM register instead. These are converted in
3483// expandPostRAPseudos.
Craig Toppereab23d32016-10-03 02:22:33 +00003484let isReMaterializable = 1, canFoldAsLoad = 1,
Craig Topperd875d6b2016-09-29 06:07:09 +00003485 isPseudo = 1, SchedRW = [WriteLoad], mayLoad = 1, hasSideEffects = 0 in {
3486def VMOVAPSZ128rm_NOVLX : I<0, Pseudo, (outs VR128X:$dst), (ins f128mem:$src),
3487 "", []>;
3488def VMOVAPSZ256rm_NOVLX : I<0, Pseudo, (outs VR256X:$dst), (ins f256mem:$src),
3489 "", []>;
3490def VMOVUPSZ128rm_NOVLX : I<0, Pseudo, (outs VR128X:$dst), (ins f128mem:$src),
3491 "", []>;
3492def VMOVUPSZ256rm_NOVLX : I<0, Pseudo, (outs VR256X:$dst), (ins f256mem:$src),
3493 "", []>;
3494}
3495
3496let isPseudo = 1, mayStore = 1, hasSideEffects = 0 in {
Craig Topperf3e671e2016-09-30 05:35:47 +00003497def VMOVAPSZ128mr_NOVLX : I<0, Pseudo, (outs), (ins f128mem:$dst, VR128X:$src),
Craig Topperd875d6b2016-09-29 06:07:09 +00003498 "", []>;
Craig Topperf3e671e2016-09-30 05:35:47 +00003499def VMOVAPSZ256mr_NOVLX : I<0, Pseudo, (outs), (ins f256mem:$dst, VR256X:$src),
Craig Topperd875d6b2016-09-29 06:07:09 +00003500 "", []>;
Craig Topperf3e671e2016-09-30 05:35:47 +00003501def VMOVUPSZ128mr_NOVLX : I<0, Pseudo, (outs), (ins f128mem:$dst, VR128X:$src),
Craig Topperd875d6b2016-09-29 06:07:09 +00003502 "", []>;
Craig Topperf3e671e2016-09-30 05:35:47 +00003503def VMOVUPSZ256mr_NOVLX : I<0, Pseudo, (outs), (ins f256mem:$dst, VR256X:$src),
Craig Topperd875d6b2016-09-29 06:07:09 +00003504 "", []>;
3505}
3506
Elena Demikhovskya30e4372014-02-05 07:05:03 +00003507def : Pat<(v8i64 (vselect VK8WM:$mask, (bc_v8i64 (v16i32 immAllZerosV)),
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003508 (v8i64 VR512:$src))),
Igor Breger7a000f52016-01-21 14:18:11 +00003509 (VMOVDQA64Zrrkz (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$mask, VK16)),
Elena Demikhovskya30e4372014-02-05 07:05:03 +00003510 VK8), VR512:$src)>;
3511
Elena Demikhovskya30e4372014-02-05 07:05:03 +00003512def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 immAllZerosV),
Robert Khasanov7ca7df02014-08-04 14:35:15 +00003513 (v16i32 VR512:$src))),
Igor Breger7a000f52016-01-21 14:18:11 +00003514 (VMOVDQA32Zrrkz (KNOTWrr VK16WM:$mask), VR512:$src)>;
Elena Demikhovskyf1de34b2014-12-04 09:40:44 +00003515
Craig Topper33c550c2016-05-22 00:39:30 +00003516// These patterns exist to prevent the above patterns from introducing a second
3517// mask inversion when one already exists.
3518def : Pat<(v8i64 (vselect (xor VK8:$mask, (v8i1 immAllOnesV)),
3519 (bc_v8i64 (v16i32 immAllZerosV)),
3520 (v8i64 VR512:$src))),
3521 (VMOVDQA64Zrrkz VK8:$mask, VR512:$src)>;
3522def : Pat<(v16i32 (vselect (xor VK16:$mask, (v16i1 immAllOnesV)),
3523 (v16i32 immAllZerosV),
3524 (v16i32 VR512:$src))),
3525 (VMOVDQA32Zrrkz VK16WM:$mask, VR512:$src)>;
3526
Craig Topper96ab6fd2017-01-09 04:19:34 +00003527// Patterns for handling v8i1 selects of 256-bit vectors when VLX isn't
3528// available. Use a 512-bit operation and extract.
3529let Predicates = [HasAVX512, NoVLX] in {
3530def : Pat<(v8f32 (vselect (v8i1 VK8WM:$mask), (v8f32 VR256X:$src1),
3531 (v8f32 VR256X:$src0))),
3532 (EXTRACT_SUBREG
3533 (v16f32
3534 (VMOVAPSZrrk
3535 (v16f32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src0, sub_ymm)),
3536 (COPY_TO_REGCLASS VK8WM:$mask, VK16WM),
3537 (v16f32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)))),
3538 sub_ymm)>;
3539
3540def : Pat<(v8i32 (vselect (v8i1 VK8WM:$mask), (v8i32 VR256X:$src1),
3541 (v8i32 VR256X:$src0))),
3542 (EXTRACT_SUBREG
3543 (v16i32
3544 (VMOVDQA32Zrrk
3545 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src0, sub_ymm)),
3546 (COPY_TO_REGCLASS VK8WM:$mask, VK16WM),
3547 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)))),
3548 sub_ymm)>;
3549}
3550
Craig Topper2462a712017-08-01 15:31:24 +00003551let Predicates = [HasAVX512] in {
3552 // 512-bit store.
3553 def : Pat<(alignedstore512 (v32i16 VR512:$src), addr:$dst),
3554 (VMOVDQA32Zmr addr:$dst, VR512:$src)>;
3555 def : Pat<(alignedstore512 (v64i8 VR512:$src), addr:$dst),
3556 (VMOVDQA32Zmr addr:$dst, VR512:$src)>;
3557 def : Pat<(store (v32i16 VR512:$src), addr:$dst),
3558 (VMOVDQU32Zmr addr:$dst, VR512:$src)>;
3559 def : Pat<(store (v64i8 VR512:$src), addr:$dst),
3560 (VMOVDQU32Zmr addr:$dst, VR512:$src)>;
3561}
3562
3563let Predicates = [HasVLX] in {
3564 // 128-bit store.
Craig Topper5ef13ba2016-12-26 07:26:07 +00003565 def : Pat<(alignedstore (v8i16 VR128X:$src), addr:$dst),
3566 (VMOVDQA32Z128mr addr:$dst, VR128X:$src)>;
3567 def : Pat<(alignedstore (v16i8 VR128X:$src), addr:$dst),
3568 (VMOVDQA32Z128mr addr:$dst, VR128X:$src)>;
3569 def : Pat<(store (v8i16 VR128X:$src), addr:$dst),
3570 (VMOVDQU32Z128mr addr:$dst, VR128X:$src)>;
3571 def : Pat<(store (v16i8 VR128X:$src), addr:$dst),
3572 (VMOVDQU32Z128mr addr:$dst, VR128X:$src)>;
Craig Topper14aa2662016-08-11 06:04:04 +00003573
Craig Topper2462a712017-08-01 15:31:24 +00003574 // 256-bit store.
Craig Topper5ef13ba2016-12-26 07:26:07 +00003575 def : Pat<(alignedstore256 (v16i16 VR256X:$src), addr:$dst),
3576 (VMOVDQA32Z256mr addr:$dst, VR256X:$src)>;
3577 def : Pat<(alignedstore256 (v32i8 VR256X:$src), addr:$dst),
3578 (VMOVDQA32Z256mr addr:$dst, VR256X:$src)>;
3579 def : Pat<(store (v16i16 VR256X:$src), addr:$dst),
3580 (VMOVDQU32Z256mr addr:$dst, VR256X:$src)>;
3581 def : Pat<(store (v32i8 VR256X:$src), addr:$dst),
3582 (VMOVDQU32Z256mr addr:$dst, VR256X:$src)>;
Craig Topper14aa2662016-08-11 06:04:04 +00003583
Craig Topper95bdabd2016-05-22 23:44:33 +00003584 // Special patterns for storing subvector extracts of lower 128-bits of 256.
3585 // Its cheaper to just use VMOVAPS/VMOVUPS instead of VEXTRACTF128mr
3586 def : Pat<(alignedstore (v2f64 (extract_subvector
3587 (v4f64 VR256X:$src), (iPTR 0))), addr:$dst),
3588 (VMOVAPDZ128mr addr:$dst, (v2f64 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
3589 def : Pat<(alignedstore (v4f32 (extract_subvector
3590 (v8f32 VR256X:$src), (iPTR 0))), addr:$dst),
3591 (VMOVAPSZ128mr addr:$dst, (v4f32 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
3592 def : Pat<(alignedstore (v2i64 (extract_subvector
3593 (v4i64 VR256X:$src), (iPTR 0))), addr:$dst),
3594 (VMOVDQA64Z128mr addr:$dst, (v2i64 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
3595 def : Pat<(alignedstore (v4i32 (extract_subvector
3596 (v8i32 VR256X:$src), (iPTR 0))), addr:$dst),
3597 (VMOVDQA32Z128mr addr:$dst, (v4i32 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
3598 def : Pat<(alignedstore (v8i16 (extract_subvector
3599 (v16i16 VR256X:$src), (iPTR 0))), addr:$dst),
3600 (VMOVDQA32Z128mr addr:$dst, (v8i16 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
3601 def : Pat<(alignedstore (v16i8 (extract_subvector
3602 (v32i8 VR256X:$src), (iPTR 0))), addr:$dst),
3603 (VMOVDQA32Z128mr addr:$dst, (v16i8 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
3604
3605 def : Pat<(store (v2f64 (extract_subvector
3606 (v4f64 VR256X:$src), (iPTR 0))), addr:$dst),
3607 (VMOVUPDZ128mr addr:$dst, (v2f64 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
3608 def : Pat<(store (v4f32 (extract_subvector
3609 (v8f32 VR256X:$src), (iPTR 0))), addr:$dst),
3610 (VMOVUPSZ128mr addr:$dst, (v4f32 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
3611 def : Pat<(store (v2i64 (extract_subvector
3612 (v4i64 VR256X:$src), (iPTR 0))), addr:$dst),
3613 (VMOVDQU64Z128mr addr:$dst, (v2i64 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
3614 def : Pat<(store (v4i32 (extract_subvector
3615 (v8i32 VR256X:$src), (iPTR 0))), addr:$dst),
3616 (VMOVDQU32Z128mr addr:$dst, (v4i32 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
3617 def : Pat<(store (v8i16 (extract_subvector
3618 (v16i16 VR256X:$src), (iPTR 0))), addr:$dst),
3619 (VMOVDQU32Z128mr addr:$dst, (v8i16 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
3620 def : Pat<(store (v16i8 (extract_subvector
3621 (v32i8 VR256X:$src), (iPTR 0))), addr:$dst),
3622 (VMOVDQU32Z128mr addr:$dst, (v16i8 (EXTRACT_SUBREG VR256X:$src,sub_xmm)))>;
3623
3624 // Special patterns for storing subvector extracts of lower 128-bits of 512.
3625 // Its cheaper to just use VMOVAPS/VMOVUPS instead of VEXTRACTF128mr
3626 def : Pat<(alignedstore (v2f64 (extract_subvector
3627 (v8f64 VR512:$src), (iPTR 0))), addr:$dst),
3628 (VMOVAPDZ128mr addr:$dst, (v2f64 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
3629 def : Pat<(alignedstore (v4f32 (extract_subvector
3630 (v16f32 VR512:$src), (iPTR 0))), addr:$dst),
3631 (VMOVAPSZ128mr addr:$dst, (v4f32 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
3632 def : Pat<(alignedstore (v2i64 (extract_subvector
3633 (v8i64 VR512:$src), (iPTR 0))), addr:$dst),
3634 (VMOVDQA64Z128mr addr:$dst, (v2i64 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
3635 def : Pat<(alignedstore (v4i32 (extract_subvector
3636 (v16i32 VR512:$src), (iPTR 0))), addr:$dst),
3637 (VMOVDQA32Z128mr addr:$dst, (v4i32 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
3638 def : Pat<(alignedstore (v8i16 (extract_subvector
3639 (v32i16 VR512:$src), (iPTR 0))), addr:$dst),
3640 (VMOVDQA32Z128mr addr:$dst, (v8i16 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
3641 def : Pat<(alignedstore (v16i8 (extract_subvector
3642 (v64i8 VR512:$src), (iPTR 0))), addr:$dst),
3643 (VMOVDQA32Z128mr addr:$dst, (v16i8 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
3644
3645 def : Pat<(store (v2f64 (extract_subvector
3646 (v8f64 VR512:$src), (iPTR 0))), addr:$dst),
3647 (VMOVUPDZ128mr addr:$dst, (v2f64 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
3648 def : Pat<(store (v4f32 (extract_subvector
3649 (v16f32 VR512:$src), (iPTR 0))), addr:$dst),
3650 (VMOVUPSZ128mr addr:$dst, (v4f32 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
3651 def : Pat<(store (v2i64 (extract_subvector
3652 (v8i64 VR512:$src), (iPTR 0))), addr:$dst),
3653 (VMOVDQU64Z128mr addr:$dst, (v2i64 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
3654 def : Pat<(store (v4i32 (extract_subvector
3655 (v16i32 VR512:$src), (iPTR 0))), addr:$dst),
3656 (VMOVDQU32Z128mr addr:$dst, (v4i32 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
3657 def : Pat<(store (v8i16 (extract_subvector
3658 (v32i16 VR512:$src), (iPTR 0))), addr:$dst),
3659 (VMOVDQU32Z128mr addr:$dst, (v8i16 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
3660 def : Pat<(store (v16i8 (extract_subvector
3661 (v64i8 VR512:$src), (iPTR 0))), addr:$dst),
3662 (VMOVDQU32Z128mr addr:$dst, (v16i8 (EXTRACT_SUBREG VR512:$src,sub_xmm)))>;
3663
3664 // Special patterns for storing subvector extracts of lower 256-bits of 512.
3665 // Its cheaper to just use VMOVAPS/VMOVUPS instead of VEXTRACTF128mr
Craig Topper28e3dfc2016-11-09 05:31:57 +00003666 def : Pat<(alignedstore256 (v4f64 (extract_subvector
3667 (v8f64 VR512:$src), (iPTR 0))), addr:$dst),
Craig Topper95bdabd2016-05-22 23:44:33 +00003668 (VMOVAPDZ256mr addr:$dst, (v4f64 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
3669 def : Pat<(alignedstore (v8f32 (extract_subvector
3670 (v16f32 VR512:$src), (iPTR 0))), addr:$dst),
3671 (VMOVAPSZ256mr addr:$dst, (v8f32 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
Craig Topper28e3dfc2016-11-09 05:31:57 +00003672 def : Pat<(alignedstore256 (v4i64 (extract_subvector
3673 (v8i64 VR512:$src), (iPTR 0))), addr:$dst),
Craig Topper95bdabd2016-05-22 23:44:33 +00003674 (VMOVDQA64Z256mr addr:$dst, (v4i64 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
Craig Topper28e3dfc2016-11-09 05:31:57 +00003675 def : Pat<(alignedstore256 (v8i32 (extract_subvector
3676 (v16i32 VR512:$src), (iPTR 0))), addr:$dst),
Craig Topper95bdabd2016-05-22 23:44:33 +00003677 (VMOVDQA32Z256mr addr:$dst, (v8i32 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
Craig Topper28e3dfc2016-11-09 05:31:57 +00003678 def : Pat<(alignedstore256 (v16i16 (extract_subvector
3679 (v32i16 VR512:$src), (iPTR 0))), addr:$dst),
Craig Topper95bdabd2016-05-22 23:44:33 +00003680 (VMOVDQA32Z256mr addr:$dst, (v16i16 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
Craig Topper28e3dfc2016-11-09 05:31:57 +00003681 def : Pat<(alignedstore256 (v32i8 (extract_subvector
3682 (v64i8 VR512:$src), (iPTR 0))), addr:$dst),
Craig Topper95bdabd2016-05-22 23:44:33 +00003683 (VMOVDQA32Z256mr addr:$dst, (v32i8 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
3684
3685 def : Pat<(store (v4f64 (extract_subvector
3686 (v8f64 VR512:$src), (iPTR 0))), addr:$dst),
3687 (VMOVUPDZ256mr addr:$dst, (v4f64 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
3688 def : Pat<(store (v8f32 (extract_subvector
3689 (v16f32 VR512:$src), (iPTR 0))), addr:$dst),
3690 (VMOVUPSZ256mr addr:$dst, (v8f32 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
3691 def : Pat<(store (v4i64 (extract_subvector
3692 (v8i64 VR512:$src), (iPTR 0))), addr:$dst),
3693 (VMOVDQU64Z256mr addr:$dst, (v4i64 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
3694 def : Pat<(store (v8i32 (extract_subvector
3695 (v16i32 VR512:$src), (iPTR 0))), addr:$dst),
3696 (VMOVDQU32Z256mr addr:$dst, (v8i32 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
3697 def : Pat<(store (v16i16 (extract_subvector
3698 (v32i16 VR512:$src), (iPTR 0))), addr:$dst),
3699 (VMOVDQU32Z256mr addr:$dst, (v16i16 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
3700 def : Pat<(store (v32i8 (extract_subvector
3701 (v64i8 VR512:$src), (iPTR 0))), addr:$dst),
3702 (VMOVDQU32Z256mr addr:$dst, (v32i8 (EXTRACT_SUBREG VR512:$src,sub_ymm)))>;
3703}
3704
Simon Pilgrimb2a80952017-01-08 16:45:39 +00003705
3706// Move Int Doubleword to Packed Double Int
3707//
3708let ExeDomain = SSEPackedInt in {
3709def VMOVDI2PDIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR32:$src),
3710 "vmovd\t{$src, $dst|$dst, $src}",
3711 [(set VR128X:$dst,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003712 (v4i32 (scalar_to_vector GR32:$src)))], IIC_SSE_MOVDQ>,
Craig Topper401675c2015-12-28 06:32:47 +00003713 EVEX;
Elena Demikhovsky767fc962014-01-14 15:10:08 +00003714def VMOVDI2PDIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst), (ins i32mem:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00003715 "vmovd\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003716 [(set VR128X:$dst,
3717 (v4i32 (scalar_to_vector (loadi32 addr:$src))))],
Craig Topper401675c2015-12-28 06:32:47 +00003718 IIC_SSE_MOVDQ>, EVEX, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky767fc962014-01-14 15:10:08 +00003719def VMOV64toPQIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR64:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00003720 "vmovq\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003721 [(set VR128X:$dst,
3722 (v2i64 (scalar_to_vector GR64:$src)))],
Craig Topper401675c2015-12-28 06:32:47 +00003723 IIC_SSE_MOVDQ>, EVEX, VEX_W;
Craig Topperc648c9b2015-12-28 06:11:42 +00003724let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayLoad = 1 in
3725def VMOV64toPQIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst),
3726 (ins i64mem:$src),
3727 "vmovq\t{$src, $dst|$dst, $src}", []>,
Craig Topper401675c2015-12-28 06:32:47 +00003728 EVEX, VEX_W, EVEX_CD8<64, CD8VT1>;
Craig Topper88adf2a2013-10-12 05:41:08 +00003729let isCodeGenOnly = 1 in {
Craig Topperaf88afb2015-12-28 06:11:45 +00003730def VMOV64toSDZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR64X:$dst), (ins GR64:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00003731 "vmovq\t{$src, $dst|$dst, $src}",
Craig Topperaf88afb2015-12-28 06:11:45 +00003732 [(set FR64X:$dst, (bitconvert GR64:$src))],
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003733 IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteMove]>;
Craig Topper5971b542017-02-12 18:47:44 +00003734def VMOV64toSDZrm : AVX512XSI<0x7E, MRMSrcMem, (outs FR64X:$dst), (ins i64mem:$src),
3735 "vmovq\t{$src, $dst|$dst, $src}",
3736 [(set FR64X:$dst, (bitconvert (loadi64 addr:$src)))]>,
3737 EVEX, VEX_W, EVEX_CD8<8, CD8VT8>;
Craig Topperaf88afb2015-12-28 06:11:45 +00003738def VMOVSDto64Zrr : AVX512BI<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64X:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00003739 "vmovq\t{$src, $dst|$dst, $src}",
Craig Topperaf88afb2015-12-28 06:11:45 +00003740 [(set GR64:$dst, (bitconvert FR64X:$src))],
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003741 IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteMove]>;
Craig Topperaf88afb2015-12-28 06:11:45 +00003742def VMOVSDto64Zmr : AVX512BI<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64X:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00003743 "vmovq\t{$src, $dst|$dst, $src}",
Craig Topperaf88afb2015-12-28 06:11:45 +00003744 [(store (i64 (bitconvert FR64X:$src)), addr:$dst)],
Simon Pilgrimb2a80952017-01-08 16:45:39 +00003745 IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteStore]>,
3746 EVEX_CD8<64, CD8VT1>;
3747}
3748} // ExeDomain = SSEPackedInt
3749
3750// Move Int Doubleword to Single Scalar
3751//
3752let ExeDomain = SSEPackedInt, isCodeGenOnly = 1 in {
3753def VMOVDI2SSZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR32X:$dst), (ins GR32:$src),
3754 "vmovd\t{$src, $dst|$dst, $src}",
3755 [(set FR32X:$dst, (bitconvert GR32:$src))],
Craig Topper401675c2015-12-28 06:32:47 +00003756 IIC_SSE_MOVDQ>, EVEX;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003757
Elena Demikhovsky767fc962014-01-14 15:10:08 +00003758def VMOVDI2SSZrm : AVX512BI<0x6E, MRMSrcMem, (outs FR32X:$dst), (ins i32mem:$src),
Simon Pilgrimb2a80952017-01-08 16:45:39 +00003759 "vmovd\t{$src, $dst|$dst, $src}",
3760 [(set FR32X:$dst, (bitconvert (loadi32 addr:$src)))],
3761 IIC_SSE_MOVDQ>, EVEX, EVEX_CD8<32, CD8VT1>;
3762} // ExeDomain = SSEPackedInt, isCodeGenOnly = 1
3763
3764// Move doubleword from xmm register to r/m32
3765//
3766let ExeDomain = SSEPackedInt in {
3767def VMOVPDI2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128X:$src),
3768 "vmovd\t{$src, $dst|$dst, $src}",
3769 [(set GR32:$dst, (extractelt (v4i32 VR128X:$src),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003770 (iPTR 0)))], IIC_SSE_MOVD_ToGP>,
Craig Topper401675c2015-12-28 06:32:47 +00003771 EVEX;
Elena Demikhovsky767fc962014-01-14 15:10:08 +00003772def VMOVPDI2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003773 (ins i32mem:$dst, VR128X:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00003774 "vmovd\t{$src, $dst|$dst, $src}",
Simon Pilgrimb2a80952017-01-08 16:45:39 +00003775 [(store (i32 (extractelt (v4i32 VR128X:$src),
3776 (iPTR 0))), addr:$dst)], IIC_SSE_MOVDQ>,
3777 EVEX, EVEX_CD8<32, CD8VT1>;
3778} // ExeDomain = SSEPackedInt
3779
3780// Move quadword from xmm1 register to r/m64
3781//
3782let ExeDomain = SSEPackedInt in {
3783def VMOVPQIto64Zrr : I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128X:$src),
3784 "vmovq\t{$src, $dst|$dst, $src}",
3785 [(set GR64:$dst, (extractelt (v2i64 VR128X:$src),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003786 (iPTR 0)))],
Craig Topper401675c2015-12-28 06:32:47 +00003787 IIC_SSE_MOVD_ToGP>, PD, EVEX, VEX_W,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003788 Requires<[HasAVX512, In64BitMode]>;
3789
Craig Topperc648c9b2015-12-28 06:11:42 +00003790let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayStore = 1 in
3791def VMOVPQIto64Zmr : I<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, VR128X:$src),
3792 "vmovq\t{$src, $dst|$dst, $src}",
Craig Topper401675c2015-12-28 06:32:47 +00003793 [], IIC_SSE_MOVD_ToGP>, PD, EVEX, VEX_W,
Craig Topperc648c9b2015-12-28 06:11:42 +00003794 Requires<[HasAVX512, In64BitMode]>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003795
Craig Topperc648c9b2015-12-28 06:11:42 +00003796def VMOVPQI2QIZmr : I<0xD6, MRMDestMem, (outs),
3797 (ins i64mem:$dst, VR128X:$src),
3798 "vmovq\t{$src, $dst|$dst, $src}",
3799 [(store (extractelt (v2i64 VR128X:$src), (iPTR 0)),
3800 addr:$dst)], IIC_SSE_MOVDQ>,
Craig Topper401675c2015-12-28 06:32:47 +00003801 EVEX, PD, VEX_W, EVEX_CD8<64, CD8VT1>,
Craig Topperc648c9b2015-12-28 06:11:42 +00003802 Sched<[WriteStore]>, Requires<[HasAVX512, In64BitMode]>;
3803
3804let hasSideEffects = 0 in
3805def VMOVPQI2QIZrr : AVX512BI<0xD6, MRMDestReg, (outs VR128X:$dst),
Simon Pilgrimb2a80952017-01-08 16:45:39 +00003806 (ins VR128X:$src),
3807 "vmovq.s\t{$src, $dst|$dst, $src}",[]>,
3808 EVEX, VEX_W;
3809} // ExeDomain = SSEPackedInt
3810
3811// Move Scalar Single to Double Int
3812//
3813let ExeDomain = SSEPackedInt, isCodeGenOnly = 1 in {
3814def VMOVSS2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst),
3815 (ins FR32X:$src),
3816 "vmovd\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003817 [(set GR32:$dst, (bitconvert FR32X:$src))],
Craig Topper401675c2015-12-28 06:32:47 +00003818 IIC_SSE_MOVD_ToGP>, EVEX;
Elena Demikhovsky767fc962014-01-14 15:10:08 +00003819def VMOVSS2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003820 (ins i32mem:$dst, FR32X:$src),
Simon Pilgrimb2a80952017-01-08 16:45:39 +00003821 "vmovd\t{$src, $dst|$dst, $src}",
3822 [(store (i32 (bitconvert FR32X:$src)), addr:$dst)],
3823 IIC_SSE_MOVDQ>, EVEX, EVEX_CD8<32, CD8VT1>;
3824} // ExeDomain = SSEPackedInt, isCodeGenOnly = 1
3825
3826// Move Quadword Int to Packed Quadword Int
3827//
3828let ExeDomain = SSEPackedInt in {
3829def VMOVQI2PQIZrm : AVX512XSI<0x7E, MRMSrcMem, (outs VR128X:$dst),
3830 (ins i64mem:$src),
3831 "vmovq\t{$src, $dst|$dst, $src}",
3832 [(set VR128X:$dst,
3833 (v2i64 (scalar_to_vector (loadi64 addr:$src))))]>,
3834 EVEX, VEX_W, EVEX_CD8<8, CD8VT8>;
3835} // ExeDomain = SSEPackedInt
3836
3837//===----------------------------------------------------------------------===//
3838// AVX-512 MOVSS, MOVSD
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003839//===----------------------------------------------------------------------===//
3840
Craig Topperc7de3a12016-07-29 02:49:08 +00003841multiclass avx512_move_scalar<string asm, SDNode OpNode,
Asaf Badouh41ecf462015-12-06 13:26:56 +00003842 X86VectorVTInfo _> {
Craig Topperc7de3a12016-07-29 02:49:08 +00003843 def rr : AVX512PI<0x10, MRMSrcReg, (outs _.RC:$dst),
3844 (ins _.RC:$src1, _.FRC:$src2),
3845 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
3846 [(set _.RC:$dst, (_.VT (OpNode _.RC:$src1,
3847 (scalar_to_vector _.FRC:$src2))))],
3848 _.ExeDomain,IIC_SSE_MOV_S_RR>, EVEX_4V;
3849 def rrkz : AVX512PI<0x10, MRMSrcReg, (outs _.RC:$dst),
Simon Pilgrim049d9c92017-03-26 12:52:28 +00003850 (ins _.KRCWM:$mask, _.RC:$src1, _.FRC:$src2),
Craig Topperc7de3a12016-07-29 02:49:08 +00003851 !strconcat(asm, "\t{$src2, $src1, $dst {${mask}} {z}|",
3852 "$dst {${mask}} {z}, $src1, $src2}"),
3853 [(set _.RC:$dst, (_.VT (X86selects _.KRCWM:$mask,
Simon Pilgrim049d9c92017-03-26 12:52:28 +00003854 (_.VT (OpNode _.RC:$src1,
3855 (scalar_to_vector _.FRC:$src2))),
Craig Topperc7de3a12016-07-29 02:49:08 +00003856 _.ImmAllZerosV)))],
3857 _.ExeDomain,IIC_SSE_MOV_S_RR>, EVEX_4V, EVEX_KZ;
3858 let Constraints = "$src0 = $dst" in
3859 def rrk : AVX512PI<0x10, MRMSrcReg, (outs _.RC:$dst),
Simon Pilgrim049d9c92017-03-26 12:52:28 +00003860 (ins _.RC:$src0, _.KRCWM:$mask, _.RC:$src1, _.FRC:$src2),
Craig Topperc7de3a12016-07-29 02:49:08 +00003861 !strconcat(asm, "\t{$src2, $src1, $dst {${mask}}|",
3862 "$dst {${mask}}, $src1, $src2}"),
3863 [(set _.RC:$dst, (_.VT (X86selects _.KRCWM:$mask,
Simon Pilgrim049d9c92017-03-26 12:52:28 +00003864 (_.VT (OpNode _.RC:$src1,
3865 (scalar_to_vector _.FRC:$src2))),
Craig Topperc7de3a12016-07-29 02:49:08 +00003866 (_.VT _.RC:$src0))))],
3867 _.ExeDomain,IIC_SSE_MOV_S_RR>, EVEX_4V, EVEX_K;
Craig Toppere4f868e2016-07-29 06:06:04 +00003868 let canFoldAsLoad = 1, isReMaterializable = 1 in
Craig Topperc7de3a12016-07-29 02:49:08 +00003869 def rm : AVX512PI<0x10, MRMSrcMem, (outs _.FRC:$dst), (ins _.ScalarMemOp:$src),
3870 !strconcat(asm, "\t{$src, $dst|$dst, $src}"),
3871 [(set _.FRC:$dst, (_.ScalarLdFrag addr:$src))],
3872 _.ExeDomain, IIC_SSE_MOV_S_RM>, EVEX;
3873 let mayLoad = 1, hasSideEffects = 0 in {
3874 let Constraints = "$src0 = $dst" in
3875 def rmk : AVX512PI<0x10, MRMSrcMem, (outs _.RC:$dst),
3876 (ins _.RC:$src0, _.KRCWM:$mask, _.ScalarMemOp:$src),
3877 !strconcat(asm, "\t{$src, $dst {${mask}}|",
3878 "$dst {${mask}}, $src}"),
3879 [], _.ExeDomain, IIC_SSE_MOV_S_RM>, EVEX, EVEX_K;
3880 def rmkz : AVX512PI<0x10, MRMSrcMem, (outs _.RC:$dst),
3881 (ins _.KRCWM:$mask, _.ScalarMemOp:$src),
3882 !strconcat(asm, "\t{$src, $dst {${mask}} {z}|",
3883 "$dst {${mask}} {z}, $src}"),
3884 [], _.ExeDomain, IIC_SSE_MOV_S_RM>, EVEX, EVEX_KZ;
Asaf Badouh41ecf462015-12-06 13:26:56 +00003885 }
Craig Toppere1cac152016-06-07 07:27:54 +00003886 def mr: AVX512PI<0x11, MRMDestMem, (outs), (ins _.ScalarMemOp:$dst, _.FRC:$src),
3887 !strconcat(asm, "\t{$src, $dst|$dst, $src}"),
3888 [(store _.FRC:$src, addr:$dst)], _.ExeDomain, IIC_SSE_MOV_S_MR>,
3889 EVEX;
Craig Topperc7de3a12016-07-29 02:49:08 +00003890 let mayStore = 1, hasSideEffects = 0 in
Craig Toppere1cac152016-06-07 07:27:54 +00003891 def mrk: AVX512PI<0x11, MRMDestMem, (outs),
3892 (ins _.ScalarMemOp:$dst, VK1WM:$mask, _.FRC:$src),
3893 !strconcat(asm, "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}"),
3894 [], _.ExeDomain, IIC_SSE_MOV_S_MR>, EVEX, EVEX_K;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003895}
3896
Asaf Badouh41ecf462015-12-06 13:26:56 +00003897defm VMOVSSZ : avx512_move_scalar<"vmovss", X86Movss, f32x_info>,
3898 VEX_LIG, XS, EVEX_CD8<32, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003899
Asaf Badouh41ecf462015-12-06 13:26:56 +00003900defm VMOVSDZ : avx512_move_scalar<"vmovsd", X86Movsd, f64x_info>,
3901 VEX_LIG, XD, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00003902
Ayman Musa46af8f92016-11-13 14:29:32 +00003903
3904multiclass avx512_move_scalar_lowering<string InstrStr, SDNode OpNode,
3905 PatLeaf ZeroFP, X86VectorVTInfo _> {
3906
3907def : Pat<(_.VT (OpNode _.RC:$src0,
Simon Pilgrim3f10e992016-11-20 14:05:23 +00003908 (_.VT (scalar_to_vector
Guy Blank548e22a2017-05-19 12:35:15 +00003909 (_.EltVT (X86selects (scalar_to_vector (and (i8 (trunc GR32:$mask)), (i8 1))),
Ayman Musa46af8f92016-11-13 14:29:32 +00003910 (_.EltVT _.FRC:$src1),
3911 (_.EltVT _.FRC:$src2))))))),
Simon Pilgrim3f10e992016-11-20 14:05:23 +00003912 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr#rrk)
Ayman Musa46af8f92016-11-13 14:29:32 +00003913 (COPY_TO_REGCLASS _.FRC:$src2, _.RC),
3914 (COPY_TO_REGCLASS GR32:$mask, VK1WM),
Simon Pilgrim049d9c92017-03-26 12:52:28 +00003915 (_.VT _.RC:$src0), _.FRC:$src1),
Ayman Musa46af8f92016-11-13 14:29:32 +00003916 _.RC)>;
3917
3918def : Pat<(_.VT (OpNode _.RC:$src0,
Simon Pilgrim3f10e992016-11-20 14:05:23 +00003919 (_.VT (scalar_to_vector
Guy Blank548e22a2017-05-19 12:35:15 +00003920 (_.EltVT (X86selects (scalar_to_vector (and (i8 (trunc GR32:$mask)), (i8 1))),
Ayman Musa46af8f92016-11-13 14:29:32 +00003921 (_.EltVT _.FRC:$src1),
3922 (_.EltVT ZeroFP))))))),
Simon Pilgrim3f10e992016-11-20 14:05:23 +00003923 (COPY_TO_REGCLASS (!cast<Instruction>(InstrStr#rrkz)
Ayman Musa46af8f92016-11-13 14:29:32 +00003924 (COPY_TO_REGCLASS GR32:$mask, VK1WM),
Simon Pilgrim049d9c92017-03-26 12:52:28 +00003925 (_.VT _.RC:$src0), _.FRC:$src1),
Ayman Musa46af8f92016-11-13 14:29:32 +00003926 _.RC)>;
Ayman Musa46af8f92016-11-13 14:29:32 +00003927}
3928
3929multiclass avx512_store_scalar_lowering<string InstrStr, AVX512VLVectorVTInfo _,
3930 dag Mask, RegisterClass MaskRC> {
3931
3932def : Pat<(masked_store addr:$dst, Mask,
Simon Pilgrim3f10e992016-11-20 14:05:23 +00003933 (_.info512.VT (insert_subvector undef,
Ayman Musa46af8f92016-11-13 14:29:32 +00003934 (_.info256.VT (insert_subvector undef,
3935 (_.info128.VT _.info128.RC:$src),
Craig Topper7a5ee1c2017-03-14 06:40:04 +00003936 (iPTR 0))),
3937 (iPTR 0)))),
Simon Pilgrim3f10e992016-11-20 14:05:23 +00003938 (!cast<Instruction>(InstrStr#mrk) addr:$dst,
Guy Blank548e22a2017-05-19 12:35:15 +00003939 (COPY_TO_REGCLASS MaskRC:$mask, VK1WM),
Simon Pilgrim3f10e992016-11-20 14:05:23 +00003940 (COPY_TO_REGCLASS _.info128.RC:$src, _.info128.FRC))>;
Ayman Musa46af8f92016-11-13 14:29:32 +00003941
3942}
3943
Craig Topper058f2f62017-03-28 16:35:29 +00003944multiclass avx512_store_scalar_lowering_subreg<string InstrStr,
3945 AVX512VLVectorVTInfo _,
3946 dag Mask, RegisterClass MaskRC,
3947 SubRegIndex subreg> {
3948
3949def : Pat<(masked_store addr:$dst, Mask,
3950 (_.info512.VT (insert_subvector undef,
3951 (_.info256.VT (insert_subvector undef,
3952 (_.info128.VT _.info128.RC:$src),
3953 (iPTR 0))),
3954 (iPTR 0)))),
3955 (!cast<Instruction>(InstrStr#mrk) addr:$dst,
Guy Blank548e22a2017-05-19 12:35:15 +00003956 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),
Craig Topper058f2f62017-03-28 16:35:29 +00003957 (COPY_TO_REGCLASS _.info128.RC:$src, _.info128.FRC))>;
3958
3959}
3960
Ayman Musa46af8f92016-11-13 14:29:32 +00003961multiclass avx512_load_scalar_lowering<string InstrStr, AVX512VLVectorVTInfo _,
3962 dag Mask, RegisterClass MaskRC> {
3963
3964def : Pat<(_.info128.VT (extract_subvector
3965 (_.info512.VT (masked_load addr:$srcAddr, Mask,
Simon Pilgrim3f10e992016-11-20 14:05:23 +00003966 (_.info512.VT (bitconvert
Ayman Musa46af8f92016-11-13 14:29:32 +00003967 (v16i32 immAllZerosV))))),
Craig Topper7a5ee1c2017-03-14 06:40:04 +00003968 (iPTR 0))),
Simon Pilgrim3f10e992016-11-20 14:05:23 +00003969 (!cast<Instruction>(InstrStr#rmkz)
Guy Blank548e22a2017-05-19 12:35:15 +00003970 (COPY_TO_REGCLASS MaskRC:$mask, VK1WM),
Ayman Musa46af8f92016-11-13 14:29:32 +00003971 addr:$srcAddr)>;
3972
3973def : Pat<(_.info128.VT (extract_subvector
3974 (_.info512.VT (masked_load addr:$srcAddr, Mask,
3975 (_.info512.VT (insert_subvector undef,
3976 (_.info256.VT (insert_subvector undef,
3977 (_.info128.VT (X86vzmovl _.info128.RC:$src)),
Craig Topper7a5ee1c2017-03-14 06:40:04 +00003978 (iPTR 0))),
3979 (iPTR 0))))),
3980 (iPTR 0))),
Ayman Musa46af8f92016-11-13 14:29:32 +00003981 (!cast<Instruction>(InstrStr#rmk) _.info128.RC:$src,
Guy Blank548e22a2017-05-19 12:35:15 +00003982 (COPY_TO_REGCLASS MaskRC:$mask, VK1WM),
Ayman Musa46af8f92016-11-13 14:29:32 +00003983 addr:$srcAddr)>;
3984
3985}
3986
Craig Topper058f2f62017-03-28 16:35:29 +00003987multiclass avx512_load_scalar_lowering_subreg<string InstrStr,
3988 AVX512VLVectorVTInfo _,
3989 dag Mask, RegisterClass MaskRC,
3990 SubRegIndex subreg> {
3991
3992def : Pat<(_.info128.VT (extract_subvector
3993 (_.info512.VT (masked_load addr:$srcAddr, Mask,
3994 (_.info512.VT (bitconvert
3995 (v16i32 immAllZerosV))))),
3996 (iPTR 0))),
3997 (!cast<Instruction>(InstrStr#rmkz)
Guy Blank548e22a2017-05-19 12:35:15 +00003998 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),
Craig Topper058f2f62017-03-28 16:35:29 +00003999 addr:$srcAddr)>;
4000
4001def : Pat<(_.info128.VT (extract_subvector
4002 (_.info512.VT (masked_load addr:$srcAddr, Mask,
4003 (_.info512.VT (insert_subvector undef,
4004 (_.info256.VT (insert_subvector undef,
4005 (_.info128.VT (X86vzmovl _.info128.RC:$src)),
4006 (iPTR 0))),
4007 (iPTR 0))))),
4008 (iPTR 0))),
4009 (!cast<Instruction>(InstrStr#rmk) _.info128.RC:$src,
Guy Blank548e22a2017-05-19 12:35:15 +00004010 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),
Craig Topper058f2f62017-03-28 16:35:29 +00004011 addr:$srcAddr)>;
4012
4013}
4014
Ayman Musa46af8f92016-11-13 14:29:32 +00004015defm : avx512_move_scalar_lowering<"VMOVSSZ", X86Movss, fp32imm0, v4f32x_info>;
4016defm : avx512_move_scalar_lowering<"VMOVSDZ", X86Movsd, fp64imm0, v2f64x_info>;
4017
4018defm : avx512_store_scalar_lowering<"VMOVSSZ", avx512vl_f32_info,
4019 (v16i1 (bitconvert (i16 (trunc (and GR32:$mask, (i32 1)))))), GR32>;
Craig Topper058f2f62017-03-28 16:35:29 +00004020defm : avx512_store_scalar_lowering_subreg<"VMOVSSZ", avx512vl_f32_info,
4021 (v16i1 (bitconvert (i16 (and GR16:$mask, (i16 1))))), GR16, sub_16bit>;
4022defm : avx512_store_scalar_lowering_subreg<"VMOVSDZ", avx512vl_f64_info,
4023 (v8i1 (bitconvert (i8 (and GR8:$mask, (i8 1))))), GR8, sub_8bit>;
Ayman Musa46af8f92016-11-13 14:29:32 +00004024
4025defm : avx512_load_scalar_lowering<"VMOVSSZ", avx512vl_f32_info,
4026 (v16i1 (bitconvert (i16 (trunc (and GR32:$mask, (i32 1)))))), GR32>;
Craig Topper058f2f62017-03-28 16:35:29 +00004027defm : avx512_load_scalar_lowering_subreg<"VMOVSSZ", avx512vl_f32_info,
4028 (v16i1 (bitconvert (i16 (and GR16:$mask, (i16 1))))), GR16, sub_16bit>;
4029defm : avx512_load_scalar_lowering_subreg<"VMOVSDZ", avx512vl_f64_info,
4030 (v8i1 (bitconvert (i8 (and GR8:$mask, (i8 1))))), GR8, sub_8bit>;
Ayman Musa46af8f92016-11-13 14:29:32 +00004031
Guy Blankb169d56d2017-07-31 08:26:14 +00004032def : Pat<(f32 (X86selects (scalar_to_vector (and GR8:$mask, (i8 1))),
4033 (f32 FR32X:$src1), (f32 FR32X:$src2))),
4034 (COPY_TO_REGCLASS
4035 (VMOVSSZrrk (COPY_TO_REGCLASS FR32X:$src2, VR128X),
4036 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF),
4037 GR8:$mask, sub_8bit)), VK1WM),
4038 (v4f32 (IMPLICIT_DEF)), FR32X:$src1), FR32X)>;
4039
Craig Topper74ed0872016-05-18 06:55:59 +00004040def : Pat<(f32 (X86selects VK1WM:$mask, (f32 FR32X:$src1), (f32 FR32X:$src2))),
Craig Topperc7de3a12016-07-29 02:49:08 +00004041 (COPY_TO_REGCLASS (VMOVSSZrrk (COPY_TO_REGCLASS FR32X:$src2, VR128X),
Simon Pilgrim049d9c92017-03-26 12:52:28 +00004042 VK1WM:$mask, (v4f32 (IMPLICIT_DEF)), FR32X:$src1), FR32X)>;
Elena Demikhovsky47fc44e2013-12-16 13:52:35 +00004043
Guy Blankb169d56d2017-07-31 08:26:14 +00004044def : Pat<(f64 (X86selects (scalar_to_vector (and GR8:$mask, (i8 1))),
4045 (f64 FR64X:$src1), (f64 FR64X:$src2))),
4046 (COPY_TO_REGCLASS
4047 (VMOVSDZrrk (COPY_TO_REGCLASS FR64X:$src2, VR128X),
4048 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF),
4049 GR8:$mask, sub_8bit)), VK1WM),
4050 (v2f64 (IMPLICIT_DEF)), FR64X:$src1), FR64X)>;
4051
Craig Topper74ed0872016-05-18 06:55:59 +00004052def : Pat<(f64 (X86selects VK1WM:$mask, (f64 FR64X:$src1), (f64 FR64X:$src2))),
Craig Topperc7de3a12016-07-29 02:49:08 +00004053 (COPY_TO_REGCLASS (VMOVSDZrrk (COPY_TO_REGCLASS FR64X:$src2, VR128X),
Simon Pilgrim049d9c92017-03-26 12:52:28 +00004054 VK1WM:$mask, (v2f64 (IMPLICIT_DEF)), FR64X:$src1), FR64X)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004055
Elena Demikhovskyff620ed2014-08-27 07:38:43 +00004056def : Pat<(int_x86_avx512_mask_store_ss addr:$dst, VR128X:$src, GR8:$mask),
Guy Blank548e22a2017-05-19 12:35:15 +00004057 (VMOVSSZmrk addr:$dst, (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), GR8:$mask, sub_8bit)), VK1WM),
Elena Demikhovskyff620ed2014-08-27 07:38:43 +00004058 (COPY_TO_REGCLASS VR128X:$src, FR32X))>;
4059
Ayman Musa0b4f97d2017-05-28 12:39:37 +00004060let hasSideEffects = 0 in {
Simon Pilgrim64fff142017-07-16 18:37:23 +00004061 def VMOVSSZrr_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),
Ayman Musa0b4f97d2017-05-28 12:39:37 +00004062 (ins VR128X:$src1, FR32X:$src2),
4063 "vmovss.s\t{$src2, $src1, $dst|$dst, $src1, $src2}",
4064 [], NoItinerary>, XS, EVEX_4V, VEX_LIG,
4065 FoldGenData<"VMOVSSZrr">;
Igor Breger4424aaa2015-11-19 07:58:33 +00004066
Ayman Musa0b4f97d2017-05-28 12:39:37 +00004067let Constraints = "$src0 = $dst" in
Simon Pilgrim64fff142017-07-16 18:37:23 +00004068 def VMOVSSZrrk_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),
4069 (ins f32x_info.RC:$src0, f32x_info.KRCWM:$mask,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00004070 VR128X:$src1, FR32X:$src2),
4071 "vmovss.s\t{$src2, $src1, $dst {${mask}}|"#
4072 "$dst {${mask}}, $src1, $src2}",
4073 [], NoItinerary>, EVEX_K, XS, EVEX_4V, VEX_LIG,
4074 FoldGenData<"VMOVSSZrrk">;
Simon Pilgrim64fff142017-07-16 18:37:23 +00004075
4076 def VMOVSSZrrkz_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),
Ayman Musa0b4f97d2017-05-28 12:39:37 +00004077 (ins f32x_info.KRCWM:$mask, VR128X:$src1, FR32X:$src2),
4078 "vmovss.s\t{$src2, $src1, $dst {${mask}} {z}|"#
4079 "$dst {${mask}} {z}, $src1, $src2}",
4080 [], NoItinerary>, EVEX_KZ, XS, EVEX_4V, VEX_LIG,
4081 FoldGenData<"VMOVSSZrrkz">;
4082
Simon Pilgrim64fff142017-07-16 18:37:23 +00004083 def VMOVSDZrr_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),
Ayman Musa0b4f97d2017-05-28 12:39:37 +00004084 (ins VR128X:$src1, FR64X:$src2),
4085 "vmovsd.s\t{$src2, $src1, $dst|$dst, $src1, $src2}",
4086 [], NoItinerary>, XD, EVEX_4V, VEX_LIG, VEX_W,
4087 FoldGenData<"VMOVSDZrr">;
4088
4089let Constraints = "$src0 = $dst" in
Simon Pilgrim64fff142017-07-16 18:37:23 +00004090 def VMOVSDZrrk_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),
4091 (ins f64x_info.RC:$src0, f64x_info.KRCWM:$mask,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00004092 VR128X:$src1, FR64X:$src2),
4093 "vmovsd.s\t{$src2, $src1, $dst {${mask}}|"#
4094 "$dst {${mask}}, $src1, $src2}",
4095 [], NoItinerary>, EVEX_K, XD, EVEX_4V, VEX_LIG,
Simon Pilgrim64fff142017-07-16 18:37:23 +00004096 VEX_W, FoldGenData<"VMOVSDZrrk">;
Ayman Musa0b4f97d2017-05-28 12:39:37 +00004097
Simon Pilgrim64fff142017-07-16 18:37:23 +00004098 def VMOVSDZrrkz_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),
4099 (ins f64x_info.KRCWM:$mask, VR128X:$src1,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00004100 FR64X:$src2),
4101 "vmovsd.s\t{$src2, $src1, $dst {${mask}} {z}|"#
4102 "$dst {${mask}} {z}, $src1, $src2}",
Simon Pilgrim64fff142017-07-16 18:37:23 +00004103 [], NoItinerary>, EVEX_KZ, XD, EVEX_4V, VEX_LIG,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00004104 VEX_W, FoldGenData<"VMOVSDZrrkz">;
4105}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004106
4107let Predicates = [HasAVX512] in {
4108 let AddedComplexity = 15 in {
4109 // Move scalar to XMM zero-extended, zeroing a VR128X then do a
4110 // MOVS{S,D} to the lower bits.
4111 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector FR32X:$src)))),
Craig Topper09b7e0f2017-01-14 07:29:24 +00004112 (VMOVSSZrr (v4f32 (AVX512_128_SET0)), FR32X:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004113 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128X:$src))),
Craig Topper09b7e0f2017-01-14 07:29:24 +00004114 (VMOVSSZrr (v4f32 (AVX512_128_SET0)), (COPY_TO_REGCLASS VR128X:$src, FR32X))>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004115 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128X:$src))),
Craig Topper09b7e0f2017-01-14 07:29:24 +00004116 (VMOVSSZrr (v4i32 (AVX512_128_SET0)), (COPY_TO_REGCLASS VR128X:$src, FR32X))>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004117 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64X:$src)))),
Craig Topper09b7e0f2017-01-14 07:29:24 +00004118 (VMOVSDZrr (v2f64 (AVX512_128_SET0)), FR64X:$src)>;
Craig Topper3f8126e2016-08-13 05:43:20 +00004119 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004120
4121 // Move low f32 and clear high bits.
4122 def : Pat<(v8f32 (X86vzmovl (v8f32 VR256X:$src))),
4123 (SUBREG_TO_REG (i32 0),
Craig Topper09b7e0f2017-01-14 07:29:24 +00004124 (VMOVSSZrr (v4f32 (AVX512_128_SET0)),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004125 (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm)), sub_xmm)>;
4126 def : Pat<(v8i32 (X86vzmovl (v8i32 VR256X:$src))),
4127 (SUBREG_TO_REG (i32 0),
Craig Topper09b7e0f2017-01-14 07:29:24 +00004128 (VMOVSSZrr (v4i32 (AVX512_128_SET0)),
Craig Topper600685d2016-08-13 05:33:12 +00004129 (EXTRACT_SUBREG (v8i32 VR256X:$src), sub_xmm)), sub_xmm)>;
Craig Topper600685d2016-08-13 05:33:12 +00004130 def : Pat<(v16f32 (X86vzmovl (v16f32 VR512:$src))),
4131 (SUBREG_TO_REG (i32 0),
Craig Topper09b7e0f2017-01-14 07:29:24 +00004132 (VMOVSSZrr (v4f32 (AVX512_128_SET0)),
Craig Topper600685d2016-08-13 05:33:12 +00004133 (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm)), sub_xmm)>;
4134 def : Pat<(v16i32 (X86vzmovl (v16i32 VR512:$src))),
4135 (SUBREG_TO_REG (i32 0),
Craig Topper09b7e0f2017-01-14 07:29:24 +00004136 (VMOVSSZrr (v4i32 (AVX512_128_SET0)),
Craig Topper600685d2016-08-13 05:33:12 +00004137 (EXTRACT_SUBREG (v16i32 VR512:$src), sub_xmm)), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004138
4139 let AddedComplexity = 20 in {
4140 // MOVSSrm zeros the high parts of the register; represent this
4141 // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0
4142 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))),
4143 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>;
4144 def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))),
4145 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>;
4146 def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))),
4147 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>;
Simon Pilgrim6392b8d2016-08-24 10:46:40 +00004148 def : Pat<(v4f32 (X86vzload addr:$src)),
4149 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004150
4151 // MOVSDrm zeros the high parts of the register; represent this
4152 // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0
4153 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))),
4154 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
4155 def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))),
4156 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
4157 def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))),
4158 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
4159 def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))),
4160 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
4161 def : Pat<(v2f64 (X86vzload addr:$src)),
4162 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
4163
4164 // Represent the same patterns above but in the form they appear for
4165 // 256-bit types
4166 def : Pat<(v8i32 (X86vzmovl (insert_subvector undef,
4167 (v4i32 (scalar_to_vector (loadi32 addr:$src))), (iPTR 0)))),
Elena Demikhovsky34586e72013-10-02 12:20:42 +00004168 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrm addr:$src), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004169 def : Pat<(v8f32 (X86vzmovl (insert_subvector undef,
4170 (v4f32 (scalar_to_vector (loadf32 addr:$src))), (iPTR 0)))),
4171 (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>;
Simon Pilgrim6392b8d2016-08-24 10:46:40 +00004172 def : Pat<(v8f32 (X86vzload addr:$src)),
4173 (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004174 def : Pat<(v4f64 (X86vzmovl (insert_subvector undef,
4175 (v2f64 (scalar_to_vector (loadf64 addr:$src))), (iPTR 0)))),
4176 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;
Simon Pilgrim7823fd22016-02-04 19:27:51 +00004177 def : Pat<(v4f64 (X86vzload addr:$src)),
4178 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;
Simon Pilgrim6788f332016-02-04 16:12:56 +00004179
4180 // Represent the same patterns above but in the form they appear for
4181 // 512-bit types
4182 def : Pat<(v16i32 (X86vzmovl (insert_subvector undef,
4183 (v4i32 (scalar_to_vector (loadi32 addr:$src))), (iPTR 0)))),
4184 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrm addr:$src), sub_xmm)>;
4185 def : Pat<(v16f32 (X86vzmovl (insert_subvector undef,
4186 (v4f32 (scalar_to_vector (loadf32 addr:$src))), (iPTR 0)))),
4187 (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>;
Simon Pilgrim6392b8d2016-08-24 10:46:40 +00004188 def : Pat<(v16f32 (X86vzload addr:$src)),
4189 (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>;
Simon Pilgrim6788f332016-02-04 16:12:56 +00004190 def : Pat<(v8f64 (X86vzmovl (insert_subvector undef,
4191 (v2f64 (scalar_to_vector (loadf64 addr:$src))), (iPTR 0)))),
4192 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;
Simon Pilgrim7823fd22016-02-04 19:27:51 +00004193 def : Pat<(v8f64 (X86vzload addr:$src)),
4194 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004195 }
4196 def : Pat<(v8f32 (X86vzmovl (insert_subvector undef,
4197 (v4f32 (scalar_to_vector FR32X:$src)), (iPTR 0)))),
Craig Topper09b7e0f2017-01-14 07:29:24 +00004198 (SUBREG_TO_REG (i32 0), (v4f32 (VMOVSSZrr (v4f32 (AVX512_128_SET0)),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004199 FR32X:$src)), sub_xmm)>;
4200 def : Pat<(v4f64 (X86vzmovl (insert_subvector undef,
4201 (v2f64 (scalar_to_vector FR64X:$src)), (iPTR 0)))),
Craig Topper09b7e0f2017-01-14 07:29:24 +00004202 (SUBREG_TO_REG (i64 0), (v2f64 (VMOVSDZrr (v2f64 (AVX512_128_SET0)),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004203 FR64X:$src)), sub_xmm)>;
4204 def : Pat<(v4i64 (X86vzmovl (insert_subvector undef,
4205 (v2i64 (scalar_to_vector (loadi64 addr:$src))), (iPTR 0)))),
Elena Demikhovsky34586e72013-10-02 12:20:42 +00004206 (SUBREG_TO_REG (i64 0), (VMOVQI2PQIZrm addr:$src), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004207
4208 // Move low f64 and clear high bits.
4209 def : Pat<(v4f64 (X86vzmovl (v4f64 VR256X:$src))),
4210 (SUBREG_TO_REG (i32 0),
Craig Topper09b7e0f2017-01-14 07:29:24 +00004211 (VMOVSDZrr (v2f64 (AVX512_128_SET0)),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004212 (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm)), sub_xmm)>;
Craig Topper600685d2016-08-13 05:33:12 +00004213 def : Pat<(v8f64 (X86vzmovl (v8f64 VR512:$src))),
4214 (SUBREG_TO_REG (i32 0),
Craig Topper09b7e0f2017-01-14 07:29:24 +00004215 (VMOVSDZrr (v2f64 (AVX512_128_SET0)),
Craig Topper600685d2016-08-13 05:33:12 +00004216 (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm)), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004217
4218 def : Pat<(v4i64 (X86vzmovl (v4i64 VR256X:$src))),
Craig Topper09b7e0f2017-01-14 07:29:24 +00004219 (SUBREG_TO_REG (i32 0), (VMOVSDZrr (v2i64 (AVX512_128_SET0)),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004220 (EXTRACT_SUBREG (v4i64 VR256X:$src), sub_xmm)), sub_xmm)>;
Craig Topper600685d2016-08-13 05:33:12 +00004221 def : Pat<(v8i64 (X86vzmovl (v8i64 VR512:$src))),
Craig Topper09b7e0f2017-01-14 07:29:24 +00004222 (SUBREG_TO_REG (i32 0), (VMOVSDZrr (v2i64 (AVX512_128_SET0)),
Craig Topper600685d2016-08-13 05:33:12 +00004223 (EXTRACT_SUBREG (v8i64 VR512:$src), sub_xmm)), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004224
4225 // Extract and store.
Matt Arsenaultfbd9bbf2015-12-11 19:20:16 +00004226 def : Pat<(store (f32 (extractelt (v4f32 VR128X:$src), (iPTR 0))),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004227 addr:$dst),
4228 (VMOVSSZmr addr:$dst, (COPY_TO_REGCLASS (v4f32 VR128X:$src), FR32X))>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004229
4230 // Shuffle with VMOVSS
4231 def : Pat<(v4i32 (X86Movss VR128X:$src1, VR128X:$src2)),
4232 (VMOVSSZrr (v4i32 VR128X:$src1),
4233 (COPY_TO_REGCLASS (v4i32 VR128X:$src2), FR32X))>;
4234 def : Pat<(v4f32 (X86Movss VR128X:$src1, VR128X:$src2)),
4235 (VMOVSSZrr (v4f32 VR128X:$src1),
4236 (COPY_TO_REGCLASS (v4f32 VR128X:$src2), FR32X))>;
4237
4238 // 256-bit variants
4239 def : Pat<(v8i32 (X86Movss VR256X:$src1, VR256X:$src2)),
4240 (SUBREG_TO_REG (i32 0),
4241 (VMOVSSZrr (EXTRACT_SUBREG (v8i32 VR256X:$src1), sub_xmm),
4242 (EXTRACT_SUBREG (v8i32 VR256X:$src2), sub_xmm)),
4243 sub_xmm)>;
4244 def : Pat<(v8f32 (X86Movss VR256X:$src1, VR256X:$src2)),
4245 (SUBREG_TO_REG (i32 0),
4246 (VMOVSSZrr (EXTRACT_SUBREG (v8f32 VR256X:$src1), sub_xmm),
4247 (EXTRACT_SUBREG (v8f32 VR256X:$src2), sub_xmm)),
4248 sub_xmm)>;
4249
4250 // Shuffle with VMOVSD
4251 def : Pat<(v2i64 (X86Movsd VR128X:$src1, VR128X:$src2)),
4252 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
4253 def : Pat<(v2f64 (X86Movsd VR128X:$src1, VR128X:$src2)),
4254 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004255
4256 // 256-bit variants
4257 def : Pat<(v4i64 (X86Movsd VR256X:$src1, VR256X:$src2)),
4258 (SUBREG_TO_REG (i32 0),
4259 (VMOVSDZrr (EXTRACT_SUBREG (v4i64 VR256X:$src1), sub_xmm),
4260 (EXTRACT_SUBREG (v4i64 VR256X:$src2), sub_xmm)),
4261 sub_xmm)>;
4262 def : Pat<(v4f64 (X86Movsd VR256X:$src1, VR256X:$src2)),
4263 (SUBREG_TO_REG (i32 0),
4264 (VMOVSDZrr (EXTRACT_SUBREG (v4f64 VR256X:$src1), sub_xmm),
4265 (EXTRACT_SUBREG (v4f64 VR256X:$src2), sub_xmm)),
4266 sub_xmm)>;
4267
4268 def : Pat<(v2f64 (X86Movlpd VR128X:$src1, VR128X:$src2)),
4269 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
4270 def : Pat<(v2i64 (X86Movlpd VR128X:$src1, VR128X:$src2)),
4271 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
4272 def : Pat<(v4f32 (X86Movlps VR128X:$src1, VR128X:$src2)),
4273 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
4274 def : Pat<(v4i32 (X86Movlps VR128X:$src1, VR128X:$src2)),
4275 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
4276}
4277
4278let AddedComplexity = 15 in
4279def VMOVZPQILo2PQIZrr : AVX512XSI<0x7E, MRMSrcReg, (outs VR128X:$dst),
4280 (ins VR128X:$src),
Elena Demikhovskycf088092013-12-11 14:31:04 +00004281 "vmovq\t{$src, $dst|$dst, $src}",
Michael Liao5bf95782014-12-04 05:20:33 +00004282 [(set VR128X:$dst, (v2i64 (X86vzmovl
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004283 (v2i64 VR128X:$src))))],
4284 IIC_SSE_MOVQ_RR>, EVEX, VEX_W;
4285
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004286let Predicates = [HasAVX512] in {
Craig Topperde549852016-05-22 06:09:34 +00004287 let AddedComplexity = 15 in {
4288 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))),
4289 (VMOVDI2PDIZrr GR32:$src)>;
4290
4291 def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector GR64:$src)))),
4292 (VMOV64toPQIZrr GR64:$src)>;
4293
4294 def : Pat<(v4i64 (X86vzmovl (insert_subvector undef,
4295 (v2i64 (scalar_to_vector GR64:$src)),(iPTR 0)))),
4296 (SUBREG_TO_REG (i64 0), (VMOV64toPQIZrr GR64:$src), sub_xmm)>;
Craig Topperf4442312016-08-07 21:52:59 +00004297
4298 def : Pat<(v8i64 (X86vzmovl (insert_subvector undef,
4299 (v2i64 (scalar_to_vector GR64:$src)),(iPTR 0)))),
4300 (SUBREG_TO_REG (i64 0), (VMOV64toPQIZrr GR64:$src), sub_xmm)>;
Craig Topperde549852016-05-22 06:09:34 +00004301 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004302 // AVX 128-bit movd/movq instruction write zeros in the high 128-bit part.
4303 let AddedComplexity = 20 in {
Simon Pilgrima4c350f2017-02-17 20:43:32 +00004304 def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector (zextloadi64i32 addr:$src))))),
4305 (VMOVDI2PDIZrm addr:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004306 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector (loadi32 addr:$src))))),
4307 (VMOVDI2PDIZrm addr:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004308 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))),
4309 (VMOVDI2PDIZrm addr:$src)>;
4310 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))),
4311 (VMOVDI2PDIZrm addr:$src)>;
Simon Pilgrim6392b8d2016-08-24 10:46:40 +00004312 def : Pat<(v4i32 (X86vzload addr:$src)),
4313 (VMOVDI2PDIZrm addr:$src)>;
4314 def : Pat<(v8i32 (X86vzload addr:$src)),
4315 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrm addr:$src), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004316 def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))),
Craig Topper3dcf45f2016-11-22 05:31:43 +00004317 (VMOVQI2PQIZrm addr:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004318 def : Pat<(v2f64 (X86vzmovl (v2f64 VR128X:$src))),
Simon Pilgrim6392b8d2016-08-24 10:46:40 +00004319 (VMOVZPQILo2PQIZrr VR128X:$src)>;
Cameron McInally30bbb212013-12-05 00:11:25 +00004320 def : Pat<(v2i64 (X86vzload addr:$src)),
Craig Topper3dcf45f2016-11-22 05:31:43 +00004321 (VMOVQI2PQIZrm addr:$src)>;
Craig Topperde549852016-05-22 06:09:34 +00004322 def : Pat<(v4i64 (X86vzload addr:$src)),
Craig Topper3dcf45f2016-11-22 05:31:43 +00004323 (SUBREG_TO_REG (i64 0), (VMOVQI2PQIZrm addr:$src), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004324 }
Elena Demikhovsky3b75f5d2013-10-01 08:38:02 +00004325
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004326 // Use regular 128-bit instructions to match 256-bit scalar_to_vec+zext.
4327 def : Pat<(v8i32 (X86vzmovl (insert_subvector undef,
4328 (v4i32 (scalar_to_vector GR32:$src)),(iPTR 0)))),
4329 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src), sub_xmm)>;
Craig Topperf4442312016-08-07 21:52:59 +00004330 def : Pat<(v16i32 (X86vzmovl (insert_subvector undef,
4331 (v4i32 (scalar_to_vector GR32:$src)),(iPTR 0)))),
4332 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src), sub_xmm)>;
4333
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004334 // Use regular 128-bit instructions to match 512-bit scalar_to_vec+zext.
Simon Pilgrim6392b8d2016-08-24 10:46:40 +00004335 def : Pat<(v16i32 (X86vzload addr:$src)),
4336 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrm addr:$src), sub_xmm)>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004337 def : Pat<(v8i64 (X86vzload addr:$src)),
Craig Topper3dcf45f2016-11-22 05:31:43 +00004338 (SUBREG_TO_REG (i64 0), (VMOVQI2PQIZrm addr:$src), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004339}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004340//===----------------------------------------------------------------------===//
Adam Nemet7f62b232014-06-10 16:39:53 +00004341// AVX-512 - Non-temporals
4342//===----------------------------------------------------------------------===//
Robert Khasanoved882972014-08-13 10:46:00 +00004343let SchedRW = [WriteLoad] in {
4344 def VMOVNTDQAZrm : AVX512PI<0x2A, MRMSrcMem, (outs VR512:$dst),
4345 (ins i512mem:$src), "vmovntdqa\t{$src, $dst|$dst, $src}",
Simon Pilgrim5a22eaa2017-04-14 15:05:35 +00004346 [], SSEPackedInt>, EVEX, T8PD, EVEX_V512,
Robert Khasanoved882972014-08-13 10:46:00 +00004347 EVEX_CD8<64, CD8VF>;
Adam Nemet7f62b232014-06-10 16:39:53 +00004348
Craig Topper2f90c1f2016-06-07 07:27:57 +00004349 let Predicates = [HasVLX] in {
Robert Khasanoved882972014-08-13 10:46:00 +00004350 def VMOVNTDQAZ256rm : AVX512PI<0x2A, MRMSrcMem, (outs VR256X:$dst),
Craig Topper2f90c1f2016-06-07 07:27:57 +00004351 (ins i256mem:$src),
4352 "vmovntdqa\t{$src, $dst|$dst, $src}",
Simon Pilgrim5a22eaa2017-04-14 15:05:35 +00004353 [], SSEPackedInt>, EVEX, T8PD, EVEX_V256,
Craig Topper2f90c1f2016-06-07 07:27:57 +00004354 EVEX_CD8<64, CD8VF>;
Adam Nemet7f62b232014-06-10 16:39:53 +00004355
Robert Khasanoved882972014-08-13 10:46:00 +00004356 def VMOVNTDQAZ128rm : AVX512PI<0x2A, MRMSrcMem, (outs VR128X:$dst),
Craig Topper2f90c1f2016-06-07 07:27:57 +00004357 (ins i128mem:$src),
4358 "vmovntdqa\t{$src, $dst|$dst, $src}",
Simon Pilgrim5a22eaa2017-04-14 15:05:35 +00004359 [], SSEPackedInt>, EVEX, T8PD, EVEX_V128,
Craig Topper2f90c1f2016-06-07 07:27:57 +00004360 EVEX_CD8<64, CD8VF>;
Robert Khasanoved882972014-08-13 10:46:00 +00004361 }
Adam Nemetefd07852014-06-18 16:51:10 +00004362}
4363
Igor Bregerd3341f52016-01-20 13:11:47 +00004364multiclass avx512_movnt<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
4365 PatFrag st_frag = alignednontemporalstore,
4366 InstrItinClass itin = IIC_SSE_MOVNT> {
Craig Toppere1cac152016-06-07 07:27:54 +00004367 let SchedRW = [WriteStore], AddedComplexity = 400 in
Igor Bregerd3341f52016-01-20 13:11:47 +00004368 def mr : AVX512PI<opc, MRMDestMem, (outs), (ins _.MemOp:$dst, _.RC:$src),
Robert Khasanoved882972014-08-13 10:46:00 +00004369 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
Igor Bregerd3341f52016-01-20 13:11:47 +00004370 [(st_frag (_.VT _.RC:$src), addr:$dst)],
4371 _.ExeDomain, itin>, EVEX, EVEX_CD8<_.EltSize, CD8VF>;
Robert Khasanoved882972014-08-13 10:46:00 +00004372}
4373
Igor Bregerd3341f52016-01-20 13:11:47 +00004374multiclass avx512_movnt_vl<bits<8> opc, string OpcodeStr,
4375 AVX512VLVectorVTInfo VTInfo> {
4376 let Predicates = [HasAVX512] in
4377 defm Z : avx512_movnt<opc, OpcodeStr, VTInfo.info512>, EVEX_V512;
Robert Khasanoved882972014-08-13 10:46:00 +00004378
Igor Bregerd3341f52016-01-20 13:11:47 +00004379 let Predicates = [HasAVX512, HasVLX] in {
4380 defm Z256 : avx512_movnt<opc, OpcodeStr, VTInfo.info256>, EVEX_V256;
4381 defm Z128 : avx512_movnt<opc, OpcodeStr, VTInfo.info128>, EVEX_V128;
Robert Khasanoved882972014-08-13 10:46:00 +00004382 }
4383}
4384
Igor Bregerd3341f52016-01-20 13:11:47 +00004385defm VMOVNTDQ : avx512_movnt_vl<0xE7, "vmovntdq", avx512vl_i64_info>, PD;
4386defm VMOVNTPD : avx512_movnt_vl<0x2B, "vmovntpd", avx512vl_f64_info>, PD, VEX_W;
4387defm VMOVNTPS : avx512_movnt_vl<0x2B, "vmovntps", avx512vl_f32_info>, PS;
Robert Khasanoved882972014-08-13 10:46:00 +00004388
Craig Topper707c89c2016-05-08 23:43:17 +00004389let Predicates = [HasAVX512], AddedComplexity = 400 in {
4390 def : Pat<(alignednontemporalstore (v16i32 VR512:$src), addr:$dst),
4391 (VMOVNTDQZmr addr:$dst, VR512:$src)>;
4392 def : Pat<(alignednontemporalstore (v32i16 VR512:$src), addr:$dst),
4393 (VMOVNTDQZmr addr:$dst, VR512:$src)>;
4394 def : Pat<(alignednontemporalstore (v64i8 VR512:$src), addr:$dst),
4395 (VMOVNTDQZmr addr:$dst, VR512:$src)>;
Simon Pilgrim9a896232016-06-07 13:34:24 +00004396
4397 def : Pat<(v8f64 (alignednontemporalload addr:$src)),
4398 (VMOVNTDQAZrm addr:$src)>;
4399 def : Pat<(v16f32 (alignednontemporalload addr:$src)),
4400 (VMOVNTDQAZrm addr:$src)>;
4401 def : Pat<(v8i64 (alignednontemporalload addr:$src)),
4402 (VMOVNTDQAZrm addr:$src)>;
Craig Toppera78b7682016-08-11 06:04:07 +00004403 def : Pat<(v16i32 (bitconvert (v8i64 (alignednontemporalload addr:$src)))),
Simon Pilgrim9a896232016-06-07 13:34:24 +00004404 (VMOVNTDQAZrm addr:$src)>;
Craig Toppera78b7682016-08-11 06:04:07 +00004405 def : Pat<(v32i16 (bitconvert (v8i64 (alignednontemporalload addr:$src)))),
Simon Pilgrim9a896232016-06-07 13:34:24 +00004406 (VMOVNTDQAZrm addr:$src)>;
Craig Toppera78b7682016-08-11 06:04:07 +00004407 def : Pat<(v64i8 (bitconvert (v8i64 (alignednontemporalload addr:$src)))),
Simon Pilgrim9a896232016-06-07 13:34:24 +00004408 (VMOVNTDQAZrm addr:$src)>;
Craig Topper707c89c2016-05-08 23:43:17 +00004409}
4410
Craig Topperc41320d2016-05-08 23:08:45 +00004411let Predicates = [HasVLX], AddedComplexity = 400 in {
4412 def : Pat<(alignednontemporalstore (v8i32 VR256X:$src), addr:$dst),
4413 (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;
4414 def : Pat<(alignednontemporalstore (v16i16 VR256X:$src), addr:$dst),
4415 (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;
4416 def : Pat<(alignednontemporalstore (v32i8 VR256X:$src), addr:$dst),
4417 (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;
4418
Simon Pilgrim9a896232016-06-07 13:34:24 +00004419 def : Pat<(v4f64 (alignednontemporalload addr:$src)),
4420 (VMOVNTDQAZ256rm addr:$src)>;
4421 def : Pat<(v8f32 (alignednontemporalload addr:$src)),
4422 (VMOVNTDQAZ256rm addr:$src)>;
4423 def : Pat<(v4i64 (alignednontemporalload addr:$src)),
4424 (VMOVNTDQAZ256rm addr:$src)>;
Craig Topper31140ad2017-07-21 00:40:42 +00004425 def : Pat<(v8i32 (bitconvert (v4i64 (alignednontemporalload addr:$src)))),
Simon Pilgrim9a896232016-06-07 13:34:24 +00004426 (VMOVNTDQAZ256rm addr:$src)>;
Craig Topper31140ad2017-07-21 00:40:42 +00004427 def : Pat<(v16i16 (bitconvert (v4i64 (alignednontemporalload addr:$src)))),
Simon Pilgrim9a896232016-06-07 13:34:24 +00004428 (VMOVNTDQAZ256rm addr:$src)>;
Craig Topper31140ad2017-07-21 00:40:42 +00004429 def : Pat<(v32i8 (bitconvert (v4i64 (alignednontemporalload addr:$src)))),
Simon Pilgrim9a896232016-06-07 13:34:24 +00004430 (VMOVNTDQAZ256rm addr:$src)>;
4431
Craig Topperc41320d2016-05-08 23:08:45 +00004432 def : Pat<(alignednontemporalstore (v4i32 VR128X:$src), addr:$dst),
4433 (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;
4434 def : Pat<(alignednontemporalstore (v8i16 VR128X:$src), addr:$dst),
4435 (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;
4436 def : Pat<(alignednontemporalstore (v16i8 VR128X:$src), addr:$dst),
4437 (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;
Simon Pilgrim9a896232016-06-07 13:34:24 +00004438
4439 def : Pat<(v2f64 (alignednontemporalload addr:$src)),
4440 (VMOVNTDQAZ128rm addr:$src)>;
4441 def : Pat<(v4f32 (alignednontemporalload addr:$src)),
4442 (VMOVNTDQAZ128rm addr:$src)>;
4443 def : Pat<(v2i64 (alignednontemporalload addr:$src)),
4444 (VMOVNTDQAZ128rm addr:$src)>;
Craig Topper3563d0f2016-08-11 06:04:00 +00004445 def : Pat<(v4i32 (bitconvert (v2i64 (alignednontemporalload addr:$src)))),
Simon Pilgrim9a896232016-06-07 13:34:24 +00004446 (VMOVNTDQAZ128rm addr:$src)>;
Craig Topper3563d0f2016-08-11 06:04:00 +00004447 def : Pat<(v8i16 (bitconvert (v2i64 (alignednontemporalload addr:$src)))),
Simon Pilgrim9a896232016-06-07 13:34:24 +00004448 (VMOVNTDQAZ128rm addr:$src)>;
Craig Topper3563d0f2016-08-11 06:04:00 +00004449 def : Pat<(v16i8 (bitconvert (v2i64 (alignednontemporalload addr:$src)))),
Simon Pilgrim9a896232016-06-07 13:34:24 +00004450 (VMOVNTDQAZ128rm addr:$src)>;
Craig Topperc41320d2016-05-08 23:08:45 +00004451}
4452
Adam Nemet7f62b232014-06-10 16:39:53 +00004453//===----------------------------------------------------------------------===//
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004454// AVX-512 - Integer arithmetic
4455//
4456multiclass avx512_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
Robert Khasanov44241442014-10-08 14:37:45 +00004457 X86VectorVTInfo _, OpndItins itins,
4458 bit IsCommutable = 0> {
Adam Nemet34801422014-10-08 23:25:39 +00004459 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
Igor Bregerf2460112015-07-26 14:41:44 +00004460 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
Robert Khasanov44241442014-10-08 14:37:45 +00004461 "$src2, $src1", "$src1, $src2",
4462 (_.VT (OpNode _.RC:$src1, _.RC:$src2)),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00004463 itins.rr, IsCommutable>,
Robert Khasanov44241442014-10-08 14:37:45 +00004464 AVX512BIBase, EVEX_4V;
Elena Demikhovskybb2f6b72014-03-27 09:45:08 +00004465
Craig Toppere1cac152016-06-07 07:27:54 +00004466 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
4467 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
4468 "$src2, $src1", "$src1, $src2",
4469 (_.VT (OpNode _.RC:$src1,
4470 (bitconvert (_.LdFrag addr:$src2)))),
4471 itins.rm>,
4472 AVX512BIBase, EVEX_4V;
Robert Khasanov545d1b72014-10-14 14:36:19 +00004473}
4474
4475multiclass avx512_binop_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
4476 X86VectorVTInfo _, OpndItins itins,
4477 bit IsCommutable = 0> :
4478 avx512_binop_rm<opc, OpcodeStr, OpNode, _, itins, IsCommutable> {
Craig Toppere1cac152016-06-07 07:27:54 +00004479 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
4480 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
4481 "${src2}"##_.BroadcastStr##", $src1",
4482 "$src1, ${src2}"##_.BroadcastStr,
4483 (_.VT (OpNode _.RC:$src1,
4484 (X86VBroadcast
4485 (_.ScalarLdFrag addr:$src2)))),
4486 itins.rm>,
4487 AVX512BIBase, EVEX_4V, EVEX_B;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004488}
Elena Demikhovskybb2f6b72014-03-27 09:45:08 +00004489
Robert Khasanovd5b14f72014-10-09 08:38:48 +00004490multiclass avx512_binop_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
4491 AVX512VLVectorVTInfo VTInfo, OpndItins itins,
4492 Predicate prd, bit IsCommutable = 0> {
4493 let Predicates = [prd] in
4494 defm Z : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info512, itins,
4495 IsCommutable>, EVEX_V512;
4496
4497 let Predicates = [prd, HasVLX] in {
4498 defm Z256 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info256, itins,
4499 IsCommutable>, EVEX_V256;
4500 defm Z128 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info128, itins,
4501 IsCommutable>, EVEX_V128;
4502 }
4503}
4504
Robert Khasanov545d1b72014-10-14 14:36:19 +00004505multiclass avx512_binop_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
4506 AVX512VLVectorVTInfo VTInfo, OpndItins itins,
4507 Predicate prd, bit IsCommutable = 0> {
4508 let Predicates = [prd] in
4509 defm Z : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info512, itins,
4510 IsCommutable>, EVEX_V512;
4511
4512 let Predicates = [prd, HasVLX] in {
4513 defm Z256 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info256, itins,
4514 IsCommutable>, EVEX_V256;
4515 defm Z128 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info128, itins,
4516 IsCommutable>, EVEX_V128;
4517 }
4518}
4519
4520multiclass avx512_binop_rm_vl_q<bits<8> opc, string OpcodeStr, SDNode OpNode,
4521 OpndItins itins, Predicate prd,
4522 bit IsCommutable = 0> {
4523 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i64_info,
4524 itins, prd, IsCommutable>,
4525 VEX_W, EVEX_CD8<64, CD8VF>;
4526}
4527
4528multiclass avx512_binop_rm_vl_d<bits<8> opc, string OpcodeStr, SDNode OpNode,
4529 OpndItins itins, Predicate prd,
4530 bit IsCommutable = 0> {
4531 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i32_info,
4532 itins, prd, IsCommutable>, EVEX_CD8<32, CD8VF>;
4533}
4534
4535multiclass avx512_binop_rm_vl_w<bits<8> opc, string OpcodeStr, SDNode OpNode,
4536 OpndItins itins, Predicate prd,
4537 bit IsCommutable = 0> {
4538 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i16_info,
4539 itins, prd, IsCommutable>, EVEX_CD8<16, CD8VF>;
4540}
4541
4542multiclass avx512_binop_rm_vl_b<bits<8> opc, string OpcodeStr, SDNode OpNode,
4543 OpndItins itins, Predicate prd,
4544 bit IsCommutable = 0> {
4545 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i8_info,
4546 itins, prd, IsCommutable>, EVEX_CD8<8, CD8VF>;
4547}
4548
4549multiclass avx512_binop_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr,
4550 SDNode OpNode, OpndItins itins, Predicate prd,
4551 bit IsCommutable = 0> {
Igor Bregerf2460112015-07-26 14:41:44 +00004552 defm Q : avx512_binop_rm_vl_q<opc_q, OpcodeStr#"q", OpNode, itins, prd,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004553 IsCommutable>;
4554
Igor Bregerf2460112015-07-26 14:41:44 +00004555 defm D : avx512_binop_rm_vl_d<opc_d, OpcodeStr#"d", OpNode, itins, prd,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004556 IsCommutable>;
4557}
4558
4559multiclass avx512_binop_rm_vl_bw<bits<8> opc_b, bits<8> opc_w, string OpcodeStr,
4560 SDNode OpNode, OpndItins itins, Predicate prd,
4561 bit IsCommutable = 0> {
Igor Bregerf2460112015-07-26 14:41:44 +00004562 defm W : avx512_binop_rm_vl_w<opc_w, OpcodeStr#"w", OpNode, itins, prd,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004563 IsCommutable>;
4564
Igor Bregerf2460112015-07-26 14:41:44 +00004565 defm B : avx512_binop_rm_vl_b<opc_b, OpcodeStr#"b", OpNode, itins, prd,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004566 IsCommutable>;
4567}
4568
4569multiclass avx512_binop_rm_vl_all<bits<8> opc_b, bits<8> opc_w,
4570 bits<8> opc_d, bits<8> opc_q,
4571 string OpcodeStr, SDNode OpNode,
4572 OpndItins itins, bit IsCommutable = 0> {
4573 defm NAME : avx512_binop_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode,
4574 itins, HasAVX512, IsCommutable>,
4575 avx512_binop_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode,
4576 itins, HasBWI, IsCommutable>;
4577}
4578
Elena Demikhovsky1eeece12015-04-02 10:51:40 +00004579multiclass avx512_binop_rm2<bits<8> opc, string OpcodeStr, OpndItins itins,
Michael Liao66233b72015-08-06 09:06:20 +00004580 SDNode OpNode,X86VectorVTInfo _Src,
Asaf Badouh5a3a0232016-02-01 15:48:21 +00004581 X86VectorVTInfo _Dst, X86VectorVTInfo _Brdct,
4582 bit IsCommutable = 0> {
Michael Liao66233b72015-08-06 09:06:20 +00004583 defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst),
Elena Demikhovsky1eeece12015-04-02 10:51:40 +00004584 (ins _Src.RC:$src1, _Src.RC:$src2), OpcodeStr,
Michael Liao66233b72015-08-06 09:06:20 +00004585 "$src2, $src1","$src1, $src2",
4586 (_Dst.VT (OpNode
4587 (_Src.VT _Src.RC:$src1),
Elena Demikhovsky1eeece12015-04-02 10:51:40 +00004588 (_Src.VT _Src.RC:$src2))),
Michael Liao66233b72015-08-06 09:06:20 +00004589 itins.rr, IsCommutable>,
Elena Demikhovsky1eeece12015-04-02 10:51:40 +00004590 AVX512BIBase, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00004591 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
4592 (ins _Src.RC:$src1, _Src.MemOp:$src2), OpcodeStr,
4593 "$src2, $src1", "$src1, $src2",
4594 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1),
4595 (bitconvert (_Src.LdFrag addr:$src2)))),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00004596 itins.rm>,
Craig Toppere1cac152016-06-07 07:27:54 +00004597 AVX512BIBase, EVEX_4V;
4598
4599 defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
Coby Tayree99a66392016-11-20 17:19:55 +00004600 (ins _Src.RC:$src1, _Brdct.ScalarMemOp:$src2),
Craig Toppere1cac152016-06-07 07:27:54 +00004601 OpcodeStr,
4602 "${src2}"##_Brdct.BroadcastStr##", $src1",
Coby Tayree99a66392016-11-20 17:19:55 +00004603 "$src1, ${src2}"##_Brdct.BroadcastStr,
Craig Toppere1cac152016-06-07 07:27:54 +00004604 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert
4605 (_Brdct.VT (X86VBroadcast
4606 (_Brdct.ScalarLdFrag addr:$src2)))))),
4607 itins.rm>,
4608 AVX512BIBase, EVEX_4V, EVEX_B;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004609}
4610
Robert Khasanov545d1b72014-10-14 14:36:19 +00004611defm VPADD : avx512_binop_rm_vl_all<0xFC, 0xFD, 0xFE, 0xD4, "vpadd", add,
4612 SSE_INTALU_ITINS_P, 1>;
4613defm VPSUB : avx512_binop_rm_vl_all<0xF8, 0xF9, 0xFA, 0xFB, "vpsub", sub,
4614 SSE_INTALU_ITINS_P, 0>;
Elena Demikhovsky52266382015-05-04 12:35:55 +00004615defm VPADDS : avx512_binop_rm_vl_bw<0xEC, 0xED, "vpadds", X86adds,
4616 SSE_INTALU_ITINS_P, HasBWI, 1>;
4617defm VPSUBS : avx512_binop_rm_vl_bw<0xE8, 0xE9, "vpsubs", X86subs,
4618 SSE_INTALU_ITINS_P, HasBWI, 0>;
4619defm VPADDUS : avx512_binop_rm_vl_bw<0xDC, 0xDD, "vpaddus", X86addus,
Michael Liao66233b72015-08-06 09:06:20 +00004620 SSE_INTALU_ITINS_P, HasBWI, 1>;
Elena Demikhovsky52266382015-05-04 12:35:55 +00004621defm VPSUBUS : avx512_binop_rm_vl_bw<0xD8, 0xD9, "vpsubus", X86subus,
Michael Liao66233b72015-08-06 09:06:20 +00004622 SSE_INTALU_ITINS_P, HasBWI, 0>;
Igor Bregerf2460112015-07-26 14:41:44 +00004623defm VPMULLD : avx512_binop_rm_vl_d<0x40, "vpmulld", mul,
Michael Liao66233b72015-08-06 09:06:20 +00004624 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
Igor Bregerf2460112015-07-26 14:41:44 +00004625defm VPMULLW : avx512_binop_rm_vl_w<0xD5, "vpmullw", mul,
Michael Liao66233b72015-08-06 09:06:20 +00004626 SSE_INTALU_ITINS_P, HasBWI, 1>;
Igor Bregerf2460112015-07-26 14:41:44 +00004627defm VPMULLQ : avx512_binop_rm_vl_q<0x40, "vpmullq", mul,
Michael Liao66233b72015-08-06 09:06:20 +00004628 SSE_INTALU_ITINS_P, HasDQI, 1>, T8PD;
Igor Bregerf2460112015-07-26 14:41:44 +00004629defm VPMULHW : avx512_binop_rm_vl_w<0xE5, "vpmulhw", mulhs, SSE_INTALU_ITINS_P,
Asaf Badouh73f26f82015-07-05 12:23:20 +00004630 HasBWI, 1>;
Igor Bregerf2460112015-07-26 14:41:44 +00004631defm VPMULHUW : avx512_binop_rm_vl_w<0xE4, "vpmulhuw", mulhu, SSE_INTMUL_ITINS_P,
Michael Liao66233b72015-08-06 09:06:20 +00004632 HasBWI, 1>;
Igor Bregerf2460112015-07-26 14:41:44 +00004633defm VPMULHRSW : avx512_binop_rm_vl_w<0x0B, "vpmulhrsw", X86mulhrs, SSE_INTMUL_ITINS_P,
Michael Liao66233b72015-08-06 09:06:20 +00004634 HasBWI, 1>, T8PD;
Asaf Badouh81f03c32015-06-18 12:30:53 +00004635defm VPAVG : avx512_binop_rm_vl_bw<0xE0, 0xE3, "vpavg", X86avg,
Michael Liao66233b72015-08-06 09:06:20 +00004636 SSE_INTALU_ITINS_P, HasBWI, 1>;
4637
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004638multiclass avx512_binop_all<bits<8> opc, string OpcodeStr, OpndItins itins,
Asaf Badouh5a3a0232016-02-01 15:48:21 +00004639 AVX512VLVectorVTInfo _SrcVTInfo, AVX512VLVectorVTInfo _DstVTInfo,
4640 SDNode OpNode, Predicate prd, bit IsCommutable = 0> {
4641 let Predicates = [prd] in
4642 defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
4643 _SrcVTInfo.info512, _DstVTInfo.info512,
4644 v8i64_info, IsCommutable>,
4645 EVEX_V512, EVEX_CD8<64, CD8VF>, VEX_W;
4646 let Predicates = [HasVLX, prd] in {
Elena Demikhovsky50b88dd2015-04-21 10:27:40 +00004647 defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004648 _SrcVTInfo.info256, _DstVTInfo.info256,
Asaf Badouh5a3a0232016-02-01 15:48:21 +00004649 v4i64x_info, IsCommutable>,
4650 EVEX_V256, EVEX_CD8<64, CD8VF>, VEX_W;
Elena Demikhovsky50b88dd2015-04-21 10:27:40 +00004651 defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004652 _SrcVTInfo.info128, _DstVTInfo.info128,
Asaf Badouh5a3a0232016-02-01 15:48:21 +00004653 v2i64x_info, IsCommutable>,
Elena Demikhovsky50b88dd2015-04-21 10:27:40 +00004654 EVEX_V128, EVEX_CD8<64, CD8VF>, VEX_W;
4655 }
Michael Liao66233b72015-08-06 09:06:20 +00004656}
Elena Demikhovsky50b88dd2015-04-21 10:27:40 +00004657
4658defm VPMULDQ : avx512_binop_all<0x28, "vpmuldq", SSE_INTALU_ITINS_P,
Asaf Badouh5a3a0232016-02-01 15:48:21 +00004659 avx512vl_i32_info, avx512vl_i64_info,
4660 X86pmuldq, HasAVX512, 1>,T8PD;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00004661defm VPMULUDQ : avx512_binop_all<0xF4, "vpmuludq", SSE_INTMUL_ITINS_P,
Asaf Badouh5a3a0232016-02-01 15:48:21 +00004662 avx512vl_i32_info, avx512vl_i64_info,
4663 X86pmuludq, HasAVX512, 1>;
4664defm VPMULTISHIFTQB : avx512_binop_all<0x83, "vpmultishiftqb", SSE_INTALU_ITINS_P,
4665 avx512vl_i8_info, avx512vl_i8_info,
4666 X86multishift, HasVBMI, 0>, T8PD;
Elena Demikhovsky172a27c2014-01-08 10:54:22 +00004667
Elena Demikhovsky2557a222015-05-04 09:14:02 +00004668multiclass avx512_packs_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
4669 X86VectorVTInfo _Src, X86VectorVTInfo _Dst> {
Craig Toppere1cac152016-06-07 07:27:54 +00004670 defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
4671 (ins _Src.RC:$src1, _Src.ScalarMemOp:$src2),
4672 OpcodeStr,
4673 "${src2}"##_Src.BroadcastStr##", $src1",
4674 "$src1, ${src2}"##_Src.BroadcastStr,
4675 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert
4676 (_Src.VT (X86VBroadcast
4677 (_Src.ScalarLdFrag addr:$src2))))))>,
4678 EVEX_4V, EVEX_B, EVEX_CD8<_Src.EltSize, CD8VF>;
Elena Demikhovsky2557a222015-05-04 09:14:02 +00004679}
4680
Michael Liao66233b72015-08-06 09:06:20 +00004681multiclass avx512_packs_rm<bits<8> opc, string OpcodeStr,
4682 SDNode OpNode,X86VectorVTInfo _Src,
Craig Topper37e8c542016-08-14 17:57:22 +00004683 X86VectorVTInfo _Dst, bit IsCommutable = 0> {
Michael Liao66233b72015-08-06 09:06:20 +00004684 defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst),
Elena Demikhovsky2557a222015-05-04 09:14:02 +00004685 (ins _Src.RC:$src1, _Src.RC:$src2), OpcodeStr,
Michael Liao66233b72015-08-06 09:06:20 +00004686 "$src2, $src1","$src1, $src2",
4687 (_Dst.VT (OpNode
4688 (_Src.VT _Src.RC:$src1),
Craig Topper37e8c542016-08-14 17:57:22 +00004689 (_Src.VT _Src.RC:$src2))),
4690 NoItinerary, IsCommutable>,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00004691 EVEX_CD8<_Src.EltSize, CD8VF>, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00004692 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
4693 (ins _Src.RC:$src1, _Src.MemOp:$src2), OpcodeStr,
4694 "$src2, $src1", "$src1, $src2",
4695 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1),
4696 (bitconvert (_Src.LdFrag addr:$src2))))>,
4697 EVEX_4V, EVEX_CD8<_Src.EltSize, CD8VF>;
Elena Demikhovsky2557a222015-05-04 09:14:02 +00004698}
4699
4700multiclass avx512_packs_all_i32_i16<bits<8> opc, string OpcodeStr,
4701 SDNode OpNode> {
Craig Topper5acb5a12016-05-01 06:24:57 +00004702 let Predicates = [HasBWI] in
Elena Demikhovsky2557a222015-05-04 09:14:02 +00004703 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i32_info,
4704 v32i16_info>,
4705 avx512_packs_rmb<opc, OpcodeStr, OpNode, v16i32_info,
4706 v32i16_info>, EVEX_V512;
Craig Topper5acb5a12016-05-01 06:24:57 +00004707 let Predicates = [HasBWI, HasVLX] in {
Elena Demikhovsky2557a222015-05-04 09:14:02 +00004708 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i32x_info,
4709 v16i16x_info>,
4710 avx512_packs_rmb<opc, OpcodeStr, OpNode, v8i32x_info,
4711 v16i16x_info>, EVEX_V256;
4712 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v4i32x_info,
4713 v8i16x_info>,
4714 avx512_packs_rmb<opc, OpcodeStr, OpNode, v4i32x_info,
4715 v8i16x_info>, EVEX_V128;
4716 }
4717}
4718multiclass avx512_packs_all_i16_i8<bits<8> opc, string OpcodeStr,
4719 SDNode OpNode> {
Craig Topper5acb5a12016-05-01 06:24:57 +00004720 let Predicates = [HasBWI] in
Elena Demikhovsky2557a222015-05-04 09:14:02 +00004721 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v32i16_info,
4722 v64i8_info>, EVEX_V512;
Craig Topper5acb5a12016-05-01 06:24:57 +00004723 let Predicates = [HasBWI, HasVLX] in {
Elena Demikhovsky2557a222015-05-04 09:14:02 +00004724 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i16x_info,
4725 v32i8x_info>, EVEX_V256;
4726 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i16x_info,
4727 v16i8x_info>, EVEX_V128;
4728 }
4729}
Igor Bregerf7fd5472015-07-21 07:11:28 +00004730
4731multiclass avx512_vpmadd<bits<8> opc, string OpcodeStr,
4732 SDNode OpNode, AVX512VLVectorVTInfo _Src,
Craig Topper37e8c542016-08-14 17:57:22 +00004733 AVX512VLVectorVTInfo _Dst, bit IsCommutable = 0> {
Craig Topper5acb5a12016-05-01 06:24:57 +00004734 let Predicates = [HasBWI] in
Igor Bregerf7fd5472015-07-21 07:11:28 +00004735 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info512,
Craig Topper37e8c542016-08-14 17:57:22 +00004736 _Dst.info512, IsCommutable>, EVEX_V512;
Craig Topper5acb5a12016-05-01 06:24:57 +00004737 let Predicates = [HasBWI, HasVLX] in {
Igor Bregerf7fd5472015-07-21 07:11:28 +00004738 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info256,
Craig Topper37e8c542016-08-14 17:57:22 +00004739 _Dst.info256, IsCommutable>, EVEX_V256;
Igor Bregerf7fd5472015-07-21 07:11:28 +00004740 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info128,
Craig Topper37e8c542016-08-14 17:57:22 +00004741 _Dst.info128, IsCommutable>, EVEX_V128;
Igor Bregerf7fd5472015-07-21 07:11:28 +00004742 }
4743}
4744
Craig Topperb6da6542016-05-01 17:38:32 +00004745defm VPACKSSDW : avx512_packs_all_i32_i16<0x6B, "vpackssdw", X86Packss>, AVX512BIBase;
4746defm VPACKUSDW : avx512_packs_all_i32_i16<0x2b, "vpackusdw", X86Packus>, AVX5128IBase;
4747defm VPACKSSWB : avx512_packs_all_i16_i8 <0x63, "vpacksswb", X86Packss>, AVX512BIBase;
4748defm VPACKUSWB : avx512_packs_all_i16_i8 <0x67, "vpackuswb", X86Packus>, AVX512BIBase;
Igor Bregerf7fd5472015-07-21 07:11:28 +00004749
Craig Topper5acb5a12016-05-01 06:24:57 +00004750defm VPMADDUBSW : avx512_vpmadd<0x04, "vpmaddubsw", X86vpmaddubsw,
4751 avx512vl_i8_info, avx512vl_i16_info>, AVX512BIBase, T8PD;
4752defm VPMADDWD : avx512_vpmadd<0xF5, "vpmaddwd", X86vpmaddwd,
Craig Topper37e8c542016-08-14 17:57:22 +00004753 avx512vl_i16_info, avx512vl_i32_info, 1>, AVX512BIBase;
Elena Demikhovsky2557a222015-05-04 09:14:02 +00004754
Igor Bregerf2460112015-07-26 14:41:44 +00004755defm VPMAXSB : avx512_binop_rm_vl_b<0x3C, "vpmaxsb", smax,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004756 SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD;
Igor Bregerf2460112015-07-26 14:41:44 +00004757defm VPMAXSW : avx512_binop_rm_vl_w<0xEE, "vpmaxsw", smax,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004758 SSE_INTALU_ITINS_P, HasBWI, 1>;
Simon Pilgrim8b756592015-07-06 20:30:47 +00004759defm VPMAXS : avx512_binop_rm_vl_dq<0x3D, 0x3D, "vpmaxs", smax,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004760 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
Elena Demikhovsky199c8232013-10-27 08:18:37 +00004761
Igor Bregerf2460112015-07-26 14:41:44 +00004762defm VPMAXUB : avx512_binop_rm_vl_b<0xDE, "vpmaxub", umax,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004763 SSE_INTALU_ITINS_P, HasBWI, 1>;
Igor Bregerf2460112015-07-26 14:41:44 +00004764defm VPMAXUW : avx512_binop_rm_vl_w<0x3E, "vpmaxuw", umax,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004765 SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD;
Simon Pilgrim8b756592015-07-06 20:30:47 +00004766defm VPMAXU : avx512_binop_rm_vl_dq<0x3F, 0x3F, "vpmaxu", umax,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004767 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
Elena Demikhovsky199c8232013-10-27 08:18:37 +00004768
Igor Bregerf2460112015-07-26 14:41:44 +00004769defm VPMINSB : avx512_binop_rm_vl_b<0x38, "vpminsb", smin,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004770 SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD;
Igor Bregerf2460112015-07-26 14:41:44 +00004771defm VPMINSW : avx512_binop_rm_vl_w<0xEA, "vpminsw", smin,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004772 SSE_INTALU_ITINS_P, HasBWI, 1>;
Simon Pilgrim8b756592015-07-06 20:30:47 +00004773defm VPMINS : avx512_binop_rm_vl_dq<0x39, 0x39, "vpmins", smin,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004774 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
Elena Demikhovsky199c8232013-10-27 08:18:37 +00004775
Igor Bregerf2460112015-07-26 14:41:44 +00004776defm VPMINUB : avx512_binop_rm_vl_b<0xDA, "vpminub", umin,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004777 SSE_INTALU_ITINS_P, HasBWI, 1>;
Igor Bregerf2460112015-07-26 14:41:44 +00004778defm VPMINUW : avx512_binop_rm_vl_w<0x3A, "vpminuw", umin,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004779 SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD;
Simon Pilgrim8b756592015-07-06 20:30:47 +00004780defm VPMINU : avx512_binop_rm_vl_dq<0x3B, 0x3B, "vpminu", umin,
Robert Khasanov545d1b72014-10-14 14:36:19 +00004781 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD;
Craig Topperabe80cc2016-08-28 06:06:28 +00004782
Simon Pilgrim47c1ff72016-10-27 17:07:40 +00004783// PMULLQ: Use 512bit version to implement 128/256 bit in case NoVLX.
4784let Predicates = [HasDQI, NoVLX] in {
4785 def : Pat<(v4i64 (mul (v4i64 VR256X:$src1), (v4i64 VR256X:$src2))),
4786 (EXTRACT_SUBREG
4787 (VPMULLQZrr
4788 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src1, sub_ymm),
4789 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src2, sub_ymm)),
4790 sub_ymm)>;
4791
4792 def : Pat<(v2i64 (mul (v2i64 VR128X:$src1), (v2i64 VR128X:$src2))),
4793 (EXTRACT_SUBREG
4794 (VPMULLQZrr
4795 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src1, sub_xmm),
4796 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src2, sub_xmm)),
4797 sub_xmm)>;
4798}
4799
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004800//===----------------------------------------------------------------------===//
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004801// AVX-512 Logical Instructions
4802//===----------------------------------------------------------------------===//
4803
Craig Topperabe80cc2016-08-28 06:06:28 +00004804multiclass avx512_logic_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topperb0cbd5b2017-01-24 06:25:34 +00004805 X86VectorVTInfo _, bit IsCommutable = 0> {
Craig Topperabe80cc2016-08-28 06:06:28 +00004806 defm rr : AVX512_maskable_logic<opc, MRMSrcReg, _, (outs _.RC:$dst),
4807 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
4808 "$src2, $src1", "$src1, $src2",
4809 (_.i64VT (OpNode (bitconvert (_.VT _.RC:$src1)),
4810 (bitconvert (_.VT _.RC:$src2)))),
4811 (_.VT (bitconvert (_.i64VT (OpNode _.RC:$src1,
4812 _.RC:$src2)))),
Craig Topperb0cbd5b2017-01-24 06:25:34 +00004813 IIC_SSE_BIT_P_RR, IsCommutable>,
Craig Topperabe80cc2016-08-28 06:06:28 +00004814 AVX512BIBase, EVEX_4V;
4815
4816 defm rm : AVX512_maskable_logic<opc, MRMSrcMem, _, (outs _.RC:$dst),
4817 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
4818 "$src2, $src1", "$src1, $src2",
4819 (_.i64VT (OpNode (bitconvert (_.VT _.RC:$src1)),
4820 (bitconvert (_.LdFrag addr:$src2)))),
4821 (_.VT (bitconvert (_.i64VT (OpNode _.RC:$src1,
4822 (bitconvert (_.LdFrag addr:$src2)))))),
Craig Topperb0cbd5b2017-01-24 06:25:34 +00004823 IIC_SSE_BIT_P_RM>,
Craig Topperabe80cc2016-08-28 06:06:28 +00004824 AVX512BIBase, EVEX_4V;
4825}
4826
4827multiclass avx512_logic_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topperb0cbd5b2017-01-24 06:25:34 +00004828 X86VectorVTInfo _, bit IsCommutable = 0> :
4829 avx512_logic_rm<opc, OpcodeStr, OpNode, _, IsCommutable> {
Craig Topperabe80cc2016-08-28 06:06:28 +00004830 defm rmb : AVX512_maskable_logic<opc, MRMSrcMem, _, (outs _.RC:$dst),
4831 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
4832 "${src2}"##_.BroadcastStr##", $src1",
4833 "$src1, ${src2}"##_.BroadcastStr,
4834 (_.i64VT (OpNode _.RC:$src1,
4835 (bitconvert
4836 (_.VT (X86VBroadcast
4837 (_.ScalarLdFrag addr:$src2)))))),
4838 (_.VT (bitconvert (_.i64VT (OpNode _.RC:$src1,
4839 (bitconvert
4840 (_.VT (X86VBroadcast
4841 (_.ScalarLdFrag addr:$src2)))))))),
Craig Topperb0cbd5b2017-01-24 06:25:34 +00004842 IIC_SSE_BIT_P_RM>,
Craig Topperabe80cc2016-08-28 06:06:28 +00004843 AVX512BIBase, EVEX_4V, EVEX_B;
4844}
4845
4846multiclass avx512_logic_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topperb0cbd5b2017-01-24 06:25:34 +00004847 AVX512VLVectorVTInfo VTInfo,
4848 bit IsCommutable = 0> {
4849 let Predicates = [HasAVX512] in
4850 defm Z : avx512_logic_rmb<opc, OpcodeStr, OpNode, VTInfo.info512,
Craig Topperabe80cc2016-08-28 06:06:28 +00004851 IsCommutable>, EVEX_V512;
4852
Craig Topperb0cbd5b2017-01-24 06:25:34 +00004853 let Predicates = [HasAVX512, HasVLX] in {
4854 defm Z256 : avx512_logic_rmb<opc, OpcodeStr, OpNode, VTInfo.info256,
Craig Topperabe80cc2016-08-28 06:06:28 +00004855 IsCommutable>, EVEX_V256;
Craig Topperb0cbd5b2017-01-24 06:25:34 +00004856 defm Z128 : avx512_logic_rmb<opc, OpcodeStr, OpNode, VTInfo.info128,
Craig Topperabe80cc2016-08-28 06:06:28 +00004857 IsCommutable>, EVEX_V128;
4858 }
4859}
4860
4861multiclass avx512_logic_rm_vl_d<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topperabe80cc2016-08-28 06:06:28 +00004862 bit IsCommutable = 0> {
4863 defm NAME : avx512_logic_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i32_info,
Craig Topperb0cbd5b2017-01-24 06:25:34 +00004864 IsCommutable>, EVEX_CD8<32, CD8VF>;
Craig Topperabe80cc2016-08-28 06:06:28 +00004865}
4866
4867multiclass avx512_logic_rm_vl_q<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topperabe80cc2016-08-28 06:06:28 +00004868 bit IsCommutable = 0> {
4869 defm NAME : avx512_logic_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i64_info,
Craig Topperb0cbd5b2017-01-24 06:25:34 +00004870 IsCommutable>,
4871 VEX_W, EVEX_CD8<64, CD8VF>;
Craig Topperabe80cc2016-08-28 06:06:28 +00004872}
4873
4874multiclass avx512_logic_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr,
Craig Topperb0cbd5b2017-01-24 06:25:34 +00004875 SDNode OpNode, bit IsCommutable = 0> {
4876 defm Q : avx512_logic_rm_vl_q<opc_q, OpcodeStr#"q", OpNode, IsCommutable>;
4877 defm D : avx512_logic_rm_vl_d<opc_d, OpcodeStr#"d", OpNode, IsCommutable>;
Craig Topperabe80cc2016-08-28 06:06:28 +00004878}
4879
Craig Topperb0cbd5b2017-01-24 06:25:34 +00004880defm VPAND : avx512_logic_rm_vl_dq<0xDB, 0xDB, "vpand", and, 1>;
4881defm VPOR : avx512_logic_rm_vl_dq<0xEB, 0xEB, "vpor", or, 1>;
4882defm VPXOR : avx512_logic_rm_vl_dq<0xEF, 0xEF, "vpxor", xor, 1>;
4883defm VPANDN : avx512_logic_rm_vl_dq<0xDF, 0xDF, "vpandn", X86andnp>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004884
4885//===----------------------------------------------------------------------===//
4886// AVX-512 FP arithmetic
4887//===----------------------------------------------------------------------===//
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004888multiclass avx512_fp_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
4889 SDNode OpNode, SDNode VecNode, OpndItins itins,
4890 bit IsCommutable> {
Craig Topper5ec33a92016-07-22 05:00:42 +00004891 let ExeDomain = _.ExeDomain in {
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004892 defm rr_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
4893 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
4894 "$src2, $src1", "$src1, $src2",
Craig Topperd9fe6642017-02-21 04:26:10 +00004895 (_.VT (VecNode _.RC:$src1, _.RC:$src2,
4896 (i32 FROUND_CURRENT))),
Craig Topper26000f82016-07-26 08:06:14 +00004897 itins.rr>;
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004898
4899 defm rm_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
Craig Topperd9fe6642017-02-21 04:26:10 +00004900 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr,
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004901 "$src2, $src1", "$src1, $src2",
Craig Topperd9fe6642017-02-21 04:26:10 +00004902 (_.VT (VecNode _.RC:$src1,
4903 _.ScalarIntMemCPat:$src2,
4904 (i32 FROUND_CURRENT))),
Craig Topper26000f82016-07-26 08:06:14 +00004905 itins.rm>;
Craig Topper79011a62016-07-26 08:06:18 +00004906 let isCodeGenOnly = 1, Predicates = [HasAVX512] in {
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004907 def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst),
Michael Liao66233b72015-08-06 09:06:20 +00004908 (ins _.FRC:$src1, _.FRC:$src2),
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004909 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
4910 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))],
Craig Topper79011a62016-07-26 08:06:18 +00004911 itins.rr> {
4912 let isCommutable = IsCommutable;
4913 }
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004914 def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst),
Michael Liao66233b72015-08-06 09:06:20 +00004915 (ins _.FRC:$src1, _.ScalarMemOp:$src2),
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004916 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
4917 [(set _.FRC:$dst, (OpNode _.FRC:$src1,
Elena Demikhovskyd84f3372016-07-11 06:08:06 +00004918 (_.ScalarLdFrag addr:$src2)))], itins.rm>;
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004919 }
Craig Topper5ec33a92016-07-22 05:00:42 +00004920 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004921}
4922
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004923multiclass avx512_fp_scalar_round<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00004924 SDNode VecNode, OpndItins itins, bit IsCommutable = 0> {
Craig Topper5ec33a92016-07-22 05:00:42 +00004925 let ExeDomain = _.ExeDomain in
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004926 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
4927 (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr,
4928 "$rc, $src2, $src1", "$src1, $src2, $rc",
4929 (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00004930 (i32 imm:$rc)), itins.rr, IsCommutable>,
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004931 EVEX_B, EVEX_RC;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004932}
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004933multiclass avx512_fp_scalar_sae<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
Craig Topper56d40222017-02-22 06:54:18 +00004934 SDNode OpNode, SDNode VecNode, SDNode SaeNode,
4935 OpndItins itins, bit IsCommutable> {
4936 let ExeDomain = _.ExeDomain in {
4937 defm rr_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
4938 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
4939 "$src2, $src1", "$src1, $src2",
4940 (_.VT (VecNode _.RC:$src1, _.RC:$src2)),
4941 itins.rr>;
4942
4943 defm rm_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
4944 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr,
4945 "$src2, $src1", "$src1, $src2",
4946 (_.VT (VecNode _.RC:$src1,
4947 _.ScalarIntMemCPat:$src2)),
4948 itins.rm>;
4949
4950 let isCodeGenOnly = 1, Predicates = [HasAVX512] in {
4951 def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst),
4952 (ins _.FRC:$src1, _.FRC:$src2),
4953 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
4954 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))],
4955 itins.rr> {
4956 let isCommutable = IsCommutable;
4957 }
4958 def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst),
4959 (ins _.FRC:$src1, _.ScalarMemOp:$src2),
4960 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
4961 [(set _.FRC:$dst, (OpNode _.FRC:$src1,
4962 (_.ScalarLdFrag addr:$src2)))], itins.rm>;
4963 }
4964
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004965 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
4966 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00004967 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
Craig Topper56d40222017-02-22 06:54:18 +00004968 (SaeNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00004969 (i32 FROUND_NO_EXC))>, EVEX_B;
Craig Topper56d40222017-02-22 06:54:18 +00004970 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00004971}
4972
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004973multiclass avx512_binop_s_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
4974 SDNode VecNode,
4975 SizeItins itins, bit IsCommutable> {
4976 defm SSZ : avx512_fp_scalar<opc, OpcodeStr#"ss", f32x_info, OpNode, VecNode,
4977 itins.s, IsCommutable>,
4978 avx512_fp_scalar_round<opc, OpcodeStr#"ss", f32x_info, VecNode,
4979 itins.s, IsCommutable>,
4980 XS, EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>;
4981 defm SDZ : avx512_fp_scalar<opc, OpcodeStr#"sd", f64x_info, OpNode, VecNode,
4982 itins.d, IsCommutable>,
4983 avx512_fp_scalar_round<opc, OpcodeStr#"sd", f64x_info, VecNode,
4984 itins.d, IsCommutable>,
4985 XD, VEX_W, EVEX_4V, VEX_LIG, EVEX_CD8<64, CD8VT1>;
4986}
4987
4988multiclass avx512_binop_s_sae<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper56d40222017-02-22 06:54:18 +00004989 SDNode VecNode, SDNode SaeNode,
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004990 SizeItins itins, bit IsCommutable> {
Craig Topper56d40222017-02-22 06:54:18 +00004991 defm SSZ : avx512_fp_scalar_sae<opc, OpcodeStr#"ss", f32x_info, OpNode,
4992 VecNode, SaeNode, itins.s, IsCommutable>,
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004993 XS, EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>;
Craig Topper56d40222017-02-22 06:54:18 +00004994 defm SDZ : avx512_fp_scalar_sae<opc, OpcodeStr#"sd", f64x_info, OpNode,
4995 VecNode, SaeNode, itins.d, IsCommutable>,
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00004996 XD, VEX_W, EVEX_4V, VEX_LIG, EVEX_CD8<64, CD8VT1>;
4997}
Craig Topper8783bbb2017-02-24 07:21:10 +00004998defm VADD : avx512_binop_s_round<0x58, "vadd", fadd, X86faddRnds, SSE_ALU_ITINS_S, 1>;
4999defm VMUL : avx512_binop_s_round<0x59, "vmul", fmul, X86fmulRnds, SSE_MUL_ITINS_S, 1>;
5000defm VSUB : avx512_binop_s_round<0x5C, "vsub", fsub, X86fsubRnds, SSE_ALU_ITINS_S, 0>;
5001defm VDIV : avx512_binop_s_round<0x5E, "vdiv", fdiv, X86fdivRnds, SSE_DIV_ITINS_S, 0>;
5002defm VMIN : avx512_binop_s_sae <0x5D, "vmin", X86fmin, X86fmins, X86fminRnds,
Craig Topper56d40222017-02-22 06:54:18 +00005003 SSE_ALU_ITINS_S, 0>;
Craig Topper8783bbb2017-02-24 07:21:10 +00005004defm VMAX : avx512_binop_s_sae <0x5F, "vmax", X86fmax, X86fmaxs, X86fmaxRnds,
Craig Topper56d40222017-02-22 06:54:18 +00005005 SSE_ALU_ITINS_S, 0>;
Elena Demikhovskyd84f3372016-07-11 06:08:06 +00005006
5007// MIN/MAX nodes are commutable under "unsafe-fp-math". In this case we use
5008// X86fminc and X86fmaxc instead of X86fmin and X86fmax
5009multiclass avx512_comutable_binop_s<bits<8> opc, string OpcodeStr,
5010 X86VectorVTInfo _, SDNode OpNode, OpndItins itins> {
Craig Topper03669332017-02-26 06:45:56 +00005011 let isCodeGenOnly = 1, Predicates = [HasAVX512], ExeDomain = _.ExeDomain in {
Elena Demikhovskyd84f3372016-07-11 06:08:06 +00005012 def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst),
5013 (ins _.FRC:$src1, _.FRC:$src2),
5014 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
5015 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))],
Craig Topper79011a62016-07-26 08:06:18 +00005016 itins.rr> {
5017 let isCommutable = 1;
5018 }
Elena Demikhovskyd84f3372016-07-11 06:08:06 +00005019 def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst),
5020 (ins _.FRC:$src1, _.ScalarMemOp:$src2),
5021 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
5022 [(set _.FRC:$dst, (OpNode _.FRC:$src1,
5023 (_.ScalarLdFrag addr:$src2)))], itins.rm>;
5024 }
5025}
5026defm VMINCSSZ : avx512_comutable_binop_s<0x5D, "vminss", f32x_info, X86fminc,
5027 SSE_ALU_ITINS_S.s>, XS, EVEX_4V, VEX_LIG,
5028 EVEX_CD8<32, CD8VT1>;
5029
5030defm VMINCSDZ : avx512_comutable_binop_s<0x5D, "vminsd", f64x_info, X86fminc,
5031 SSE_ALU_ITINS_S.d>, XD, VEX_W, EVEX_4V, VEX_LIG,
5032 EVEX_CD8<64, CD8VT1>;
5033
5034defm VMAXCSSZ : avx512_comutable_binop_s<0x5F, "vmaxss", f32x_info, X86fmaxc,
5035 SSE_ALU_ITINS_S.s>, XS, EVEX_4V, VEX_LIG,
5036 EVEX_CD8<32, CD8VT1>;
5037
5038defm VMAXCSDZ : avx512_comutable_binop_s<0x5F, "vmaxsd", f64x_info, X86fmaxc,
5039 SSE_ALU_ITINS_S.d>, XD, VEX_W, EVEX_4V, VEX_LIG,
5040 EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky02ffd262015-03-01 07:44:04 +00005041
Craig Topper375aa902016-12-19 00:42:28 +00005042multiclass avx512_fp_packed<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,
Craig Topper9433f972016-08-02 06:16:53 +00005043 X86VectorVTInfo _, OpndItins itins,
5044 bit IsCommutable> {
Craig Topper375aa902016-12-19 00:42:28 +00005045 let ExeDomain = _.ExeDomain, hasSideEffects = 0 in {
Robert Khasanov595e5982014-10-29 15:43:02 +00005046 defm rr: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5047 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
5048 "$src2, $src1", "$src1, $src2",
Craig Topper9433f972016-08-02 06:16:53 +00005049 (_.VT (OpNode _.RC:$src1, _.RC:$src2)), itins.rr,
5050 IsCommutable>, EVEX_4V;
Craig Topper375aa902016-12-19 00:42:28 +00005051 let mayLoad = 1 in {
5052 defm rm: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5053 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix,
5054 "$src2, $src1", "$src1, $src2",
5055 (OpNode _.RC:$src1, (_.LdFrag addr:$src2)), itins.rm>,
5056 EVEX_4V;
5057 defm rmb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5058 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix,
5059 "${src2}"##_.BroadcastStr##", $src1",
5060 "$src1, ${src2}"##_.BroadcastStr,
5061 (OpNode _.RC:$src1, (_.VT (X86VBroadcast
5062 (_.ScalarLdFrag addr:$src2)))),
5063 itins.rm>, EVEX_4V, EVEX_B;
5064 }
Craig Topper5ec33a92016-07-22 05:00:42 +00005065 }
Robert Khasanov595e5982014-10-29 15:43:02 +00005066}
Elena Demikhovskyf7c1b162014-03-06 08:45:30 +00005067
Craig Topper375aa902016-12-19 00:42:28 +00005068multiclass avx512_fp_round_packed<bits<8> opc, string OpcodeStr, SDPatternOperator OpNodeRnd,
Craig Topper5ec33a92016-07-22 05:00:42 +00005069 X86VectorVTInfo _> {
5070 let ExeDomain = _.ExeDomain in
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00005071 defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5072 (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr##_.Suffix,
5073 "$rc, $src2, $src1", "$src1, $src2, $rc",
5074 (_.VT (OpNodeRnd _.RC:$src1, _.RC:$src2, (i32 imm:$rc)))>,
5075 EVEX_4V, EVEX_B, EVEX_RC;
5076}
5077
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00005078
Craig Topper375aa902016-12-19 00:42:28 +00005079multiclass avx512_fp_sae_packed<bits<8> opc, string OpcodeStr, SDPatternOperator OpNodeRnd,
Craig Topper5ec33a92016-07-22 05:00:42 +00005080 X86VectorVTInfo _> {
5081 let ExeDomain = _.ExeDomain in
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00005082 defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5083 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
5084 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
5085 (_.VT (OpNodeRnd _.RC:$src1, _.RC:$src2, (i32 FROUND_NO_EXC)))>,
5086 EVEX_4V, EVEX_B;
5087}
5088
Craig Topper375aa902016-12-19 00:42:28 +00005089multiclass avx512_fp_binop_p<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,
Craig Topper9433f972016-08-02 06:16:53 +00005090 Predicate prd, SizeItins itins,
5091 bit IsCommutable = 0> {
Craig Topperdb290662016-05-01 05:57:06 +00005092 let Predicates = [prd] in {
Robert Khasanov595e5982014-10-29 15:43:02 +00005093 defm PSZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v16f32_info,
Craig Topper9433f972016-08-02 06:16:53 +00005094 itins.s, IsCommutable>, EVEX_V512, PS,
Robert Khasanov595e5982014-10-29 15:43:02 +00005095 EVEX_CD8<32, CD8VF>;
5096 defm PDZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v8f64_info,
Craig Topper9433f972016-08-02 06:16:53 +00005097 itins.d, IsCommutable>, EVEX_V512, PD, VEX_W,
Robert Khasanov595e5982014-10-29 15:43:02 +00005098 EVEX_CD8<64, CD8VF>;
Craig Topperdb290662016-05-01 05:57:06 +00005099 }
Elena Demikhovskyf7c1b162014-03-06 08:45:30 +00005100
Robert Khasanov595e5982014-10-29 15:43:02 +00005101 // Define only if AVX512VL feature is present.
Craig Topperdb290662016-05-01 05:57:06 +00005102 let Predicates = [prd, HasVLX] in {
Robert Khasanov595e5982014-10-29 15:43:02 +00005103 defm PSZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, v4f32x_info,
Craig Topper9433f972016-08-02 06:16:53 +00005104 itins.s, IsCommutable>, EVEX_V128, PS,
Robert Khasanov595e5982014-10-29 15:43:02 +00005105 EVEX_CD8<32, CD8VF>;
5106 defm PSZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, v8f32x_info,
Craig Topper9433f972016-08-02 06:16:53 +00005107 itins.s, IsCommutable>, EVEX_V256, PS,
Robert Khasanov595e5982014-10-29 15:43:02 +00005108 EVEX_CD8<32, CD8VF>;
5109 defm PDZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, v2f64x_info,
Craig Topper9433f972016-08-02 06:16:53 +00005110 itins.d, IsCommutable>, EVEX_V128, PD, VEX_W,
Robert Khasanov595e5982014-10-29 15:43:02 +00005111 EVEX_CD8<64, CD8VF>;
5112 defm PDZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, v4f64x_info,
Craig Topper9433f972016-08-02 06:16:53 +00005113 itins.d, IsCommutable>, EVEX_V256, PD, VEX_W,
Robert Khasanov595e5982014-10-29 15:43:02 +00005114 EVEX_CD8<64, CD8VF>;
Elena Demikhovskyf7c1b162014-03-06 08:45:30 +00005115 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005116}
5117
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00005118multiclass avx512_fp_binop_p_round<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd> {
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00005119 defm PSZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, v16f32_info>,
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00005120 EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00005121 defm PDZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, v8f64_info>,
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00005122 EVEX_V512, PD, VEX_W,EVEX_CD8<64, CD8VF>;
5123}
5124
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00005125multiclass avx512_fp_binop_p_sae<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd> {
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00005126 defm PSZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, v16f32_info>,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00005127 EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00005128 defm PDZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, v8f64_info>,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00005129 EVEX_V512, PD, VEX_W,EVEX_CD8<64, CD8VF>;
5130}
5131
Craig Topper9433f972016-08-02 06:16:53 +00005132defm VADD : avx512_fp_binop_p<0x58, "vadd", fadd, HasAVX512,
5133 SSE_ALU_ITINS_P, 1>,
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00005134 avx512_fp_binop_p_round<0x58, "vadd", X86faddRnd>;
Craig Topper9433f972016-08-02 06:16:53 +00005135defm VMUL : avx512_fp_binop_p<0x59, "vmul", fmul, HasAVX512,
5136 SSE_MUL_ITINS_P, 1>,
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00005137 avx512_fp_binop_p_round<0x59, "vmul", X86fmulRnd>;
Craig Topper9433f972016-08-02 06:16:53 +00005138defm VSUB : avx512_fp_binop_p<0x5C, "vsub", fsub, HasAVX512, SSE_ALU_ITINS_P>,
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00005139 avx512_fp_binop_p_round<0x5C, "vsub", X86fsubRnd>;
Craig Topper9433f972016-08-02 06:16:53 +00005140defm VDIV : avx512_fp_binop_p<0x5E, "vdiv", fdiv, HasAVX512, SSE_DIV_ITINS_P>,
Elena Demikhovsky714f23b2015-02-18 07:59:20 +00005141 avx512_fp_binop_p_round<0x5E, "vdiv", X86fdivRnd>;
Craig Topper9433f972016-08-02 06:16:53 +00005142defm VMIN : avx512_fp_binop_p<0x5D, "vmin", X86fmin, HasAVX512,
5143 SSE_ALU_ITINS_P, 0>,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00005144 avx512_fp_binop_p_sae<0x5D, "vmin", X86fminRnd>;
Craig Topper9433f972016-08-02 06:16:53 +00005145defm VMAX : avx512_fp_binop_p<0x5F, "vmax", X86fmax, HasAVX512,
5146 SSE_ALU_ITINS_P, 0>,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00005147 avx512_fp_binop_p_sae<0x5F, "vmax", X86fmaxRnd>;
Igor Breger58c07802016-05-03 11:51:45 +00005148let isCodeGenOnly = 1 in {
Craig Topper9433f972016-08-02 06:16:53 +00005149 defm VMINC : avx512_fp_binop_p<0x5D, "vmin", X86fminc, HasAVX512,
5150 SSE_ALU_ITINS_P, 1>;
5151 defm VMAXC : avx512_fp_binop_p<0x5F, "vmax", X86fmaxc, HasAVX512,
5152 SSE_ALU_ITINS_P, 1>;
Igor Breger58c07802016-05-03 11:51:45 +00005153}
Craig Topper375aa902016-12-19 00:42:28 +00005154defm VAND : avx512_fp_binop_p<0x54, "vand", null_frag, HasDQI,
Craig Topper9433f972016-08-02 06:16:53 +00005155 SSE_ALU_ITINS_P, 1>;
Craig Topper375aa902016-12-19 00:42:28 +00005156defm VANDN : avx512_fp_binop_p<0x55, "vandn", null_frag, HasDQI,
Craig Topper9433f972016-08-02 06:16:53 +00005157 SSE_ALU_ITINS_P, 0>;
Craig Topper375aa902016-12-19 00:42:28 +00005158defm VOR : avx512_fp_binop_p<0x56, "vor", null_frag, HasDQI,
Craig Topper9433f972016-08-02 06:16:53 +00005159 SSE_ALU_ITINS_P, 1>;
Craig Topper375aa902016-12-19 00:42:28 +00005160defm VXOR : avx512_fp_binop_p<0x57, "vxor", null_frag, HasDQI,
Craig Topper9433f972016-08-02 06:16:53 +00005161 SSE_ALU_ITINS_P, 1>;
Elena Demikhovsky52e4a0e2014-01-05 10:46:09 +00005162
Craig Topper8f6827c2016-08-31 05:37:52 +00005163// Patterns catch floating point selects with bitcasted integer logic ops.
Craig Topper45d65032016-09-02 05:29:13 +00005164multiclass avx512_fp_logical_lowering<string InstrStr, SDNode OpNode,
5165 X86VectorVTInfo _, Predicate prd> {
5166let Predicates = [prd] in {
5167 // Masked register-register logical operations.
5168 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5169 (bitconvert (_.i64VT (OpNode _.RC:$src1, _.RC:$src2))),
5170 _.RC:$src0)),
5171 (!cast<Instruction>(InstrStr#rrk) _.RC:$src0, _.KRCWM:$mask,
5172 _.RC:$src1, _.RC:$src2)>;
5173 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5174 (bitconvert (_.i64VT (OpNode _.RC:$src1, _.RC:$src2))),
5175 _.ImmAllZerosV)),
5176 (!cast<Instruction>(InstrStr#rrkz) _.KRCWM:$mask, _.RC:$src1,
5177 _.RC:$src2)>;
5178 // Masked register-memory logical operations.
5179 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5180 (bitconvert (_.i64VT (OpNode _.RC:$src1,
5181 (load addr:$src2)))),
5182 _.RC:$src0)),
5183 (!cast<Instruction>(InstrStr#rmk) _.RC:$src0, _.KRCWM:$mask,
5184 _.RC:$src1, addr:$src2)>;
5185 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5186 (bitconvert (_.i64VT (OpNode _.RC:$src1, (load addr:$src2)))),
5187 _.ImmAllZerosV)),
5188 (!cast<Instruction>(InstrStr#rmkz) _.KRCWM:$mask, _.RC:$src1,
5189 addr:$src2)>;
5190 // Register-broadcast logical operations.
5191 def : Pat<(_.i64VT (OpNode _.RC:$src1,
5192 (bitconvert (_.VT (X86VBroadcast
5193 (_.ScalarLdFrag addr:$src2)))))),
5194 (!cast<Instruction>(InstrStr#rmb) _.RC:$src1, addr:$src2)>;
5195 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5196 (bitconvert
5197 (_.i64VT (OpNode _.RC:$src1,
5198 (bitconvert (_.VT
5199 (X86VBroadcast
5200 (_.ScalarLdFrag addr:$src2))))))),
5201 _.RC:$src0)),
5202 (!cast<Instruction>(InstrStr#rmbk) _.RC:$src0, _.KRCWM:$mask,
5203 _.RC:$src1, addr:$src2)>;
5204 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5205 (bitconvert
5206 (_.i64VT (OpNode _.RC:$src1,
5207 (bitconvert (_.VT
5208 (X86VBroadcast
5209 (_.ScalarLdFrag addr:$src2))))))),
5210 _.ImmAllZerosV)),
5211 (!cast<Instruction>(InstrStr#rmbkz) _.KRCWM:$mask,
5212 _.RC:$src1, addr:$src2)>;
5213}
Craig Topper8f6827c2016-08-31 05:37:52 +00005214}
5215
Craig Topper45d65032016-09-02 05:29:13 +00005216multiclass avx512_fp_logical_lowering_sizes<string InstrStr, SDNode OpNode> {
5217 defm : avx512_fp_logical_lowering<InstrStr#DZ128, OpNode, v4f32x_info, HasVLX>;
5218 defm : avx512_fp_logical_lowering<InstrStr#QZ128, OpNode, v2f64x_info, HasVLX>;
5219 defm : avx512_fp_logical_lowering<InstrStr#DZ256, OpNode, v8f32x_info, HasVLX>;
5220 defm : avx512_fp_logical_lowering<InstrStr#QZ256, OpNode, v4f64x_info, HasVLX>;
5221 defm : avx512_fp_logical_lowering<InstrStr#DZ, OpNode, v16f32_info, HasAVX512>;
5222 defm : avx512_fp_logical_lowering<InstrStr#QZ, OpNode, v8f64_info, HasAVX512>;
Craig Topper8f6827c2016-08-31 05:37:52 +00005223}
5224
Craig Topper45d65032016-09-02 05:29:13 +00005225defm : avx512_fp_logical_lowering_sizes<"VPAND", and>;
5226defm : avx512_fp_logical_lowering_sizes<"VPOR", or>;
5227defm : avx512_fp_logical_lowering_sizes<"VPXOR", xor>;
5228defm : avx512_fp_logical_lowering_sizes<"VPANDN", X86andnp>;
5229
Craig Topper2baef8f2016-12-18 04:17:00 +00005230let Predicates = [HasVLX,HasDQI] in {
Craig Topperd3295c62016-12-17 19:26:00 +00005231 // Use packed logical operations for scalar ops.
5232 def : Pat<(f64 (X86fand FR64X:$src1, FR64X:$src2)),
5233 (COPY_TO_REGCLASS (VANDPDZ128rr
5234 (COPY_TO_REGCLASS FR64X:$src1, VR128X),
5235 (COPY_TO_REGCLASS FR64X:$src2, VR128X)), FR64X)>;
5236 def : Pat<(f64 (X86for FR64X:$src1, FR64X:$src2)),
5237 (COPY_TO_REGCLASS (VORPDZ128rr
5238 (COPY_TO_REGCLASS FR64X:$src1, VR128X),
5239 (COPY_TO_REGCLASS FR64X:$src2, VR128X)), FR64X)>;
5240 def : Pat<(f64 (X86fxor FR64X:$src1, FR64X:$src2)),
5241 (COPY_TO_REGCLASS (VXORPDZ128rr
5242 (COPY_TO_REGCLASS FR64X:$src1, VR128X),
5243 (COPY_TO_REGCLASS FR64X:$src2, VR128X)), FR64X)>;
5244 def : Pat<(f64 (X86fandn FR64X:$src1, FR64X:$src2)),
5245 (COPY_TO_REGCLASS (VANDNPDZ128rr
5246 (COPY_TO_REGCLASS FR64X:$src1, VR128X),
5247 (COPY_TO_REGCLASS FR64X:$src2, VR128X)), FR64X)>;
5248
5249 def : Pat<(f32 (X86fand FR32X:$src1, FR32X:$src2)),
5250 (COPY_TO_REGCLASS (VANDPSZ128rr
5251 (COPY_TO_REGCLASS FR32X:$src1, VR128X),
5252 (COPY_TO_REGCLASS FR32X:$src2, VR128X)), FR32X)>;
5253 def : Pat<(f32 (X86for FR32X:$src1, FR32X:$src2)),
5254 (COPY_TO_REGCLASS (VORPSZ128rr
5255 (COPY_TO_REGCLASS FR32X:$src1, VR128X),
5256 (COPY_TO_REGCLASS FR32X:$src2, VR128X)), FR32X)>;
5257 def : Pat<(f32 (X86fxor FR32X:$src1, FR32X:$src2)),
5258 (COPY_TO_REGCLASS (VXORPSZ128rr
5259 (COPY_TO_REGCLASS FR32X:$src1, VR128X),
5260 (COPY_TO_REGCLASS FR32X:$src2, VR128X)), FR32X)>;
5261 def : Pat<(f32 (X86fandn FR32X:$src1, FR32X:$src2)),
5262 (COPY_TO_REGCLASS (VANDNPSZ128rr
5263 (COPY_TO_REGCLASS FR32X:$src1, VR128X),
5264 (COPY_TO_REGCLASS FR32X:$src2, VR128X)), FR32X)>;
5265}
5266
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00005267multiclass avx512_fp_scalef_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
5268 X86VectorVTInfo _> {
Craig Topperaa8e9032017-02-26 06:45:40 +00005269 let ExeDomain = _.ExeDomain in {
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00005270 defm rr: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5271 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
5272 "$src2, $src1", "$src1, $src2",
5273 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (i32 FROUND_CURRENT)))>, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00005274 defm rm: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5275 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix,
5276 "$src2, $src1", "$src1, $src2",
5277 (OpNode _.RC:$src1, (_.LdFrag addr:$src2), (i32 FROUND_CURRENT))>, EVEX_4V;
5278 defm rmb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5279 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix,
5280 "${src2}"##_.BroadcastStr##", $src1",
5281 "$src1, ${src2}"##_.BroadcastStr,
5282 (OpNode _.RC:$src1, (_.VT (X86VBroadcast
5283 (_.ScalarLdFrag addr:$src2))), (i32 FROUND_CURRENT))>,
5284 EVEX_4V, EVEX_B;
Craig Topperaa8e9032017-02-26 06:45:40 +00005285 }
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00005286}
5287
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00005288multiclass avx512_fp_scalef_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode,
5289 X86VectorVTInfo _> {
Craig Topperaa8e9032017-02-26 06:45:40 +00005290 let ExeDomain = _.ExeDomain in {
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00005291 defm rr: AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5292 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
5293 "$src2, $src1", "$src1, $src2",
5294 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (i32 FROUND_CURRENT)))>;
Craig Toppere1cac152016-06-07 07:27:54 +00005295 defm rm: AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
5296 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix,
5297 "$src2, $src1", "$src1, $src2",
Simon Pilgrimb13961d2016-06-11 14:34:10 +00005298 (OpNode _.RC:$src1,
Craig Toppere1cac152016-06-07 07:27:54 +00005299 (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))),
5300 (i32 FROUND_CURRENT))>;
Craig Topperaa8e9032017-02-26 06:45:40 +00005301 }
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00005302}
5303
Michael Zuckerman11b55b22016-05-21 11:09:53 +00005304multiclass avx512_fp_scalef_all<bits<8> opc, bits<8> opcScaler, string OpcodeStr, SDNode OpNode, SDNode OpNodeScal> {
Michael Liao66233b72015-08-06 09:06:20 +00005305 defm PSZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v16f32_info>,
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00005306 avx512_fp_round_packed<opc, OpcodeStr, OpNode, v16f32_info>,
5307 EVEX_V512, EVEX_CD8<32, CD8VF>;
Michael Liao66233b72015-08-06 09:06:20 +00005308 defm PDZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v8f64_info>,
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00005309 avx512_fp_round_packed<opc, OpcodeStr, OpNode, v8f64_info>,
5310 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
Michael Zuckerman11b55b22016-05-21 11:09:53 +00005311 defm SSZ128 : avx512_fp_scalef_scalar<opcScaler, OpcodeStr, OpNodeScal, f32x_info>,
5312 avx512_fp_scalar_round<opcScaler, OpcodeStr##"ss", f32x_info, OpNodeScal, SSE_ALU_ITINS_S.s>,
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00005313 EVEX_4V,EVEX_CD8<32, CD8VT1>;
Michael Zuckerman11b55b22016-05-21 11:09:53 +00005314 defm SDZ128 : avx512_fp_scalef_scalar<opcScaler, OpcodeStr, OpNodeScal, f64x_info>,
5315 avx512_fp_scalar_round<opcScaler, OpcodeStr##"sd", f64x_info, OpNodeScal, SSE_ALU_ITINS_S.d>,
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00005316 EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
5317
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00005318 // Define only if AVX512VL feature is present.
5319 let Predicates = [HasVLX] in {
5320 defm PSZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v4f32x_info>,
5321 EVEX_V128, EVEX_CD8<32, CD8VF>;
5322 defm PSZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v8f32x_info>,
5323 EVEX_V256, EVEX_CD8<32, CD8VF>;
5324 defm PDZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v2f64x_info>,
5325 EVEX_V128, VEX_W, EVEX_CD8<64, CD8VF>;
5326 defm PDZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v4f64x_info>,
5327 EVEX_V256, VEX_W, EVEX_CD8<64, CD8VF>;
5328 }
5329}
Michael Zuckerman11b55b22016-05-21 11:09:53 +00005330defm VSCALEF : avx512_fp_scalef_all<0x2C, 0x2D, "vscalef", X86scalef, X86scalefs>, T8PD;
Asaf Badouh7ec4b7a2015-06-28 14:30:39 +00005331
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005332//===----------------------------------------------------------------------===//
5333// AVX-512 VPTESTM instructions
5334//===----------------------------------------------------------------------===//
5335
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00005336multiclass avx512_vptest<bits<8> opc, string OpcodeStr, SDNode OpNode,
5337 X86VectorVTInfo _> {
Igor Breger639fde72016-03-03 14:18:38 +00005338 let isCommutable = 1 in
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00005339 defm rr : AVX512_maskable_cmp<opc, MRMSrcReg, _, (outs _.KRC:$dst),
5340 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
5341 "$src2, $src1", "$src1, $src2",
5342 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>,
5343 EVEX_4V;
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00005344 defm rm : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst),
5345 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
5346 "$src2, $src1", "$src1, $src2",
Michael Liao66233b72015-08-06 09:06:20 +00005347 (OpNode (_.VT _.RC:$src1),
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00005348 (_.VT (bitconvert (_.LdFrag addr:$src2))))>,
5349 EVEX_4V,
5350 EVEX_CD8<_.EltSize, CD8VF>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005351}
5352
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00005353multiclass avx512_vptest_mb<bits<8> opc, string OpcodeStr, SDNode OpNode,
5354 X86VectorVTInfo _> {
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00005355 defm rmb : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst),
5356 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
5357 "${src2}"##_.BroadcastStr##", $src1",
5358 "$src1, ${src2}"##_.BroadcastStr,
5359 (OpNode (_.VT _.RC:$src1), (_.VT (X86VBroadcast
5360 (_.ScalarLdFrag addr:$src2))))>,
5361 EVEX_B, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>;
Elena Demikhovskya30e4372014-02-05 07:05:03 +00005362}
Igor Bregerfca0a342016-01-28 13:19:25 +00005363
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005364// Use 512bit version to implement 128/256 bit in case NoVLX.
Igor Bregerfca0a342016-01-28 13:19:25 +00005365multiclass avx512_vptest_lowering<SDNode OpNode, X86VectorVTInfo ExtendInfo,
5366 X86VectorVTInfo _, string Suffix> {
5367 def : Pat<(_.KVT (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))),
5368 (_.KVT (COPY_TO_REGCLASS
5369 (!cast<Instruction>(NAME # Suffix # "Zrr")
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005370 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
Igor Bregerfca0a342016-01-28 13:19:25 +00005371 _.RC:$src1, _.SubRegIdx),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005372 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
Igor Bregerfca0a342016-01-28 13:19:25 +00005373 _.RC:$src2, _.SubRegIdx)),
5374 _.KRC))>;
5375}
5376
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00005377multiclass avx512_vptest_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
Igor Bregerfca0a342016-01-28 13:19:25 +00005378 AVX512VLVectorVTInfo _, string Suffix> {
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00005379 let Predicates = [HasAVX512] in
5380 defm Z : avx512_vptest<opc, OpcodeStr, OpNode, _.info512>,
5381 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
5382
5383 let Predicates = [HasAVX512, HasVLX] in {
5384 defm Z256 : avx512_vptest<opc, OpcodeStr, OpNode, _.info256>,
5385 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
5386 defm Z128 : avx512_vptest<opc, OpcodeStr, OpNode, _.info128>,
5387 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128;
5388 }
Igor Bregerfca0a342016-01-28 13:19:25 +00005389 let Predicates = [HasAVX512, NoVLX] in {
5390 defm Z256_Alt : avx512_vptest_lowering< OpNode, _.info512, _.info256, Suffix>;
5391 defm Z128_Alt : avx512_vptest_lowering< OpNode, _.info512, _.info128, Suffix>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005392 }
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00005393}
5394
5395multiclass avx512_vptest_dq<bits<8> opc, string OpcodeStr, SDNode OpNode> {
5396 defm D : avx512_vptest_dq_sizes<opc, OpcodeStr#"d", OpNode,
Igor Bregerfca0a342016-01-28 13:19:25 +00005397 avx512vl_i32_info, "D">;
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00005398 defm Q : avx512_vptest_dq_sizes<opc, OpcodeStr#"q", OpNode,
Igor Bregerfca0a342016-01-28 13:19:25 +00005399 avx512vl_i64_info, "Q">, VEX_W;
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00005400}
5401
5402multiclass avx512_vptest_wb<bits<8> opc, string OpcodeStr,
5403 SDNode OpNode> {
5404 let Predicates = [HasBWI] in {
5405 defm WZ: avx512_vptest<opc, OpcodeStr#"w", OpNode, v32i16_info>,
5406 EVEX_V512, VEX_W;
5407 defm BZ: avx512_vptest<opc, OpcodeStr#"b", OpNode, v64i8_info>,
5408 EVEX_V512;
5409 }
5410 let Predicates = [HasVLX, HasBWI] in {
5411
5412 defm WZ256: avx512_vptest<opc, OpcodeStr#"w", OpNode, v16i16x_info>,
5413 EVEX_V256, VEX_W;
5414 defm WZ128: avx512_vptest<opc, OpcodeStr#"w", OpNode, v8i16x_info>,
5415 EVEX_V128, VEX_W;
5416 defm BZ256: avx512_vptest<opc, OpcodeStr#"b", OpNode, v32i8x_info>,
5417 EVEX_V256;
5418 defm BZ128: avx512_vptest<opc, OpcodeStr#"b", OpNode, v16i8x_info>,
5419 EVEX_V128;
5420 }
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005421
Igor Bregerfca0a342016-01-28 13:19:25 +00005422 let Predicates = [HasAVX512, NoVLX] in {
5423 defm BZ256_Alt : avx512_vptest_lowering< OpNode, v64i8_info, v32i8x_info, "B">;
5424 defm BZ128_Alt : avx512_vptest_lowering< OpNode, v64i8_info, v16i8x_info, "B">;
5425 defm WZ256_Alt : avx512_vptest_lowering< OpNode, v32i16_info, v16i16x_info, "W">;
5426 defm WZ128_Alt : avx512_vptest_lowering< OpNode, v32i16_info, v8i16x_info, "W">;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005427 }
Igor Bregerfca0a342016-01-28 13:19:25 +00005428
Elena Demikhovsky431b81e2015-04-21 13:13:46 +00005429}
5430
5431multiclass avx512_vptest_all_forms<bits<8> opc_wb, bits<8> opc_dq, string OpcodeStr,
5432 SDNode OpNode> :
5433 avx512_vptest_wb <opc_wb, OpcodeStr, OpNode>,
5434 avx512_vptest_dq<opc_dq, OpcodeStr, OpNode>;
5435
5436defm VPTESTM : avx512_vptest_all_forms<0x26, 0x27, "vptestm", X86testm>, T8PD;
5437defm VPTESTNM : avx512_vptest_all_forms<0x26, 0x27, "vptestnm", X86testnm>, T8XS;
Elena Demikhovskya30e4372014-02-05 07:05:03 +00005438
Cameron McInally9b7c15a2014-11-25 20:41:51 +00005439
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005440//===----------------------------------------------------------------------===//
5441// AVX-512 Shift instructions
5442//===----------------------------------------------------------------------===//
5443multiclass avx512_shift_rmi<bits<8> opc, Format ImmFormR, Format ImmFormM,
Michael Liao5bf95782014-12-04 05:20:33 +00005444 string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> {
Craig Topper05948fb2016-08-02 05:11:15 +00005445 let ExeDomain = _.ExeDomain in {
Cameron McInally04400442014-11-14 15:43:00 +00005446 defm ri : AVX512_maskable<opc, ImmFormR, _, (outs _.RC:$dst),
Craig Topper7ff6ab32015-01-21 08:43:49 +00005447 (ins _.RC:$src1, u8imm:$src2), OpcodeStr,
Cameron McInally04400442014-11-14 15:43:00 +00005448 "$src2, $src1", "$src1, $src2",
5449 (_.VT (OpNode _.RC:$src1, (i8 imm:$src2))),
Elena Demikhovsky75ede682015-06-01 07:17:23 +00005450 SSE_INTSHIFT_ITINS_P.rr>;
Cameron McInally04400442014-11-14 15:43:00 +00005451 defm mi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst),
Craig Topper7ff6ab32015-01-21 08:43:49 +00005452 (ins _.MemOp:$src1, u8imm:$src2), OpcodeStr,
Cameron McInally04400442014-11-14 15:43:00 +00005453 "$src2, $src1", "$src1, $src2",
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005454 (_.VT (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
5455 (i8 imm:$src2))),
Elena Demikhovsky75ede682015-06-01 07:17:23 +00005456 SSE_INTSHIFT_ITINS_P.rm>;
Craig Topper05948fb2016-08-02 05:11:15 +00005457 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005458}
5459
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005460multiclass avx512_shift_rmbi<bits<8> opc, Format ImmFormM,
5461 string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> {
Craig Topper05948fb2016-08-02 05:11:15 +00005462 let ExeDomain = _.ExeDomain in
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005463 defm mbi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst),
5464 (ins _.ScalarMemOp:$src1, u8imm:$src2), OpcodeStr,
5465 "$src2, ${src1}"##_.BroadcastStr, "${src1}"##_.BroadcastStr##", $src2",
5466 (_.VT (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src1)), (i8 imm:$src2))),
Elena Demikhovsky75ede682015-06-01 07:17:23 +00005467 SSE_INTSHIFT_ITINS_P.rm>, EVEX_B;
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005468}
5469
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005470multiclass avx512_shift_rrm<bits<8> opc, string OpcodeStr, SDNode OpNode,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005471 ValueType SrcVT, PatFrag bc_frag, X86VectorVTInfo _> {
Cameron McInally9b7c15a2014-11-25 20:41:51 +00005472 // src2 is always 128-bit
Craig Topper05948fb2016-08-02 05:11:15 +00005473 let ExeDomain = _.ExeDomain in {
Cameron McInally9b7c15a2014-11-25 20:41:51 +00005474 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5475 (ins _.RC:$src1, VR128X:$src2), OpcodeStr,
5476 "$src2, $src1", "$src1, $src2",
5477 (_.VT (OpNode _.RC:$src1, (SrcVT VR128X:$src2))),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00005478 SSE_INTSHIFT_ITINS_P.rr>, AVX512BIBase, EVEX_4V;
Cameron McInally9b7c15a2014-11-25 20:41:51 +00005479 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5480 (ins _.RC:$src1, i128mem:$src2), OpcodeStr,
5481 "$src2, $src1", "$src1, $src2",
Craig Topper820d4922015-02-09 04:04:50 +00005482 (_.VT (OpNode _.RC:$src1, (bc_frag (loadv2i64 addr:$src2)))),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00005483 SSE_INTSHIFT_ITINS_P.rm>, AVX512BIBase,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005484 EVEX_4V;
Craig Topper05948fb2016-08-02 05:11:15 +00005485 }
Cameron McInally9b7c15a2014-11-25 20:41:51 +00005486}
5487
Cameron McInally5fb084e2014-12-11 17:13:05 +00005488multiclass avx512_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005489 ValueType SrcVT, PatFrag bc_frag,
5490 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
5491 let Predicates = [prd] in
5492 defm Z : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag,
5493 VTInfo.info512>, EVEX_V512,
5494 EVEX_CD8<VTInfo.info512.EltSize, CD8VQ> ;
5495 let Predicates = [prd, HasVLX] in {
5496 defm Z256 : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag,
5497 VTInfo.info256>, EVEX_V256,
5498 EVEX_CD8<VTInfo.info256.EltSize, CD8VH>;
5499 defm Z128 : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag,
5500 VTInfo.info128>, EVEX_V128,
5501 EVEX_CD8<VTInfo.info128.EltSize, CD8VF>;
5502 }
Cameron McInally9b7c15a2014-11-25 20:41:51 +00005503}
5504
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005505multiclass avx512_shift_types<bits<8> opcd, bits<8> opcq, bits<8> opcw,
5506 string OpcodeStr, SDNode OpNode> {
Cameron McInally5fb084e2014-12-11 17:13:05 +00005507 defm D : avx512_shift_sizes<opcd, OpcodeStr#"d", OpNode, v4i32, bc_v4i32,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005508 avx512vl_i32_info, HasAVX512>;
Cameron McInally5fb084e2014-12-11 17:13:05 +00005509 defm Q : avx512_shift_sizes<opcq, OpcodeStr#"q", OpNode, v2i64, bc_v2i64,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005510 avx512vl_i64_info, HasAVX512>, VEX_W;
5511 defm W : avx512_shift_sizes<opcw, OpcodeStr#"w", OpNode, v8i16, bc_v8i16,
5512 avx512vl_i16_info, HasBWI>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005513}
5514
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005515multiclass avx512_shift_rmi_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM,
5516 string OpcodeStr, SDNode OpNode,
5517 AVX512VLVectorVTInfo VTInfo> {
5518 let Predicates = [HasAVX512] in
5519 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5520 VTInfo.info512>,
5521 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
5522 VTInfo.info512>, EVEX_V512;
5523 let Predicates = [HasAVX512, HasVLX] in {
5524 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5525 VTInfo.info256>,
5526 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
5527 VTInfo.info256>, EVEX_V256;
5528 defm Z128: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5529 VTInfo.info128>,
Michael Liao66233b72015-08-06 09:06:20 +00005530 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005531 VTInfo.info128>, EVEX_V128;
5532 }
5533}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005534
Michael Liao66233b72015-08-06 09:06:20 +00005535multiclass avx512_shift_rmi_w<bits<8> opcw,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005536 Format ImmFormR, Format ImmFormM,
5537 string OpcodeStr, SDNode OpNode> {
5538 let Predicates = [HasBWI] in
5539 defm WZ: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5540 v32i16_info>, EVEX_V512;
5541 let Predicates = [HasVLX, HasBWI] in {
5542 defm WZ256: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5543 v16i16x_info>, EVEX_V256;
5544 defm WZ128: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5545 v8i16x_info>, EVEX_V128;
5546 }
5547}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005548
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005549multiclass avx512_shift_rmi_dq<bits<8> opcd, bits<8> opcq,
5550 Format ImmFormR, Format ImmFormM,
5551 string OpcodeStr, SDNode OpNode> {
5552 defm D: avx512_shift_rmi_sizes<opcd, ImmFormR, ImmFormM, OpcodeStr#"d", OpNode,
5553 avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
5554 defm Q: avx512_shift_rmi_sizes<opcq, ImmFormR, ImmFormM, OpcodeStr#"q", OpNode,
5555 avx512vl_i64_info>, EVEX_CD8<64, CD8VF>, VEX_W;
5556}
Cameron McInally9b7c15a2014-11-25 20:41:51 +00005557
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005558defm VPSRL : avx512_shift_rmi_dq<0x72, 0x73, MRM2r, MRM2m, "vpsrl", X86vsrli>,
Elena Demikhovsky75ede682015-06-01 07:17:23 +00005559 avx512_shift_rmi_w<0x71, MRM2r, MRM2m, "vpsrlw", X86vsrli>, AVX512BIi8Base, EVEX_4V;
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005560
5561defm VPSLL : avx512_shift_rmi_dq<0x72, 0x73, MRM6r, MRM6m, "vpsll", X86vshli>,
Elena Demikhovsky75ede682015-06-01 07:17:23 +00005562 avx512_shift_rmi_w<0x71, MRM6r, MRM6m, "vpsllw", X86vshli>, AVX512BIi8Base, EVEX_4V;
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005563
Elena Demikhovsky1b2f2f12015-05-13 07:35:05 +00005564defm VPSRA : avx512_shift_rmi_dq<0x72, 0x72, MRM4r, MRM4m, "vpsra", X86vsrai>,
Elena Demikhovsky75ede682015-06-01 07:17:23 +00005565 avx512_shift_rmi_w<0x71, MRM4r, MRM4m, "vpsraw", X86vsrai>, AVX512BIi8Base, EVEX_4V;
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005566
Michael Zuckerman298a6802016-01-13 12:39:33 +00005567defm VPROR : avx512_shift_rmi_dq<0x72, 0x72, MRM0r, MRM0m, "vpror", X86vrotri>, AVX512BIi8Base, EVEX_4V;
Michael Zuckerman2ddcbcf2016-01-12 21:19:17 +00005568defm VPROL : avx512_shift_rmi_dq<0x72, 0x72, MRM1r, MRM1m, "vprol", X86vrotli>, AVX512BIi8Base, EVEX_4V;
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005569
5570defm VPSLL : avx512_shift_types<0xF2, 0xF3, 0xF1, "vpsll", X86vshl>;
5571defm VPSRA : avx512_shift_types<0xE2, 0xE2, 0xE1, "vpsra", X86vsra>;
5572defm VPSRL : avx512_shift_types<0xD2, 0xD3, 0xD1, "vpsrl", X86vsrl>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005573
Simon Pilgrim5910ebe2017-02-20 12:16:38 +00005574// Use 512bit VPSRA/VPSRAI version to implement v2i64/v4i64 in case NoVLX.
5575let Predicates = [HasAVX512, NoVLX] in {
5576 def : Pat<(v4i64 (X86vsra (v4i64 VR256X:$src1), (v2i64 VR128X:$src2))),
5577 (EXTRACT_SUBREG (v8i64
5578 (VPSRAQZrr
5579 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
5580 VR128X:$src2)), sub_ymm)>;
5581
5582 def : Pat<(v2i64 (X86vsra (v2i64 VR128X:$src1), (v2i64 VR128X:$src2))),
5583 (EXTRACT_SUBREG (v8i64
5584 (VPSRAQZrr
5585 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
5586 VR128X:$src2)), sub_xmm)>;
5587
5588 def : Pat<(v4i64 (X86vsrai (v4i64 VR256X:$src1), (i8 imm:$src2))),
5589 (EXTRACT_SUBREG (v8i64
5590 (VPSRAQZri
5591 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
5592 imm:$src2)), sub_ymm)>;
5593
5594 def : Pat<(v2i64 (X86vsrai (v2i64 VR128X:$src1), (i8 imm:$src2))),
5595 (EXTRACT_SUBREG (v8i64
5596 (VPSRAQZri
5597 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
5598 imm:$src2)), sub_xmm)>;
5599}
5600
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005601//===-------------------------------------------------------------------===//
5602// Variable Bit Shifts
5603//===-------------------------------------------------------------------===//
5604multiclass avx512_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode,
Cameron McInally5fb084e2014-12-11 17:13:05 +00005605 X86VectorVTInfo _> {
Craig Topper05948fb2016-08-02 05:11:15 +00005606 let ExeDomain = _.ExeDomain in {
Cameron McInally5fb084e2014-12-11 17:13:05 +00005607 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5608 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
5609 "$src2, $src1", "$src1, $src2",
5610 (_.VT (OpNode _.RC:$src1, (_.VT _.RC:$src2))),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00005611 SSE_INTSHIFT_ITINS_P.rr>, AVX5128IBase, EVEX_4V;
Cameron McInally5fb084e2014-12-11 17:13:05 +00005612 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5613 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
5614 "$src2, $src1", "$src1, $src2",
Elena Demikhovsky4078c752015-06-04 07:07:13 +00005615 (_.VT (OpNode _.RC:$src1,
5616 (_.VT (bitconvert (_.LdFrag addr:$src2))))),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00005617 SSE_INTSHIFT_ITINS_P.rm>, AVX5128IBase, EVEX_4V,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005618 EVEX_CD8<_.EltSize, CD8VF>;
Craig Topper05948fb2016-08-02 05:11:15 +00005619 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005620}
5621
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005622multiclass avx512_var_shift_mb<bits<8> opc, string OpcodeStr, SDNode OpNode,
5623 X86VectorVTInfo _> {
Craig Topper05948fb2016-08-02 05:11:15 +00005624 let ExeDomain = _.ExeDomain in
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005625 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5626 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
5627 "${src2}"##_.BroadcastStr##", $src1",
5628 "$src1, ${src2}"##_.BroadcastStr,
5629 (_.VT (OpNode _.RC:$src1, (_.VT (X86VBroadcast
5630 (_.ScalarLdFrag addr:$src2))))),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00005631 SSE_INTSHIFT_ITINS_P.rm>, AVX5128IBase, EVEX_B,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005632 EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>;
5633}
Simon Pilgrim7f2a6d52017-01-13 13:16:19 +00005634
Cameron McInally5fb084e2014-12-11 17:13:05 +00005635multiclass avx512_var_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
5636 AVX512VLVectorVTInfo _> {
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005637 let Predicates = [HasAVX512] in
5638 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>,
5639 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
5640
5641 let Predicates = [HasAVX512, HasVLX] in {
5642 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>,
5643 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
5644 defm Z128 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info128>,
5645 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128;
5646 }
Cameron McInally5fb084e2014-12-11 17:13:05 +00005647}
5648
5649multiclass avx512_var_shift_types<bits<8> opc, string OpcodeStr,
5650 SDNode OpNode> {
5651 defm D : avx512_var_shift_sizes<opc, OpcodeStr#"d", OpNode,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005652 avx512vl_i32_info>;
Cameron McInally5fb084e2014-12-11 17:13:05 +00005653 defm Q : avx512_var_shift_sizes<opc, OpcodeStr#"q", OpNode,
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005654 avx512vl_i64_info>, VEX_W;
Cameron McInally5fb084e2014-12-11 17:13:05 +00005655}
5656
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005657// Use 512bit version to implement 128/256 bit in case NoVLX.
Simon Pilgrim7f2a6d52017-01-13 13:16:19 +00005658multiclass avx512_var_shift_lowering<AVX512VLVectorVTInfo _, string OpcodeStr,
5659 SDNode OpNode, list<Predicate> p> {
5660 let Predicates = p in {
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005661 def : Pat<(_.info256.VT (OpNode (_.info256.VT _.info256.RC:$src1),
Igor Breger7b46b4e2015-12-23 08:06:50 +00005662 (_.info256.VT _.info256.RC:$src2))),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005663 (EXTRACT_SUBREG
Simon Pilgrim7f2a6d52017-01-13 13:16:19 +00005664 (!cast<Instruction>(OpcodeStr#"Zrr")
Igor Breger7b46b4e2015-12-23 08:06:50 +00005665 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR256X:$src1, sub_ymm),
5666 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR256X:$src2, sub_ymm)),
5667 sub_ymm)>;
5668
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005669 def : Pat<(_.info128.VT (OpNode (_.info128.VT _.info128.RC:$src1),
Igor Breger7b46b4e2015-12-23 08:06:50 +00005670 (_.info128.VT _.info128.RC:$src2))),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005671 (EXTRACT_SUBREG
Simon Pilgrim7f2a6d52017-01-13 13:16:19 +00005672 (!cast<Instruction>(OpcodeStr#"Zrr")
Igor Breger7b46b4e2015-12-23 08:06:50 +00005673 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR128X:$src1, sub_xmm),
5674 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR128X:$src2, sub_xmm)),
5675 sub_xmm)>;
5676 }
5677}
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005678multiclass avx512_var_shift_w<bits<8> opc, string OpcodeStr,
5679 SDNode OpNode> {
5680 let Predicates = [HasBWI] in
5681 defm WZ: avx512_var_shift<opc, OpcodeStr, OpNode, v32i16_info>,
5682 EVEX_V512, VEX_W;
5683 let Predicates = [HasVLX, HasBWI] in {
5684
5685 defm WZ256: avx512_var_shift<opc, OpcodeStr, OpNode, v16i16x_info>,
5686 EVEX_V256, VEX_W;
5687 defm WZ128: avx512_var_shift<opc, OpcodeStr, OpNode, v8i16x_info>,
5688 EVEX_V128, VEX_W;
5689 }
5690}
5691
5692defm VPSLLV : avx512_var_shift_types<0x47, "vpsllv", shl>,
Simon Pilgrim7f2a6d52017-01-13 13:16:19 +00005693 avx512_var_shift_w<0x12, "vpsllvw", shl>;
Igor Bregere59165c2016-06-20 07:05:43 +00005694
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005695defm VPSRAV : avx512_var_shift_types<0x46, "vpsrav", sra>,
Simon Pilgrim7f2a6d52017-01-13 13:16:19 +00005696 avx512_var_shift_w<0x11, "vpsravw", sra>;
Igor Bregere59165c2016-06-20 07:05:43 +00005697
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005698defm VPSRLV : avx512_var_shift_types<0x45, "vpsrlv", srl>,
Simon Pilgrim7f2a6d52017-01-13 13:16:19 +00005699 avx512_var_shift_w<0x10, "vpsrlvw", srl>;
5700
Elena Demikhovsky0b9dbe32015-03-11 10:25:42 +00005701defm VPRORV : avx512_var_shift_types<0x14, "vprorv", rotr>;
5702defm VPROLV : avx512_var_shift_types<0x15, "vprolv", rotl>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005703
Simon Pilgrim7f2a6d52017-01-13 13:16:19 +00005704defm : avx512_var_shift_lowering<avx512vl_i64_info, "VPSRAVQ", sra, [HasAVX512, NoVLX]>;
5705defm : avx512_var_shift_lowering<avx512vl_i16_info, "VPSLLVW", shl, [HasBWI, NoVLX]>;
5706defm : avx512_var_shift_lowering<avx512vl_i16_info, "VPSRAVW", sra, [HasBWI, NoVLX]>;
5707defm : avx512_var_shift_lowering<avx512vl_i16_info, "VPSRLVW", srl, [HasBWI, NoVLX]>;
5708
Craig Topper05629d02016-07-24 07:32:45 +00005709// Special handing for handling VPSRAV intrinsics.
5710multiclass avx512_var_shift_int_lowering<string InstrStr, X86VectorVTInfo _,
5711 list<Predicate> p> {
5712 let Predicates = p in {
5713 def : Pat<(_.VT (X86vsrav _.RC:$src1, _.RC:$src2)),
5714 (!cast<Instruction>(InstrStr#_.ZSuffix#rr) _.RC:$src1,
5715 _.RC:$src2)>;
5716 def : Pat<(_.VT (X86vsrav _.RC:$src1, (bitconvert (_.LdFrag addr:$src2)))),
5717 (!cast<Instruction>(InstrStr#_.ZSuffix##rm)
5718 _.RC:$src1, addr:$src2)>;
Craig Topper05629d02016-07-24 07:32:45 +00005719 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5720 (X86vsrav _.RC:$src1, _.RC:$src2), _.RC:$src0)),
5721 (!cast<Instruction>(InstrStr#_.ZSuffix#rrk) _.RC:$src0,
5722 _.KRC:$mask, _.RC:$src1, _.RC:$src2)>;
5723 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5724 (X86vsrav _.RC:$src1, (bitconvert (_.LdFrag addr:$src2))),
5725 _.RC:$src0)),
5726 (!cast<Instruction>(InstrStr#_.ZSuffix##rmk) _.RC:$src0,
5727 _.KRC:$mask, _.RC:$src1, addr:$src2)>;
Craig Topper05629d02016-07-24 07:32:45 +00005728 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5729 (X86vsrav _.RC:$src1, _.RC:$src2), _.ImmAllZerosV)),
5730 (!cast<Instruction>(InstrStr#_.ZSuffix#rrkz) _.KRC:$mask,
5731 _.RC:$src1, _.RC:$src2)>;
5732 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5733 (X86vsrav _.RC:$src1, (bitconvert (_.LdFrag addr:$src2))),
5734 _.ImmAllZerosV)),
5735 (!cast<Instruction>(InstrStr#_.ZSuffix##rmkz) _.KRC:$mask,
5736 _.RC:$src1, addr:$src2)>;
Craig Topper05629d02016-07-24 07:32:45 +00005737 }
5738}
5739
5740multiclass avx512_var_shift_int_lowering_mb<string InstrStr, X86VectorVTInfo _,
5741 list<Predicate> p> :
5742 avx512_var_shift_int_lowering<InstrStr, _, p> {
5743 let Predicates = p in {
5744 def : Pat<(_.VT (X86vsrav _.RC:$src1,
5745 (X86VBroadcast (_.ScalarLdFrag addr:$src2)))),
5746 (!cast<Instruction>(InstrStr#_.ZSuffix##rmb)
5747 _.RC:$src1, addr:$src2)>;
Craig Topper05629d02016-07-24 07:32:45 +00005748 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5749 (X86vsrav _.RC:$src1,
5750 (X86VBroadcast (_.ScalarLdFrag addr:$src2))),
5751 _.RC:$src0)),
5752 (!cast<Instruction>(InstrStr#_.ZSuffix##rmbk) _.RC:$src0,
5753 _.KRC:$mask, _.RC:$src1, addr:$src2)>;
Craig Topper05629d02016-07-24 07:32:45 +00005754 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5755 (X86vsrav _.RC:$src1,
5756 (X86VBroadcast (_.ScalarLdFrag addr:$src2))),
5757 _.ImmAllZerosV)),
5758 (!cast<Instruction>(InstrStr#_.ZSuffix##rmbkz) _.KRC:$mask,
5759 _.RC:$src1, addr:$src2)>;
5760 }
5761}
5762
5763defm : avx512_var_shift_int_lowering<"VPSRAVW", v8i16x_info, [HasVLX, HasBWI]>;
5764defm : avx512_var_shift_int_lowering<"VPSRAVW", v16i16x_info, [HasVLX, HasBWI]>;
5765defm : avx512_var_shift_int_lowering<"VPSRAVW", v32i16_info, [HasBWI]>;
5766defm : avx512_var_shift_int_lowering_mb<"VPSRAVD", v4i32x_info, [HasVLX]>;
5767defm : avx512_var_shift_int_lowering_mb<"VPSRAVD", v8i32x_info, [HasVLX]>;
5768defm : avx512_var_shift_int_lowering_mb<"VPSRAVD", v16i32_info, [HasAVX512]>;
5769defm : avx512_var_shift_int_lowering_mb<"VPSRAVQ", v2i64x_info, [HasVLX]>;
5770defm : avx512_var_shift_int_lowering_mb<"VPSRAVQ", v4i64x_info, [HasVLX]>;
5771defm : avx512_var_shift_int_lowering_mb<"VPSRAVQ", v8i64_info, [HasAVX512]>;
5772
Simon Pilgrim1cbe8c22017-07-17 14:11:30 +00005773
5774// Use 512bit VPROL/VPROLI version to implement v2i64/v4i64 + v4i32/v8i32 in case NoVLX.
5775let Predicates = [HasAVX512, NoVLX] in {
5776 def : Pat<(v2i64 (rotl (v2i64 VR128X:$src1), (v2i64 VR128X:$src2))),
5777 (EXTRACT_SUBREG (v8i64
5778 (VPROLVQZrr
5779 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
5780 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src2, sub_xmm))),
5781 sub_xmm)>;
5782 def : Pat<(v4i64 (rotl (v4i64 VR256X:$src1), (v4i64 VR256X:$src2))),
5783 (EXTRACT_SUBREG (v8i64
5784 (VPROLVQZrr
5785 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
5786 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm))),
5787 sub_ymm)>;
5788
5789 def : Pat<(v4i32 (rotl (v4i32 VR128X:$src1), (v4i32 VR128X:$src2))),
5790 (EXTRACT_SUBREG (v16i32
5791 (VPROLVDZrr
5792 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
5793 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src2, sub_xmm))),
5794 sub_xmm)>;
5795 def : Pat<(v8i32 (rotl (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))),
5796 (EXTRACT_SUBREG (v16i32
5797 (VPROLVDZrr
5798 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
5799 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm))),
5800 sub_ymm)>;
5801
5802 def : Pat<(v2i64 (X86vrotli (v2i64 VR128X:$src1), (i8 imm:$src2))),
5803 (EXTRACT_SUBREG (v8i64
5804 (VPROLQZri
5805 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
5806 imm:$src2)), sub_xmm)>;
5807 def : Pat<(v4i64 (X86vrotli (v4i64 VR256X:$src1), (i8 imm:$src2))),
5808 (EXTRACT_SUBREG (v8i64
5809 (VPROLQZri
5810 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
5811 imm:$src2)), sub_ymm)>;
5812
5813 def : Pat<(v4i32 (X86vrotli (v4i32 VR128X:$src1), (i8 imm:$src2))),
5814 (EXTRACT_SUBREG (v16i32
5815 (VPROLDZri
5816 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
5817 imm:$src2)), sub_xmm)>;
5818 def : Pat<(v8i32 (X86vrotli (v8i32 VR256X:$src1), (i8 imm:$src2))),
5819 (EXTRACT_SUBREG (v16i32
5820 (VPROLDZri
5821 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
5822 imm:$src2)), sub_ymm)>;
5823}
5824
5825// Use 512bit VPROR/VPRORI version to implement v2i64/v4i64 + v4i32/v8i32 in case NoVLX.
5826let Predicates = [HasAVX512, NoVLX] in {
5827 def : Pat<(v2i64 (rotr (v2i64 VR128X:$src1), (v2i64 VR128X:$src2))),
5828 (EXTRACT_SUBREG (v8i64
5829 (VPRORVQZrr
5830 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
5831 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src2, sub_xmm))),
5832 sub_xmm)>;
5833 def : Pat<(v4i64 (rotr (v4i64 VR256X:$src1), (v4i64 VR256X:$src2))),
5834 (EXTRACT_SUBREG (v8i64
5835 (VPRORVQZrr
5836 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
5837 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm))),
5838 sub_ymm)>;
5839
5840 def : Pat<(v4i32 (rotr (v4i32 VR128X:$src1), (v4i32 VR128X:$src2))),
5841 (EXTRACT_SUBREG (v16i32
5842 (VPRORVDZrr
5843 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
5844 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src2, sub_xmm))),
5845 sub_xmm)>;
5846 def : Pat<(v8i32 (rotr (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))),
5847 (EXTRACT_SUBREG (v16i32
5848 (VPRORVDZrr
5849 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
5850 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm))),
5851 sub_ymm)>;
5852
5853 def : Pat<(v2i64 (X86vrotri (v2i64 VR128X:$src1), (i8 imm:$src2))),
5854 (EXTRACT_SUBREG (v8i64
5855 (VPRORQZri
5856 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
5857 imm:$src2)), sub_xmm)>;
5858 def : Pat<(v4i64 (X86vrotri (v4i64 VR256X:$src1), (i8 imm:$src2))),
5859 (EXTRACT_SUBREG (v8i64
5860 (VPRORQZri
5861 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
5862 imm:$src2)), sub_ymm)>;
5863
5864 def : Pat<(v4i32 (X86vrotri (v4i32 VR128X:$src1), (i8 imm:$src2))),
5865 (EXTRACT_SUBREG (v16i32
5866 (VPRORDZri
5867 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
5868 imm:$src2)), sub_xmm)>;
5869 def : Pat<(v8i32 (X86vrotri (v8i32 VR256X:$src1), (i8 imm:$src2))),
5870 (EXTRACT_SUBREG (v16i32
5871 (VPRORDZri
5872 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
5873 imm:$src2)), sub_ymm)>;
5874}
5875
Elena Demikhovsky4078c752015-06-04 07:07:13 +00005876//===-------------------------------------------------------------------===//
5877// 1-src variable permutation VPERMW/D/Q
5878//===-------------------------------------------------------------------===//
5879multiclass avx512_vperm_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
5880 AVX512VLVectorVTInfo _> {
5881 let Predicates = [HasAVX512] in
5882 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>,
5883 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
5884
5885 let Predicates = [HasAVX512, HasVLX] in
5886 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>,
5887 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
5888}
5889
5890multiclass avx512_vpermi_dq_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM,
5891 string OpcodeStr, SDNode OpNode,
5892 AVX512VLVectorVTInfo VTInfo> {
5893 let Predicates = [HasAVX512] in
5894 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5895 VTInfo.info512>,
5896 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
5897 VTInfo.info512>, EVEX_V512;
5898 let Predicates = [HasAVX512, HasVLX] in
5899 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5900 VTInfo.info256>,
5901 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
5902 VTInfo.info256>, EVEX_V256;
5903}
5904
Michael Zuckermand9cac592016-01-19 17:07:43 +00005905multiclass avx512_vperm_bw<bits<8> opc, string OpcodeStr,
5906 Predicate prd, SDNode OpNode,
5907 AVX512VLVectorVTInfo _> {
5908 let Predicates = [prd] in
5909 defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>,
5910 EVEX_V512 ;
5911 let Predicates = [HasVLX, prd] in {
5912 defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>,
5913 EVEX_V256 ;
5914 defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, _.info128>,
5915 EVEX_V128 ;
5916 }
5917}
Elena Demikhovsky4078c752015-06-04 07:07:13 +00005918
Michael Zuckermand9cac592016-01-19 17:07:43 +00005919defm VPERMW : avx512_vperm_bw<0x8D, "vpermw", HasBWI, X86VPermv,
5920 avx512vl_i16_info>, VEX_W;
5921defm VPERMB : avx512_vperm_bw<0x8D, "vpermb", HasVBMI, X86VPermv,
5922 avx512vl_i8_info>;
Elena Demikhovsky4078c752015-06-04 07:07:13 +00005923
5924defm VPERMD : avx512_vperm_dq_sizes<0x36, "vpermd", X86VPermv,
5925 avx512vl_i32_info>;
5926defm VPERMQ : avx512_vperm_dq_sizes<0x36, "vpermq", X86VPermv,
5927 avx512vl_i64_info>, VEX_W;
5928defm VPERMPS : avx512_vperm_dq_sizes<0x16, "vpermps", X86VPermv,
5929 avx512vl_f32_info>;
5930defm VPERMPD : avx512_vperm_dq_sizes<0x16, "vpermpd", X86VPermv,
5931 avx512vl_f64_info>, VEX_W;
5932
5933defm VPERMQ : avx512_vpermi_dq_sizes<0x00, MRMSrcReg, MRMSrcMem, "vpermq",
5934 X86VPermi, avx512vl_i64_info>,
5935 EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, VEX_W;
5936defm VPERMPD : avx512_vpermi_dq_sizes<0x01, MRMSrcReg, MRMSrcMem, "vpermpd",
5937 X86VPermi, avx512vl_f64_info>,
5938 EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, VEX_W;
Igor Breger78741a12015-10-04 07:20:41 +00005939//===----------------------------------------------------------------------===//
Simon Pilgrim18bcf932016-02-03 09:41:59 +00005940// AVX-512 - VPERMIL
Igor Breger78741a12015-10-04 07:20:41 +00005941//===----------------------------------------------------------------------===//
Elena Demikhovsky4078c752015-06-04 07:07:13 +00005942
Igor Breger78741a12015-10-04 07:20:41 +00005943multiclass avx512_permil_vec<bits<8> OpcVar, string OpcodeStr, SDNode OpNode,
5944 X86VectorVTInfo _, X86VectorVTInfo Ctrl> {
5945 defm rr: AVX512_maskable<OpcVar, MRMSrcReg, _, (outs _.RC:$dst),
5946 (ins _.RC:$src1, Ctrl.RC:$src2), OpcodeStr,
5947 "$src2, $src1", "$src1, $src2",
5948 (_.VT (OpNode _.RC:$src1,
5949 (Ctrl.VT Ctrl.RC:$src2)))>,
5950 T8PD, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00005951 defm rm: AVX512_maskable<OpcVar, MRMSrcMem, _, (outs _.RC:$dst),
5952 (ins _.RC:$src1, Ctrl.MemOp:$src2), OpcodeStr,
5953 "$src2, $src1", "$src1, $src2",
5954 (_.VT (OpNode
5955 _.RC:$src1,
5956 (Ctrl.VT (bitconvert(Ctrl.LdFrag addr:$src2)))))>,
5957 T8PD, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>;
5958 defm rmb: AVX512_maskable<OpcVar, MRMSrcMem, _, (outs _.RC:$dst),
5959 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
5960 "${src2}"##_.BroadcastStr##", $src1",
5961 "$src1, ${src2}"##_.BroadcastStr,
5962 (_.VT (OpNode
5963 _.RC:$src1,
5964 (Ctrl.VT (X86VBroadcast
5965 (Ctrl.ScalarLdFrag addr:$src2)))))>,
5966 T8PD, EVEX_4V, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>;
Igor Breger78741a12015-10-04 07:20:41 +00005967}
5968
5969multiclass avx512_permil_vec_common<string OpcodeStr, bits<8> OpcVar,
5970 AVX512VLVectorVTInfo _, AVX512VLVectorVTInfo Ctrl>{
5971 let Predicates = [HasAVX512] in {
5972 defm Z : avx512_permil_vec<OpcVar, OpcodeStr, X86VPermilpv, _.info512,
5973 Ctrl.info512>, EVEX_V512;
5974 }
5975 let Predicates = [HasAVX512, HasVLX] in {
5976 defm Z128 : avx512_permil_vec<OpcVar, OpcodeStr, X86VPermilpv, _.info128,
5977 Ctrl.info128>, EVEX_V128;
5978 defm Z256 : avx512_permil_vec<OpcVar, OpcodeStr, X86VPermilpv, _.info256,
5979 Ctrl.info256>, EVEX_V256;
5980 }
5981}
5982
5983multiclass avx512_permil<string OpcodeStr, bits<8> OpcImm, bits<8> OpcVar,
5984 AVX512VLVectorVTInfo _, AVX512VLVectorVTInfo Ctrl>{
5985
5986 defm NAME: avx512_permil_vec_common<OpcodeStr, OpcVar, _, Ctrl>;
5987 defm NAME: avx512_shift_rmi_sizes<OpcImm, MRMSrcReg, MRMSrcMem, OpcodeStr,
5988 X86VPermilpi, _>,
5989 EVEX, AVX512AIi8Base, EVEX_CD8<_.info128.EltSize, CD8VF>;
Igor Breger78741a12015-10-04 07:20:41 +00005990}
5991
Craig Topper05948fb2016-08-02 05:11:15 +00005992let ExeDomain = SSEPackedSingle in
Igor Breger78741a12015-10-04 07:20:41 +00005993defm VPERMILPS : avx512_permil<"vpermilps", 0x04, 0x0C, avx512vl_f32_info,
5994 avx512vl_i32_info>;
Craig Topper05948fb2016-08-02 05:11:15 +00005995let ExeDomain = SSEPackedDouble in
Igor Breger78741a12015-10-04 07:20:41 +00005996defm VPERMILPD : avx512_permil<"vpermilpd", 0x05, 0x0D, avx512vl_f64_info,
5997 avx512vl_i64_info>, VEX_W;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00005998//===----------------------------------------------------------------------===//
Elena Demikhovsky75ede682015-06-01 07:17:23 +00005999// AVX-512 - VPSHUFD, VPSHUFLW, VPSHUFHW
6000//===----------------------------------------------------------------------===//
6001
6002defm VPSHUFD : avx512_shift_rmi_sizes<0x70, MRMSrcReg, MRMSrcMem, "vpshufd",
Michael Liao66233b72015-08-06 09:06:20 +00006003 X86PShufd, avx512vl_i32_info>,
Elena Demikhovsky75ede682015-06-01 07:17:23 +00006004 EVEX, AVX512BIi8Base, EVEX_CD8<32, CD8VF>;
6005defm VPSHUFH : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshufhw",
Igor Breger1a6fd1c2015-10-07 06:31:18 +00006006 X86PShufhw>, EVEX, AVX512XSIi8Base;
Elena Demikhovsky75ede682015-06-01 07:17:23 +00006007defm VPSHUFL : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshuflw",
Igor Breger1a6fd1c2015-10-07 06:31:18 +00006008 X86PShuflw>, EVEX, AVX512XDIi8Base;
Michael Liao66233b72015-08-06 09:06:20 +00006009
Elena Demikhovsky55a99742015-06-22 13:00:42 +00006010multiclass avx512_pshufb_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6011 let Predicates = [HasBWI] in
6012 defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, v64i8_info>, EVEX_V512;
6013
6014 let Predicates = [HasVLX, HasBWI] in {
6015 defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, v32i8x_info>, EVEX_V256;
6016 defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, v16i8x_info>, EVEX_V128;
6017 }
6018}
6019
6020defm VPSHUFB: avx512_pshufb_sizes<0x00, "vpshufb", X86pshufb>;
6021
Elena Demikhovsky75ede682015-06-01 07:17:23 +00006022//===----------------------------------------------------------------------===//
Elena Demikhovsky0a74b7d2013-11-14 11:29:27 +00006023// Move Low to High and High to Low packed FP Instructions
6024//===----------------------------------------------------------------------===//
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006025def VMOVLHPSZrr : AVX512PSI<0x16, MRMSrcReg, (outs VR128X:$dst),
6026 (ins VR128X:$src1, VR128X:$src2),
Elena Demikhovskycf088092013-12-11 14:31:04 +00006027 "vmovlhps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006028 [(set VR128X:$dst, (v4f32 (X86Movlhps VR128X:$src1, VR128X:$src2)))],
6029 IIC_SSE_MOV_LH>, EVEX_4V;
6030def VMOVHLPSZrr : AVX512PSI<0x12, MRMSrcReg, (outs VR128X:$dst),
6031 (ins VR128X:$src1, VR128X:$src2),
Elena Demikhovskycf088092013-12-11 14:31:04 +00006032 "vmovhlps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006033 [(set VR128X:$dst, (v4f32 (X86Movhlps VR128X:$src1, VR128X:$src2)))],
6034 IIC_SSE_MOV_LH>, EVEX_4V;
6035
Craig Topperdbe8b7d2013-09-27 07:20:47 +00006036let Predicates = [HasAVX512] in {
6037 // MOVLHPS patterns
6038 def : Pat<(v4i32 (X86Movlhps VR128X:$src1, VR128X:$src2)),
6039 (VMOVLHPSZrr VR128X:$src1, VR128X:$src2)>;
6040 def : Pat<(v2i64 (X86Movlhps VR128X:$src1, VR128X:$src2)),
6041 (VMOVLHPSZrr (v2i64 VR128X:$src1), VR128X:$src2)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006042
Craig Topperdbe8b7d2013-09-27 07:20:47 +00006043 // MOVHLPS patterns
6044 def : Pat<(v4i32 (X86Movhlps VR128X:$src1, VR128X:$src2)),
6045 (VMOVHLPSZrr VR128X:$src1, VR128X:$src2)>;
6046}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006047
6048//===----------------------------------------------------------------------===//
Igor Bregerb6b27af2015-11-10 07:09:07 +00006049// VMOVHPS/PD VMOVLPS Instructions
6050// All patterns was taken from SSS implementation.
6051//===----------------------------------------------------------------------===//
6052multiclass avx512_mov_hilo_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
6053 X86VectorVTInfo _> {
Craig Toppere70231b2017-02-26 06:45:54 +00006054 let ExeDomain = _.ExeDomain in
Craig Toppere1cac152016-06-07 07:27:54 +00006055 def rm : AVX512<opc, MRMSrcMem, (outs _.RC:$dst),
6056 (ins _.RC:$src1, f64mem:$src2),
6057 !strconcat(OpcodeStr,
6058 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
6059 [(set _.RC:$dst,
6060 (OpNode _.RC:$src1,
6061 (_.VT (bitconvert
6062 (v2f64 (scalar_to_vector (loadf64 addr:$src2)))))))],
6063 IIC_SSE_MOV_LH>, EVEX_4V;
Igor Bregerb6b27af2015-11-10 07:09:07 +00006064}
6065
6066defm VMOVHPSZ128 : avx512_mov_hilo_packed<0x16, "vmovhps", X86Movlhps,
6067 v4f32x_info>, EVEX_CD8<32, CD8VT2>, PS;
6068defm VMOVHPDZ128 : avx512_mov_hilo_packed<0x16, "vmovhpd", X86Movlhpd,
6069 v2f64x_info>, EVEX_CD8<64, CD8VT1>, PD, VEX_W;
6070defm VMOVLPSZ128 : avx512_mov_hilo_packed<0x12, "vmovlps", X86Movlps,
6071 v4f32x_info>, EVEX_CD8<32, CD8VT2>, PS;
6072defm VMOVLPDZ128 : avx512_mov_hilo_packed<0x12, "vmovlpd", X86Movlpd,
6073 v2f64x_info>, EVEX_CD8<64, CD8VT1>, PD, VEX_W;
6074
6075let Predicates = [HasAVX512] in {
6076 // VMOVHPS patterns
6077 def : Pat<(X86Movlhps VR128X:$src1,
6078 (bc_v4f32 (v2i64 (scalar_to_vector (loadi64 addr:$src2))))),
6079 (VMOVHPSZ128rm VR128X:$src1, addr:$src2)>;
6080 def : Pat<(X86Movlhps VR128X:$src1,
6081 (bc_v4i32 (v2i64 (X86vzload addr:$src2)))),
6082 (VMOVHPSZ128rm VR128X:$src1, addr:$src2)>;
6083 // VMOVHPD patterns
6084 def : Pat<(v2f64 (X86Unpckl VR128X:$src1,
6085 (scalar_to_vector (loadf64 addr:$src2)))),
6086 (VMOVHPDZ128rm VR128X:$src1, addr:$src2)>;
6087 def : Pat<(v2f64 (X86Unpckl VR128X:$src1,
6088 (bc_v2f64 (v2i64 (scalar_to_vector (loadi64 addr:$src2)))))),
6089 (VMOVHPDZ128rm VR128X:$src1, addr:$src2)>;
6090 // VMOVLPS patterns
6091 def : Pat<(v4f32 (X86Movlps VR128X:$src1, (load addr:$src2))),
6092 (VMOVLPSZ128rm VR128X:$src1, addr:$src2)>;
6093 def : Pat<(v4i32 (X86Movlps VR128X:$src1, (load addr:$src2))),
6094 (VMOVLPSZ128rm VR128X:$src1, addr:$src2)>;
6095 // VMOVLPD patterns
6096 def : Pat<(v2f64 (X86Movlpd VR128X:$src1, (load addr:$src2))),
6097 (VMOVLPDZ128rm VR128X:$src1, addr:$src2)>;
6098 def : Pat<(v2i64 (X86Movlpd VR128X:$src1, (load addr:$src2))),
6099 (VMOVLPDZ128rm VR128X:$src1, addr:$src2)>;
6100 def : Pat<(v2f64 (X86Movsd VR128X:$src1,
6101 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))),
6102 (VMOVLPDZ128rm VR128X:$src1, addr:$src2)>;
6103}
6104
Igor Bregerb6b27af2015-11-10 07:09:07 +00006105def VMOVHPSZ128mr : AVX512PSI<0x17, MRMDestMem, (outs),
6106 (ins f64mem:$dst, VR128X:$src),
6107 "vmovhps\t{$src, $dst|$dst, $src}",
Craig Topperc9b19232016-05-01 04:59:44 +00006108 [(store (f64 (extractelt
Igor Bregerb6b27af2015-11-10 07:09:07 +00006109 (X86Unpckh (bc_v2f64 (v4f32 VR128X:$src)),
6110 (bc_v2f64 (v4f32 VR128X:$src))),
6111 (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>,
6112 EVEX, EVEX_CD8<32, CD8VT2>;
6113def VMOVHPDZ128mr : AVX512PDI<0x17, MRMDestMem, (outs),
6114 (ins f64mem:$dst, VR128X:$src),
6115 "vmovhpd\t{$src, $dst|$dst, $src}",
Craig Topperc9b19232016-05-01 04:59:44 +00006116 [(store (f64 (extractelt
Igor Bregerb6b27af2015-11-10 07:09:07 +00006117 (v2f64 (X86Unpckh VR128X:$src, VR128X:$src)),
6118 (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>,
6119 EVEX, EVEX_CD8<64, CD8VT1>, VEX_W;
6120def VMOVLPSZ128mr : AVX512PSI<0x13, MRMDestMem, (outs),
6121 (ins f64mem:$dst, VR128X:$src),
6122 "vmovlps\t{$src, $dst|$dst, $src}",
Craig Topperc9b19232016-05-01 04:59:44 +00006123 [(store (f64 (extractelt (bc_v2f64 (v4f32 VR128X:$src)),
Igor Bregerb6b27af2015-11-10 07:09:07 +00006124 (iPTR 0))), addr:$dst)],
6125 IIC_SSE_MOV_LH>,
6126 EVEX, EVEX_CD8<32, CD8VT2>;
6127def VMOVLPDZ128mr : AVX512PDI<0x13, MRMDestMem, (outs),
6128 (ins f64mem:$dst, VR128X:$src),
6129 "vmovlpd\t{$src, $dst|$dst, $src}",
Craig Topperc9b19232016-05-01 04:59:44 +00006130 [(store (f64 (extractelt (v2f64 VR128X:$src),
Igor Bregerb6b27af2015-11-10 07:09:07 +00006131 (iPTR 0))), addr:$dst)],
6132 IIC_SSE_MOV_LH>,
6133 EVEX, EVEX_CD8<64, CD8VT1>, VEX_W;
Craig Toppere1cac152016-06-07 07:27:54 +00006134
Igor Bregerb6b27af2015-11-10 07:09:07 +00006135let Predicates = [HasAVX512] in {
6136 // VMOVHPD patterns
Craig Topperc9b19232016-05-01 04:59:44 +00006137 def : Pat<(store (f64 (extractelt
Igor Bregerb6b27af2015-11-10 07:09:07 +00006138 (v2f64 (X86VPermilpi VR128X:$src, (i8 1))),
6139 (iPTR 0))), addr:$dst),
6140 (VMOVHPDZ128mr addr:$dst, VR128X:$src)>;
6141 // VMOVLPS patterns
6142 def : Pat<(store (v4f32 (X86Movlps (load addr:$src1), VR128X:$src2)),
6143 addr:$src1),
6144 (VMOVLPSZ128mr addr:$src1, VR128X:$src2)>;
6145 def : Pat<(store (v4i32 (X86Movlps
6146 (bc_v4i32 (loadv2i64 addr:$src1)), VR128X:$src2)), addr:$src1),
6147 (VMOVLPSZ128mr addr:$src1, VR128X:$src2)>;
6148 // VMOVLPD patterns
6149 def : Pat<(store (v2f64 (X86Movlpd (load addr:$src1), VR128X:$src2)),
6150 addr:$src1),
6151 (VMOVLPDZ128mr addr:$src1, VR128X:$src2)>;
6152 def : Pat<(store (v2i64 (X86Movlpd (load addr:$src1), VR128X:$src2)),
6153 addr:$src1),
6154 (VMOVLPDZ128mr addr:$src1, VR128X:$src2)>;
6155}
6156//===----------------------------------------------------------------------===//
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006157// FMA - Fused Multiply Operations
6158//
Adam Nemet26371ce2014-10-24 00:02:55 +00006159
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006160multiclass avx512_fma3p_213_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper318e40b2016-07-25 07:20:31 +00006161 X86VectorVTInfo _, string Suff> {
6162 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {
Adam Nemet34801422014-10-08 23:25:39 +00006163 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
Adam Nemet6bddb8c2014-09-29 22:54:41 +00006164 (ins _.RC:$src2, _.RC:$src3),
Adam Nemet2e91ee52014-08-14 17:13:19 +00006165 OpcodeStr, "$src3, $src2", "$src2, $src3",
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +00006166 (_.VT (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3)), 1, 1>,
Adam Nemet2e91ee52014-08-14 17:13:19 +00006167 AVX512FMA3Base;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006168
Craig Toppere1cac152016-06-07 07:27:54 +00006169 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
6170 (ins _.RC:$src2, _.MemOp:$src3),
6171 OpcodeStr, "$src3, $src2", "$src2, $src3",
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +00006172 (_.VT (OpNode _.RC:$src2, _.RC:$src1, (_.LdFrag addr:$src3))), 1, 0>,
Craig Toppere1cac152016-06-07 07:27:54 +00006173 AVX512FMA3Base;
Elena Demikhovskyfcea06a2014-12-23 10:30:39 +00006174
Craig Toppere1cac152016-06-07 07:27:54 +00006175 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
6176 (ins _.RC:$src2, _.ScalarMemOp:$src3),
6177 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),
6178 !strconcat("$src2, ${src3}", _.BroadcastStr ),
Craig Topper6bcbf532016-07-25 07:20:28 +00006179 (OpNode _.RC:$src2,
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +00006180 _.RC:$src1,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3)))), 1, 0>,
Craig Toppere1cac152016-06-07 07:27:54 +00006181 AVX512FMA3Base, EVEX_B;
Craig Topper5ec33a92016-07-22 05:00:42 +00006182 }
Craig Topper318e40b2016-07-25 07:20:31 +00006183
6184 // Additional pattern for folding broadcast nodes in other orders.
6185 def : Pat<(_.VT (vselect _.KRCWM:$mask,
6186 (OpNode _.RC:$src1, _.RC:$src2,
6187 (X86VBroadcast (_.ScalarLdFrag addr:$src3))),
6188 _.RC:$src1)),
6189 (!cast<Instruction>(NAME#Suff#_.ZSuffix#mbk) _.RC:$src1,
6190 _.KRCWM:$mask, _.RC:$src2, addr:$src3)>;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006191}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006192
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006193multiclass avx512_fma3_213_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper318e40b2016-07-25 07:20:31 +00006194 X86VectorVTInfo _, string Suff> {
6195 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006196 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
Elena Demikhovsky7b0dd392015-01-28 10:21:27 +00006197 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
6198 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +00006199 (_.VT ( OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3, (i32 imm:$rc))), 1, 1>,
Elena Demikhovsky7b0dd392015-01-28 10:21:27 +00006200 AVX512FMA3Base, EVEX_B, EVEX_RC;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006201}
Elena Demikhovsky7b0dd392015-01-28 10:21:27 +00006202
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006203multiclass avx512_fma3p_213_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper318e40b2016-07-25 07:20:31 +00006204 SDNode OpNodeRnd, AVX512VLVectorVTInfo _,
6205 string Suff> {
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006206 let Predicates = [HasAVX512] in {
Craig Topper318e40b2016-07-25 07:20:31 +00006207 defm Z : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info512, Suff>,
6208 avx512_fma3_213_round<opc, OpcodeStr, OpNodeRnd, _.info512,
6209 Suff>, EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
Elena Demikhovskyfcea06a2014-12-23 10:30:39 +00006210 }
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006211 let Predicates = [HasVLX, HasAVX512] in {
Craig Topper318e40b2016-07-25 07:20:31 +00006212 defm Z256 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info256, Suff>,
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006213 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
Craig Topper318e40b2016-07-25 07:20:31 +00006214 defm Z128 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info128, Suff>,
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006215 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
Elena Demikhovskyfcea06a2014-12-23 10:30:39 +00006216 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006217}
6218
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006219multiclass avx512_fma3p_213_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper318e40b2016-07-25 07:20:31 +00006220 SDNode OpNodeRnd > {
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006221 defm PS : avx512_fma3p_213_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
Craig Topper318e40b2016-07-25 07:20:31 +00006222 avx512vl_f32_info, "PS">;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006223 defm PD : avx512_fma3p_213_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
Craig Topper318e40b2016-07-25 07:20:31 +00006224 avx512vl_f64_info, "PD">, VEX_W;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006225}
6226
6227defm VFMADD213 : avx512_fma3p_213_f<0xA8, "vfmadd213", X86Fmadd, X86FmaddRnd>;
6228defm VFMSUB213 : avx512_fma3p_213_f<0xAA, "vfmsub213", X86Fmsub, X86FmsubRnd>;
6229defm VFMADDSUB213 : avx512_fma3p_213_f<0xA6, "vfmaddsub213", X86Fmaddsub, X86FmaddsubRnd>;
6230defm VFMSUBADD213 : avx512_fma3p_213_f<0xA7, "vfmsubadd213", X86Fmsubadd, X86FmsubaddRnd>;
6231defm VFNMADD213 : avx512_fma3p_213_f<0xAC, "vfnmadd213", X86Fnmadd, X86FnmaddRnd>;
6232defm VFNMSUB213 : avx512_fma3p_213_f<0xAE, "vfnmsub213", X86Fnmsub, X86FnmsubRnd>;
6233
Elena Demikhovskyfcea06a2014-12-23 10:30:39 +00006234
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006235multiclass avx512_fma3p_231_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper318e40b2016-07-25 07:20:31 +00006236 X86VectorVTInfo _, string Suff> {
6237 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006238 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
6239 (ins _.RC:$src2, _.RC:$src3),
6240 OpcodeStr, "$src3, $src2", "$src2, $src3",
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +00006241 (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1)), 1, 1>,
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006242 AVX512FMA3Base;
6243
Craig Toppere1cac152016-06-07 07:27:54 +00006244 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
6245 (ins _.RC:$src2, _.MemOp:$src3),
6246 OpcodeStr, "$src3, $src2", "$src2, $src3",
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +00006247 (_.VT (OpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1)), 1, 0>,
Craig Toppere1cac152016-06-07 07:27:54 +00006248 AVX512FMA3Base;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006249
Craig Toppere1cac152016-06-07 07:27:54 +00006250 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
6251 (ins _.RC:$src2, _.ScalarMemOp:$src3),
6252 OpcodeStr, "${src3}"##_.BroadcastStr##", $src2",
6253 "$src2, ${src3}"##_.BroadcastStr,
6254 (_.VT (OpNode _.RC:$src2,
6255 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +00006256 _.RC:$src1)), 1, 0>, AVX512FMA3Base, EVEX_B;
Craig Topper5ec33a92016-07-22 05:00:42 +00006257 }
Craig Topper318e40b2016-07-25 07:20:31 +00006258
6259 // Additional patterns for folding broadcast nodes in other orders.
6260 def : Pat<(_.VT (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
6261 _.RC:$src2, _.RC:$src1)),
6262 (!cast<Instruction>(NAME#Suff#_.ZSuffix#mb) _.RC:$src1,
6263 _.RC:$src2, addr:$src3)>;
6264 def : Pat<(_.VT (vselect _.KRCWM:$mask,
6265 (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
6266 _.RC:$src2, _.RC:$src1),
6267 _.RC:$src1)),
6268 (!cast<Instruction>(NAME#Suff#_.ZSuffix#mbk) _.RC:$src1,
6269 _.KRCWM:$mask, _.RC:$src2, addr:$src3)>;
6270 def : Pat<(_.VT (vselect _.KRCWM:$mask,
6271 (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
6272 _.RC:$src2, _.RC:$src1),
6273 _.ImmAllZerosV)),
6274 (!cast<Instruction>(NAME#Suff#_.ZSuffix#mbkz) _.RC:$src1,
6275 _.KRCWM:$mask, _.RC:$src2, addr:$src3)>;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006276}
6277
6278multiclass avx512_fma3_231_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper318e40b2016-07-25 07:20:31 +00006279 X86VectorVTInfo _, string Suff> {
6280 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006281 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
6282 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
6283 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +00006284 (_.VT ( OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1, (i32 imm:$rc))), 1, 1>,
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006285 AVX512FMA3Base, EVEX_B, EVEX_RC;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006286}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006287
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006288multiclass avx512_fma3p_231_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper318e40b2016-07-25 07:20:31 +00006289 SDNode OpNodeRnd, AVX512VLVectorVTInfo _,
6290 string Suff> {
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006291 let Predicates = [HasAVX512] in {
Craig Topper318e40b2016-07-25 07:20:31 +00006292 defm Z : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info512, Suff>,
6293 avx512_fma3_231_round<opc, OpcodeStr, OpNodeRnd, _.info512,
6294 Suff>, EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
Elena Demikhovskyfcea06a2014-12-23 10:30:39 +00006295 }
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006296 let Predicates = [HasVLX, HasAVX512] in {
Craig Topper318e40b2016-07-25 07:20:31 +00006297 defm Z256 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info256, Suff>,
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006298 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
Craig Topper318e40b2016-07-25 07:20:31 +00006299 defm Z128 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info128, Suff>,
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006300 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
Elena Demikhovskyfcea06a2014-12-23 10:30:39 +00006301 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006302}
6303
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006304multiclass avx512_fma3p_231_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper318e40b2016-07-25 07:20:31 +00006305 SDNode OpNodeRnd > {
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006306 defm PS : avx512_fma3p_231_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
Craig Topper318e40b2016-07-25 07:20:31 +00006307 avx512vl_f32_info, "PS">;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006308 defm PD : avx512_fma3p_231_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
Craig Topper318e40b2016-07-25 07:20:31 +00006309 avx512vl_f64_info, "PD">, VEX_W;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006310}
6311
6312defm VFMADD231 : avx512_fma3p_231_f<0xB8, "vfmadd231", X86Fmadd, X86FmaddRnd>;
6313defm VFMSUB231 : avx512_fma3p_231_f<0xBA, "vfmsub231", X86Fmsub, X86FmsubRnd>;
6314defm VFMADDSUB231 : avx512_fma3p_231_f<0xB6, "vfmaddsub231", X86Fmaddsub, X86FmaddsubRnd>;
6315defm VFMSUBADD231 : avx512_fma3p_231_f<0xB7, "vfmsubadd231", X86Fmsubadd, X86FmsubaddRnd>;
6316defm VFNMADD231 : avx512_fma3p_231_f<0xBC, "vfnmadd231", X86Fnmadd, X86FnmaddRnd>;
6317defm VFNMSUB231 : avx512_fma3p_231_f<0xBE, "vfnmsub231", X86Fnmsub, X86FnmsubRnd>;
6318
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006319multiclass avx512_fma3p_132_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper318e40b2016-07-25 07:20:31 +00006320 X86VectorVTInfo _, string Suff> {
6321 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006322 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
Craig Topper6bcbf532016-07-25 07:20:28 +00006323 (ins _.RC:$src2, _.RC:$src3),
6324 OpcodeStr, "$src3, $src2", "$src2, $src3",
Craig Topper5f2441d2016-08-13 06:48:39 +00006325 (_.VT (OpNode _.RC:$src1, _.RC:$src3, _.RC:$src2)), 1, 1>,
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006326 AVX512FMA3Base;
6327
Craig Toppere1cac152016-06-07 07:27:54 +00006328 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
Craig Topper6bcbf532016-07-25 07:20:28 +00006329 (ins _.RC:$src2, _.MemOp:$src3),
6330 OpcodeStr, "$src3, $src2", "$src2, $src3",
Craig Topper5f2441d2016-08-13 06:48:39 +00006331 (_.VT (OpNode _.RC:$src1, (_.LdFrag addr:$src3), _.RC:$src2)), 1, 0>,
Craig Toppere1cac152016-06-07 07:27:54 +00006332 AVX512FMA3Base;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006333
Craig Toppere1cac152016-06-07 07:27:54 +00006334 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
Craig Topper6bcbf532016-07-25 07:20:28 +00006335 (ins _.RC:$src2, _.ScalarMemOp:$src3),
6336 OpcodeStr, "${src3}"##_.BroadcastStr##", $src2",
6337 "$src2, ${src3}"##_.BroadcastStr,
Craig Toppere1cac152016-06-07 07:27:54 +00006338 (_.VT (OpNode _.RC:$src1,
Craig Topper6bcbf532016-07-25 07:20:28 +00006339 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
Craig Topper5f2441d2016-08-13 06:48:39 +00006340 _.RC:$src2)), 1, 0>, AVX512FMA3Base, EVEX_B;
Craig Topper5ec33a92016-07-22 05:00:42 +00006341 }
Craig Topper318e40b2016-07-25 07:20:31 +00006342
6343 // Additional patterns for folding broadcast nodes in other orders.
6344 def : Pat<(_.VT (vselect _.KRCWM:$mask,
6345 (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
6346 _.RC:$src1, _.RC:$src2),
6347 _.RC:$src1)),
6348 (!cast<Instruction>(NAME#Suff#_.ZSuffix#mbk) _.RC:$src1,
6349 _.KRCWM:$mask, _.RC:$src2, addr:$src3)>;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006350}
6351
6352multiclass avx512_fma3_132_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper318e40b2016-07-25 07:20:31 +00006353 X86VectorVTInfo _, string Suff> {
6354 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006355 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
Craig Topper6bcbf532016-07-25 07:20:28 +00006356 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
6357 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",
Craig Toppereafdbec2016-08-13 06:48:41 +00006358 (_.VT ( OpNode _.RC:$src1, _.RC:$src3, _.RC:$src2, (i32 imm:$rc))), 1, 1>,
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006359 AVX512FMA3Base, EVEX_B, EVEX_RC;
6360}
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006361
6362multiclass avx512_fma3p_132_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper318e40b2016-07-25 07:20:31 +00006363 SDNode OpNodeRnd, AVX512VLVectorVTInfo _,
6364 string Suff> {
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006365 let Predicates = [HasAVX512] in {
Craig Topper318e40b2016-07-25 07:20:31 +00006366 defm Z : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info512, Suff>,
6367 avx512_fma3_132_round<opc, OpcodeStr, OpNodeRnd, _.info512,
6368 Suff>, EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006369 }
6370 let Predicates = [HasVLX, HasAVX512] in {
Craig Topper318e40b2016-07-25 07:20:31 +00006371 defm Z256 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info256, Suff>,
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006372 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
Craig Topper318e40b2016-07-25 07:20:31 +00006373 defm Z128 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info128, Suff>,
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006374 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
6375 }
6376}
6377
6378multiclass avx512_fma3p_132_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper318e40b2016-07-25 07:20:31 +00006379 SDNode OpNodeRnd > {
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006380 defm PS : avx512_fma3p_132_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
Craig Topper318e40b2016-07-25 07:20:31 +00006381 avx512vl_f32_info, "PS">;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006382 defm PD : avx512_fma3p_132_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
Craig Topper318e40b2016-07-25 07:20:31 +00006383 avx512vl_f64_info, "PD">, VEX_W;
Igor Bregera7a8e9a2015-06-29 09:10:00 +00006384}
6385
6386defm VFMADD132 : avx512_fma3p_132_f<0x98, "vfmadd132", X86Fmadd, X86FmaddRnd>;
6387defm VFMSUB132 : avx512_fma3p_132_f<0x9A, "vfmsub132", X86Fmsub, X86FmsubRnd>;
6388defm VFMADDSUB132 : avx512_fma3p_132_f<0x96, "vfmaddsub132", X86Fmaddsub, X86FmaddsubRnd>;
6389defm VFMSUBADD132 : avx512_fma3p_132_f<0x97, "vfmsubadd132", X86Fmsubadd, X86FmsubaddRnd>;
6390defm VFNMADD132 : avx512_fma3p_132_f<0x9C, "vfnmadd132", X86Fnmadd, X86FnmaddRnd>;
6391defm VFNMSUB132 : avx512_fma3p_132_f<0x9E, "vfnmsub132", X86Fnmsub, X86FnmsubRnd>;
Elena Demikhovskyfcea06a2014-12-23 10:30:39 +00006392
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006393// Scalar FMA
6394let Constraints = "$src1 = $dst" in {
Igor Breger15820b02015-07-01 13:24:28 +00006395multiclass avx512_fma3s_common<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
6396 dag RHS_VEC_r, dag RHS_VEC_m, dag RHS_VEC_rb,
6397 dag RHS_r, dag RHS_m > {
6398 defm r_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
6399 (ins _.RC:$src2, _.RC:$src3), OpcodeStr,
Craig Toppereafdbec2016-08-13 06:48:41 +00006400 "$src3, $src2", "$src2, $src3", RHS_VEC_r, 1, 1>, AVX512FMA3Base;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006401
Craig Toppere1cac152016-06-07 07:27:54 +00006402 defm m_Int: AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
Craig Topperd9fe6642017-02-21 04:26:10 +00006403 (ins _.RC:$src2, _.IntScalarMemOp:$src3), OpcodeStr,
Craig Toppereafdbec2016-08-13 06:48:41 +00006404 "$src3, $src2", "$src2, $src3", RHS_VEC_m, 1, 1>, AVX512FMA3Base;
Igor Breger15820b02015-07-01 13:24:28 +00006405
6406 defm rb_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
6407 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
Craig Toppereafdbec2016-08-13 06:48:41 +00006408 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc", RHS_VEC_rb, 1, 1>,
Igor Breger15820b02015-07-01 13:24:28 +00006409 AVX512FMA3Base, EVEX_B, EVEX_RC;
6410
Craig Toppereafdbec2016-08-13 06:48:41 +00006411 let isCodeGenOnly = 1, isCommutable = 1 in {
Igor Breger15820b02015-07-01 13:24:28 +00006412 def r : AVX512FMA3<opc, MRMSrcReg, (outs _.FRC:$dst),
6413 (ins _.FRC:$src1, _.FRC:$src2, _.FRC:$src3),
6414 !strconcat(OpcodeStr,
6415 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
6416 [RHS_r]>;
Craig Toppere1cac152016-06-07 07:27:54 +00006417 def m : AVX512FMA3<opc, MRMSrcMem, (outs _.FRC:$dst),
6418 (ins _.FRC:$src1, _.FRC:$src2, _.ScalarMemOp:$src3),
6419 !strconcat(OpcodeStr,
6420 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
6421 [RHS_m]>;
Igor Breger15820b02015-07-01 13:24:28 +00006422 }// isCodeGenOnly = 1
6423}
6424}// Constraints = "$src1 = $dst"
6425
6426multiclass avx512_fma3s_all<bits<8> opc213, bits<8> opc231, bits<8> opc132,
Craig Toppera55b4832016-12-09 06:42:28 +00006427 string OpcodeStr, SDNode OpNode, SDNode OpNodeRnds1,
6428 SDNode OpNodeRnds3, X86VectorVTInfo _ , string SUFF> {
Craig Topper2caa97c2017-02-25 19:36:28 +00006429 let ExeDomain = _.ExeDomain in {
Craig Topper2dca3b22016-07-24 08:26:38 +00006430 defm NAME#213#SUFF#Z: avx512_fma3s_common<opc213, OpcodeStr#"213"#_.Suffix , _ ,
Craig Toppera55b4832016-12-09 06:42:28 +00006431 // Operands for intrinsic are in 123 order to preserve passthu
6432 // semantics.
6433 (_.VT (OpNodeRnds1 _.RC:$src1, _.RC:$src2, _.RC:$src3, (i32 FROUND_CURRENT))),
6434 (_.VT (OpNodeRnds1 _.RC:$src1, _.RC:$src2,
Craig Topperd9fe6642017-02-21 04:26:10 +00006435 _.ScalarIntMemCPat:$src3, (i32 FROUND_CURRENT))),
Craig Toppera55b4832016-12-09 06:42:28 +00006436 (_.VT (OpNodeRnds1 _.RC:$src1, _.RC:$src2, _.RC:$src3,
Igor Breger15820b02015-07-01 13:24:28 +00006437 (i32 imm:$rc))),
6438 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1,
6439 _.FRC:$src3))),
6440 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1,
6441 (_.ScalarLdFrag addr:$src3))))>;
6442
Craig Topper2dca3b22016-07-24 08:26:38 +00006443 defm NAME#231#SUFF#Z: avx512_fma3s_common<opc231, OpcodeStr#"231"#_.Suffix , _ ,
Craig Toppera55b4832016-12-09 06:42:28 +00006444 (_.VT (OpNodeRnds3 _.RC:$src2, _.RC:$src3, _.RC:$src1, (i32 FROUND_CURRENT))),
Craig Topperd9fe6642017-02-21 04:26:10 +00006445 (_.VT (OpNodeRnds3 _.RC:$src2, _.ScalarIntMemCPat:$src3,
Michael Zuckerman7d733602016-02-04 14:41:08 +00006446 _.RC:$src1, (i32 FROUND_CURRENT))),
Craig Toppera55b4832016-12-09 06:42:28 +00006447 (_.VT ( OpNodeRnds3 _.RC:$src2, _.RC:$src3, _.RC:$src1,
Igor Breger15820b02015-07-01 13:24:28 +00006448 (i32 imm:$rc))),
6449 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src3,
6450 _.FRC:$src1))),
6451 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2,
6452 (_.ScalarLdFrag addr:$src3), _.FRC:$src1)))>;
6453
Craig Topper2dca3b22016-07-24 08:26:38 +00006454 defm NAME#132#SUFF#Z: avx512_fma3s_common<opc132, OpcodeStr#"132"#_.Suffix , _ ,
Craig Toppera55b4832016-12-09 06:42:28 +00006455 (_.VT (OpNodeRnds1 _.RC:$src1, _.RC:$src3, _.RC:$src2, (i32 FROUND_CURRENT))),
Craig Topperd9fe6642017-02-21 04:26:10 +00006456 (_.VT (OpNodeRnds1 _.RC:$src1, _.ScalarIntMemCPat:$src3,
Michael Zuckerman7d733602016-02-04 14:41:08 +00006457 _.RC:$src2, (i32 FROUND_CURRENT))),
Craig Toppera55b4832016-12-09 06:42:28 +00006458 (_.VT (OpNodeRnds1 _.RC:$src1, _.RC:$src3, _.RC:$src2,
Igor Breger15820b02015-07-01 13:24:28 +00006459 (i32 imm:$rc))),
6460 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1, _.FRC:$src3,
6461 _.FRC:$src2))),
6462 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1,
6463 (_.ScalarLdFrag addr:$src3), _.FRC:$src2)))>;
Craig Topper2caa97c2017-02-25 19:36:28 +00006464 }
Igor Breger15820b02015-07-01 13:24:28 +00006465}
6466
6467multiclass avx512_fma3s<bits<8> opc213, bits<8> opc231, bits<8> opc132,
Craig Toppera55b4832016-12-09 06:42:28 +00006468 string OpcodeStr, SDNode OpNode, SDNode OpNodeRnds1,
6469 SDNode OpNodeRnds3> {
Igor Breger15820b02015-07-01 13:24:28 +00006470 let Predicates = [HasAVX512] in {
6471 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,
Craig Toppera55b4832016-12-09 06:42:28 +00006472 OpNodeRnds1, OpNodeRnds3, f32x_info, "SS">,
6473 EVEX_CD8<32, CD8VT1>, VEX_LIG;
Igor Breger15820b02015-07-01 13:24:28 +00006474 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,
Craig Toppera55b4832016-12-09 06:42:28 +00006475 OpNodeRnds1, OpNodeRnds3, f64x_info, "SD">,
6476 EVEX_CD8<64, CD8VT1>, VEX_LIG, VEX_W;
Igor Breger15820b02015-07-01 13:24:28 +00006477 }
6478}
6479
Craig Toppera55b4832016-12-09 06:42:28 +00006480defm VFMADD : avx512_fma3s<0xA9, 0xB9, 0x99, "vfmadd", X86Fmadd, X86FmaddRnds1,
6481 X86FmaddRnds3>;
6482defm VFMSUB : avx512_fma3s<0xAB, 0xBB, 0x9B, "vfmsub", X86Fmsub, X86FmsubRnds1,
6483 X86FmsubRnds3>;
6484defm VFNMADD : avx512_fma3s<0xAD, 0xBD, 0x9D, "vfnmadd", X86Fnmadd,
6485 X86FnmaddRnds1, X86FnmaddRnds3>;
6486defm VFNMSUB : avx512_fma3s<0xAF, 0xBF, 0x9F, "vfnmsub", X86Fnmsub,
6487 X86FnmsubRnds1, X86FnmsubRnds3>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006488
6489//===----------------------------------------------------------------------===//
Asaf Badouh655822a2016-01-25 11:14:24 +00006490// AVX-512 Packed Multiply of Unsigned 52-bit Integers and Add the Low 52-bit IFMA
6491//===----------------------------------------------------------------------===//
6492let Constraints = "$src1 = $dst" in {
6493multiclass avx512_pmadd52_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
6494 X86VectorVTInfo _> {
Craig Topper6bf9b802017-02-26 06:45:45 +00006495 let ExeDomain = _.ExeDomain in {
Asaf Badouh655822a2016-01-25 11:14:24 +00006496 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
6497 (ins _.RC:$src2, _.RC:$src3),
6498 OpcodeStr, "$src3, $src2", "$src2, $src3",
6499 (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>,
6500 AVX512FMA3Base;
6501
Craig Toppere1cac152016-06-07 07:27:54 +00006502 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
6503 (ins _.RC:$src2, _.MemOp:$src3),
6504 OpcodeStr, "$src3, $src2", "$src2, $src3",
6505 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (_.LdFrag addr:$src3)))>,
6506 AVX512FMA3Base;
Asaf Badouh655822a2016-01-25 11:14:24 +00006507
Craig Toppere1cac152016-06-07 07:27:54 +00006508 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
6509 (ins _.RC:$src2, _.ScalarMemOp:$src3),
6510 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),
6511 !strconcat("$src2, ${src3}", _.BroadcastStr ),
6512 (OpNode _.RC:$src1,
6513 _.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))))>,
6514 AVX512FMA3Base, EVEX_B;
Craig Topper6bf9b802017-02-26 06:45:45 +00006515 }
Asaf Badouh655822a2016-01-25 11:14:24 +00006516}
6517} // Constraints = "$src1 = $dst"
6518
6519multiclass avx512_pmadd52_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
6520 AVX512VLVectorVTInfo _> {
6521 let Predicates = [HasIFMA] in {
6522 defm Z : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, _.info512>,
6523 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
6524 }
6525 let Predicates = [HasVLX, HasIFMA] in {
6526 defm Z256 : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, _.info256>,
6527 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
6528 defm Z128 : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, _.info128>,
6529 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
6530 }
6531}
6532
6533defm VPMADD52LUQ : avx512_pmadd52_common<0xb4, "vpmadd52luq", x86vpmadd52l,
6534 avx512vl_i64_info>, VEX_W;
6535defm VPMADD52HUQ : avx512_pmadd52_common<0xb5, "vpmadd52huq", x86vpmadd52h,
6536 avx512vl_i64_info>, VEX_W;
6537
6538//===----------------------------------------------------------------------===//
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006539// AVX-512 Scalar convert from sign integer to float/double
6540//===----------------------------------------------------------------------===//
6541
Igor Bregerdfcc3d32015-06-17 07:23:57 +00006542multiclass avx512_vcvtsi<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
6543 X86VectorVTInfo DstVT, X86MemOperand x86memop,
6544 PatFrag ld_frag, string asm> {
6545 let hasSideEffects = 0 in {
6546 def rr : SI<opc, MRMSrcReg, (outs DstVT.FRC:$dst),
6547 (ins DstVT.FRC:$src1, SrcRC:$src),
Craig Topperedb09112014-11-25 20:11:23 +00006548 !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006549 EVEX_4V;
Igor Bregerdfcc3d32015-06-17 07:23:57 +00006550 let mayLoad = 1 in
6551 def rm : SI<opc, MRMSrcMem, (outs DstVT.FRC:$dst),
6552 (ins DstVT.FRC:$src1, x86memop:$src),
Craig Topperedb09112014-11-25 20:11:23 +00006553 !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006554 EVEX_4V;
Igor Bregerdfcc3d32015-06-17 07:23:57 +00006555 } // hasSideEffects = 0
6556 let isCodeGenOnly = 1 in {
6557 def rr_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst),
6558 (ins DstVT.RC:$src1, SrcRC:$src2),
6559 !strconcat(asm,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
6560 [(set DstVT.RC:$dst,
6561 (OpNode (DstVT.VT DstVT.RC:$src1),
6562 SrcRC:$src2,
6563 (i32 FROUND_CURRENT)))]>, EVEX_4V;
6564
6565 def rm_Int : SI<opc, MRMSrcMem, (outs DstVT.RC:$dst),
6566 (ins DstVT.RC:$src1, x86memop:$src2),
6567 !strconcat(asm,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
6568 [(set DstVT.RC:$dst,
6569 (OpNode (DstVT.VT DstVT.RC:$src1),
6570 (ld_frag addr:$src2),
6571 (i32 FROUND_CURRENT)))]>, EVEX_4V;
6572 }//isCodeGenOnly = 1
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006573}
Elena Demikhovskyd8fda622015-03-30 09:29:28 +00006574
Igor Bregerabe4a792015-06-14 12:44:55 +00006575multiclass avx512_vcvtsi_round<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00006576 X86VectorVTInfo DstVT, string asm> {
Igor Bregerabe4a792015-06-14 12:44:55 +00006577 def rrb_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst),
6578 (ins DstVT.RC:$src1, SrcRC:$src2, AVX512RC:$rc),
Igor Bregerdfcc3d32015-06-17 07:23:57 +00006579 !strconcat(asm,
6580 "\t{$src2, $rc, $src1, $dst|$dst, $src1, $rc, $src2}"),
Igor Bregerabe4a792015-06-14 12:44:55 +00006581 [(set DstVT.RC:$dst,
6582 (OpNode (DstVT.VT DstVT.RC:$src1),
6583 SrcRC:$src2,
6584 (i32 imm:$rc)))]>, EVEX_4V, EVEX_B, EVEX_RC;
6585}
6586
6587multiclass avx512_vcvtsi_common<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00006588 X86VectorVTInfo DstVT, X86MemOperand x86memop,
6589 PatFrag ld_frag, string asm> {
6590 defm NAME : avx512_vcvtsi_round<opc, OpNode, SrcRC, DstVT, asm>,
6591 avx512_vcvtsi<opc, OpNode, SrcRC, DstVT, x86memop, ld_frag, asm>,
6592 VEX_LIG;
Igor Bregerabe4a792015-06-14 12:44:55 +00006593}
6594
Andrew Trick15a47742013-10-09 05:11:10 +00006595let Predicates = [HasAVX512] in {
Igor Bregerabe4a792015-06-14 12:44:55 +00006596defm VCVTSI2SSZ : avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR32,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00006597 v4f32x_info, i32mem, loadi32, "cvtsi2ss{l}">,
6598 XS, EVEX_CD8<32, CD8VT1>;
Igor Bregerabe4a792015-06-14 12:44:55 +00006599defm VCVTSI642SSZ: avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR64,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00006600 v4f32x_info, i64mem, loadi64, "cvtsi2ss{q}">,
6601 XS, VEX_W, EVEX_CD8<64, CD8VT1>;
Igor Bregerabe4a792015-06-14 12:44:55 +00006602defm VCVTSI2SDZ : avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR32,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00006603 v2f64x_info, i32mem, loadi32, "cvtsi2sd{l}">,
6604 XD, EVEX_CD8<32, CD8VT1>;
Igor Bregerabe4a792015-06-14 12:44:55 +00006605defm VCVTSI642SDZ: avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR64,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00006606 v2f64x_info, i64mem, loadi64, "cvtsi2sd{q}">,
6607 XD, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006608
Craig Topper8f85ad12016-11-14 02:46:58 +00006609def : InstAlias<"vcvtsi2ss\t{$src, $src1, $dst|$dst, $src1, $src}",
6610 (VCVTSI2SSZrm FR64X:$dst, FR64X:$src1, i32mem:$src), 0>;
6611def : InstAlias<"vcvtsi2sd\t{$src, $src1, $dst|$dst, $src1, $src}",
6612 (VCVTSI2SDZrm FR64X:$dst, FR64X:$src1, i32mem:$src), 0>;
6613
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006614def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))),
6615 (VCVTSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
6616def : Pat<(f32 (sint_to_fp (loadi64 addr:$src))),
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006617 (VCVTSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006618def : Pat<(f64 (sint_to_fp (loadi32 addr:$src))),
6619 (VCVTSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
6620def : Pat<(f64 (sint_to_fp (loadi64 addr:$src))),
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006621 (VCVTSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006622
6623def : Pat<(f32 (sint_to_fp GR32:$src)),
6624 (VCVTSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>;
6625def : Pat<(f32 (sint_to_fp GR64:$src)),
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006626 (VCVTSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006627def : Pat<(f64 (sint_to_fp GR32:$src)),
6628 (VCVTSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>;
6629def : Pat<(f64 (sint_to_fp GR64:$src)),
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006630 (VCVTSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>;
6631
Elena Demikhovsky0f370932015-07-13 13:26:20 +00006632defm VCVTUSI2SSZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR32,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00006633 v4f32x_info, i32mem, loadi32,
6634 "cvtusi2ss{l}">, XS, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00006635defm VCVTUSI642SSZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR64,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00006636 v4f32x_info, i64mem, loadi64, "cvtusi2ss{q}">,
6637 XS, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00006638defm VCVTUSI2SDZ : avx512_vcvtsi<0x7B, X86UintToFpRnd, GR32, v2f64x_info,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00006639 i32mem, loadi32, "cvtusi2sd{l}">,
6640 XD, VEX_LIG, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00006641defm VCVTUSI642SDZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR64,
Igor Bregerdfcc3d32015-06-17 07:23:57 +00006642 v2f64x_info, i64mem, loadi64, "cvtusi2sd{q}">,
6643 XD, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006644
Craig Topper8f85ad12016-11-14 02:46:58 +00006645def : InstAlias<"vcvtusi2ss\t{$src, $src1, $dst|$dst, $src1, $src}",
6646 (VCVTUSI2SSZrm FR64X:$dst, FR64X:$src1, i32mem:$src), 0>;
6647def : InstAlias<"vcvtusi2sd\t{$src, $src1, $dst|$dst, $src1, $src}",
6648 (VCVTUSI2SDZrm FR64X:$dst, FR64X:$src1, i32mem:$src), 0>;
6649
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006650def : Pat<(f32 (uint_to_fp (loadi32 addr:$src))),
6651 (VCVTUSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
6652def : Pat<(f32 (uint_to_fp (loadi64 addr:$src))),
6653 (VCVTUSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
6654def : Pat<(f64 (uint_to_fp (loadi32 addr:$src))),
6655 (VCVTUSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
6656def : Pat<(f64 (uint_to_fp (loadi64 addr:$src))),
6657 (VCVTUSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
6658
6659def : Pat<(f32 (uint_to_fp GR32:$src)),
6660 (VCVTUSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>;
6661def : Pat<(f32 (uint_to_fp GR64:$src)),
6662 (VCVTUSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>;
6663def : Pat<(f64 (uint_to_fp GR32:$src)),
6664 (VCVTUSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>;
6665def : Pat<(f64 (uint_to_fp GR64:$src)),
6666 (VCVTUSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>;
Andrew Trick15a47742013-10-09 05:11:10 +00006667}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006668
6669//===----------------------------------------------------------------------===//
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006670// AVX-512 Scalar convert from float/double to integer
6671//===----------------------------------------------------------------------===//
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00006672multiclass avx512_cvt_s_int_round<bits<8> opc, X86VectorVTInfo SrcVT ,
6673 X86VectorVTInfo DstVT, SDNode OpNode, string asm> {
Craig Toppere1cac152016-06-07 07:27:54 +00006674 let Predicates = [HasAVX512] in {
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00006675 def rr : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst), (ins SrcVT.RC:$src),
Asaf Badouh2744d212015-09-20 14:31:19 +00006676 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00006677 [(set DstVT.RC:$dst, (OpNode (SrcVT.VT SrcVT.RC:$src),(i32 FROUND_CURRENT)))]>,
6678 EVEX, VEX_LIG;
6679 def rb : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst), (ins SrcVT.RC:$src, AVX512RC:$rc),
6680 !strconcat(asm,"\t{$rc, $src, $dst|$dst, $src, $rc}"),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006681 [(set DstVT.RC:$dst, (OpNode (SrcVT.VT SrcVT.RC:$src),(i32 imm:$rc)))]>,
Asaf Badouh2744d212015-09-20 14:31:19 +00006682 EVEX, VEX_LIG, EVEX_B, EVEX_RC;
Craig Topper5a63ca22017-03-13 03:59:06 +00006683 def rm : SI<opc, MRMSrcMem, (outs DstVT.RC:$dst), (ins SrcVT.IntScalarMemOp:$src),
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00006684 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006685 [(set DstVT.RC:$dst, (OpNode
Craig Topper5a63ca22017-03-13 03:59:06 +00006686 (SrcVT.VT SrcVT.ScalarIntMemCPat:$src),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006687 (i32 FROUND_CURRENT)))]>,
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00006688 EVEX, VEX_LIG;
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006689 } // Predicates = [HasAVX512]
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006690}
Asaf Badouh2744d212015-09-20 14:31:19 +00006691
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006692// Convert float/double to signed/unsigned int 32/64
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006693defm VCVTSS2SIZ: avx512_cvt_s_int_round<0x2D, f32x_info, i32x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00006694 X86cvts2si, "cvtss2si">,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006695 XS, EVEX_CD8<32, CD8VT1>;
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006696defm VCVTSS2SI64Z: avx512_cvt_s_int_round<0x2D, f32x_info, i64x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00006697 X86cvts2si, "cvtss2si">,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006698 XS, VEX_W, EVEX_CD8<32, CD8VT1>;
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006699defm VCVTSS2USIZ: avx512_cvt_s_int_round<0x79, f32x_info, i32x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00006700 X86cvts2usi, "cvtss2usi">,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006701 XS, EVEX_CD8<32, CD8VT1>;
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006702defm VCVTSS2USI64Z: avx512_cvt_s_int_round<0x79, f32x_info, i64x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00006703 X86cvts2usi, "cvtss2usi">, XS, VEX_W,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006704 EVEX_CD8<32, CD8VT1>;
Simon Pilgrimb13961d2016-06-11 14:34:10 +00006705defm VCVTSD2SIZ: avx512_cvt_s_int_round<0x2D, f64x_info, i32x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00006706 X86cvts2si, "cvtsd2si">,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006707 XD, EVEX_CD8<64, CD8VT1>;
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00006708defm VCVTSD2SI64Z: avx512_cvt_s_int_round<0x2D, f64x_info, i64x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00006709 X86cvts2si, "cvtsd2si">,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006710 XD, VEX_W, EVEX_CD8<64, CD8VT1>;
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00006711defm VCVTSD2USIZ: avx512_cvt_s_int_round<0x79, f64x_info, i32x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00006712 X86cvts2usi, "cvtsd2usi">,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006713 XD, EVEX_CD8<64, CD8VT1>;
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00006714defm VCVTSD2USI64Z: avx512_cvt_s_int_round<0x79, f64x_info, i64x_info,
Craig Topper19e04b62016-05-19 06:13:58 +00006715 X86cvts2usi, "cvtsd2usi">, XD, VEX_W,
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006716 EVEX_CD8<64, CD8VT1>;
6717
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00006718// The SSE version of these instructions are disabled for AVX512.
6719// Therefore, the SSE intrinsics are mapped to the AVX512 instructions.
6720let Predicates = [HasAVX512] in {
6721 def : Pat<(i32 (int_x86_sse_cvtss2si (v4f32 VR128X:$src))),
Craig Topper8c252bc2016-09-18 18:59:33 +00006722 (VCVTSS2SIZrr VR128X:$src)>;
Craig Topper5a63ca22017-03-13 03:59:06 +00006723 def : Pat<(i32 (int_x86_sse_cvtss2si sse_load_f32:$src)),
6724 (VCVTSS2SIZrm sse_load_f32:$src)>;
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00006725 def : Pat<(i64 (int_x86_sse_cvtss2si64 (v4f32 VR128X:$src))),
Craig Topper8c252bc2016-09-18 18:59:33 +00006726 (VCVTSS2SI64Zrr VR128X:$src)>;
Craig Topper5a63ca22017-03-13 03:59:06 +00006727 def : Pat<(i64 (int_x86_sse_cvtss2si64 sse_load_f32:$src)),
6728 (VCVTSS2SI64Zrm sse_load_f32:$src)>;
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00006729 def : Pat<(i32 (int_x86_sse2_cvtsd2si (v2f64 VR128X:$src))),
Craig Topper8c252bc2016-09-18 18:59:33 +00006730 (VCVTSD2SIZrr VR128X:$src)>;
Craig Topper5a63ca22017-03-13 03:59:06 +00006731 def : Pat<(i32 (int_x86_sse2_cvtsd2si sse_load_f64:$src)),
6732 (VCVTSD2SIZrm sse_load_f64:$src)>;
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00006733 def : Pat<(i64 (int_x86_sse2_cvtsd2si64 (v2f64 VR128X:$src))),
Craig Topper8c252bc2016-09-18 18:59:33 +00006734 (VCVTSD2SI64Zrr VR128X:$src)>;
Craig Topper5a63ca22017-03-13 03:59:06 +00006735 def : Pat<(i64 (int_x86_sse2_cvtsd2si64 sse_load_f64:$src)),
6736 (VCVTSD2SI64Zrm sse_load_f64:$src)>;
Asaf Badouhad5c3fc2016-02-07 14:59:13 +00006737} // HasAVX512
6738
Craig Topperac941b92016-09-25 16:33:53 +00006739let Predicates = [HasAVX512] in {
6740 def : Pat<(int_x86_sse_cvtsi2ss VR128X:$src1, GR32:$src2),
6741 (VCVTSI2SSZrr_Int VR128X:$src1, GR32:$src2)>;
6742 def : Pat<(int_x86_sse_cvtsi2ss VR128X:$src1, (loadi32 addr:$src2)),
6743 (VCVTSI2SSZrm_Int VR128X:$src1, addr:$src2)>;
6744 def : Pat<(int_x86_sse_cvtsi642ss VR128X:$src1, GR64:$src2),
6745 (VCVTSI642SSZrr_Int VR128X:$src1, GR64:$src2)>;
6746 def : Pat<(int_x86_sse_cvtsi642ss VR128X:$src1, (loadi64 addr:$src2)),
6747 (VCVTSI642SSZrm_Int VR128X:$src1, addr:$src2)>;
6748 def : Pat<(int_x86_sse2_cvtsi2sd VR128X:$src1, GR32:$src2),
6749 (VCVTSI2SDZrr_Int VR128X:$src1, GR32:$src2)>;
6750 def : Pat<(int_x86_sse2_cvtsi2sd VR128X:$src1, (loadi32 addr:$src2)),
6751 (VCVTSI2SDZrm_Int VR128X:$src1, addr:$src2)>;
6752 def : Pat<(int_x86_sse2_cvtsi642sd VR128X:$src1, GR64:$src2),
6753 (VCVTSI642SDZrr_Int VR128X:$src1, GR64:$src2)>;
6754 def : Pat<(int_x86_sse2_cvtsi642sd VR128X:$src1, (loadi64 addr:$src2)),
6755 (VCVTSI642SDZrm_Int VR128X:$src1, addr:$src2)>;
6756 def : Pat<(int_x86_avx512_cvtusi2sd VR128X:$src1, GR32:$src2),
6757 (VCVTUSI2SDZrr_Int VR128X:$src1, GR32:$src2)>;
6758 def : Pat<(int_x86_avx512_cvtusi2sd VR128X:$src1, (loadi32 addr:$src2)),
6759 (VCVTUSI2SDZrm_Int VR128X:$src1, addr:$src2)>;
6760} // Predicates = [HasAVX512]
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006761
Elad Cohen0c260102017-01-11 09:11:48 +00006762// Patterns used for matching vcvtsi2s{s,d} intrinsic sequences from clang
6763// which produce unnecessary vmovs{s,d} instructions
6764let Predicates = [HasAVX512] in {
6765def : Pat<(v4f32 (X86Movss
6766 (v4f32 VR128X:$dst),
6767 (v4f32 (scalar_to_vector (f32 (sint_to_fp GR64:$src)))))),
6768 (VCVTSI642SSZrr_Int VR128X:$dst, GR64:$src)>;
6769
6770def : Pat<(v4f32 (X86Movss
6771 (v4f32 VR128X:$dst),
6772 (v4f32 (scalar_to_vector (f32 (sint_to_fp GR32:$src)))))),
6773 (VCVTSI2SSZrr_Int VR128X:$dst, GR32:$src)>;
6774
6775def : Pat<(v2f64 (X86Movsd
6776 (v2f64 VR128X:$dst),
6777 (v2f64 (scalar_to_vector (f64 (sint_to_fp GR64:$src)))))),
6778 (VCVTSI642SDZrr_Int VR128X:$dst, GR64:$src)>;
6779
6780def : Pat<(v2f64 (X86Movsd
6781 (v2f64 VR128X:$dst),
6782 (v2f64 (scalar_to_vector (f64 (sint_to_fp GR32:$src)))))),
6783 (VCVTSI2SDZrr_Int VR128X:$dst, GR32:$src)>;
6784} // Predicates = [HasAVX512]
6785
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006786// Convert float/double to signed/unsigned int 32/64 with truncation
Simon Pilgrim18bcf932016-02-03 09:41:59 +00006787multiclass avx512_cvt_s_all<bits<8> opc, string asm, X86VectorVTInfo _SrcRC,
6788 X86VectorVTInfo _DstRC, SDNode OpNode,
Igor Bregerc59b3a22016-08-03 10:58:05 +00006789 SDNode OpNodeRnd, string aliasStr>{
Asaf Badouh2744d212015-09-20 14:31:19 +00006790let Predicates = [HasAVX512] in {
Igor Bregerc59b3a22016-08-03 10:58:05 +00006791 def rr : AVX512<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.FRC:$src),
Asaf Badouh2744d212015-09-20 14:31:19 +00006792 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
6793 [(set _DstRC.RC:$dst, (OpNode _SrcRC.FRC:$src))]>, EVEX;
Craig Topper0e473952016-09-07 04:46:15 +00006794 let hasSideEffects = 0 in
Igor Bregerc59b3a22016-08-03 10:58:05 +00006795 def rb : AVX512<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.FRC:$src),
Asaf Badouh2744d212015-09-20 14:31:19 +00006796 !strconcat(asm,"\t{{sae}, $src, $dst|$dst, $src, {sae}}"),
6797 []>, EVEX, EVEX_B;
Igor Bregerc59b3a22016-08-03 10:58:05 +00006798 def rm : AVX512<opc, MRMSrcMem, (outs _DstRC.RC:$dst), (ins _SrcRC.ScalarMemOp:$src),
Asaf Badouh2744d212015-09-20 14:31:19 +00006799 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
Simon Pilgrim18bcf932016-02-03 09:41:59 +00006800 [(set _DstRC.RC:$dst, (OpNode (_SrcRC.ScalarLdFrag addr:$src)))]>,
Asaf Badouh2744d212015-09-20 14:31:19 +00006801 EVEX;
Simon Pilgrim916485c2016-08-18 11:22:22 +00006802
Igor Bregerc59b3a22016-08-03 10:58:05 +00006803 def : InstAlias<asm # aliasStr # "\t{$src, $dst|$dst, $src}",
6804 (!cast<Instruction>(NAME # "rr") _DstRC.RC:$dst, _SrcRC.FRC:$src), 0>;
6805 def : InstAlias<asm # aliasStr # "\t\t{{sae}, $src, $dst|$dst, $src, {sae}}",
6806 (!cast<Instruction>(NAME # "rb") _DstRC.RC:$dst, _SrcRC.FRC:$src), 0>;
6807 def : InstAlias<asm # aliasStr # "\t{$src, $dst|$dst, $src}",
Simon Pilgrim916485c2016-08-18 11:22:22 +00006808 (!cast<Instruction>(NAME # "rm") _DstRC.RC:$dst,
6809 _SrcRC.ScalarMemOp:$src), 0>;
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006810
Craig Toppere1cac152016-06-07 07:27:54 +00006811 let isCodeGenOnly = 1 in {
Igor Bregerc59b3a22016-08-03 10:58:05 +00006812 def rr_Int : AVX512<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.RC:$src),
6813 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
6814 [(set _DstRC.RC:$dst, (OpNodeRnd (_SrcRC.VT _SrcRC.RC:$src),
6815 (i32 FROUND_CURRENT)))]>, EVEX, VEX_LIG;
6816 def rb_Int : AVX512<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.RC:$src),
6817 !strconcat(asm,"\t{{sae}, $src, $dst|$dst, $src, {sae}}"),
6818 [(set _DstRC.RC:$dst, (OpNodeRnd (_SrcRC.VT _SrcRC.RC:$src),
6819 (i32 FROUND_NO_EXC)))]>,
6820 EVEX,VEX_LIG , EVEX_B;
6821 let mayLoad = 1, hasSideEffects = 0 in
6822 def rm_Int : AVX512<opc, MRMSrcMem, (outs _DstRC.RC:$dst),
Ayman Musaf77219e2017-02-13 09:55:48 +00006823 (ins _SrcRC.IntScalarMemOp:$src),
Igor Bregerc59b3a22016-08-03 10:58:05 +00006824 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
6825 []>, EVEX, VEX_LIG;
Asaf Badouh2744d212015-09-20 14:31:19 +00006826
Craig Toppere1cac152016-06-07 07:27:54 +00006827 } // isCodeGenOnly = 1
Asaf Badouh2744d212015-09-20 14:31:19 +00006828} //HasAVX512
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006829}
6830
Asaf Badouh2744d212015-09-20 14:31:19 +00006831
Igor Bregerc59b3a22016-08-03 10:58:05 +00006832defm VCVTTSS2SIZ: avx512_cvt_s_all<0x2C, "vcvttss2si", f32x_info, i32x_info,
6833 fp_to_sint, X86cvtts2IntRnd, "{l}">,
Asaf Badouh2744d212015-09-20 14:31:19 +00006834 XS, EVEX_CD8<32, CD8VT1>;
Igor Bregerc59b3a22016-08-03 10:58:05 +00006835defm VCVTTSS2SI64Z: avx512_cvt_s_all<0x2C, "vcvttss2si", f32x_info, i64x_info,
6836 fp_to_sint, X86cvtts2IntRnd, "{q}">,
Asaf Badouh2744d212015-09-20 14:31:19 +00006837 VEX_W, XS, EVEX_CD8<32, CD8VT1>;
Igor Bregerc59b3a22016-08-03 10:58:05 +00006838defm VCVTTSD2SIZ: avx512_cvt_s_all<0x2C, "vcvttsd2si", f64x_info, i32x_info,
6839 fp_to_sint, X86cvtts2IntRnd, "{l}">,
Asaf Badouh2744d212015-09-20 14:31:19 +00006840 XD, EVEX_CD8<64, CD8VT1>;
Igor Bregerc59b3a22016-08-03 10:58:05 +00006841defm VCVTTSD2SI64Z: avx512_cvt_s_all<0x2C, "vcvttsd2si", f64x_info, i64x_info,
6842 fp_to_sint, X86cvtts2IntRnd, "{q}">,
Asaf Badouh2744d212015-09-20 14:31:19 +00006843 VEX_W, XD, EVEX_CD8<64, CD8VT1>;
6844
Igor Bregerc59b3a22016-08-03 10:58:05 +00006845defm VCVTTSS2USIZ: avx512_cvt_s_all<0x78, "vcvttss2usi", f32x_info, i32x_info,
6846 fp_to_uint, X86cvtts2UIntRnd, "{l}">,
Asaf Badouh2744d212015-09-20 14:31:19 +00006847 XS, EVEX_CD8<32, CD8VT1>;
Igor Bregerc59b3a22016-08-03 10:58:05 +00006848defm VCVTTSS2USI64Z: avx512_cvt_s_all<0x78, "vcvttss2usi", f32x_info, i64x_info,
6849 fp_to_uint, X86cvtts2UIntRnd, "{q}">,
Asaf Badouh2744d212015-09-20 14:31:19 +00006850 XS,VEX_W, EVEX_CD8<32, CD8VT1>;
Igor Bregerc59b3a22016-08-03 10:58:05 +00006851defm VCVTTSD2USIZ: avx512_cvt_s_all<0x78, "vcvttsd2usi", f64x_info, i32x_info,
6852 fp_to_uint, X86cvtts2UIntRnd, "{l}">,
Asaf Badouh2744d212015-09-20 14:31:19 +00006853 XD, EVEX_CD8<64, CD8VT1>;
Igor Bregerc59b3a22016-08-03 10:58:05 +00006854defm VCVTTSD2USI64Z: avx512_cvt_s_all<0x78, "vcvttsd2usi", f64x_info, i64x_info,
6855 fp_to_uint, X86cvtts2UIntRnd, "{q}">,
Asaf Badouh2744d212015-09-20 14:31:19 +00006856 XD, VEX_W, EVEX_CD8<64, CD8VT1>;
6857let Predicates = [HasAVX512] in {
6858 def : Pat<(i32 (int_x86_sse_cvttss2si (v4f32 VR128X:$src))),
Craig Topper8c252bc2016-09-18 18:59:33 +00006859 (VCVTTSS2SIZrr_Int VR128X:$src)>;
Ayman Musaf77219e2017-02-13 09:55:48 +00006860 def : Pat<(i32 (int_x86_sse_cvttss2si sse_load_f32:$src)),
6861 (VCVTTSS2SIZrm_Int ssmem:$src)>;
Asaf Badouh2744d212015-09-20 14:31:19 +00006862 def : Pat<(i64 (int_x86_sse_cvttss2si64 (v4f32 VR128X:$src))),
Craig Topper8c252bc2016-09-18 18:59:33 +00006863 (VCVTTSS2SI64Zrr_Int VR128X:$src)>;
Ayman Musaf77219e2017-02-13 09:55:48 +00006864 def : Pat<(i64 (int_x86_sse_cvttss2si64 sse_load_f32:$src)),
6865 (VCVTTSS2SI64Zrm_Int ssmem:$src)>;
Asaf Badouh2744d212015-09-20 14:31:19 +00006866 def : Pat<(i32 (int_x86_sse2_cvttsd2si (v2f64 VR128X:$src))),
Craig Topper8c252bc2016-09-18 18:59:33 +00006867 (VCVTTSD2SIZrr_Int VR128X:$src)>;
Ayman Musaf77219e2017-02-13 09:55:48 +00006868 def : Pat<(i32 (int_x86_sse2_cvttsd2si sse_load_f64:$src)),
6869 (VCVTTSD2SIZrm_Int sdmem:$src)>;
Asaf Badouh2744d212015-09-20 14:31:19 +00006870 def : Pat<(i64 (int_x86_sse2_cvttsd2si64 (v2f64 VR128X:$src))),
Craig Topper8c252bc2016-09-18 18:59:33 +00006871 (VCVTTSD2SI64Zrr_Int VR128X:$src)>;
Ayman Musaf77219e2017-02-13 09:55:48 +00006872 def : Pat<(i64 (int_x86_sse2_cvttsd2si64 sse_load_f64:$src)),
6873 (VCVTTSD2SI64Zrm_Int sdmem:$src)>;
Elena Demikhovskycf088092013-12-11 14:31:04 +00006874} // HasAVX512
Elena Demikhovsky2e408ae2013-10-06 13:11:09 +00006875//===----------------------------------------------------------------------===//
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006876// AVX-512 Convert form float to double and back
6877//===----------------------------------------------------------------------===//
Asaf Badouh2744d212015-09-20 14:31:19 +00006878multiclass avx512_cvt_fp_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
6879 X86VectorVTInfo _Src, SDNode OpNode> {
Ayman Musa6e670cf2017-02-23 07:24:21 +00006880 defm rr_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
Craig Toppera58abd12016-05-09 05:34:12 +00006881 (ins _.RC:$src1, _Src.RC:$src2), OpcodeStr,
Asaf Badouh2744d212015-09-20 14:31:19 +00006882 "$src2, $src1", "$src1, $src2",
Craig Toppera58abd12016-05-09 05:34:12 +00006883 (_.VT (OpNode (_.VT _.RC:$src1),
Craig Toppera02e3942016-09-23 06:24:43 +00006884 (_Src.VT _Src.RC:$src2),
6885 (i32 FROUND_CURRENT)))>,
Asaf Badouh2744d212015-09-20 14:31:19 +00006886 EVEX_4V, VEX_LIG, Sched<[WriteCvtF2F]>;
Ayman Musa6e670cf2017-02-23 07:24:21 +00006887 defm rm_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
Craig Topper08b413a2017-03-13 05:14:44 +00006888 (ins _.RC:$src1, _Src.IntScalarMemOp:$src2), OpcodeStr,
Asaf Badouh2744d212015-09-20 14:31:19 +00006889 "$src2, $src1", "$src1, $src2",
Craig Toppera58abd12016-05-09 05:34:12 +00006890 (_.VT (OpNode (_.VT _.RC:$src1),
Craig Topper08b413a2017-03-13 05:14:44 +00006891 (_Src.VT _Src.ScalarIntMemCPat:$src2),
Craig Toppera02e3942016-09-23 06:24:43 +00006892 (i32 FROUND_CURRENT)))>,
Asaf Badouh2744d212015-09-20 14:31:19 +00006893 EVEX_4V, VEX_LIG, Sched<[WriteCvtF2FLd, ReadAfterLd]>;
Ayman Musa6e670cf2017-02-23 07:24:21 +00006894
Craig Topperd2011e32017-02-25 18:43:42 +00006895 let isCodeGenOnly = 1, hasSideEffects = 0 in {
6896 def rr : I<opc, MRMSrcReg, (outs _.FRC:$dst),
6897 (ins _.FRC:$src1, _Src.FRC:$src2),
6898 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
6899 EVEX_4V, VEX_LIG, Sched<[WriteCvtF2F]>;
6900 let mayLoad = 1 in
6901 def rm : I<opc, MRMSrcMem, (outs _.FRC:$dst),
6902 (ins _.FRC:$src1, _Src.ScalarMemOp:$src2),
6903 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
6904 EVEX_4V, VEX_LIG, Sched<[WriteCvtF2FLd, ReadAfterLd]>;
6905 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006906}
6907
Asaf Badouh2744d212015-09-20 14:31:19 +00006908// Scalar Coversion with SAE - suppress all exceptions
6909multiclass avx512_cvt_fp_sae_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
6910 X86VectorVTInfo _Src, SDNode OpNodeRnd> {
Ayman Musa6e670cf2017-02-23 07:24:21 +00006911 defm rrb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
Craig Toppera58abd12016-05-09 05:34:12 +00006912 (ins _.RC:$src1, _Src.RC:$src2), OpcodeStr,
Asaf Badouh2744d212015-09-20 14:31:19 +00006913 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
Craig Toppera58abd12016-05-09 05:34:12 +00006914 (_.VT (OpNodeRnd (_.VT _.RC:$src1),
Asaf Badouh2744d212015-09-20 14:31:19 +00006915 (_Src.VT _Src.RC:$src2),
6916 (i32 FROUND_NO_EXC)))>,
6917 EVEX_4V, VEX_LIG, EVEX_B;
6918}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006919
Asaf Badouh2744d212015-09-20 14:31:19 +00006920// Scalar Conversion with rounding control (RC)
6921multiclass avx512_cvt_fp_rc_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
6922 X86VectorVTInfo _Src, SDNode OpNodeRnd> {
Ayman Musa6e670cf2017-02-23 07:24:21 +00006923 defm rrb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
Craig Toppera58abd12016-05-09 05:34:12 +00006924 (ins _.RC:$src1, _Src.RC:$src2, AVX512RC:$rc), OpcodeStr,
Asaf Badouh2744d212015-09-20 14:31:19 +00006925 "$rc, $src2, $src1", "$src1, $src2, $rc",
Craig Toppera58abd12016-05-09 05:34:12 +00006926 (_.VT (OpNodeRnd (_.VT _.RC:$src1),
Asaf Badouh2744d212015-09-20 14:31:19 +00006927 (_Src.VT _Src.RC:$src2), (i32 imm:$rc)))>,
6928 EVEX_4V, VEX_LIG, Sched<[WriteCvtF2FLd, ReadAfterLd]>,
6929 EVEX_B, EVEX_RC;
6930}
Craig Toppera02e3942016-09-23 06:24:43 +00006931multiclass avx512_cvt_fp_scalar_sd2ss<bits<8> opc, string OpcodeStr,
Simon Pilgrim18bcf932016-02-03 09:41:59 +00006932 SDNode OpNodeRnd, X86VectorVTInfo _src,
Asaf Badouh2744d212015-09-20 14:31:19 +00006933 X86VectorVTInfo _dst> {
6934 let Predicates = [HasAVX512] in {
Craig Toppera02e3942016-09-23 06:24:43 +00006935 defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNodeRnd>,
Asaf Badouh2744d212015-09-20 14:31:19 +00006936 avx512_cvt_fp_rc_scalar<opc, OpcodeStr, _dst, _src,
Michael Zuckerman4b88a772016-12-18 14:29:00 +00006937 OpNodeRnd>, VEX_W, EVEX_CD8<64, CD8VT1>, XD;
Asaf Badouh2744d212015-09-20 14:31:19 +00006938 }
6939}
6940
Craig Toppera02e3942016-09-23 06:24:43 +00006941multiclass avx512_cvt_fp_scalar_ss2sd<bits<8> opc, string OpcodeStr,
Simon Pilgrim18bcf932016-02-03 09:41:59 +00006942 SDNode OpNodeRnd, X86VectorVTInfo _src,
Asaf Badouh2744d212015-09-20 14:31:19 +00006943 X86VectorVTInfo _dst> {
6944 let Predicates = [HasAVX512] in {
Craig Toppera02e3942016-09-23 06:24:43 +00006945 defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNodeRnd>,
Simon Pilgrim18bcf932016-02-03 09:41:59 +00006946 avx512_cvt_fp_sae_scalar<opc, OpcodeStr, _dst, _src, OpNodeRnd>,
Michael Zuckerman4b88a772016-12-18 14:29:00 +00006947 EVEX_CD8<32, CD8VT1>, XS;
Asaf Badouh2744d212015-09-20 14:31:19 +00006948 }
6949}
Craig Toppera02e3942016-09-23 06:24:43 +00006950defm VCVTSD2SS : avx512_cvt_fp_scalar_sd2ss<0x5A, "vcvtsd2ss",
Asaf Badouh2744d212015-09-20 14:31:19 +00006951 X86froundRnd, f64x_info, f32x_info>;
Craig Toppera02e3942016-09-23 06:24:43 +00006952defm VCVTSS2SD : avx512_cvt_fp_scalar_ss2sd<0x5A, "vcvtss2sd",
Asaf Badouh2744d212015-09-20 14:31:19 +00006953 X86fpextRnd,f32x_info, f64x_info >;
6954
Michael Kuperstein2bc3d4d2016-08-18 20:08:15 +00006955def : Pat<(f64 (fpextend FR32X:$src)),
Ayman Musa6e670cf2017-02-23 07:24:21 +00006956 (VCVTSS2SDZrr (COPY_TO_REGCLASS FR32X:$src, FR64X), FR32X:$src)>,
Asaf Badouh2744d212015-09-20 14:31:19 +00006957 Requires<[HasAVX512]>;
Michael Kuperstein2bc3d4d2016-08-18 20:08:15 +00006958def : Pat<(f64 (fpextend (loadf32 addr:$src))),
Ayman Musa6e670cf2017-02-23 07:24:21 +00006959 (VCVTSS2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>,
Asaf Badouh2744d212015-09-20 14:31:19 +00006960 Requires<[HasAVX512]>;
6961
6962def : Pat<(f64 (extloadf32 addr:$src)),
Ayman Musa6e670cf2017-02-23 07:24:21 +00006963 (VCVTSS2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006964 Requires<[HasAVX512, OptForSize]>;
6965
Asaf Badouh2744d212015-09-20 14:31:19 +00006966def : Pat<(f64 (extloadf32 addr:$src)),
Ayman Musa6e670cf2017-02-23 07:24:21 +00006967 (VCVTSS2SDZrr (f64 (IMPLICIT_DEF)), (VMOVSSZrm addr:$src))>,
Asaf Badouh2744d212015-09-20 14:31:19 +00006968 Requires<[HasAVX512, OptForSpeed]>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006969
Michael Kuperstein2bc3d4d2016-08-18 20:08:15 +00006970def : Pat<(f32 (fpround FR64X:$src)),
Ayman Musa6e670cf2017-02-23 07:24:21 +00006971 (VCVTSD2SSZrr (COPY_TO_REGCLASS FR64X:$src, FR32X), FR64X:$src)>,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00006972 Requires<[HasAVX512]>;
Elad Cohen0c260102017-01-11 09:11:48 +00006973
6974def : Pat<(v4f32 (X86Movss
6975 (v4f32 VR128X:$dst),
6976 (v4f32 (scalar_to_vector
6977 (f32 (fpround (f64 (extractelt VR128X:$src, (iPTR 0))))))))),
Ayman Musa6e670cf2017-02-23 07:24:21 +00006978 (VCVTSD2SSZrr_Int VR128X:$dst, VR128X:$src)>,
Elad Cohen0c260102017-01-11 09:11:48 +00006979 Requires<[HasAVX512]>;
6980
6981def : Pat<(v2f64 (X86Movsd
6982 (v2f64 VR128X:$dst),
6983 (v2f64 (scalar_to_vector
6984 (f64 (fpextend (f32 (extractelt VR128X:$src, (iPTR 0))))))))),
Ayman Musa6e670cf2017-02-23 07:24:21 +00006985 (VCVTSS2SDZrr_Int VR128X:$dst, VR128X:$src)>,
Elad Cohen0c260102017-01-11 09:11:48 +00006986 Requires<[HasAVX512]>;
6987
Elena Demikhovsky0f370932015-07-13 13:26:20 +00006988//===----------------------------------------------------------------------===//
6989// AVX-512 Vector convert from signed/unsigned integer to float/double
6990// and from float/double to signed/unsigned integer
6991//===----------------------------------------------------------------------===//
6992
6993multiclass avx512_vcvt_fp<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
6994 X86VectorVTInfo _Src, SDNode OpNode,
6995 string Broadcast = _.BroadcastStr,
Coby Tayree97e9cf62016-11-20 17:09:56 +00006996 string Alias = "", X86MemOperand MemOp = _Src.MemOp> {
Elena Demikhovsky0f370932015-07-13 13:26:20 +00006997
6998 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
6999 (ins _Src.RC:$src), OpcodeStr, "$src", "$src",
7000 (_.VT (OpNode (_Src.VT _Src.RC:$src)))>, EVEX;
7001
7002 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
Coby Tayree97e9cf62016-11-20 17:09:56 +00007003 (ins MemOp:$src), OpcodeStr#Alias, "$src", "$src",
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007004 (_.VT (OpNode (_Src.VT
7005 (bitconvert (_Src.LdFrag addr:$src)))))>, EVEX;
7006
7007 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
Igor Breger4511e762016-02-22 11:48:27 +00007008 (ins _Src.ScalarMemOp:$src), OpcodeStr,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007009 "${src}"##Broadcast, "${src}"##Broadcast,
7010 (_.VT (OpNode (_Src.VT
7011 (X86VBroadcast (_Src.ScalarLdFrag addr:$src)))
7012 ))>, EVEX, EVEX_B;
7013}
7014// Coversion with SAE - suppress all exceptions
7015multiclass avx512_vcvt_fp_sae<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
7016 X86VectorVTInfo _Src, SDNode OpNodeRnd> {
7017 defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
7018 (ins _Src.RC:$src), OpcodeStr,
7019 "{sae}, $src", "$src, {sae}",
7020 (_.VT (OpNodeRnd (_Src.VT _Src.RC:$src),
7021 (i32 FROUND_NO_EXC)))>,
7022 EVEX, EVEX_B;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007023}
7024
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007025// Conversion with rounding control (RC)
7026multiclass avx512_vcvt_fp_rc<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
7027 X86VectorVTInfo _Src, SDNode OpNodeRnd> {
7028 defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
7029 (ins _Src.RC:$src, AVX512RC:$rc), OpcodeStr,
7030 "$rc, $src", "$src, $rc",
7031 (_.VT (OpNodeRnd (_Src.VT _Src.RC:$src), (i32 imm:$rc)))>,
7032 EVEX, EVEX_B, EVEX_RC;
Elena Demikhovskyde3f7512014-01-01 15:12:34 +00007033}
7034
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007035// Extend Float to Double
7036multiclass avx512_cvtps2pd<bits<8> opc, string OpcodeStr> {
7037 let Predicates = [HasAVX512] in {
Michael Kuperstein2bc3d4d2016-08-18 20:08:15 +00007038 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8f32x_info, fpextend>,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007039 avx512_vcvt_fp_sae<opc, OpcodeStr, v8f64_info, v8f32x_info,
7040 X86vfpextRnd>, EVEX_V512;
7041 }
7042 let Predicates = [HasVLX] in {
7043 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4f32x_info,
Coby Tayree97e9cf62016-11-20 17:09:56 +00007044 X86vfpext, "{1to2}", "", f64mem>, EVEX_V128;
Michael Kuperstein2bc3d4d2016-08-18 20:08:15 +00007045 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4f32x_info, fpextend>,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007046 EVEX_V256;
7047 }
7048}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007049
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007050// Truncate Double to Float
7051multiclass avx512_cvtpd2ps<bits<8> opc, string OpcodeStr> {
7052 let Predicates = [HasAVX512] in {
Michael Kuperstein2bc3d4d2016-08-18 20:08:15 +00007053 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8f64_info, fpround>,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007054 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8f64_info,
7055 X86vfproundRnd>, EVEX_V512;
7056 }
7057 let Predicates = [HasVLX] in {
7058 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2f64x_info,
7059 X86vfpround, "{1to2}", "{x}">, EVEX_V128;
Michael Kuperstein2bc3d4d2016-08-18 20:08:15 +00007060 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4f64x_info, fpround,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007061 "{1to4}", "{y}">, EVEX_V256;
Craig Topperb8596e42016-11-14 01:53:29 +00007062
7063 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7064 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
7065 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7066 (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, f128mem:$src), 0>;
7067 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7068 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
7069 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7070 (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, f256mem:$src), 0>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007071 }
7072}
7073
7074defm VCVTPD2PS : avx512_cvtpd2ps<0x5A, "vcvtpd2ps">,
7075 VEX_W, PD, EVEX_CD8<64, CD8VF>;
7076defm VCVTPS2PD : avx512_cvtps2pd<0x5A, "vcvtps2pd">,
7077 PS, EVEX_CD8<32, CD8VH>;
7078
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007079def : Pat<(v8f64 (extloadv8f32 addr:$src)),
7080 (VCVTPS2PDZrm addr:$src)>;
Michael Liao5bf95782014-12-04 05:20:33 +00007081
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007082let Predicates = [HasVLX] in {
Craig Topper731bf9c2016-11-09 07:31:32 +00007083 let AddedComplexity = 15 in
7084 def : Pat<(X86vzmovl (v2f64 (bitconvert
7085 (v4f32 (X86vfpround (v2f64 VR128X:$src)))))),
7086 (VCVTPD2PSZ128rr VR128X:$src)>;
Craig Topper5471fc22016-11-06 04:12:52 +00007087 def : Pat<(v2f64 (extloadv2f32 addr:$src)),
7088 (VCVTPS2PDZ128rm addr:$src)>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007089 def : Pat<(v4f64 (extloadv4f32 addr:$src)),
7090 (VCVTPS2PDZ256rm addr:$src)>;
7091}
Elena Demikhovsky3629b4a2014-01-06 08:45:54 +00007092
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007093// Convert Signed/Unsigned Doubleword to Double
7094multiclass avx512_cvtdq2pd<bits<8> opc, string OpcodeStr, SDNode OpNode,
7095 SDNode OpNode128> {
7096 // No rounding in this op
7097 let Predicates = [HasAVX512] in
7098 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i32x_info, OpNode>,
7099 EVEX_V512;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007100
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007101 let Predicates = [HasVLX] in {
7102 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4i32x_info,
Coby Tayree97e9cf62016-11-20 17:09:56 +00007103 OpNode128, "{1to2}", "", i64mem>, EVEX_V128;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007104 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i32x_info, OpNode>,
7105 EVEX_V256;
7106 }
7107}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007108
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007109// Convert Signed/Unsigned Doubleword to Float
7110multiclass avx512_cvtdq2ps<bits<8> opc, string OpcodeStr, SDNode OpNode,
7111 SDNode OpNodeRnd> {
7112 let Predicates = [HasAVX512] in
7113 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16f32_info, v16i32_info, OpNode>,
7114 avx512_vcvt_fp_rc<opc, OpcodeStr, v16f32_info, v16i32_info,
7115 OpNodeRnd>, EVEX_V512;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007116
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007117 let Predicates = [HasVLX] in {
7118 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i32x_info, OpNode>,
7119 EVEX_V128;
7120 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i32x_info, OpNode>,
7121 EVEX_V256;
7122 }
7123}
7124
7125// Convert Float to Signed/Unsigned Doubleword with truncation
7126multiclass avx512_cvttps2dq<bits<8> opc, string OpcodeStr,
7127 SDNode OpNode, SDNode OpNodeRnd> {
7128 let Predicates = [HasAVX512] in {
7129 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode>,
7130 avx512_vcvt_fp_sae<opc, OpcodeStr, v16i32_info, v16f32_info,
7131 OpNodeRnd>, EVEX_V512;
7132 }
7133 let Predicates = [HasVLX] in {
7134 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode>,
7135 EVEX_V128;
7136 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode>,
7137 EVEX_V256;
7138 }
7139}
7140
7141// Convert Float to Signed/Unsigned Doubleword
7142multiclass avx512_cvtps2dq<bits<8> opc, string OpcodeStr,
7143 SDNode OpNode, SDNode OpNodeRnd> {
7144 let Predicates = [HasAVX512] in {
7145 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode>,
7146 avx512_vcvt_fp_rc<opc, OpcodeStr, v16i32_info, v16f32_info,
7147 OpNodeRnd>, EVEX_V512;
7148 }
7149 let Predicates = [HasVLX] in {
7150 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode>,
7151 EVEX_V128;
7152 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode>,
7153 EVEX_V256;
7154 }
7155}
7156
7157// Convert Double to Signed/Unsigned Doubleword with truncation
Craig Topper731bf9c2016-11-09 07:31:32 +00007158multiclass avx512_cvttpd2dq<bits<8> opc, string OpcodeStr, SDNode OpNode,
7159 SDNode OpNode128, SDNode OpNodeRnd> {
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007160 let Predicates = [HasAVX512] in {
7161 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode>,
7162 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i32x_info, v8f64_info,
7163 OpNodeRnd>, EVEX_V512;
7164 }
7165 let Predicates = [HasVLX] in {
7166 // we need "x"/"y" suffixes in order to distinguish between 128 and 256
Craig Topper731bf9c2016-11-09 07:31:32 +00007167 // memory forms of these instructions in Asm Parser. They have the same
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007168 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
7169 // due to the same reason.
Craig Topper731bf9c2016-11-09 07:31:32 +00007170 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info,
7171 OpNode128, "{1to2}", "{x}">, EVEX_V128;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007172 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
7173 "{1to4}", "{y}">, EVEX_V256;
Craig Topperb8596e42016-11-14 01:53:29 +00007174
7175 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7176 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
7177 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7178 (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, i128mem:$src), 0>;
7179 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7180 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
7181 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7182 (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, i256mem:$src), 0>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007183 }
7184}
7185
7186// Convert Double to Signed/Unsigned Doubleword
7187multiclass avx512_cvtpd2dq<bits<8> opc, string OpcodeStr,
7188 SDNode OpNode, SDNode OpNodeRnd> {
7189 let Predicates = [HasAVX512] in {
7190 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode>,
7191 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i32x_info, v8f64_info,
7192 OpNodeRnd>, EVEX_V512;
7193 }
7194 let Predicates = [HasVLX] in {
7195 // we need "x"/"y" suffixes in order to distinguish between 128 and 256
7196 // memory forms of these instructions in Asm Parcer. They have the same
7197 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
7198 // due to the same reason.
7199 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode,
7200 "{1to2}", "{x}">, EVEX_V128;
7201 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
7202 "{1to4}", "{y}">, EVEX_V256;
Craig Topperb8596e42016-11-14 01:53:29 +00007203
7204 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7205 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
7206 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7207 (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, f128mem:$src), 0>;
7208 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7209 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
7210 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7211 (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, f256mem:$src), 0>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007212 }
7213}
7214
7215// Convert Double to Signed/Unsigned Quardword
7216multiclass avx512_cvtpd2qq<bits<8> opc, string OpcodeStr,
7217 SDNode OpNode, SDNode OpNodeRnd> {
7218 let Predicates = [HasDQI] in {
7219 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode>,
7220 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f64_info,
7221 OpNodeRnd>, EVEX_V512;
7222 }
7223 let Predicates = [HasDQI, HasVLX] in {
7224 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode>,
7225 EVEX_V128;
7226 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode>,
7227 EVEX_V256;
7228 }
7229}
7230
7231// Convert Double to Signed/Unsigned Quardword with truncation
7232multiclass avx512_cvttpd2qq<bits<8> opc, string OpcodeStr,
7233 SDNode OpNode, SDNode OpNodeRnd> {
7234 let Predicates = [HasDQI] in {
7235 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode>,
7236 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f64_info,
7237 OpNodeRnd>, EVEX_V512;
7238 }
7239 let Predicates = [HasDQI, HasVLX] in {
7240 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode>,
7241 EVEX_V128;
7242 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode>,
7243 EVEX_V256;
7244 }
7245}
7246
7247// Convert Signed/Unsigned Quardword to Double
7248multiclass avx512_cvtqq2pd<bits<8> opc, string OpcodeStr,
7249 SDNode OpNode, SDNode OpNodeRnd> {
7250 let Predicates = [HasDQI] in {
7251 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i64_info, OpNode>,
7252 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f64_info, v8i64_info,
7253 OpNodeRnd>, EVEX_V512;
7254 }
7255 let Predicates = [HasDQI, HasVLX] in {
7256 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v2i64x_info, OpNode>,
7257 EVEX_V128;
7258 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i64x_info, OpNode>,
7259 EVEX_V256;
7260 }
7261}
7262
7263// Convert Float to Signed/Unsigned Quardword
7264multiclass avx512_cvtps2qq<bits<8> opc, string OpcodeStr,
7265 SDNode OpNode, SDNode OpNodeRnd> {
7266 let Predicates = [HasDQI] in {
7267 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode>,
7268 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f32x_info,
7269 OpNodeRnd>, EVEX_V512;
7270 }
7271 let Predicates = [HasDQI, HasVLX] in {
7272 // Explicitly specified broadcast string, since we take only 2 elements
7273 // from v4f32x_info source
7274 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,
Coby Tayree97e9cf62016-11-20 17:09:56 +00007275 "{1to2}", "", f64mem>, EVEX_V128;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007276 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode>,
7277 EVEX_V256;
7278 }
7279}
7280
7281// Convert Float to Signed/Unsigned Quardword with truncation
Craig Toppera39b6502016-12-10 06:02:48 +00007282multiclass avx512_cvttps2qq<bits<8> opc, string OpcodeStr, SDNode OpNode,
7283 SDNode OpNode128, SDNode OpNodeRnd> {
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007284 let Predicates = [HasDQI] in {
7285 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode>,
7286 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f32x_info,
7287 OpNodeRnd>, EVEX_V512;
7288 }
7289 let Predicates = [HasDQI, HasVLX] in {
7290 // Explicitly specified broadcast string, since we take only 2 elements
7291 // from v4f32x_info source
Craig Toppera39b6502016-12-10 06:02:48 +00007292 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode128,
Coby Tayree97e9cf62016-11-20 17:09:56 +00007293 "{1to2}", "", f64mem>, EVEX_V128;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007294 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode>,
7295 EVEX_V256;
7296 }
7297}
7298
7299// Convert Signed/Unsigned Quardword to Float
Simon Pilgrima3af7962016-11-24 12:13:46 +00007300multiclass avx512_cvtqq2ps<bits<8> opc, string OpcodeStr, SDNode OpNode,
7301 SDNode OpNode128, SDNode OpNodeRnd> {
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007302 let Predicates = [HasDQI] in {
7303 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i64_info, OpNode>,
7304 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8i64_info,
7305 OpNodeRnd>, EVEX_V512;
7306 }
7307 let Predicates = [HasDQI, HasVLX] in {
7308 // we need "x"/"y" suffixes in order to distinguish between 128 and 256
7309 // memory forms of these instructions in Asm Parcer. They have the same
7310 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
7311 // due to the same reason.
Simon Pilgrima3af7962016-11-24 12:13:46 +00007312 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2i64x_info, OpNode128,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007313 "{1to2}", "{x}">, EVEX_V128;
7314 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i64x_info, OpNode,
7315 "{1to4}", "{y}">, EVEX_V256;
Craig Topperb8596e42016-11-14 01:53:29 +00007316
7317 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7318 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
7319 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7320 (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, i128mem:$src), 0>;
7321 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7322 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
7323 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7324 (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, i256mem:$src), 0>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007325 }
7326}
7327
Simon Pilgrima3af7962016-11-24 12:13:46 +00007328defm VCVTDQ2PD : avx512_cvtdq2pd<0xE6, "vcvtdq2pd", sint_to_fp, X86VSintToFP>,
Coby Tayree97e9cf62016-11-20 17:09:56 +00007329 XS, EVEX_CD8<32, CD8VH>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007330
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007331defm VCVTDQ2PS : avx512_cvtdq2ps<0x5B, "vcvtdq2ps", sint_to_fp,
7332 X86VSintToFpRnd>,
7333 PS, EVEX_CD8<32, CD8VF>;
7334
7335defm VCVTTPS2DQ : avx512_cvttps2dq<0x5B, "vcvttps2dq", fp_to_sint,
Craig Topper3174b6e2016-09-23 06:24:39 +00007336 X86cvttp2siRnd>,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007337 XS, EVEX_CD8<32, CD8VF>;
7338
Simon Pilgrima3af7962016-11-24 12:13:46 +00007339defm VCVTTPD2DQ : avx512_cvttpd2dq<0xE6, "vcvttpd2dq", fp_to_sint, X86cvttp2si,
Craig Topper3174b6e2016-09-23 06:24:39 +00007340 X86cvttp2siRnd>,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007341 PD, VEX_W, EVEX_CD8<64, CD8VF>;
7342
7343defm VCVTTPS2UDQ : avx512_cvttps2dq<0x78, "vcvttps2udq", fp_to_uint,
Craig Topper3174b6e2016-09-23 06:24:39 +00007344 X86cvttp2uiRnd>, PS,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007345 EVEX_CD8<32, CD8VF>;
7346
Craig Topperf334ac192016-11-09 07:48:51 +00007347defm VCVTTPD2UDQ : avx512_cvttpd2dq<0x78, "vcvttpd2udq", fp_to_uint,
Simon Pilgrima3af7962016-11-24 12:13:46 +00007348 X86cvttp2ui, X86cvttp2uiRnd>, PS, VEX_W,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007349 EVEX_CD8<64, CD8VF>;
7350
Simon Pilgrima3af7962016-11-24 12:13:46 +00007351defm VCVTUDQ2PD : avx512_cvtdq2pd<0x7A, "vcvtudq2pd", uint_to_fp, X86VUintToFP>,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007352 XS, EVEX_CD8<32, CD8VH>;
7353
7354defm VCVTUDQ2PS : avx512_cvtdq2ps<0x7A, "vcvtudq2ps", uint_to_fp,
7355 X86VUintToFpRnd>, XD,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007356 EVEX_CD8<32, CD8VF>;
7357
Craig Topper19e04b62016-05-19 06:13:58 +00007358defm VCVTPS2DQ : avx512_cvtps2dq<0x5B, "vcvtps2dq", X86cvtp2Int,
7359 X86cvtp2IntRnd>, PD, EVEX_CD8<32, CD8VF>;
Elena Demikhovskyde3f7512014-01-01 15:12:34 +00007360
Craig Topper19e04b62016-05-19 06:13:58 +00007361defm VCVTPD2DQ : avx512_cvtpd2dq<0xE6, "vcvtpd2dq", X86cvtp2Int,
7362 X86cvtp2IntRnd>, XD, VEX_W,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007363 EVEX_CD8<64, CD8VF>;
Michael Liao5bf95782014-12-04 05:20:33 +00007364
Craig Topper19e04b62016-05-19 06:13:58 +00007365defm VCVTPS2UDQ : avx512_cvtps2dq<0x79, "vcvtps2udq", X86cvtp2UInt,
7366 X86cvtp2UIntRnd>,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007367 PS, EVEX_CD8<32, CD8VF>;
Craig Topper19e04b62016-05-19 06:13:58 +00007368defm VCVTPD2UDQ : avx512_cvtpd2dq<0x79, "vcvtpd2udq", X86cvtp2UInt,
7369 X86cvtp2UIntRnd>, VEX_W,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007370 PS, EVEX_CD8<64, CD8VF>;
Elena Demikhovskyde3f7512014-01-01 15:12:34 +00007371
Craig Topper19e04b62016-05-19 06:13:58 +00007372defm VCVTPD2QQ : avx512_cvtpd2qq<0x7B, "vcvtpd2qq", X86cvtp2Int,
7373 X86cvtp2IntRnd>, VEX_W,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007374 PD, EVEX_CD8<64, CD8VF>;
Michael Liao5bf95782014-12-04 05:20:33 +00007375
Craig Topper19e04b62016-05-19 06:13:58 +00007376defm VCVTPS2QQ : avx512_cvtps2qq<0x7B, "vcvtps2qq", X86cvtp2Int,
7377 X86cvtp2IntRnd>, PD, EVEX_CD8<32, CD8VH>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007378
Craig Topper19e04b62016-05-19 06:13:58 +00007379defm VCVTPD2UQQ : avx512_cvtpd2qq<0x79, "vcvtpd2uqq", X86cvtp2UInt,
7380 X86cvtp2UIntRnd>, VEX_W,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007381 PD, EVEX_CD8<64, CD8VF>;
7382
Craig Topper19e04b62016-05-19 06:13:58 +00007383defm VCVTPS2UQQ : avx512_cvtps2qq<0x79, "vcvtps2uqq", X86cvtp2UInt,
7384 X86cvtp2UIntRnd>, PD, EVEX_CD8<32, CD8VH>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007385
7386defm VCVTTPD2QQ : avx512_cvttpd2qq<0x7A, "vcvttpd2qq", fp_to_sint,
Craig Topper3174b6e2016-09-23 06:24:39 +00007387 X86cvttp2siRnd>, VEX_W,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007388 PD, EVEX_CD8<64, CD8VF>;
7389
Craig Toppera39b6502016-12-10 06:02:48 +00007390defm VCVTTPS2QQ : avx512_cvttps2qq<0x7A, "vcvttps2qq", fp_to_sint, X86cvttp2si,
Craig Topper3174b6e2016-09-23 06:24:39 +00007391 X86cvttp2siRnd>, PD, EVEX_CD8<32, CD8VH>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007392
7393defm VCVTTPD2UQQ : avx512_cvttpd2qq<0x78, "vcvttpd2uqq", fp_to_uint,
Craig Topper3174b6e2016-09-23 06:24:39 +00007394 X86cvttp2uiRnd>, VEX_W,
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007395 PD, EVEX_CD8<64, CD8VF>;
7396
Craig Toppera39b6502016-12-10 06:02:48 +00007397defm VCVTTPS2UQQ : avx512_cvttps2qq<0x78, "vcvttps2uqq", fp_to_uint, X86cvttp2ui,
Craig Topper3174b6e2016-09-23 06:24:39 +00007398 X86cvttp2uiRnd>, PD, EVEX_CD8<32, CD8VH>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007399
7400defm VCVTQQ2PD : avx512_cvtqq2pd<0xE6, "vcvtqq2pd", sint_to_fp,
Craig Topper19e04b62016-05-19 06:13:58 +00007401 X86VSintToFpRnd>, VEX_W, XS, EVEX_CD8<64, CD8VF>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007402
7403defm VCVTUQQ2PD : avx512_cvtqq2pd<0x7A, "vcvtuqq2pd", uint_to_fp,
Craig Topper19e04b62016-05-19 06:13:58 +00007404 X86VUintToFpRnd>, VEX_W, XS, EVEX_CD8<64, CD8VF>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007405
Simon Pilgrima3af7962016-11-24 12:13:46 +00007406defm VCVTQQ2PS : avx512_cvtqq2ps<0x5B, "vcvtqq2ps", sint_to_fp, X86VSintToFP,
Craig Topper19e04b62016-05-19 06:13:58 +00007407 X86VSintToFpRnd>, VEX_W, PS, EVEX_CD8<64, CD8VF>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007408
Simon Pilgrima3af7962016-11-24 12:13:46 +00007409defm VCVTUQQ2PS : avx512_cvtqq2ps<0x7A, "vcvtuqq2ps", uint_to_fp, X86VUintToFP,
Craig Topper19e04b62016-05-19 06:13:58 +00007410 X86VUintToFpRnd>, VEX_W, XD, EVEX_CD8<64, CD8VF>;
Elena Demikhovsky0f370932015-07-13 13:26:20 +00007411
Craig Toppere38c57a2015-11-27 05:44:02 +00007412let Predicates = [HasAVX512, NoVLX] in {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007413def : Pat<(v8i32 (fp_to_uint (v8f32 VR256X:$src1))),
Michael Liao5bf95782014-12-04 05:20:33 +00007414 (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr
Craig Topper61403202016-09-19 02:53:43 +00007415 (v16f32 (INSERT_SUBREG (IMPLICIT_DEF),
7416 VR256X:$src1, sub_ymm)))), sub_ymm)>;
Michael Liao5bf95782014-12-04 05:20:33 +00007417
Elena Demikhovsky3dcfbdf2014-04-08 07:24:02 +00007418def : Pat<(v4i32 (fp_to_uint (v4f32 VR128X:$src1))),
7419 (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr
Craig Topper61403202016-09-19 02:53:43 +00007420 (v16f32 (INSERT_SUBREG (IMPLICIT_DEF),
7421 VR128X:$src1, sub_xmm)))), sub_xmm)>;
Elena Demikhovsky3dcfbdf2014-04-08 07:24:02 +00007422
Elena Demikhovsky95629ca2016-03-29 06:33:41 +00007423def : Pat<(v4i32 (fp_to_uint (v4f64 VR256X:$src1))),
7424 (EXTRACT_SUBREG (v8i32 (VCVTTPD2UDQZrr
Craig Topper61403202016-09-19 02:53:43 +00007425 (v8f64 (INSERT_SUBREG (IMPLICIT_DEF),
7426 VR256X:$src1, sub_ymm)))), sub_xmm)>;
Elena Demikhovsky95629ca2016-03-29 06:33:41 +00007427
Simon Pilgrima3af7962016-11-24 12:13:46 +00007428def : Pat<(v4i32 (X86cvttp2ui (v2f64 VR128X:$src))),
Craig Topperf334ac192016-11-09 07:48:51 +00007429 (EXTRACT_SUBREG (v8i32 (VCVTTPD2UDQZrr
7430 (v8f64 (INSERT_SUBREG (IMPLICIT_DEF),
7431 VR128X:$src, sub_xmm)))), sub_xmm)>;
7432
Elena Demikhovsky3dcfbdf2014-04-08 07:24:02 +00007433def : Pat<(v8f32 (uint_to_fp (v8i32 VR256X:$src1))),
7434 (EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr
Craig Topper61403202016-09-19 02:53:43 +00007435 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF),
7436 VR256X:$src1, sub_ymm)))), sub_ymm)>;
Michael Liao5bf95782014-12-04 05:20:33 +00007437
Elena Demikhovsky3dcfbdf2014-04-08 07:24:02 +00007438def : Pat<(v4f32 (uint_to_fp (v4i32 VR128X:$src1))),
7439 (EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr
Craig Topper61403202016-09-19 02:53:43 +00007440 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF),
7441 VR128X:$src1, sub_xmm)))), sub_xmm)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007442
Cameron McInallyf10a7c92014-06-18 14:04:37 +00007443def : Pat<(v4f64 (uint_to_fp (v4i32 VR128X:$src1))),
7444 (EXTRACT_SUBREG (v8f64 (VCVTUDQ2PDZrr
Craig Topper61403202016-09-19 02:53:43 +00007445 (v8i32 (INSERT_SUBREG (IMPLICIT_DEF),
7446 VR128X:$src1, sub_xmm)))), sub_ymm)>;
Simon Pilgrim096b6d42016-11-20 14:03:23 +00007447
Simon Pilgrima3af7962016-11-24 12:13:46 +00007448def : Pat<(v2f64 (X86VUintToFP (v4i32 VR128X:$src1))),
Simon Pilgrim096b6d42016-11-20 14:03:23 +00007449 (EXTRACT_SUBREG (v8f64 (VCVTUDQ2PDZrr
7450 (v8i32 (INSERT_SUBREG (IMPLICIT_DEF),
7451 VR128X:$src1, sub_xmm)))), sub_xmm)>;
Elena Demikhovskyde3f7512014-01-01 15:12:34 +00007452}
7453
Simon Pilgrim4ddc92b2016-10-18 07:42:15 +00007454let Predicates = [HasAVX512, HasVLX] in {
Simon Pilgrim3ce6a542016-11-23 22:35:06 +00007455 let AddedComplexity = 15 in {
7456 def : Pat<(X86vzmovl (v2i64 (bitconvert
7457 (v4i32 (X86cvtp2Int (v2f64 VR128X:$src)))))),
Craig Topper5ef13ba2016-12-26 07:26:07 +00007458 (VCVTPD2DQZ128rr VR128X:$src)>;
Simon Pilgrim3ce6a542016-11-23 22:35:06 +00007459 def : Pat<(v4i32 (bitconvert (X86vzmovl (v2i64 (bitconvert
7460 (v4i32 (X86cvtp2UInt (v2f64 VR128X:$src)))))))),
Craig Topper5ef13ba2016-12-26 07:26:07 +00007461 (VCVTPD2UDQZ128rr VR128X:$src)>;
Simon Pilgrim3ce6a542016-11-23 22:35:06 +00007462 def : Pat<(X86vzmovl (v2i64 (bitconvert
Simon Pilgrima3af7962016-11-24 12:13:46 +00007463 (v4i32 (X86cvttp2si (v2f64 VR128X:$src)))))),
Craig Topper5ef13ba2016-12-26 07:26:07 +00007464 (VCVTTPD2DQZ128rr VR128X:$src)>;
Simon Pilgrim3ce6a542016-11-23 22:35:06 +00007465 def : Pat<(v4i32 (bitconvert (X86vzmovl (v2i64 (bitconvert
Simon Pilgrima3af7962016-11-24 12:13:46 +00007466 (v4i32 (X86cvttp2ui (v2f64 VR128X:$src)))))))),
Craig Topper5ef13ba2016-12-26 07:26:07 +00007467 (VCVTTPD2UDQZ128rr VR128X:$src)>;
Simon Pilgrim3ce6a542016-11-23 22:35:06 +00007468 }
Simon Pilgrim4ddc92b2016-10-18 07:42:15 +00007469}
7470
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007471let Predicates = [HasAVX512] in {
Michael Kuperstein2bc3d4d2016-08-18 20:08:15 +00007472 def : Pat<(v8f32 (fpround (loadv8f64 addr:$src))),
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007473 (VCVTPD2PSZrm addr:$src)>;
7474 def : Pat<(v8f64 (extloadv8f32 addr:$src)),
7475 (VCVTPS2PDZrm addr:$src)>;
7476}
7477
Simon Pilgrim7c26a6f2016-11-24 14:02:30 +00007478let Predicates = [HasDQI, HasVLX] in {
7479 let AddedComplexity = 15 in {
7480 def : Pat<(X86vzmovl (v2f64 (bitconvert
7481 (v4f32 (X86VSintToFP (v2i64 VR128X:$src)))))),
Craig Topper5ef13ba2016-12-26 07:26:07 +00007482 (VCVTQQ2PSZ128rr VR128X:$src)>;
Simon Pilgrim7c26a6f2016-11-24 14:02:30 +00007483 def : Pat<(X86vzmovl (v2f64 (bitconvert
7484 (v4f32 (X86VUintToFP (v2i64 VR128X:$src)))))),
Craig Topper5ef13ba2016-12-26 07:26:07 +00007485 (VCVTUQQ2PSZ128rr VR128X:$src)>;
Simon Pilgrim7c26a6f2016-11-24 14:02:30 +00007486 }
7487}
7488
Simon Pilgrim4e9b9cb2016-11-23 14:01:18 +00007489let Predicates = [HasDQI, NoVLX] in {
Simon Pilgrim841d7ca2016-11-24 14:46:55 +00007490def : Pat<(v2i64 (fp_to_sint (v2f64 VR128X:$src1))),
7491 (EXTRACT_SUBREG (v8i64 (VCVTTPD2QQZrr
7492 (v8f64 (INSERT_SUBREG (IMPLICIT_DEF),
7493 VR128X:$src1, sub_xmm)))), sub_xmm)>;
7494
Simon Pilgrim4e9b9cb2016-11-23 14:01:18 +00007495def : Pat<(v4i64 (fp_to_sint (v4f32 VR128X:$src1))),
7496 (EXTRACT_SUBREG (v8i64 (VCVTTPS2QQZrr
7497 (v8f32 (INSERT_SUBREG (IMPLICIT_DEF),
7498 VR128X:$src1, sub_xmm)))), sub_ymm)>;
7499
7500def : Pat<(v4i64 (fp_to_sint (v4f64 VR256X:$src1))),
7501 (EXTRACT_SUBREG (v8i64 (VCVTTPD2QQZrr
7502 (v8f64 (INSERT_SUBREG (IMPLICIT_DEF),
7503 VR256X:$src1, sub_ymm)))), sub_ymm)>;
7504
Simon Pilgrim841d7ca2016-11-24 14:46:55 +00007505def : Pat<(v2i64 (fp_to_uint (v2f64 VR128X:$src1))),
7506 (EXTRACT_SUBREG (v8i64 (VCVTTPD2UQQZrr
7507 (v8f64 (INSERT_SUBREG (IMPLICIT_DEF),
7508 VR128X:$src1, sub_xmm)))), sub_xmm)>;
7509
Simon Pilgrim4e9b9cb2016-11-23 14:01:18 +00007510def : Pat<(v4i64 (fp_to_uint (v4f32 VR128X:$src1))),
7511 (EXTRACT_SUBREG (v8i64 (VCVTTPS2UQQZrr
7512 (v8f32 (INSERT_SUBREG (IMPLICIT_DEF),
7513 VR128X:$src1, sub_xmm)))), sub_ymm)>;
7514
7515def : Pat<(v4i64 (fp_to_uint (v4f64 VR256X:$src1))),
7516 (EXTRACT_SUBREG (v8i64 (VCVTTPD2UQQZrr
7517 (v8f64 (INSERT_SUBREG (IMPLICIT_DEF),
7518 VR256X:$src1, sub_ymm)))), sub_ymm)>;
7519
7520def : Pat<(v4f32 (sint_to_fp (v4i64 VR256X:$src1))),
7521 (EXTRACT_SUBREG (v8f32 (VCVTQQ2PSZrr
7522 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
7523 VR256X:$src1, sub_ymm)))), sub_xmm)>;
7524
Simon Pilgrim841d7ca2016-11-24 14:46:55 +00007525def : Pat<(v2f64 (sint_to_fp (v2i64 VR128X:$src1))),
7526 (EXTRACT_SUBREG (v8f64 (VCVTQQ2PDZrr
7527 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
7528 VR128X:$src1, sub_xmm)))), sub_xmm)>;
7529
Simon Pilgrim4e9b9cb2016-11-23 14:01:18 +00007530def : Pat<(v4f64 (sint_to_fp (v4i64 VR256X:$src1))),
7531 (EXTRACT_SUBREG (v8f64 (VCVTQQ2PDZrr
7532 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
7533 VR256X:$src1, sub_ymm)))), sub_ymm)>;
7534
7535def : Pat<(v4f32 (uint_to_fp (v4i64 VR256X:$src1))),
7536 (EXTRACT_SUBREG (v8f32 (VCVTUQQ2PSZrr
7537 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
7538 VR256X:$src1, sub_ymm)))), sub_xmm)>;
7539
Simon Pilgrim841d7ca2016-11-24 14:46:55 +00007540def : Pat<(v2f64 (uint_to_fp (v2i64 VR128X:$src1))),
7541 (EXTRACT_SUBREG (v8f64 (VCVTUQQ2PDZrr
7542 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
7543 VR128X:$src1, sub_xmm)))), sub_xmm)>;
7544
Simon Pilgrim4e9b9cb2016-11-23 14:01:18 +00007545def : Pat<(v4f64 (uint_to_fp (v4i64 VR256X:$src1))),
7546 (EXTRACT_SUBREG (v8f64 (VCVTUQQ2PDZrr
7547 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
7548 VR256X:$src1, sub_ymm)))), sub_ymm)>;
7549}
7550
Elena Demikhovskydd0794e2013-10-24 07:16:35 +00007551//===----------------------------------------------------------------------===//
7552// Half precision conversion instructions
7553//===----------------------------------------------------------------------===//
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007554multiclass avx512_cvtph2ps<X86VectorVTInfo _dest, X86VectorVTInfo _src,
Asaf Badouh7c522452015-10-22 14:01:16 +00007555 X86MemOperand x86memop, PatFrag ld_frag> {
7556 defm rr : AVX512_maskable<0x13, MRMSrcReg, _dest ,(outs _dest.RC:$dst), (ins _src.RC:$src),
7557 "vcvtph2ps", "$src", "$src",
7558 (X86cvtph2ps (_src.VT _src.RC:$src),
7559 (i32 FROUND_CURRENT))>, T8PD;
Craig Toppere1cac152016-06-07 07:27:54 +00007560 defm rm : AVX512_maskable<0x13, MRMSrcMem, _dest, (outs _dest.RC:$dst), (ins x86memop:$src),
7561 "vcvtph2ps", "$src", "$src",
7562 (X86cvtph2ps (_src.VT (bitconvert (ld_frag addr:$src))),
7563 (i32 FROUND_CURRENT))>, T8PD;
Asaf Badouh7c522452015-10-22 14:01:16 +00007564}
7565
Asaf Badouhc7cb8802015-10-27 15:37:17 +00007566multiclass avx512_cvtph2ps_sae<X86VectorVTInfo _dest, X86VectorVTInfo _src> {
Asaf Badouh7c522452015-10-22 14:01:16 +00007567 defm rb : AVX512_maskable<0x13, MRMSrcReg, _dest ,(outs _dest.RC:$dst), (ins _src.RC:$src),
7568 "vcvtph2ps", "{sae}, $src", "$src, {sae}",
7569 (X86cvtph2ps (_src.VT _src.RC:$src),
7570 (i32 FROUND_NO_EXC))>, T8PD, EVEX_B;
7571
7572}
7573
7574let Predicates = [HasAVX512] in {
7575 defm VCVTPH2PSZ : avx512_cvtph2ps<v16f32_info, v16i16x_info, f256mem, loadv4i64>,
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007576 avx512_cvtph2ps_sae<v16f32_info, v16i16x_info>,
Asaf Badouh7c522452015-10-22 14:01:16 +00007577 EVEX, EVEX_V512, EVEX_CD8<32, CD8VH>;
7578 let Predicates = [HasVLX] in {
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007579 defm VCVTPH2PSZ256 : avx512_cvtph2ps<v8f32x_info, v8i16x_info, f128mem,
Asaf Badouh7c522452015-10-22 14:01:16 +00007580 loadv2i64>,EVEX, EVEX_V256, EVEX_CD8<32, CD8VH>;
7581 defm VCVTPH2PSZ128 : avx512_cvtph2ps<v4f32x_info, v8i16x_info, f64mem,
7582 loadv2i64>, EVEX, EVEX_V128, EVEX_CD8<32, CD8VH>;
7583 }
Elena Demikhovskydd0794e2013-10-24 07:16:35 +00007584}
7585
Simon Pilgrim18bcf932016-02-03 09:41:59 +00007586multiclass avx512_cvtps2ph<X86VectorVTInfo _dest, X86VectorVTInfo _src,
Asaf Badouhc7cb8802015-10-27 15:37:17 +00007587 X86MemOperand x86memop> {
7588 defm rr : AVX512_maskable<0x1D, MRMDestReg, _dest ,(outs _dest.RC:$dst),
Igor Breger73ee8ba2016-05-31 08:04:21 +00007589 (ins _src.RC:$src1, i32u8imm:$src2),
7590 "vcvtps2ph", "$src2, $src1", "$src1, $src2",
Asaf Badouhc7cb8802015-10-27 15:37:17 +00007591 (X86cvtps2ph (_src.VT _src.RC:$src1),
Craig Topperd8688702016-09-21 03:58:44 +00007592 (i32 imm:$src2)),
Vyacheslav Klochkov6daefcf2016-08-11 22:07:33 +00007593 NoItinerary, 0, 0, X86select>, AVX512AIi8Base;
Craig Toppere1cac152016-06-07 07:27:54 +00007594 def mr : AVX512AIi8<0x1D, MRMDestMem, (outs),
7595 (ins x86memop:$dst, _src.RC:$src1, i32u8imm:$src2),
7596 "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}",
7597 [(store (_dest.VT (X86cvtps2ph (_src.VT _src.RC:$src1),
Craig Topperd8688702016-09-21 03:58:44 +00007598 (i32 imm:$src2))),
Craig Toppere1cac152016-06-07 07:27:54 +00007599 addr:$dst)]>;
7600 let hasSideEffects = 0, mayStore = 1 in
7601 def mrk : AVX512AIi8<0x1D, MRMDestMem, (outs),
7602 (ins x86memop:$dst, _dest.KRCWM:$mask, _src.RC:$src1, i32u8imm:$src2),
7603 "vcvtps2ph\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
7604 []>, EVEX_K;
Elena Demikhovskydd0794e2013-10-24 07:16:35 +00007605}
Asaf Badouhc7cb8802015-10-27 15:37:17 +00007606multiclass avx512_cvtps2ph_sae<X86VectorVTInfo _dest, X86VectorVTInfo _src> {
Craig Topperd8688702016-09-21 03:58:44 +00007607 let hasSideEffects = 0 in
7608 defm rb : AVX512_maskable_in_asm<0x1D, MRMDestReg, _dest,
7609 (outs _dest.RC:$dst),
Igor Breger73ee8ba2016-05-31 08:04:21 +00007610 (ins _src.RC:$src1, i32u8imm:$src2),
7611 "vcvtps2ph", "$src2, {sae}, $src1", "$src1, {sae}, $src2",
Craig Topperd8688702016-09-21 03:58:44 +00007612 []>, EVEX_B, AVX512AIi8Base;
Asaf Badouhc7cb8802015-10-27 15:37:17 +00007613}
7614let Predicates = [HasAVX512] in {
7615 defm VCVTPS2PHZ : avx512_cvtps2ph<v16i16x_info, v16f32_info, f256mem>,
7616 avx512_cvtps2ph_sae<v16i16x_info, v16f32_info>,
7617 EVEX, EVEX_V512, EVEX_CD8<32, CD8VH>;
7618 let Predicates = [HasVLX] in {
7619 defm VCVTPS2PHZ256 : avx512_cvtps2ph<v8i16x_info, v8f32x_info, f128mem>,
7620 EVEX, EVEX_V256, EVEX_CD8<32, CD8VH>;
Ayman Musaf77219e2017-02-13 09:55:48 +00007621 defm VCVTPS2PHZ128 : avx512_cvtps2ph<v8i16x_info, v4f32x_info, f64mem>,
Asaf Badouhc7cb8802015-10-27 15:37:17 +00007622 EVEX, EVEX_V128, EVEX_CD8<32, CD8VH>;
7623 }
7624}
Asaf Badouh2489f352015-12-02 08:17:51 +00007625
Craig Topper9820e342016-09-20 05:44:47 +00007626// Patterns for matching conversions from float to half-float and vice versa.
Craig Topperb3b50332016-09-19 02:53:37 +00007627let Predicates = [HasVLX] in {
7628 // Use MXCSR.RC for rounding instead of explicitly specifying the default
7629 // rounding mode (Nearest-Even, encoded as 0). Both are equivalent in the
7630 // configurations we support (the default). However, falling back to MXCSR is
7631 // more consistent with other instructions, which are always controlled by it.
7632 // It's encoded as 0b100.
7633 def : Pat<(fp_to_f16 FR32X:$src),
7634 (i16 (EXTRACT_SUBREG (VMOVPDI2DIZrr (VCVTPS2PHZ128rr
7635 (COPY_TO_REGCLASS FR32X:$src, VR128X), 4)), sub_16bit))>;
7636
7637 def : Pat<(f16_to_fp GR16:$src),
7638 (f32 (COPY_TO_REGCLASS (VCVTPH2PSZ128rr
7639 (COPY_TO_REGCLASS (MOVSX32rr16 GR16:$src), VR128X)), FR32X)) >;
7640
7641 def : Pat<(f16_to_fp (i16 (fp_to_f16 FR32X:$src))),
7642 (f32 (COPY_TO_REGCLASS (VCVTPH2PSZ128rr
7643 (VCVTPS2PHZ128rr (COPY_TO_REGCLASS FR32X:$src, VR128X), 4)), FR32X)) >;
7644}
7645
Craig Topper9820e342016-09-20 05:44:47 +00007646// Patterns for matching float to half-float conversion when AVX512 is supported
7647// but F16C isn't. In that case we have to use 512-bit vectors.
7648let Predicates = [HasAVX512, NoVLX, NoF16C] in {
7649 def : Pat<(fp_to_f16 FR32X:$src),
7650 (i16 (EXTRACT_SUBREG
7651 (VMOVPDI2DIZrr
7652 (v8i16 (EXTRACT_SUBREG
7653 (VCVTPS2PHZrr
7654 (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)),
7655 (v4f32 (COPY_TO_REGCLASS FR32X:$src, VR128X)),
7656 sub_xmm), 4), sub_xmm))), sub_16bit))>;
7657
7658 def : Pat<(f16_to_fp GR16:$src),
7659 (f32 (COPY_TO_REGCLASS
7660 (v4f32 (EXTRACT_SUBREG
7661 (VCVTPH2PSZrr
7662 (INSERT_SUBREG (v16i16 (IMPLICIT_DEF)),
7663 (v8i16 (COPY_TO_REGCLASS (MOVSX32rr16 GR16:$src), VR128X)),
7664 sub_xmm)), sub_xmm)), FR32X))>;
7665
7666 def : Pat<(f16_to_fp (i16 (fp_to_f16 FR32X:$src))),
7667 (f32 (COPY_TO_REGCLASS
7668 (v4f32 (EXTRACT_SUBREG
7669 (VCVTPH2PSZrr
7670 (VCVTPS2PHZrr (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)),
7671 (v4f32 (COPY_TO_REGCLASS FR32X:$src, VR128X)),
7672 sub_xmm), 4)), sub_xmm)), FR32X))>;
7673}
7674
Asaf Badouh2489f352015-12-02 08:17:51 +00007675// Unordered/Ordered scalar fp compare with Sea and set EFLAGS
Craig Topper7e664da2016-09-24 21:42:43 +00007676multiclass avx512_ord_cmp_sae<bits<8> opc, X86VectorVTInfo _,
Asaf Badouh2489f352015-12-02 08:17:51 +00007677 string OpcodeStr> {
Craig Topper07a7d562017-07-23 03:59:39 +00007678 let hasSideEffects = 0 in
Asaf Badouh2489f352015-12-02 08:17:51 +00007679 def rb: AVX512<opc, MRMSrcReg, (outs), (ins _.RC:$src1, _.RC:$src2),
7680 !strconcat(OpcodeStr, "\t{{sae}, $src2, $src1|$src1, $src2, {sae}}"),
Craig Topper7e664da2016-09-24 21:42:43 +00007681 [], IIC_SSE_COMIS_RR>, EVEX, EVEX_B, VEX_LIG, EVEX_V128,
Asaf Badouh2489f352015-12-02 08:17:51 +00007682 Sched<[WriteFAdd]>;
7683}
7684
7685let Defs = [EFLAGS], Predicates = [HasAVX512] in {
Craig Topper7e664da2016-09-24 21:42:43 +00007686 defm VUCOMISSZ : avx512_ord_cmp_sae<0x2E, v4f32x_info, "vucomiss">,
Asaf Badouh2489f352015-12-02 08:17:51 +00007687 AVX512PSIi8Base, EVEX_CD8<32, CD8VT1>;
Craig Topper7e664da2016-09-24 21:42:43 +00007688 defm VUCOMISDZ : avx512_ord_cmp_sae<0x2E, v2f64x_info, "vucomisd">,
Asaf Badouh2489f352015-12-02 08:17:51 +00007689 AVX512PDIi8Base, VEX_W, EVEX_CD8<64, CD8VT1>;
Craig Topper7e664da2016-09-24 21:42:43 +00007690 defm VCOMISSZ : avx512_ord_cmp_sae<0x2F, v4f32x_info, "vcomiss">,
Asaf Badouh2489f352015-12-02 08:17:51 +00007691 AVX512PSIi8Base, EVEX_CD8<32, CD8VT1>;
Craig Topper7e664da2016-09-24 21:42:43 +00007692 defm VCOMISDZ : avx512_ord_cmp_sae<0x2F, v2f64x_info, "vcomisd">,
Asaf Badouh2489f352015-12-02 08:17:51 +00007693 AVX512PDIi8Base, VEX_W, EVEX_CD8<64, CD8VT1>;
7694}
7695
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007696let Defs = [EFLAGS], Predicates = [HasAVX512] in {
7697 defm VUCOMISSZ : sse12_ord_cmp<0x2E, FR32X, X86cmp, f32, f32mem, loadf32,
Craig Topper5ccb6172014-02-18 00:21:49 +00007698 "ucomiss">, PS, EVEX, VEX_LIG,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007699 EVEX_CD8<32, CD8VT1>;
7700 defm VUCOMISDZ : sse12_ord_cmp<0x2E, FR64X, X86cmp, f64, f64mem, loadf64,
Craig Topperae11aed2014-01-14 07:41:20 +00007701 "ucomisd">, PD, EVEX,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007702 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
7703 let Pattern = []<dag> in {
Marina Yatsina7a4e1ba2015-08-20 11:21:36 +00007704 defm VCOMISSZ : sse12_ord_cmp<0x2F, FR32X, undef, f32, f32mem, loadf32,
Craig Topper5ccb6172014-02-18 00:21:49 +00007705 "comiss">, PS, EVEX, VEX_LIG,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007706 EVEX_CD8<32, CD8VT1>;
Marina Yatsina7a4e1ba2015-08-20 11:21:36 +00007707 defm VCOMISDZ : sse12_ord_cmp<0x2F, FR64X, undef, f64, f64mem, loadf64,
Craig Topperae11aed2014-01-14 07:41:20 +00007708 "comisd">, PD, EVEX,
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007709 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
7710 }
Craig Topper9dd48c82014-01-02 17:28:14 +00007711 let isCodeGenOnly = 1 in {
Ayman Musa02f95332017-01-04 08:21:54 +00007712 defm Int_VUCOMISSZ : sse12_ord_cmp_int<0x2E, VR128X, X86ucomi, v4f32, ssmem,
7713 sse_load_f32, "ucomiss">, PS, EVEX, VEX_LIG,
Craig Topper9dd48c82014-01-02 17:28:14 +00007714 EVEX_CD8<32, CD8VT1>;
Ayman Musa02f95332017-01-04 08:21:54 +00007715 defm Int_VUCOMISDZ : sse12_ord_cmp_int<0x2E, VR128X, X86ucomi, v2f64, sdmem,
7716 sse_load_f64, "ucomisd">, PD, EVEX,
Craig Topper9dd48c82014-01-02 17:28:14 +00007717 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007718
Ayman Musa02f95332017-01-04 08:21:54 +00007719 defm Int_VCOMISSZ : sse12_ord_cmp_int<0x2F, VR128X, X86comi, v4f32, ssmem,
7720 sse_load_f32, "comiss">, PS, EVEX, VEX_LIG,
Craig Topper9dd48c82014-01-02 17:28:14 +00007721 EVEX_CD8<32, CD8VT1>;
Ayman Musa02f95332017-01-04 08:21:54 +00007722 defm Int_VCOMISDZ : sse12_ord_cmp_int<0x2F, VR128X, X86comi, v2f64, sdmem,
7723 sse_load_f64, "comisd">, PD, EVEX,
Craig Topper9dd48c82014-01-02 17:28:14 +00007724 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
7725 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007726}
Michael Liao5bf95782014-12-04 05:20:33 +00007727
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00007728/// avx512_fp14_s rcp14ss, rcp14sd, rsqrt14ss, rsqrt14sd
Asaf Badouheaf2da12015-09-21 10:23:53 +00007729multiclass avx512_fp14_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
7730 X86VectorVTInfo _> {
Craig Topper176f3312017-02-25 19:18:11 +00007731 let Predicates = [HasAVX512], ExeDomain = _.ExeDomain in {
Asaf Badouheaf2da12015-09-21 10:23:53 +00007732 defm rr : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
7733 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
7734 "$src2, $src1", "$src1, $src2",
7735 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>, EVEX_4V;
Asaf Badouheaf2da12015-09-21 10:23:53 +00007736 defm rm : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
Igor Breger4511e762016-02-22 11:48:27 +00007737 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
Asaf Badouheaf2da12015-09-21 10:23:53 +00007738 "$src2, $src1", "$src1, $src2",
7739 (OpNode (_.VT _.RC:$src1),
7740 (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))))>, EVEX_4V;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007741}
7742}
7743
Asaf Badouheaf2da12015-09-21 10:23:53 +00007744defm VRCP14SS : avx512_fp14_s<0x4D, "vrcp14ss", X86frcp14s, f32x_info>,
7745 EVEX_CD8<32, CD8VT1>, T8PD;
7746defm VRCP14SD : avx512_fp14_s<0x4D, "vrcp14sd", X86frcp14s, f64x_info>,
7747 VEX_W, EVEX_CD8<64, CD8VT1>, T8PD;
7748defm VRSQRT14SS : avx512_fp14_s<0x4F, "vrsqrt14ss", X86frsqrt14s, f32x_info>,
7749 EVEX_CD8<32, CD8VT1>, T8PD;
7750defm VRSQRT14SD : avx512_fp14_s<0x4F, "vrsqrt14sd", X86frsqrt14s, f64x_info>,
7751 VEX_W, EVEX_CD8<64, CD8VT1>, T8PD;
Elena Demikhovskya3a71402013-10-09 08:16:14 +00007752
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00007753/// avx512_fp14_p rcp14ps, rcp14pd, rsqrt14ps, rsqrt14pd
7754multiclass avx512_fp14_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
Robert Khasanov3e534c92014-10-28 16:37:13 +00007755 X86VectorVTInfo _> {
Craig Topper176f3312017-02-25 19:18:11 +00007756 let ExeDomain = _.ExeDomain in {
Robert Khasanov3e534c92014-10-28 16:37:13 +00007757 defm r: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
7758 (ins _.RC:$src), OpcodeStr, "$src", "$src",
7759 (_.FloatVT (OpNode _.RC:$src))>, EVEX, T8PD;
Craig Toppere1cac152016-06-07 07:27:54 +00007760 defm m: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7761 (ins _.MemOp:$src), OpcodeStr, "$src", "$src",
7762 (OpNode (_.FloatVT
7763 (bitconvert (_.LdFrag addr:$src))))>, EVEX, T8PD;
7764 defm mb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7765 (ins _.ScalarMemOp:$src), OpcodeStr,
7766 "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr,
7767 (OpNode (_.FloatVT
7768 (X86VBroadcast (_.ScalarLdFrag addr:$src))))>,
7769 EVEX, T8PD, EVEX_B;
Craig Topper176f3312017-02-25 19:18:11 +00007770 }
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00007771}
Robert Khasanov3e534c92014-10-28 16:37:13 +00007772
7773multiclass avx512_fp14_p_vl_all<bits<8> opc, string OpcodeStr, SDNode OpNode> {
7774 defm PSZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"), OpNode, v16f32_info>,
7775 EVEX_V512, EVEX_CD8<32, CD8VF>;
7776 defm PDZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"), OpNode, v8f64_info>,
7777 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
7778
7779 // Define only if AVX512VL feature is present.
7780 let Predicates = [HasVLX] in {
7781 defm PSZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"),
7782 OpNode, v4f32x_info>,
7783 EVEX_V128, EVEX_CD8<32, CD8VF>;
7784 defm PSZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"),
7785 OpNode, v8f32x_info>,
7786 EVEX_V256, EVEX_CD8<32, CD8VF>;
7787 defm PDZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"),
7788 OpNode, v2f64x_info>,
7789 EVEX_V128, VEX_W, EVEX_CD8<64, CD8VF>;
7790 defm PDZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"),
7791 OpNode, v4f64x_info>,
7792 EVEX_V256, VEX_W, EVEX_CD8<64, CD8VF>;
7793 }
7794}
7795
7796defm VRSQRT14 : avx512_fp14_p_vl_all<0x4E, "vrsqrt14", X86frsqrt>;
7797defm VRCP14 : avx512_fp14_p_vl_all<0x4C, "vrcp14", X86frcp>;
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00007798
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00007799/// avx512_fp28_s rcp28ss, rcp28sd, rsqrt28ss, rsqrt28sd
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00007800multiclass avx512_fp28_s<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
7801 SDNode OpNode> {
Craig Topper176f3312017-02-25 19:18:11 +00007802 let ExeDomain = _.ExeDomain in {
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00007803 defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
7804 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
7805 "$src2, $src1", "$src1, $src2",
7806 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
7807 (i32 FROUND_CURRENT))>;
7808
7809 defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
7810 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00007811 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00007812 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00007813 (i32 FROUND_NO_EXC))>, EVEX_B;
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00007814
7815 defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
Igor Breger4511e762016-02-22 11:48:27 +00007816 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00007817 "$src2, $src1", "$src1, $src2",
7818 (OpNode (_.VT _.RC:$src1),
7819 (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))),
7820 (i32 FROUND_CURRENT))>;
Craig Topper176f3312017-02-25 19:18:11 +00007821 }
Elena Demikhovskya3a71402013-10-09 08:16:14 +00007822}
7823
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00007824multiclass avx512_eri_s<bits<8> opc, string OpcodeStr, SDNode OpNode> {
7825 defm SS : avx512_fp28_s<opc, OpcodeStr#"ss", f32x_info, OpNode>,
7826 EVEX_CD8<32, CD8VT1>;
7827 defm SD : avx512_fp28_s<opc, OpcodeStr#"sd", f64x_info, OpNode>,
7828 EVEX_CD8<64, CD8VT1>, VEX_W;
7829}
Elena Demikhovskya3a71402013-10-09 08:16:14 +00007830
Craig Toppere1cac152016-06-07 07:27:54 +00007831let Predicates = [HasERI] in {
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00007832 defm VRCP28 : avx512_eri_s<0xCB, "vrcp28", X86rcp28s>, T8PD, EVEX_4V;
7833 defm VRSQRT28 : avx512_eri_s<0xCD, "vrsqrt28", X86rsqrt28s>, T8PD, EVEX_4V;
7834}
Igor Breger8352a0d2015-07-28 06:53:28 +00007835
7836defm VGETEXP : avx512_eri_s<0x43, "vgetexp", X86fgetexpRnds>, T8PD, EVEX_4V;
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00007837/// avx512_fp28_p rcp28ps, rcp28pd, rsqrt28ps, rsqrt28pd
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00007838
7839multiclass avx512_fp28_p<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
7840 SDNode OpNode> {
Craig Topper176f3312017-02-25 19:18:11 +00007841 let ExeDomain = _.ExeDomain in {
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00007842 defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
7843 (ins _.RC:$src), OpcodeStr, "$src", "$src",
7844 (OpNode (_.VT _.RC:$src), (i32 FROUND_CURRENT))>;
7845
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00007846 defm m : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7847 (ins _.MemOp:$src), OpcodeStr, "$src", "$src",
7848 (OpNode (_.FloatVT
Elena Demikhovsky905a5a62014-11-26 10:46:49 +00007849 (bitconvert (_.LdFrag addr:$src))),
7850 (i32 FROUND_CURRENT))>;
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00007851
7852 defm mb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
Igor Breger4511e762016-02-22 11:48:27 +00007853 (ins _.ScalarMemOp:$src), OpcodeStr,
Asaf Badouh402ebb32015-06-03 13:41:48 +00007854 "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr,
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00007855 (OpNode (_.FloatVT
7856 (X86VBroadcast (_.ScalarLdFrag addr:$src))),
7857 (i32 FROUND_CURRENT))>, EVEX_B;
Craig Topper176f3312017-02-25 19:18:11 +00007858 }
Elena Demikhovskya3a71402013-10-09 08:16:14 +00007859}
Asaf Badouh402ebb32015-06-03 13:41:48 +00007860multiclass avx512_fp28_p_round<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
7861 SDNode OpNode> {
Craig Topper176f3312017-02-25 19:18:11 +00007862 let ExeDomain = _.ExeDomain in
Asaf Badouh402ebb32015-06-03 13:41:48 +00007863 defm rb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
7864 (ins _.RC:$src), OpcodeStr,
7865 "{sae}, $src", "$src, {sae}",
7866 (OpNode (_.VT _.RC:$src), (i32 FROUND_NO_EXC))>, EVEX_B;
7867}
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00007868
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00007869multiclass avx512_eri<bits<8> opc, string OpcodeStr, SDNode OpNode> {
7870 defm PS : avx512_fp28_p<opc, OpcodeStr#"ps", v16f32_info, OpNode>,
Asaf Badouh402ebb32015-06-03 13:41:48 +00007871 avx512_fp28_p_round<opc, OpcodeStr#"ps", v16f32_info, OpNode>,
7872 T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>;
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00007873 defm PD : avx512_fp28_p<opc, OpcodeStr#"pd", v8f64_info, OpNode>,
Asaf Badouh402ebb32015-06-03 13:41:48 +00007874 avx512_fp28_p_round<opc, OpcodeStr#"pd", v8f64_info, OpNode>,
7875 T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00007876}
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00007877
Asaf Badouh402ebb32015-06-03 13:41:48 +00007878multiclass avx512_fp_unaryop_packed<bits<8> opc, string OpcodeStr,
7879 SDNode OpNode> {
7880 // Define only if AVX512VL feature is present.
7881 let Predicates = [HasVLX] in {
7882 defm PSZ128 : avx512_fp28_p<opc, OpcodeStr#"ps", v4f32x_info, OpNode>,
7883 EVEX_V128, T8PD, EVEX_CD8<32, CD8VF>;
7884 defm PSZ256 : avx512_fp28_p<opc, OpcodeStr#"ps", v8f32x_info, OpNode>,
7885 EVEX_V256, T8PD, EVEX_CD8<32, CD8VF>;
7886 defm PDZ128 : avx512_fp28_p<opc, OpcodeStr#"pd", v2f64x_info, OpNode>,
7887 EVEX_V128, VEX_W, T8PD, EVEX_CD8<64, CD8VF>;
7888 defm PDZ256 : avx512_fp28_p<opc, OpcodeStr#"pd", v4f64x_info, OpNode>,
7889 EVEX_V256, VEX_W, T8PD, EVEX_CD8<64, CD8VF>;
7890 }
7891}
Craig Toppere1cac152016-06-07 07:27:54 +00007892let Predicates = [HasERI] in {
Michael Liao5bf95782014-12-04 05:20:33 +00007893
Asaf Badouh402ebb32015-06-03 13:41:48 +00007894 defm VRSQRT28 : avx512_eri<0xCC, "vrsqrt28", X86rsqrt28>, EVEX;
7895 defm VRCP28 : avx512_eri<0xCA, "vrcp28", X86rcp28>, EVEX;
7896 defm VEXP2 : avx512_eri<0xC8, "vexp2", X86exp2>, EVEX;
7897}
7898defm VGETEXP : avx512_eri<0x42, "vgetexp", X86fgetexpRnd>,
7899 avx512_fp_unaryop_packed<0x42, "vgetexp", X86fgetexpRnd> , EVEX;
7900
7901multiclass avx512_sqrt_packed_round<bits<8> opc, string OpcodeStr,
7902 SDNode OpNodeRnd, X86VectorVTInfo _>{
Craig Topper176f3312017-02-25 19:18:11 +00007903 let ExeDomain = _.ExeDomain in
Asaf Badouh402ebb32015-06-03 13:41:48 +00007904 defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
7905 (ins _.RC:$src, AVX512RC:$rc), OpcodeStr, "$rc, $src", "$src, $rc",
7906 (_.VT (OpNodeRnd _.RC:$src, (i32 imm:$rc)))>,
7907 EVEX, EVEX_B, EVEX_RC;
Elena Demikhovskybe8808d2014-11-12 07:31:03 +00007908}
Elena Demikhovskyb19c9dc2014-01-13 12:55:03 +00007909
Robert Khasanoveb126392014-10-28 18:15:20 +00007910multiclass avx512_sqrt_packed<bits<8> opc, string OpcodeStr,
7911 SDNode OpNode, X86VectorVTInfo _>{
Craig Topper176f3312017-02-25 19:18:11 +00007912 let ExeDomain = _.ExeDomain in {
Robert Khasanov1cf354c2014-10-28 18:22:41 +00007913 defm r: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
Robert Khasanoveb126392014-10-28 18:15:20 +00007914 (ins _.RC:$src), OpcodeStr, "$src", "$src",
7915 (_.FloatVT (OpNode _.RC:$src))>, EVEX;
Craig Toppere1cac152016-06-07 07:27:54 +00007916 defm m: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7917 (ins _.MemOp:$src), OpcodeStr, "$src", "$src",
7918 (OpNode (_.FloatVT
7919 (bitconvert (_.LdFrag addr:$src))))>, EVEX;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007920
Craig Toppere1cac152016-06-07 07:27:54 +00007921 defm mb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7922 (ins _.ScalarMemOp:$src), OpcodeStr,
7923 "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr,
7924 (OpNode (_.FloatVT
7925 (X86VBroadcast (_.ScalarLdFrag addr:$src))))>,
7926 EVEX, EVEX_B;
Craig Topper176f3312017-02-25 19:18:11 +00007927 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00007928}
7929
Robert Khasanoveb126392014-10-28 18:15:20 +00007930multiclass avx512_sqrt_packed_all<bits<8> opc, string OpcodeStr,
7931 SDNode OpNode> {
7932 defm PSZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode,
7933 v16f32_info>,
7934 EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
7935 defm PDZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode,
7936 v8f64_info>,
7937 EVEX_V512, VEX_W, PD, EVEX_CD8<64, CD8VF>;
7938 // Define only if AVX512VL feature is present.
7939 let Predicates = [HasVLX] in {
7940 defm PSZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"),
7941 OpNode, v4f32x_info>,
7942 EVEX_V128, PS, EVEX_CD8<32, CD8VF>;
7943 defm PSZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"),
7944 OpNode, v8f32x_info>,
7945 EVEX_V256, PS, EVEX_CD8<32, CD8VF>;
7946 defm PDZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"),
7947 OpNode, v2f64x_info>,
7948 EVEX_V128, VEX_W, PD, EVEX_CD8<64, CD8VF>;
7949 defm PDZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"),
7950 OpNode, v4f64x_info>,
7951 EVEX_V256, VEX_W, PD, EVEX_CD8<64, CD8VF>;
7952 }
7953}
7954
Asaf Badouh402ebb32015-06-03 13:41:48 +00007955multiclass avx512_sqrt_packed_all_round<bits<8> opc, string OpcodeStr,
7956 SDNode OpNodeRnd> {
7957 defm PSZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "ps"), OpNodeRnd,
7958 v16f32_info>, EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
7959 defm PDZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "pd"), OpNodeRnd,
7960 v8f64_info>, EVEX_V512, VEX_W, PD, EVEX_CD8<64, CD8VF>;
7961}
7962
Igor Breger4c4cd782015-09-20 09:13:41 +00007963multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
7964 string SUFF, SDNode OpNode, SDNode OpNodeRnd> {
Craig Topper176f3312017-02-25 19:18:11 +00007965 let ExeDomain = _.ExeDomain in {
Igor Breger4c4cd782015-09-20 09:13:41 +00007966 defm r_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
7967 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
7968 "$src2, $src1", "$src1, $src2",
7969 (OpNodeRnd (_.VT _.RC:$src1),
7970 (_.VT _.RC:$src2),
7971 (i32 FROUND_CURRENT))>;
Craig Toppere1cac152016-06-07 07:27:54 +00007972 defm m_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
7973 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
7974 "$src2, $src1", "$src1, $src2",
7975 (OpNodeRnd (_.VT _.RC:$src1),
7976 (_.VT (scalar_to_vector
7977 (_.ScalarLdFrag addr:$src2))),
7978 (i32 FROUND_CURRENT))>;
Igor Breger4c4cd782015-09-20 09:13:41 +00007979
7980 defm rb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
7981 (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr,
7982 "$rc, $src2, $src1", "$src1, $src2, $rc",
7983 (OpNodeRnd (_.VT _.RC:$src1),
7984 (_.VT _.RC:$src2),
7985 (i32 imm:$rc))>,
7986 EVEX_B, EVEX_RC;
7987
Craig Toppere1cac152016-06-07 07:27:54 +00007988 let isCodeGenOnly = 1, hasSideEffects = 0 in {
Elena Demikhovsky0d0692d2015-12-01 12:43:46 +00007989 def r : I<opc, MRMSrcReg, (outs _.FRC:$dst),
Igor Breger4c4cd782015-09-20 09:13:41 +00007990 (ins _.FRC:$src1, _.FRC:$src2),
7991 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>;
7992
7993 let mayLoad = 1 in
Elena Demikhovsky0d0692d2015-12-01 12:43:46 +00007994 def m : I<opc, MRMSrcMem, (outs _.FRC:$dst),
Igor Breger4c4cd782015-09-20 09:13:41 +00007995 (ins _.FRC:$src1, _.ScalarMemOp:$src2),
7996 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>;
7997 }
Craig Topper176f3312017-02-25 19:18:11 +00007998 }
Igor Breger4c4cd782015-09-20 09:13:41 +00007999
8000 def : Pat<(_.EltVT (OpNode _.FRC:$src)),
8001 (!cast<Instruction>(NAME#SUFF#Zr)
8002 (_.EltVT (IMPLICIT_DEF)), _.FRC:$src)>;
8003
8004 def : Pat<(_.EltVT (OpNode (load addr:$src))),
8005 (!cast<Instruction>(NAME#SUFF#Zm)
Dimitry Andricdb417b62016-02-19 20:14:11 +00008006 (_.EltVT (IMPLICIT_DEF)), addr:$src)>, Requires<[HasAVX512, OptForSize]>;
Igor Breger4c4cd782015-09-20 09:13:41 +00008007}
8008
8009multiclass avx512_sqrt_scalar_all<bits<8> opc, string OpcodeStr> {
8010 defm SSZ : avx512_sqrt_scalar<opc, OpcodeStr#"ss", f32x_info, "SS", fsqrt,
8011 X86fsqrtRnds>, EVEX_CD8<32, CD8VT1>, EVEX_4V, XS;
8012 defm SDZ : avx512_sqrt_scalar<opc, OpcodeStr#"sd", f64x_info, "SD", fsqrt,
8013 X86fsqrtRnds>, EVEX_CD8<64, CD8VT1>, EVEX_4V, XD, VEX_W;
8014}
8015
Asaf Badouh402ebb32015-06-03 13:41:48 +00008016defm VSQRT : avx512_sqrt_packed_all<0x51, "vsqrt", fsqrt>,
8017 avx512_sqrt_packed_all_round<0x51, "vsqrt", X86fsqrtRnd>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008018
Igor Breger4c4cd782015-09-20 09:13:41 +00008019defm VSQRT : avx512_sqrt_scalar_all<0x51, "vsqrt">, VEX_LIG;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008020
Elena Demikhovskya3a71402013-10-09 08:16:14 +00008021let Predicates = [HasAVX512] in {
Elena Demikhovskya3a71402013-10-09 08:16:14 +00008022 def : Pat<(f32 (X86frsqrt FR32X:$src)),
Asaf Badouheaf2da12015-09-21 10:23:53 +00008023 (COPY_TO_REGCLASS (VRSQRT14SSrr (v4f32 (IMPLICIT_DEF)), (COPY_TO_REGCLASS FR32X:$src, VR128X)), VR128X)>;
Elena Demikhovskya3a71402013-10-09 08:16:14 +00008024 def : Pat<(f32 (X86frsqrt (load addr:$src))),
Asaf Badouheaf2da12015-09-21 10:23:53 +00008025 (COPY_TO_REGCLASS (VRSQRT14SSrm (v4f32 (IMPLICIT_DEF)), addr:$src), VR128X)>,
Elena Demikhovskya3a71402013-10-09 08:16:14 +00008026 Requires<[OptForSize]>;
Elena Demikhovskya3a71402013-10-09 08:16:14 +00008027 def : Pat<(f32 (X86frcp FR32X:$src)),
Asaf Badouheaf2da12015-09-21 10:23:53 +00008028 (COPY_TO_REGCLASS (VRCP14SSrr (v4f32 (IMPLICIT_DEF)), (COPY_TO_REGCLASS FR32X:$src, VR128X)), VR128X )>;
Elena Demikhovskya3a71402013-10-09 08:16:14 +00008029 def : Pat<(f32 (X86frcp (load addr:$src))),
Asaf Badouheaf2da12015-09-21 10:23:53 +00008030 (COPY_TO_REGCLASS (VRCP14SSrm (v4f32 (IMPLICIT_DEF)), addr:$src), VR128X)>,
Elena Demikhovskya3a71402013-10-09 08:16:14 +00008031 Requires<[OptForSize]>;
Elena Demikhovskya3a71402013-10-09 08:16:14 +00008032}
8033
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008034multiclass
8035avx512_rndscale_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> {
Elena Demikhovskyde3f7512014-01-01 15:12:34 +00008036
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008037 let ExeDomain = _.ExeDomain in {
8038 defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
8039 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr,
8040 "$src3, $src2, $src1", "$src1, $src2, $src3",
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00008041 (_.VT (X86RndScales (_.VT _.RC:$src1), (_.VT _.RC:$src2),
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008042 (i32 imm:$src3), (i32 FROUND_CURRENT)))>;
8043
8044 defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
8045 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr,
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00008046 "$src3, {sae}, $src2, $src1", "$src1, $src2, {sae}, $src3",
8047 (_.VT (X86RndScales (_.VT _.RC:$src1), (_.VT _.RC:$src2),
Elena Demikhovsky0d7e9362015-05-11 06:05:05 +00008048 (i32 imm:$src3), (i32 FROUND_NO_EXC)))>, EVEX_B;
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008049
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008050 defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
Simon Pilgrimb13961d2016-06-11 14:34:10 +00008051 (ins _.RC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3),
8052 OpcodeStr,
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008053 "$src3, $src2, $src1", "$src1, $src2, $src3",
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00008054 (_.VT (X86RndScales (_.VT _.RC:$src1),
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008055 (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))),
8056 (i32 imm:$src3), (i32 FROUND_CURRENT)))>;
8057 }
8058 let Predicates = [HasAVX512] in {
8059 def : Pat<(ffloor _.FRC:$src), (COPY_TO_REGCLASS
8060 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)),
Ahmed Bougacha58a19742017-06-26 16:00:24 +00008061 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x9))), _.FRC)>;
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008062 def : Pat<(fceil _.FRC:$src), (COPY_TO_REGCLASS
8063 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)),
Ahmed Bougacha58a19742017-06-26 16:00:24 +00008064 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0xa))), _.FRC)>;
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008065 def : Pat<(ftrunc _.FRC:$src), (COPY_TO_REGCLASS
8066 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)),
Ahmed Bougacha58a19742017-06-26 16:00:24 +00008067 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0xb))), _.FRC)>;
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008068 def : Pat<(frint _.FRC:$src), (COPY_TO_REGCLASS
8069 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)),
8070 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x4))), _.FRC)>;
8071 def : Pat<(fnearbyint _.FRC:$src), (COPY_TO_REGCLASS
8072 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)),
8073 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0xc))), _.FRC)>;
8074
8075 def : Pat<(ffloor (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
8076 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)),
Ahmed Bougacha58a19742017-06-26 16:00:24 +00008077 addr:$src, (i32 0x9))), _.FRC)>;
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008078 def : Pat<(fceil (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
8079 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)),
Ahmed Bougacha58a19742017-06-26 16:00:24 +00008080 addr:$src, (i32 0xa))), _.FRC)>;
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008081 def : Pat<(ftrunc (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
8082 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)),
Ahmed Bougacha58a19742017-06-26 16:00:24 +00008083 addr:$src, (i32 0xb))), _.FRC)>;
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008084 def : Pat<(frint (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
8085 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)),
8086 addr:$src, (i32 0x4))), _.FRC)>;
8087 def : Pat<(fnearbyint (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
8088 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)),
8089 addr:$src, (i32 0xc))), _.FRC)>;
8090 }
Elena Demikhovskyde3f7512014-01-01 15:12:34 +00008091}
8092
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008093defm VRNDSCALESS : avx512_rndscale_scalar<0x0A, "vrndscaless", f32x_info>,
8094 AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VT1>;
Michael Liao5bf95782014-12-04 05:20:33 +00008095
Elena Demikhovsky52e81bc2015-02-23 15:12:31 +00008096defm VRNDSCALESD : avx512_rndscale_scalar<0x0B, "vrndscalesd", f64x_info>, VEX_W,
8097 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VT1>;
Eric Christopher0d94fa92015-02-20 00:45:28 +00008098
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008099//-------------------------------------------------
8100// Integer truncate and extend operations
8101//-------------------------------------------------
8102
Igor Breger074a64e2015-07-24 17:24:15 +00008103multiclass avx512_trunc_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
8104 X86VectorVTInfo SrcInfo, X86VectorVTInfo DestInfo,
8105 X86MemOperand x86memop> {
Craig Topper52e2e832016-07-22 05:46:44 +00008106 let ExeDomain = DestInfo.ExeDomain in
Igor Breger074a64e2015-07-24 17:24:15 +00008107 defm rr : AVX512_maskable<opc, MRMDestReg, DestInfo, (outs DestInfo.RC:$dst),
8108 (ins SrcInfo.RC:$src1), OpcodeStr ,"$src1", "$src1",
8109 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1)))>,
8110 EVEX, T8XS;
8111
8112 // for intrinsic patter match
8113 def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask,
8114 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))),
8115 undef)),
8116 (!cast<Instruction>(NAME#SrcInfo.ZSuffix##rrkz) DestInfo.KRCWM:$mask ,
8117 SrcInfo.RC:$src1)>;
8118
8119 def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask,
8120 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))),
8121 DestInfo.ImmAllZerosV)),
8122 (!cast<Instruction>(NAME#SrcInfo.ZSuffix##rrkz) DestInfo.KRCWM:$mask ,
8123 SrcInfo.RC:$src1)>;
8124
8125 def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask,
8126 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))),
8127 DestInfo.RC:$src0)),
8128 (!cast<Instruction>(NAME#SrcInfo.ZSuffix##rrk) DestInfo.RC:$src0,
8129 DestInfo.KRCWM:$mask ,
8130 SrcInfo.RC:$src1)>;
8131
Craig Topper52e2e832016-07-22 05:46:44 +00008132 let mayStore = 1, mayLoad = 1, hasSideEffects = 0,
8133 ExeDomain = DestInfo.ExeDomain in {
Igor Breger074a64e2015-07-24 17:24:15 +00008134 def mr : AVX512XS8I<opc, MRMDestMem, (outs),
8135 (ins x86memop:$dst, SrcInfo.RC:$src),
Craig Topper9feea572016-01-11 00:44:58 +00008136 OpcodeStr # "\t{$src, $dst|$dst, $src}",
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008137 []>, EVEX;
8138
Igor Breger074a64e2015-07-24 17:24:15 +00008139 def mrk : AVX512XS8I<opc, MRMDestMem, (outs),
8140 (ins x86memop:$dst, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src),
Craig Topper9feea572016-01-11 00:44:58 +00008141 OpcodeStr # "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",
Robert Khasanov189e7fd2014-04-22 11:36:19 +00008142 []>, EVEX, EVEX_K;
Craig Topper99f6b622016-05-01 01:03:56 +00008143 }//mayStore = 1, mayLoad = 1, hasSideEffects = 0
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008144}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008145
Igor Breger074a64e2015-07-24 17:24:15 +00008146multiclass avx512_trunc_mr_lowering<X86VectorVTInfo SrcInfo,
8147 X86VectorVTInfo DestInfo,
8148 PatFrag truncFrag, PatFrag mtruncFrag > {
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008149
Igor Breger074a64e2015-07-24 17:24:15 +00008150 def : Pat<(truncFrag (SrcInfo.VT SrcInfo.RC:$src), addr:$dst),
8151 (!cast<Instruction>(NAME#SrcInfo.ZSuffix##mr)
8152 addr:$dst, SrcInfo.RC:$src)>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008153
Igor Breger074a64e2015-07-24 17:24:15 +00008154 def : Pat<(mtruncFrag addr:$dst, SrcInfo.KRCWM:$mask,
8155 (SrcInfo.VT SrcInfo.RC:$src)),
8156 (!cast<Instruction>(NAME#SrcInfo.ZSuffix##mrk)
8157 addr:$dst, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src)>;
8158}
8159
Igor Breger074a64e2015-07-24 17:24:15 +00008160multiclass avx512_trunc<bits<8> opc, string OpcodeStr, SDNode OpNode,
8161 AVX512VLVectorVTInfo VTSrcInfo, X86VectorVTInfo DestInfoZ128,
8162 X86VectorVTInfo DestInfoZ256, X86VectorVTInfo DestInfoZ,
8163 X86MemOperand x86memopZ128, X86MemOperand x86memopZ256,
8164 X86MemOperand x86memopZ, PatFrag truncFrag, PatFrag mtruncFrag,
8165 Predicate prd = HasAVX512>{
8166
8167 let Predicates = [HasVLX, prd] in {
8168 defm Z128: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info128,
8169 DestInfoZ128, x86memopZ128>,
8170 avx512_trunc_mr_lowering<VTSrcInfo.info128, DestInfoZ128,
8171 truncFrag, mtruncFrag>, EVEX_V128;
8172
8173 defm Z256: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info256,
8174 DestInfoZ256, x86memopZ256>,
8175 avx512_trunc_mr_lowering<VTSrcInfo.info256, DestInfoZ256,
8176 truncFrag, mtruncFrag>, EVEX_V256;
8177 }
8178 let Predicates = [prd] in
8179 defm Z: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info512,
8180 DestInfoZ, x86memopZ>,
8181 avx512_trunc_mr_lowering<VTSrcInfo.info512, DestInfoZ,
8182 truncFrag, mtruncFrag>, EVEX_V512;
8183}
8184
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008185multiclass avx512_trunc_qb<bits<8> opc, string OpcodeStr, SDNode OpNode,
8186 PatFrag StoreNode, PatFrag MaskedStoreNode> {
Igor Breger074a64e2015-07-24 17:24:15 +00008187 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
8188 v16i8x_info, v16i8x_info, v16i8x_info, i16mem, i32mem, i64mem,
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008189 StoreNode, MaskedStoreNode>, EVEX_CD8<8, CD8VO>;
Igor Breger074a64e2015-07-24 17:24:15 +00008190}
8191
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008192multiclass avx512_trunc_qw<bits<8> opc, string OpcodeStr, SDNode OpNode,
8193 PatFrag StoreNode, PatFrag MaskedStoreNode> {
Igor Breger074a64e2015-07-24 17:24:15 +00008194 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
8195 v8i16x_info, v8i16x_info, v8i16x_info, i32mem, i64mem, i128mem,
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008196 StoreNode, MaskedStoreNode>, EVEX_CD8<16, CD8VQ>;
Igor Breger074a64e2015-07-24 17:24:15 +00008197}
8198
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008199multiclass avx512_trunc_qd<bits<8> opc, string OpcodeStr, SDNode OpNode,
8200 PatFrag StoreNode, PatFrag MaskedStoreNode> {
Igor Breger074a64e2015-07-24 17:24:15 +00008201 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
8202 v4i32x_info, v4i32x_info, v8i32x_info, i64mem, i128mem, i256mem,
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008203 StoreNode, MaskedStoreNode>, EVEX_CD8<32, CD8VH>;
Igor Breger074a64e2015-07-24 17:24:15 +00008204}
8205
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008206multiclass avx512_trunc_db<bits<8> opc, string OpcodeStr, SDNode OpNode,
8207 PatFrag StoreNode, PatFrag MaskedStoreNode> {
Igor Breger074a64e2015-07-24 17:24:15 +00008208 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i32_info,
8209 v16i8x_info, v16i8x_info, v16i8x_info, i32mem, i64mem, i128mem,
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008210 StoreNode, MaskedStoreNode>, EVEX_CD8<8, CD8VQ>;
Igor Breger074a64e2015-07-24 17:24:15 +00008211}
8212
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008213multiclass avx512_trunc_dw<bits<8> opc, string OpcodeStr, SDNode OpNode,
8214 PatFrag StoreNode, PatFrag MaskedStoreNode> {
Igor Breger074a64e2015-07-24 17:24:15 +00008215 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i32_info,
8216 v8i16x_info, v8i16x_info, v16i16x_info, i64mem, i128mem, i256mem,
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008217 StoreNode, MaskedStoreNode>, EVEX_CD8<16, CD8VH>;
Igor Breger074a64e2015-07-24 17:24:15 +00008218}
8219
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008220multiclass avx512_trunc_wb<bits<8> opc, string OpcodeStr, SDNode OpNode,
8221 PatFrag StoreNode, PatFrag MaskedStoreNode> {
Igor Breger074a64e2015-07-24 17:24:15 +00008222 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i16_info,
8223 v16i8x_info, v16i8x_info, v32i8x_info, i64mem, i128mem, i256mem,
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008224 StoreNode, MaskedStoreNode, HasBWI>, EVEX_CD8<16, CD8VH>;
Igor Breger074a64e2015-07-24 17:24:15 +00008225}
8226
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008227defm VPMOVQB : avx512_trunc_qb<0x32, "vpmovqb", X86vtrunc,
8228 truncstorevi8, masked_truncstorevi8>;
8229defm VPMOVSQB : avx512_trunc_qb<0x22, "vpmovsqb", X86vtruncs,
8230 truncstore_s_vi8, masked_truncstore_s_vi8>;
8231defm VPMOVUSQB : avx512_trunc_qb<0x12, "vpmovusqb", X86vtruncus,
8232 truncstore_us_vi8, masked_truncstore_us_vi8>;
Igor Breger074a64e2015-07-24 17:24:15 +00008233
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008234defm VPMOVQW : avx512_trunc_qw<0x34, "vpmovqw", X86vtrunc,
8235 truncstorevi16, masked_truncstorevi16>;
8236defm VPMOVSQW : avx512_trunc_qw<0x24, "vpmovsqw", X86vtruncs,
8237 truncstore_s_vi16, masked_truncstore_s_vi16>;
8238defm VPMOVUSQW : avx512_trunc_qw<0x14, "vpmovusqw", X86vtruncus,
8239 truncstore_us_vi16, masked_truncstore_us_vi16>;
Igor Breger074a64e2015-07-24 17:24:15 +00008240
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008241defm VPMOVQD : avx512_trunc_qd<0x35, "vpmovqd", X86vtrunc,
8242 truncstorevi32, masked_truncstorevi32>;
8243defm VPMOVSQD : avx512_trunc_qd<0x25, "vpmovsqd", X86vtruncs,
8244 truncstore_s_vi32, masked_truncstore_s_vi32>;
8245defm VPMOVUSQD : avx512_trunc_qd<0x15, "vpmovusqd", X86vtruncus,
8246 truncstore_us_vi32, masked_truncstore_us_vi32>;
Igor Breger074a64e2015-07-24 17:24:15 +00008247
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008248defm VPMOVDB : avx512_trunc_db<0x31, "vpmovdb", X86vtrunc,
8249 truncstorevi8, masked_truncstorevi8>;
8250defm VPMOVSDB : avx512_trunc_db<0x21, "vpmovsdb", X86vtruncs,
8251 truncstore_s_vi8, masked_truncstore_s_vi8>;
8252defm VPMOVUSDB : avx512_trunc_db<0x11, "vpmovusdb", X86vtruncus,
8253 truncstore_us_vi8, masked_truncstore_us_vi8>;
Igor Breger074a64e2015-07-24 17:24:15 +00008254
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008255defm VPMOVDW : avx512_trunc_dw<0x33, "vpmovdw", X86vtrunc,
8256 truncstorevi16, masked_truncstorevi16>;
8257defm VPMOVSDW : avx512_trunc_dw<0x23, "vpmovsdw", X86vtruncs,
8258 truncstore_s_vi16, masked_truncstore_s_vi16>;
8259defm VPMOVUSDW : avx512_trunc_dw<0x13, "vpmovusdw", X86vtruncus,
8260 truncstore_us_vi16, masked_truncstore_us_vi16>;
Igor Breger074a64e2015-07-24 17:24:15 +00008261
Elena Demikhovsky7c7bf1b2016-12-21 10:43:36 +00008262defm VPMOVWB : avx512_trunc_wb<0x30, "vpmovwb", X86vtrunc,
8263 truncstorevi8, masked_truncstorevi8>;
8264defm VPMOVSWB : avx512_trunc_wb<0x20, "vpmovswb", X86vtruncs,
8265 truncstore_s_vi8, masked_truncstore_s_vi8>;
8266defm VPMOVUSWB : avx512_trunc_wb<0x10, "vpmovuswb", X86vtruncus,
8267 truncstore_us_vi8, masked_truncstore_us_vi8>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008268
Elena Demikhovskydb738d92015-11-01 11:45:47 +00008269let Predicates = [HasAVX512, NoVLX] in {
8270def: Pat<(v8i16 (X86vtrunc (v8i32 VR256X:$src))),
8271 (v8i16 (EXTRACT_SUBREG
Craig Topper61403202016-09-19 02:53:43 +00008272 (v16i16 (VPMOVDWZrr (v16i32 (INSERT_SUBREG (IMPLICIT_DEF),
Elena Demikhovskydb738d92015-11-01 11:45:47 +00008273 VR256X:$src, sub_ymm)))), sub_xmm))>;
8274def: Pat<(v4i32 (X86vtrunc (v4i64 VR256X:$src))),
8275 (v4i32 (EXTRACT_SUBREG
Craig Topper61403202016-09-19 02:53:43 +00008276 (v8i32 (VPMOVQDZrr (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
Elena Demikhovskydb738d92015-11-01 11:45:47 +00008277 VR256X:$src, sub_ymm)))), sub_xmm))>;
8278}
8279
8280let Predicates = [HasBWI, NoVLX] in {
8281def: Pat<(v16i8 (X86vtrunc (v16i16 VR256X:$src))),
Craig Topper61403202016-09-19 02:53:43 +00008282 (v16i8 (EXTRACT_SUBREG (VPMOVWBZrr (v32i16 (INSERT_SUBREG (IMPLICIT_DEF),
Elena Demikhovskydb738d92015-11-01 11:45:47 +00008283 VR256X:$src, sub_ymm))), sub_xmm))>;
8284}
8285
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008286multiclass avx512_extend_common<bits<8> opc, string OpcodeStr,
Igor Breger2ba64ab2016-05-22 10:21:04 +00008287 X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo,
Craig Topper6840f112016-07-14 06:41:34 +00008288 X86MemOperand x86memop, PatFrag LdFrag, SDPatternOperator OpNode>{
Craig Topper52e2e832016-07-22 05:46:44 +00008289 let ExeDomain = DestInfo.ExeDomain in {
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008290 defm rr : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),
8291 (ins SrcInfo.RC:$src), OpcodeStr ,"$src", "$src",
8292 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src)))>,
8293 EVEX;
Robert Khasanov189e7fd2014-04-22 11:36:19 +00008294
Craig Toppere1cac152016-06-07 07:27:54 +00008295 defm rm : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
8296 (ins x86memop:$src), OpcodeStr ,"$src", "$src",
8297 (DestInfo.VT (LdFrag addr:$src))>,
8298 EVEX;
Craig Topper52e2e832016-07-22 05:46:44 +00008299 }
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008300}
8301
Simon Pilgrimb13961d2016-06-11 14:34:10 +00008302multiclass avx512_extend_BW<bits<8> opc, string OpcodeStr,
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008303 SDPatternOperator OpNode, SDPatternOperator InVecNode,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008304 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
8305 let Predicates = [HasVLX, HasBWI] in {
8306 defm Z128: avx512_extend_common<opc, OpcodeStr, v8i16x_info,
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008307 v16i8x_info, i64mem, LdFrag, InVecNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008308 EVEX_CD8<8, CD8VH>, T8PD, EVEX_V128;
Robert Khasanov189e7fd2014-04-22 11:36:19 +00008309
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008310 defm Z256: avx512_extend_common<opc, OpcodeStr, v16i16x_info,
Craig Topper6840f112016-07-14 06:41:34 +00008311 v16i8x_info, i128mem, LdFrag, OpNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008312 EVEX_CD8<8, CD8VH>, T8PD, EVEX_V256;
8313 }
8314 let Predicates = [HasBWI] in {
8315 defm Z : avx512_extend_common<opc, OpcodeStr, v32i16_info,
Craig Topper6840f112016-07-14 06:41:34 +00008316 v32i8x_info, i256mem, LdFrag, OpNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008317 EVEX_CD8<8, CD8VH>, T8PD, EVEX_V512;
8318 }
8319}
8320
Simon Pilgrimb13961d2016-06-11 14:34:10 +00008321multiclass avx512_extend_BD<bits<8> opc, string OpcodeStr,
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008322 SDPatternOperator OpNode, SDPatternOperator InVecNode,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008323 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
8324 let Predicates = [HasVLX, HasAVX512] in {
8325 defm Z128: avx512_extend_common<opc, OpcodeStr, v4i32x_info,
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008326 v16i8x_info, i32mem, LdFrag, InVecNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008327 EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V128;
8328
8329 defm Z256: avx512_extend_common<opc, OpcodeStr, v8i32x_info,
Craig Topper6840f112016-07-14 06:41:34 +00008330 v16i8x_info, i64mem, LdFrag, OpNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008331 EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V256;
8332 }
8333 let Predicates = [HasAVX512] in {
8334 defm Z : avx512_extend_common<opc, OpcodeStr, v16i32_info,
Craig Topper6840f112016-07-14 06:41:34 +00008335 v16i8x_info, i128mem, LdFrag, OpNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008336 EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V512;
8337 }
8338}
8339
Simon Pilgrimb13961d2016-06-11 14:34:10 +00008340multiclass avx512_extend_BQ<bits<8> opc, string OpcodeStr,
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008341 SDPatternOperator OpNode, SDPatternOperator InVecNode,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008342 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
8343 let Predicates = [HasVLX, HasAVX512] in {
8344 defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info,
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008345 v16i8x_info, i16mem, LdFrag, InVecNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008346 EVEX_CD8<8, CD8VO>, T8PD, EVEX_V128;
8347
8348 defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info,
Craig Topper6840f112016-07-14 06:41:34 +00008349 v16i8x_info, i32mem, LdFrag, OpNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008350 EVEX_CD8<8, CD8VO>, T8PD, EVEX_V256;
8351 }
8352 let Predicates = [HasAVX512] in {
8353 defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info,
Craig Topper6840f112016-07-14 06:41:34 +00008354 v16i8x_info, i64mem, LdFrag, OpNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008355 EVEX_CD8<8, CD8VO>, T8PD, EVEX_V512;
8356 }
8357}
8358
Simon Pilgrimb13961d2016-06-11 14:34:10 +00008359multiclass avx512_extend_WD<bits<8> opc, string OpcodeStr,
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008360 SDPatternOperator OpNode, SDPatternOperator InVecNode,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008361 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> {
8362 let Predicates = [HasVLX, HasAVX512] in {
8363 defm Z128: avx512_extend_common<opc, OpcodeStr, v4i32x_info,
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008364 v8i16x_info, i64mem, LdFrag, InVecNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008365 EVEX_CD8<16, CD8VH>, T8PD, EVEX_V128;
8366
8367 defm Z256: avx512_extend_common<opc, OpcodeStr, v8i32x_info,
Craig Topper6840f112016-07-14 06:41:34 +00008368 v8i16x_info, i128mem, LdFrag, OpNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008369 EVEX_CD8<16, CD8VH>, T8PD, EVEX_V256;
8370 }
8371 let Predicates = [HasAVX512] in {
8372 defm Z : avx512_extend_common<opc, OpcodeStr, v16i32_info,
Craig Topper6840f112016-07-14 06:41:34 +00008373 v16i16x_info, i256mem, LdFrag, OpNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008374 EVEX_CD8<16, CD8VH>, T8PD, EVEX_V512;
8375 }
8376}
8377
Simon Pilgrimb13961d2016-06-11 14:34:10 +00008378multiclass avx512_extend_WQ<bits<8> opc, string OpcodeStr,
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008379 SDPatternOperator OpNode, SDPatternOperator InVecNode,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008380 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> {
8381 let Predicates = [HasVLX, HasAVX512] in {
8382 defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info,
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008383 v8i16x_info, i32mem, LdFrag, InVecNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008384 EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V128;
8385
8386 defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info,
Craig Topper6840f112016-07-14 06:41:34 +00008387 v8i16x_info, i64mem, LdFrag, OpNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008388 EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V256;
8389 }
8390 let Predicates = [HasAVX512] in {
8391 defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info,
Craig Topper6840f112016-07-14 06:41:34 +00008392 v8i16x_info, i128mem, LdFrag, OpNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008393 EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V512;
8394 }
8395}
8396
Simon Pilgrimb13961d2016-06-11 14:34:10 +00008397multiclass avx512_extend_DQ<bits<8> opc, string OpcodeStr,
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008398 SDPatternOperator OpNode, SDPatternOperator InVecNode,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008399 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi32")> {
8400
8401 let Predicates = [HasVLX, HasAVX512] in {
8402 defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info,
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008403 v4i32x_info, i64mem, LdFrag, InVecNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008404 EVEX_CD8<32, CD8VH>, T8PD, EVEX_V128;
8405
8406 defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info,
Craig Topper6840f112016-07-14 06:41:34 +00008407 v4i32x_info, i128mem, LdFrag, OpNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008408 EVEX_CD8<32, CD8VH>, T8PD, EVEX_V256;
8409 }
8410 let Predicates = [HasAVX512] in {
8411 defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info,
Craig Topper6840f112016-07-14 06:41:34 +00008412 v8i32x_info, i256mem, LdFrag, OpNode>,
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008413 EVEX_CD8<32, CD8VH>, T8PD, EVEX_V512;
8414 }
8415}
8416
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008417defm VPMOVZXBW : avx512_extend_BW<0x30, "vpmovzxbw", X86vzext, zext_invec, "z">;
8418defm VPMOVZXBD : avx512_extend_BD<0x31, "vpmovzxbd", X86vzext, zext_invec, "z">;
8419defm VPMOVZXBQ : avx512_extend_BQ<0x32, "vpmovzxbq", X86vzext, zext_invec, "z">;
8420defm VPMOVZXWD : avx512_extend_WD<0x33, "vpmovzxwd", X86vzext, zext_invec, "z">;
8421defm VPMOVZXWQ : avx512_extend_WQ<0x34, "vpmovzxwq", X86vzext, zext_invec, "z">;
8422defm VPMOVZXDQ : avx512_extend_DQ<0x35, "vpmovzxdq", X86vzext, zext_invec, "z">;
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008423
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008424defm VPMOVSXBW: avx512_extend_BW<0x20, "vpmovsxbw", X86vsext, sext_invec, "s">;
8425defm VPMOVSXBD: avx512_extend_BD<0x21, "vpmovsxbd", X86vsext, sext_invec, "s">;
8426defm VPMOVSXBQ: avx512_extend_BQ<0x22, "vpmovsxbq", X86vsext, sext_invec, "s">;
8427defm VPMOVSXWD: avx512_extend_WD<0x23, "vpmovsxwd", X86vsext, sext_invec, "s">;
8428defm VPMOVSXWQ: avx512_extend_WQ<0x24, "vpmovsxwq", X86vsext, sext_invec, "s">;
8429defm VPMOVSXDQ: avx512_extend_DQ<0x25, "vpmovsxdq", X86vsext, sext_invec, "s">;
Elena Demikhovsky3948c592015-05-27 08:15:19 +00008430
Igor Breger2ba64ab2016-05-22 10:21:04 +00008431// EXTLOAD patterns, implemented using vpmovz
Craig Topper6840f112016-07-14 06:41:34 +00008432multiclass avx512_ext_lowering<string InstrStr, X86VectorVTInfo To,
8433 X86VectorVTInfo From, PatFrag LdFrag> {
8434 def : Pat<(To.VT (LdFrag addr:$src)),
8435 (!cast<Instruction>("VPMOVZX"#InstrStr#"rm") addr:$src)>;
8436 def : Pat<(To.VT (vselect To.KRCWM:$mask, (LdFrag addr:$src), To.RC:$src0)),
8437 (!cast<Instruction>("VPMOVZX"#InstrStr#"rmk") To.RC:$src0,
8438 To.KRC:$mask, addr:$src)>;
8439 def : Pat<(To.VT (vselect To.KRCWM:$mask, (LdFrag addr:$src),
8440 To.ImmAllZerosV)),
8441 (!cast<Instruction>("VPMOVZX"#InstrStr#"rmkz") To.KRC:$mask,
8442 addr:$src)>;
8443}
8444
8445let Predicates = [HasVLX, HasBWI] in {
8446 defm : avx512_ext_lowering<"BWZ128", v8i16x_info, v16i8x_info, extloadvi8>;
8447 defm : avx512_ext_lowering<"BWZ256", v16i16x_info, v16i8x_info, extloadvi8>;
8448}
8449let Predicates = [HasBWI] in {
8450 defm : avx512_ext_lowering<"BWZ", v32i16_info, v32i8x_info, extloadvi8>;
8451}
8452let Predicates = [HasVLX, HasAVX512] in {
8453 defm : avx512_ext_lowering<"BDZ128", v4i32x_info, v16i8x_info, extloadvi8>;
8454 defm : avx512_ext_lowering<"BDZ256", v8i32x_info, v16i8x_info, extloadvi8>;
8455 defm : avx512_ext_lowering<"BQZ128", v2i64x_info, v16i8x_info, extloadvi8>;
8456 defm : avx512_ext_lowering<"BQZ256", v4i64x_info, v16i8x_info, extloadvi8>;
8457 defm : avx512_ext_lowering<"WDZ128", v4i32x_info, v8i16x_info, extloadvi16>;
8458 defm : avx512_ext_lowering<"WDZ256", v8i32x_info, v8i16x_info, extloadvi16>;
8459 defm : avx512_ext_lowering<"WQZ128", v2i64x_info, v8i16x_info, extloadvi16>;
8460 defm : avx512_ext_lowering<"WQZ256", v4i64x_info, v8i16x_info, extloadvi16>;
8461 defm : avx512_ext_lowering<"DQZ128", v2i64x_info, v4i32x_info, extloadvi32>;
8462 defm : avx512_ext_lowering<"DQZ256", v4i64x_info, v4i32x_info, extloadvi32>;
8463}
8464let Predicates = [HasAVX512] in {
8465 defm : avx512_ext_lowering<"BDZ", v16i32_info, v16i8x_info, extloadvi8>;
8466 defm : avx512_ext_lowering<"BQZ", v8i64_info, v16i8x_info, extloadvi8>;
8467 defm : avx512_ext_lowering<"WDZ", v16i32_info, v16i16x_info, extloadvi16>;
8468 defm : avx512_ext_lowering<"WQZ", v8i64_info, v8i16x_info, extloadvi16>;
8469 defm : avx512_ext_lowering<"DQZ", v8i64_info, v8i32x_info, extloadvi32>;
8470}
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008471
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008472multiclass AVX512_pmovx_patterns<string OpcPrefix, SDNode ExtOp,
8473 SDNode InVecOp, PatFrag ExtLoad16> {
Craig Topper64378f42016-10-09 23:08:39 +00008474 // 128-bit patterns
8475 let Predicates = [HasVLX, HasBWI] in {
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008476 def : Pat<(v8i16 (InVecOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
Craig Topper64378f42016-10-09 23:08:39 +00008477 (!cast<I>(OpcPrefix#BWZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008478 def : Pat<(v8i16 (InVecOp (bc_v16i8 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),
Craig Topper64378f42016-10-09 23:08:39 +00008479 (!cast<I>(OpcPrefix#BWZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008480 def : Pat<(v8i16 (InVecOp (v16i8 (vzmovl_v2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008481 (!cast<I>(OpcPrefix#BWZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008482 def : Pat<(v8i16 (InVecOp (v16i8 (vzload_v2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008483 (!cast<I>(OpcPrefix#BWZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008484 def : Pat<(v8i16 (InVecOp (bc_v16i8 (loadv2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008485 (!cast<I>(OpcPrefix#BWZ128rm) addr:$src)>;
8486 }
8487 let Predicates = [HasVLX] in {
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008488 def : Pat<(v4i32 (InVecOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))),
Craig Topper64378f42016-10-09 23:08:39 +00008489 (!cast<I>(OpcPrefix#BDZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008490 def : Pat<(v4i32 (InVecOp (v16i8 (vzmovl_v4i32 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008491 (!cast<I>(OpcPrefix#BDZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008492 def : Pat<(v4i32 (InVecOp (v16i8 (vzload_v2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008493 (!cast<I>(OpcPrefix#BDZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008494 def : Pat<(v4i32 (InVecOp (bc_v16i8 (loadv2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008495 (!cast<I>(OpcPrefix#BDZ128rm) addr:$src)>;
8496
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008497 def : Pat<(v2i64 (InVecOp (bc_v16i8 (v4i32 (scalar_to_vector (ExtLoad16 addr:$src)))))),
Craig Topper64378f42016-10-09 23:08:39 +00008498 (!cast<I>(OpcPrefix#BQZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008499 def : Pat<(v2i64 (InVecOp (v16i8 (vzmovl_v4i32 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008500 (!cast<I>(OpcPrefix#BQZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008501 def : Pat<(v2i64 (InVecOp (v16i8 (vzload_v2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008502 (!cast<I>(OpcPrefix#BQZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008503 def : Pat<(v2i64 (InVecOp (bc_v16i8 (loadv2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008504 (!cast<I>(OpcPrefix#BQZ128rm) addr:$src)>;
8505
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008506 def : Pat<(v4i32 (InVecOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
Craig Topper64378f42016-10-09 23:08:39 +00008507 (!cast<I>(OpcPrefix#WDZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008508 def : Pat<(v4i32 (InVecOp (bc_v8i16 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),
Craig Topper64378f42016-10-09 23:08:39 +00008509 (!cast<I>(OpcPrefix#WDZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008510 def : Pat<(v4i32 (InVecOp (v8i16 (vzmovl_v2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008511 (!cast<I>(OpcPrefix#WDZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008512 def : Pat<(v4i32 (InVecOp (v8i16 (vzload_v2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008513 (!cast<I>(OpcPrefix#WDZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008514 def : Pat<(v4i32 (InVecOp (bc_v8i16 (loadv2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008515 (!cast<I>(OpcPrefix#WDZ128rm) addr:$src)>;
8516
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008517 def : Pat<(v2i64 (InVecOp (bc_v8i16 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))),
Craig Topper64378f42016-10-09 23:08:39 +00008518 (!cast<I>(OpcPrefix#WQZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008519 def : Pat<(v2i64 (InVecOp (v8i16 (vzmovl_v4i32 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008520 (!cast<I>(OpcPrefix#WQZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008521 def : Pat<(v2i64 (InVecOp (v8i16 (vzload_v2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008522 (!cast<I>(OpcPrefix#WQZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008523 def : Pat<(v2i64 (InVecOp (bc_v8i16 (loadv2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008524 (!cast<I>(OpcPrefix#WQZ128rm) addr:$src)>;
8525
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008526 def : Pat<(v2i64 (InVecOp (bc_v4i32 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
Craig Topper64378f42016-10-09 23:08:39 +00008527 (!cast<I>(OpcPrefix#DQZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008528 def : Pat<(v2i64 (InVecOp (bc_v4i32 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),
Craig Topper64378f42016-10-09 23:08:39 +00008529 (!cast<I>(OpcPrefix#DQZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008530 def : Pat<(v2i64 (InVecOp (v4i32 (vzmovl_v2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008531 (!cast<I>(OpcPrefix#DQZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008532 def : Pat<(v2i64 (InVecOp (v4i32 (vzload_v2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008533 (!cast<I>(OpcPrefix#DQZ128rm) addr:$src)>;
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008534 def : Pat<(v2i64 (InVecOp (bc_v4i32 (loadv2i64 addr:$src)))),
Craig Topper64378f42016-10-09 23:08:39 +00008535 (!cast<I>(OpcPrefix#DQZ128rm) addr:$src)>;
8536 }
8537 // 256-bit patterns
8538 let Predicates = [HasVLX, HasBWI] in {
8539 def : Pat<(v16i16 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))),
8540 (!cast<I>(OpcPrefix#BWZ256rm) addr:$src)>;
8541 def : Pat<(v16i16 (ExtOp (v16i8 (vzmovl_v2i64 addr:$src)))),
8542 (!cast<I>(OpcPrefix#BWZ256rm) addr:$src)>;
8543 def : Pat<(v16i16 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))),
8544 (!cast<I>(OpcPrefix#BWZ256rm) addr:$src)>;
8545 }
8546 let Predicates = [HasVLX] in {
8547 def : Pat<(v8i32 (ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
8548 (!cast<I>(OpcPrefix#BDZ256rm) addr:$src)>;
8549 def : Pat<(v8i32 (ExtOp (v16i8 (vzmovl_v2i64 addr:$src)))),
8550 (!cast<I>(OpcPrefix#BDZ256rm) addr:$src)>;
8551 def : Pat<(v8i32 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))),
8552 (!cast<I>(OpcPrefix#BDZ256rm) addr:$src)>;
8553 def : Pat<(v8i32 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))),
8554 (!cast<I>(OpcPrefix#BDZ256rm) addr:$src)>;
8555
8556 def : Pat<(v4i64 (ExtOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))),
8557 (!cast<I>(OpcPrefix#BQZ256rm) addr:$src)>;
8558 def : Pat<(v4i64 (ExtOp (v16i8 (vzmovl_v4i32 addr:$src)))),
8559 (!cast<I>(OpcPrefix#BQZ256rm) addr:$src)>;
8560 def : Pat<(v4i64 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))),
8561 (!cast<I>(OpcPrefix#BQZ256rm) addr:$src)>;
8562 def : Pat<(v4i64 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))),
8563 (!cast<I>(OpcPrefix#BQZ256rm) addr:$src)>;
8564
8565 def : Pat<(v8i32 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))),
8566 (!cast<I>(OpcPrefix#WDZ256rm) addr:$src)>;
8567 def : Pat<(v8i32 (ExtOp (v8i16 (vzmovl_v2i64 addr:$src)))),
8568 (!cast<I>(OpcPrefix#WDZ256rm) addr:$src)>;
8569 def : Pat<(v8i32 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))),
8570 (!cast<I>(OpcPrefix#WDZ256rm) addr:$src)>;
8571
8572 def : Pat<(v4i64 (ExtOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
8573 (!cast<I>(OpcPrefix#WQZ256rm) addr:$src)>;
8574 def : Pat<(v4i64 (ExtOp (v8i16 (vzmovl_v2i64 addr:$src)))),
8575 (!cast<I>(OpcPrefix#WQZ256rm) addr:$src)>;
8576 def : Pat<(v4i64 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))),
8577 (!cast<I>(OpcPrefix#WQZ256rm) addr:$src)>;
8578 def : Pat<(v4i64 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))),
8579 (!cast<I>(OpcPrefix#WQZ256rm) addr:$src)>;
8580
8581 def : Pat<(v4i64 (ExtOp (bc_v4i32 (loadv2i64 addr:$src)))),
8582 (!cast<I>(OpcPrefix#DQZ256rm) addr:$src)>;
8583 def : Pat<(v4i64 (ExtOp (v4i32 (vzmovl_v2i64 addr:$src)))),
8584 (!cast<I>(OpcPrefix#DQZ256rm) addr:$src)>;
8585 def : Pat<(v4i64 (ExtOp (v4i32 (vzload_v2i64 addr:$src)))),
8586 (!cast<I>(OpcPrefix#DQZ256rm) addr:$src)>;
8587 }
8588 // 512-bit patterns
8589 let Predicates = [HasBWI] in {
8590 def : Pat<(v32i16 (ExtOp (bc_v32i8 (loadv4i64 addr:$src)))),
8591 (!cast<I>(OpcPrefix#BWZrm) addr:$src)>;
8592 }
8593 let Predicates = [HasAVX512] in {
8594 def : Pat<(v16i32 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))),
8595 (!cast<I>(OpcPrefix#BDZrm) addr:$src)>;
8596
8597 def : Pat<(v8i64 (ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
8598 (!cast<I>(OpcPrefix#BQZrm) addr:$src)>;
Craig Topper9ece2f72016-10-10 06:25:48 +00008599 def : Pat<(v8i64 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))),
8600 (!cast<I>(OpcPrefix#BQZrm) addr:$src)>;
Craig Topper64378f42016-10-09 23:08:39 +00008601
8602 def : Pat<(v16i32 (ExtOp (bc_v16i16 (loadv4i64 addr:$src)))),
8603 (!cast<I>(OpcPrefix#WDZrm) addr:$src)>;
8604
8605 def : Pat<(v8i64 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))),
8606 (!cast<I>(OpcPrefix#WQZrm) addr:$src)>;
8607
8608 def : Pat<(v8i64 (ExtOp (bc_v8i32 (loadv4i64 addr:$src)))),
8609 (!cast<I>(OpcPrefix#DQZrm) addr:$src)>;
8610 }
8611}
8612
Simon Pilgrim9f5c2512017-03-05 09:57:20 +00008613defm : AVX512_pmovx_patterns<"VPMOVSX", X86vsext, sext_invec, extloadi32i16>;
8614defm : AVX512_pmovx_patterns<"VPMOVZX", X86vzext, zext_invec, loadi16_anyext>;
Craig Topper64378f42016-10-09 23:08:39 +00008615
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008616//===----------------------------------------------------------------------===//
8617// GATHER - SCATTER Operations
8618
Elena Demikhovskye1eda8a2015-04-30 08:38:48 +00008619multiclass avx512_gather<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
8620 X86MemOperand memop, PatFrag GatherNode> {
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008621 let Constraints = "@earlyclobber $dst, $src1 = $dst, $mask = $mask_wb",
8622 ExeDomain = _.ExeDomain in
Elena Demikhovskye1eda8a2015-04-30 08:38:48 +00008623 def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst, _.KRCWM:$mask_wb),
8624 (ins _.RC:$src1, _.KRCWM:$mask, memop:$src2),
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008625 !strconcat(OpcodeStr#_.Suffix,
Craig Topperedb09112014-11-25 20:11:23 +00008626 "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"),
Elena Demikhovskye1eda8a2015-04-30 08:38:48 +00008627 [(set _.RC:$dst, _.KRCWM:$mask_wb,
8628 (GatherNode (_.VT _.RC:$src1), _.KRCWM:$mask,
8629 vectoraddr:$src2))]>, EVEX, EVEX_K,
8630 EVEX_CD8<_.EltSize, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008631}
Cameron McInally45325962014-03-26 13:50:50 +00008632
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008633multiclass avx512_gather_q_pd<bits<8> dopc, bits<8> qopc,
8634 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
8635 defm NAME##D##SUFF##Z: avx512_gather<dopc, OpcodeStr##"d", _.info512,
Igor Breger45ef10f2016-02-25 13:30:17 +00008636 vy512mem, mgatherv8i32>, EVEX_V512, VEX_W;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008637 defm NAME##Q##SUFF##Z: avx512_gather<qopc, OpcodeStr##"q", _.info512,
Igor Breger45ef10f2016-02-25 13:30:17 +00008638 vz512mem, mgatherv8i64>, EVEX_V512, VEX_W;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008639let Predicates = [HasVLX] in {
8640 defm NAME##D##SUFF##Z256: avx512_gather<dopc, OpcodeStr##"d", _.info256,
Igor Breger45ef10f2016-02-25 13:30:17 +00008641 vx256xmem, mgatherv4i32>, EVEX_V256, VEX_W;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008642 defm NAME##Q##SUFF##Z256: avx512_gather<qopc, OpcodeStr##"q", _.info256,
Igor Breger45ef10f2016-02-25 13:30:17 +00008643 vy256xmem, mgatherv4i64>, EVEX_V256, VEX_W;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008644 defm NAME##D##SUFF##Z128: avx512_gather<dopc, OpcodeStr##"d", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00008645 vx128xmem, mgatherv4i32>, EVEX_V128, VEX_W;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008646 defm NAME##Q##SUFF##Z128: avx512_gather<qopc, OpcodeStr##"q", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00008647 vx128xmem, mgatherv2i64>, EVEX_V128, VEX_W;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008648}
Cameron McInally45325962014-03-26 13:50:50 +00008649}
8650
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008651multiclass avx512_gather_d_ps<bits<8> dopc, bits<8> qopc,
8652 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
Igor Breger45ef10f2016-02-25 13:30:17 +00008653 defm NAME##D##SUFF##Z: avx512_gather<dopc, OpcodeStr##"d", _.info512, vz512mem,
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008654 mgatherv16i32>, EVEX_V512;
Craig Topper7dfd5832017-01-16 00:55:58 +00008655 defm NAME##Q##SUFF##Z: avx512_gather<qopc, OpcodeStr##"q", _.info256, vz256xmem,
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008656 mgatherv8i64>, EVEX_V512;
8657let Predicates = [HasVLX] in {
8658 defm NAME##D##SUFF##Z256: avx512_gather<dopc, OpcodeStr##"d", _.info256,
Igor Breger45ef10f2016-02-25 13:30:17 +00008659 vy256xmem, mgatherv8i32>, EVEX_V256;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008660 defm NAME##Q##SUFF##Z256: avx512_gather<qopc, OpcodeStr##"q", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00008661 vy128xmem, mgatherv4i64>, EVEX_V256;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008662 defm NAME##D##SUFF##Z128: avx512_gather<dopc, OpcodeStr##"d", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00008663 vx128xmem, mgatherv4i32>, EVEX_V128;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008664 defm NAME##Q##SUFF##Z128: avx512_gather<qopc, OpcodeStr##"q", _.info128,
Elena Demikhovsky2dac0b42017-06-22 06:47:41 +00008665 vx64xmem, X86mgatherv2i64>, EVEX_V128;
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008666}
Cameron McInally45325962014-03-26 13:50:50 +00008667}
Michael Liao5bf95782014-12-04 05:20:33 +00008668
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008669
Elena Demikhovsky6a1a3572015-06-28 10:53:29 +00008670defm VGATHER : avx512_gather_q_pd<0x92, 0x93, avx512vl_f64_info, "vgather", "PD">,
8671 avx512_gather_d_ps<0x92, 0x93, avx512vl_f32_info, "vgather", "PS">;
8672
8673defm VPGATHER : avx512_gather_q_pd<0x90, 0x91, avx512vl_i64_info, "vpgather", "Q">,
8674 avx512_gather_d_ps<0x90, 0x91, avx512vl_i32_info, "vpgather", "D">;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008675
Elena Demikhovskye1eda8a2015-04-30 08:38:48 +00008676multiclass avx512_scatter<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
8677 X86MemOperand memop, PatFrag ScatterNode> {
8678
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008679let mayStore = 1, Constraints = "$mask = $mask_wb", ExeDomain = _.ExeDomain in
Elena Demikhovskye1eda8a2015-04-30 08:38:48 +00008680
8681 def mr : AVX5128I<opc, MRMDestMem, (outs _.KRCWM:$mask_wb),
8682 (ins memop:$dst, _.KRCWM:$mask, _.RC:$src),
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008683 !strconcat(OpcodeStr#_.Suffix,
Elena Demikhovskye1eda8a2015-04-30 08:38:48 +00008684 "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"),
8685 [(set _.KRCWM:$mask_wb, (ScatterNode (_.VT _.RC:$src),
8686 _.KRCWM:$mask, vectoraddr:$dst))]>,
8687 EVEX, EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008688}
8689
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008690multiclass avx512_scatter_q_pd<bits<8> dopc, bits<8> qopc,
8691 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
8692 defm NAME##D##SUFF##Z: avx512_scatter<dopc, OpcodeStr##"d", _.info512,
Igor Breger45ef10f2016-02-25 13:30:17 +00008693 vy512mem, mscatterv8i32>, EVEX_V512, VEX_W;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008694 defm NAME##Q##SUFF##Z: avx512_scatter<qopc, OpcodeStr##"q", _.info512,
Igor Breger45ef10f2016-02-25 13:30:17 +00008695 vz512mem, mscatterv8i64>, EVEX_V512, VEX_W;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008696let Predicates = [HasVLX] in {
8697 defm NAME##D##SUFF##Z256: avx512_scatter<dopc, OpcodeStr##"d", _.info256,
Igor Breger45ef10f2016-02-25 13:30:17 +00008698 vx256xmem, mscatterv4i32>, EVEX_V256, VEX_W;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008699 defm NAME##Q##SUFF##Z256: avx512_scatter<qopc, OpcodeStr##"q", _.info256,
Igor Breger45ef10f2016-02-25 13:30:17 +00008700 vy256xmem, mscatterv4i64>, EVEX_V256, VEX_W;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008701 defm NAME##D##SUFF##Z128: avx512_scatter<dopc, OpcodeStr##"d", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00008702 vx128xmem, mscatterv4i32>, EVEX_V128, VEX_W;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008703 defm NAME##Q##SUFF##Z128: avx512_scatter<qopc, OpcodeStr##"q", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00008704 vx128xmem, mscatterv2i64>, EVEX_V128, VEX_W;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008705}
Cameron McInally45325962014-03-26 13:50:50 +00008706}
8707
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008708multiclass avx512_scatter_d_ps<bits<8> dopc, bits<8> qopc,
8709 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
Igor Breger45ef10f2016-02-25 13:30:17 +00008710 defm NAME##D##SUFF##Z: avx512_scatter<dopc, OpcodeStr##"d", _.info512, vz512mem,
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008711 mscatterv16i32>, EVEX_V512;
Craig Topper7dfd5832017-01-16 00:55:58 +00008712 defm NAME##Q##SUFF##Z: avx512_scatter<qopc, OpcodeStr##"q", _.info256, vz256xmem,
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008713 mscatterv8i64>, EVEX_V512;
8714let Predicates = [HasVLX] in {
8715 defm NAME##D##SUFF##Z256: avx512_scatter<dopc, OpcodeStr##"d", _.info256,
Igor Breger45ef10f2016-02-25 13:30:17 +00008716 vy256xmem, mscatterv8i32>, EVEX_V256;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008717 defm NAME##Q##SUFF##Z256: avx512_scatter<qopc, OpcodeStr##"q", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00008718 vy128xmem, mscatterv4i64>, EVEX_V256;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008719 defm NAME##D##SUFF##Z128: avx512_scatter<dopc, OpcodeStr##"d", _.info128,
Igor Breger45ef10f2016-02-25 13:30:17 +00008720 vx128xmem, mscatterv4i32>, EVEX_V128;
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008721 defm NAME##Q##SUFF##Z128: avx512_scatter<qopc, OpcodeStr##"q", _.info128,
8722 vx64xmem, mscatterv2i64>, EVEX_V128;
8723}
Cameron McInally45325962014-03-26 13:50:50 +00008724}
8725
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008726defm VSCATTER : avx512_scatter_q_pd<0xA2, 0xA3, avx512vl_f64_info, "vscatter", "PD">,
8727 avx512_scatter_d_ps<0xA2, 0xA3, avx512vl_f32_info, "vscatter", "PS">;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008728
Elena Demikhovsky30bc4ca2015-06-29 12:14:24 +00008729defm VPSCATTER : avx512_scatter_q_pd<0xA0, 0xA1, avx512vl_i64_info, "vpscatter", "Q">,
8730 avx512_scatter_d_ps<0xA0, 0xA1, avx512vl_i32_info, "vpscatter", "D">;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008731
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008732// prefetch
8733multiclass avx512_gather_scatter_prefetch<bits<8> opc, Format F, string OpcodeStr,
8734 RegisterClass KRC, X86MemOperand memop> {
8735 let Predicates = [HasPFI], hasSideEffects = 1 in
8736 def m : AVX5128I<opc, F, (outs), (ins KRC:$mask, memop:$src),
Craig Topperedb09112014-11-25 20:11:23 +00008737 !strconcat(OpcodeStr, "\t{$src {${mask}}|{${mask}}, $src}"),
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008738 []>, EVEX, EVEX_K;
8739}
8740
8741defm VGATHERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dps",
Igor Breger45ef10f2016-02-25 13:30:17 +00008742 VK16WM, vz512mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008743
8744defm VGATHERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qps",
Craig Topper7dfd5832017-01-16 00:55:58 +00008745 VK8WM, vz256xmem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008746
8747defm VGATHERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00008748 VK8WM, vy512mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008749
8750defm VGATHERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00008751 VK8WM, vz512mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
Michael Liao5bf95782014-12-04 05:20:33 +00008752
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008753defm VGATHERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dps",
Igor Breger45ef10f2016-02-25 13:30:17 +00008754 VK16WM, vz512mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008755
8756defm VGATHERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qps",
Craig Topper7dfd5832017-01-16 00:55:58 +00008757 VK8WM, vz256xmem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008758
8759defm VGATHERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00008760 VK8WM, vy512mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008761
8762defm VGATHERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00008763 VK8WM, vz512mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008764
8765defm VSCATTERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dps",
Igor Breger45ef10f2016-02-25 13:30:17 +00008766 VK16WM, vz512mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008767
8768defm VSCATTERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qps",
Craig Topper7dfd5832017-01-16 00:55:58 +00008769 VK8WM, vz256xmem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008770
8771defm VSCATTERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00008772 VK8WM, vy512mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008773
8774defm VSCATTERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00008775 VK8WM, vz512mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008776
8777defm VSCATTERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dps",
Igor Breger45ef10f2016-02-25 13:30:17 +00008778 VK16WM, vz512mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008779
8780defm VSCATTERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qps",
Craig Topper7dfd5832017-01-16 00:55:58 +00008781 VK8WM, vz256xmem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008782
8783defm VSCATTERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00008784 VK8WM, vy512mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky8e8fde82014-05-12 07:18:51 +00008785
8786defm VSCATTERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qpd",
Igor Breger45ef10f2016-02-25 13:30:17 +00008787 VK8WM, vz512mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
Elena Demikhovskyac3e8eb2013-09-17 07:34:34 +00008788
Elena Demikhovskybb2f6b72014-03-27 09:45:08 +00008789// Helper fragments to match sext vXi1 to vXiY.
Craig Topper850feaf2016-08-28 22:20:51 +00008790def v64i1sextv64i8 : PatLeaf<(v64i8
8791 (X86vsext
8792 (v64i1 (X86pcmpgtm
8793 (bc_v64i8 (v16i32 immAllZerosV)),
8794 VR512:$src))))>;
8795def v32i1sextv32i16 : PatLeaf<(v32i16 (X86vsrai VR512:$src, (i8 15)))>;
8796def v16i1sextv16i32 : PatLeaf<(v16i32 (X86vsrai VR512:$src, (i8 31)))>;
8797def v8i1sextv8i64 : PatLeaf<(v8i64 (X86vsrai VR512:$src, (i8 63)))>;
Elena Demikhovskybb2f6b72014-03-27 09:45:08 +00008798
Elena Demikhovsky44bf0632014-10-05 14:11:08 +00008799multiclass cvt_by_vec_width<bits<8> opc, X86VectorVTInfo Vec, string OpcodeStr > {
Elena Demikhovsky0e6d6d52015-04-21 14:38:31 +00008800def rr : AVX512XS8I<opc, MRMSrcReg, (outs Vec.RC:$dst), (ins Vec.KRC:$src),
Craig Topperedb09112014-11-25 20:11:23 +00008801 !strconcat(OpcodeStr##Vec.Suffix, "\t{$src, $dst|$dst, $src}"),
Elena Demikhovsky44bf0632014-10-05 14:11:08 +00008802 [(set Vec.RC:$dst, (Vec.VT (X86vsext Vec.KRC:$src)))]>, EVEX;
8803}
Michael Liao5bf95782014-12-04 05:20:33 +00008804
Michael Zuckerman85436ec2017-03-23 09:57:01 +00008805// Use 512bit version to implement 128/256 bit in case NoVLX.
8806multiclass avx512_convert_mask_to_vector_lowering<X86VectorVTInfo X86Info,
8807 X86VectorVTInfo _> {
8808
8809 def : Pat<(X86Info.VT (X86vsext (X86Info.KVT X86Info.KRC:$src))),
8810 (X86Info.VT (EXTRACT_SUBREG
8811 (_.VT (!cast<Instruction>(NAME#"Zrr")
8812 (_.KVT (COPY_TO_REGCLASS X86Info.KRC:$src,_.KRC)))),
8813 X86Info.SubRegIdx))>;
8814}
8815
Elena Demikhovsky44bf0632014-10-05 14:11:08 +00008816multiclass cvt_mask_by_elt_width<bits<8> opc, AVX512VLVectorVTInfo VTInfo,
8817 string OpcodeStr, Predicate prd> {
8818let Predicates = [prd] in
8819 defm Z : cvt_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512;
8820
8821 let Predicates = [prd, HasVLX] in {
8822 defm Z256 : cvt_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256;
8823 defm Z128 : cvt_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128;
8824 }
Michael Zuckerman85436ec2017-03-23 09:57:01 +00008825let Predicates = [prd, NoVLX] in {
8826 defm Z256_Alt : avx512_convert_mask_to_vector_lowering<VTInfo.info256,VTInfo.info512>;
8827 defm Z128_Alt : avx512_convert_mask_to_vector_lowering<VTInfo.info128,VTInfo.info512>;
8828 }
8829
Elena Demikhovsky44bf0632014-10-05 14:11:08 +00008830}
8831
Michael Zuckerman85436ec2017-03-23 09:57:01 +00008832defm VPMOVM2B : cvt_mask_by_elt_width<0x28, avx512vl_i8_info, "vpmovm2" , HasBWI>;
8833defm VPMOVM2W : cvt_mask_by_elt_width<0x28, avx512vl_i16_info, "vpmovm2", HasBWI> , VEX_W;
8834defm VPMOVM2D : cvt_mask_by_elt_width<0x38, avx512vl_i32_info, "vpmovm2", HasDQI>;
8835defm VPMOVM2Q : cvt_mask_by_elt_width<0x38, avx512vl_i64_info, "vpmovm2", HasDQI> , VEX_W;
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00008836
Elena Demikhovsky0e6d6d52015-04-21 14:38:31 +00008837multiclass convert_vector_to_mask_common<bits<8> opc, X86VectorVTInfo _, string OpcodeStr > {
Igor Bregerfca0a342016-01-28 13:19:25 +00008838 def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.KRC:$dst), (ins _.RC:$src),
8839 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
8840 [(set _.KRC:$dst, (X86cvt2mask (_.VT _.RC:$src)))]>, EVEX;
8841}
8842
Simon Pilgrim18bcf932016-02-03 09:41:59 +00008843// Use 512bit version to implement 128/256 bit in case NoVLX.
8844multiclass convert_vector_to_mask_lowering<X86VectorVTInfo ExtendInfo,
Igor Bregerfca0a342016-01-28 13:19:25 +00008845 X86VectorVTInfo _> {
8846
8847 def : Pat<(_.KVT (X86cvt2mask (_.VT _.RC:$src))),
8848 (_.KVT (COPY_TO_REGCLASS
8849 (!cast<Instruction>(NAME#"Zrr")
Simon Pilgrim18bcf932016-02-03 09:41:59 +00008850 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
Igor Bregerfca0a342016-01-28 13:19:25 +00008851 _.RC:$src, _.SubRegIdx)),
8852 _.KRC))>;
Elena Demikhovsky0e6d6d52015-04-21 14:38:31 +00008853}
8854
8855multiclass avx512_convert_vector_to_mask<bits<8> opc, string OpcodeStr,
Igor Bregerfca0a342016-01-28 13:19:25 +00008856 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
8857 let Predicates = [prd] in
8858 defm Z : convert_vector_to_mask_common <opc, VTInfo.info512, OpcodeStr>,
8859 EVEX_V512;
Elena Demikhovsky0e6d6d52015-04-21 14:38:31 +00008860
8861 let Predicates = [prd, HasVLX] in {
8862 defm Z256 : convert_vector_to_mask_common<opc, VTInfo.info256, OpcodeStr>,
Igor Bregerfca0a342016-01-28 13:19:25 +00008863 EVEX_V256;
Elena Demikhovsky0e6d6d52015-04-21 14:38:31 +00008864 defm Z128 : convert_vector_to_mask_common<opc, VTInfo.info128, OpcodeStr>,
Igor Bregerfca0a342016-01-28 13:19:25 +00008865 EVEX_V128;
8866 }
8867 let Predicates = [prd, NoVLX] in {
8868 defm Z256_Alt : convert_vector_to_mask_lowering<VTInfo.info512, VTInfo.info256>;
8869 defm Z128_Alt : convert_vector_to_mask_lowering<VTInfo.info512, VTInfo.info128>;
Elena Demikhovsky0e6d6d52015-04-21 14:38:31 +00008870 }
8871}
8872
8873defm VPMOVB2M : avx512_convert_vector_to_mask<0x29, "vpmovb2m",
8874 avx512vl_i8_info, HasBWI>;
8875defm VPMOVW2M : avx512_convert_vector_to_mask<0x29, "vpmovw2m",
8876 avx512vl_i16_info, HasBWI>, VEX_W;
8877defm VPMOVD2M : avx512_convert_vector_to_mask<0x39, "vpmovd2m",
8878 avx512vl_i32_info, HasDQI>;
8879defm VPMOVQ2M : avx512_convert_vector_to_mask<0x39, "vpmovq2m",
8880 avx512vl_i64_info, HasDQI>, VEX_W;
8881
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00008882//===----------------------------------------------------------------------===//
8883// AVX-512 - COMPRESS and EXPAND
8884//
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00008885
Ayman Musad7a5ed42016-09-26 06:22:08 +00008886multiclass compress_by_vec_width_common<bits<8> opc, X86VectorVTInfo _,
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00008887 string OpcodeStr> {
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00008888 defm rr : AVX512_maskable<opc, MRMDestReg, _, (outs _.RC:$dst),
Michael Liao66233b72015-08-06 09:06:20 +00008889 (ins _.RC:$src1), OpcodeStr, "$src1", "$src1",
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00008890 (_.VT (X86compress _.RC:$src1))>, AVX5128IBase;
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00008891
Craig Toppere1cac152016-06-07 07:27:54 +00008892 let mayStore = 1, hasSideEffects = 0 in
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00008893 def mr : AVX5128I<opc, MRMDestMem, (outs),
8894 (ins _.MemOp:$dst, _.RC:$src),
Craig Topper9feea572016-01-11 00:44:58 +00008895 OpcodeStr # "\t{$src, $dst|$dst, $src}",
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00008896 []>, EVEX_CD8<_.EltSize, CD8VT1>;
8897
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00008898 def mrk : AVX5128I<opc, MRMDestMem, (outs),
8899 (ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src),
Craig Topper9feea572016-01-11 00:44:58 +00008900 OpcodeStr # "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",
Ayman Musad7a5ed42016-09-26 06:22:08 +00008901 []>,
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00008902 EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>;
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00008903}
8904
Ayman Musad7a5ed42016-09-26 06:22:08 +00008905multiclass compress_by_vec_width_lowering<X86VectorVTInfo _ > {
8906
8907 def : Pat<(X86mCompressingStore addr:$dst, _.KRCWM:$mask,
8908 (_.VT _.RC:$src)),
8909 (!cast<Instruction>(NAME#_.ZSuffix##mrk)
8910 addr:$dst, _.KRCWM:$mask, _.RC:$src)>;
8911}
8912
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00008913multiclass compress_by_elt_width<bits<8> opc, string OpcodeStr,
8914 AVX512VLVectorVTInfo VTInfo> {
Ayman Musad7a5ed42016-09-26 06:22:08 +00008915 defm Z : compress_by_vec_width_common<opc, VTInfo.info512, OpcodeStr>,
8916 compress_by_vec_width_lowering<VTInfo.info512>, EVEX_V512;
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00008917
8918 let Predicates = [HasVLX] in {
Ayman Musad7a5ed42016-09-26 06:22:08 +00008919 defm Z256 : compress_by_vec_width_common<opc, VTInfo.info256, OpcodeStr>,
8920 compress_by_vec_width_lowering<VTInfo.info256>, EVEX_V256;
8921 defm Z128 : compress_by_vec_width_common<opc, VTInfo.info128, OpcodeStr>,
8922 compress_by_vec_width_lowering<VTInfo.info128>, EVEX_V128;
Elena Demikhovsky908dbf42014-12-11 15:02:24 +00008923 }
8924}
8925
8926defm VPCOMPRESSD : compress_by_elt_width <0x8B, "vpcompressd", avx512vl_i32_info>,
8927 EVEX;
8928defm VPCOMPRESSQ : compress_by_elt_width <0x8B, "vpcompressq", avx512vl_i64_info>,
8929 EVEX, VEX_W;
8930defm VCOMPRESSPS : compress_by_elt_width <0x8A, "vcompressps", avx512vl_f32_info>,
8931 EVEX;
8932defm VCOMPRESSPD : compress_by_elt_width <0x8A, "vcompresspd", avx512vl_f64_info>,
8933 EVEX, VEX_W;
8934
Elena Demikhovsky72860c32014-12-15 10:03:52 +00008935// expand
8936multiclass expand_by_vec_width<bits<8> opc, X86VectorVTInfo _,
8937 string OpcodeStr> {
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00008938 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
Michael Liao66233b72015-08-06 09:06:20 +00008939 (ins _.RC:$src1), OpcodeStr, "$src1", "$src1",
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00008940 (_.VT (X86expand _.RC:$src1))>, AVX5128IBase;
Elena Demikhovsky75ede682015-06-01 07:17:23 +00008941
Elena Demikhovskyba5ab322015-06-22 11:16:30 +00008942 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
8943 (ins _.MemOp:$src1), OpcodeStr, "$src1", "$src1",
8944 (_.VT (X86expand (_.VT (bitconvert
8945 (_.LdFrag addr:$src1)))))>,
8946 AVX5128IBase, EVEX_CD8<_.EltSize, CD8VT1>;
Elena Demikhovsky72860c32014-12-15 10:03:52 +00008947}
8948
Elena Demikhovsky5b10aa12016-10-09 10:48:52 +00008949multiclass expand_by_vec_width_lowering<X86VectorVTInfo _ > {
8950
8951 def : Pat<(_.VT (X86mExpandingLoad addr:$src, _.KRCWM:$mask, undef)),
8952 (!cast<Instruction>(NAME#_.ZSuffix##rmkz)
8953 _.KRCWM:$mask, addr:$src)>;
8954
8955 def : Pat<(_.VT (X86mExpandingLoad addr:$src, _.KRCWM:$mask,
8956 (_.VT _.RC:$src0))),
8957 (!cast<Instruction>(NAME#_.ZSuffix##rmk)
8958 _.RC:$src0, _.KRCWM:$mask, addr:$src)>;
8959}
8960
Elena Demikhovsky72860c32014-12-15 10:03:52 +00008961multiclass expand_by_elt_width<bits<8> opc, string OpcodeStr,
8962 AVX512VLVectorVTInfo VTInfo> {
Elena Demikhovsky5b10aa12016-10-09 10:48:52 +00008963 defm Z : expand_by_vec_width<opc, VTInfo.info512, OpcodeStr>,
8964 expand_by_vec_width_lowering<VTInfo.info512>, EVEX_V512;
Elena Demikhovsky72860c32014-12-15 10:03:52 +00008965
8966 let Predicates = [HasVLX] in {
Elena Demikhovsky5b10aa12016-10-09 10:48:52 +00008967 defm Z256 : expand_by_vec_width<opc, VTInfo.info256, OpcodeStr>,
8968 expand_by_vec_width_lowering<VTInfo.info256>, EVEX_V256;
8969 defm Z128 : expand_by_vec_width<opc, VTInfo.info128, OpcodeStr>,
8970 expand_by_vec_width_lowering<VTInfo.info128>, EVEX_V128;
Elena Demikhovsky72860c32014-12-15 10:03:52 +00008971 }
8972}
8973
8974defm VPEXPANDD : expand_by_elt_width <0x89, "vpexpandd", avx512vl_i32_info>,
8975 EVEX;
8976defm VPEXPANDQ : expand_by_elt_width <0x89, "vpexpandq", avx512vl_i64_info>,
8977 EVEX, VEX_W;
8978defm VEXPANDPS : expand_by_elt_width <0x88, "vexpandps", avx512vl_f32_info>,
8979 EVEX;
8980defm VEXPANDPD : expand_by_elt_width <0x88, "vexpandpd", avx512vl_f64_info>,
8981 EVEX, VEX_W;
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00008982
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00008983//handle instruction reg_vec1 = op(reg_vec,imm)
8984// op(mem_vec,imm)
8985// op(broadcast(eltVt),imm)
8986//all instruction created with FROUND_CURRENT
8987multiclass avx512_unary_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper05948fb2016-08-02 05:11:15 +00008988 X86VectorVTInfo _>{
8989 let ExeDomain = _.ExeDomain in {
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00008990 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
8991 (ins _.RC:$src1, i32u8imm:$src2),
Igor Breger252c2d92016-02-22 12:37:41 +00008992 OpcodeStr##_.Suffix, "$src2, $src1", "$src1, $src2",
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00008993 (OpNode (_.VT _.RC:$src1),
8994 (i32 imm:$src2),
8995 (i32 FROUND_CURRENT))>;
Craig Toppere1cac152016-06-07 07:27:54 +00008996 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
8997 (ins _.MemOp:$src1, i32u8imm:$src2),
8998 OpcodeStr##_.Suffix, "$src2, $src1", "$src1, $src2",
8999 (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
9000 (i32 imm:$src2),
9001 (i32 FROUND_CURRENT))>;
9002 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
9003 (ins _.ScalarMemOp:$src1, i32u8imm:$src2),
9004 OpcodeStr##_.Suffix, "$src2, ${src1}"##_.BroadcastStr,
9005 "${src1}"##_.BroadcastStr##", $src2",
9006 (OpNode (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src1))),
9007 (i32 imm:$src2),
9008 (i32 FROUND_CURRENT))>, EVEX_B;
Craig Topper05948fb2016-08-02 05:11:15 +00009009 }
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009010}
9011
9012//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}
9013multiclass avx512_unary_fp_sae_packed_imm<bits<8> opc, string OpcodeStr,
9014 SDNode OpNode, X86VectorVTInfo _>{
Craig Topper05948fb2016-08-02 05:11:15 +00009015 let ExeDomain = _.ExeDomain in
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009016 defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
9017 (ins _.RC:$src1, i32u8imm:$src2),
Craig Topperbfe13ff2016-01-11 00:44:52 +00009018 OpcodeStr##_.Suffix, "$src2, {sae}, $src1",
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009019 "$src1, {sae}, $src2",
9020 (OpNode (_.VT _.RC:$src1),
9021 (i32 imm:$src2),
9022 (i32 FROUND_NO_EXC))>, EVEX_B;
9023}
9024
9025multiclass avx512_common_unary_fp_sae_packed_imm<string OpcodeStr,
9026 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
9027 let Predicates = [prd] in {
9028 defm Z : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
9029 avx512_unary_fp_sae_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
9030 EVEX_V512;
9031 }
9032 let Predicates = [prd, HasVLX] in {
9033 defm Z128 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info128>,
9034 EVEX_V128;
9035 defm Z256 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info256>,
9036 EVEX_V256;
9037 }
9038}
9039
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00009040//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)
9041// op(reg_vec2,mem_vec,imm)
9042// op(reg_vec2,broadcast(eltVt),imm)
9043//all instruction created with FROUND_CURRENT
9044multiclass avx512_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper05948fb2016-08-02 05:11:15 +00009045 X86VectorVTInfo _>{
9046 let ExeDomain = _.ExeDomain in {
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00009047 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009048 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00009049 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
9050 (OpNode (_.VT _.RC:$src1),
9051 (_.VT _.RC:$src2),
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009052 (i32 imm:$src3),
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00009053 (i32 FROUND_CURRENT))>;
Craig Toppere1cac152016-06-07 07:27:54 +00009054 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
9055 (ins _.RC:$src1, _.MemOp:$src2, i32u8imm:$src3),
9056 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
9057 (OpNode (_.VT _.RC:$src1),
9058 (_.VT (bitconvert (_.LdFrag addr:$src2))),
9059 (i32 imm:$src3),
9060 (i32 FROUND_CURRENT))>;
9061 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
9062 (ins _.RC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3),
9063 OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1",
9064 "$src1, ${src2}"##_.BroadcastStr##", $src3",
9065 (OpNode (_.VT _.RC:$src1),
9066 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
9067 (i32 imm:$src3),
9068 (i32 FROUND_CURRENT))>, EVEX_B;
Craig Topper05948fb2016-08-02 05:11:15 +00009069 }
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00009070}
9071
Elena Demikhovsky9e380862015-06-03 10:56:40 +00009072//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)
9073// op(reg_vec2,mem_vec,imm)
Igor Breger2ae0fe32015-08-31 11:14:02 +00009074multiclass avx512_3Op_rm_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
9075 X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo>{
Craig Topper05948fb2016-08-02 05:11:15 +00009076 let ExeDomain = DestInfo.ExeDomain in {
Igor Breger2ae0fe32015-08-31 11:14:02 +00009077 defm rri : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),
9078 (ins SrcInfo.RC:$src1, SrcInfo.RC:$src2, u8imm:$src3),
9079 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
9080 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1),
9081 (SrcInfo.VT SrcInfo.RC:$src2),
9082 (i8 imm:$src3)))>;
Craig Toppere1cac152016-06-07 07:27:54 +00009083 defm rmi : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
9084 (ins SrcInfo.RC:$src1, SrcInfo.MemOp:$src2, u8imm:$src3),
9085 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
9086 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1),
9087 (SrcInfo.VT (bitconvert
9088 (SrcInfo.LdFrag addr:$src2))),
9089 (i8 imm:$src3)))>;
Craig Topper05948fb2016-08-02 05:11:15 +00009090 }
Igor Breger2ae0fe32015-08-31 11:14:02 +00009091}
9092
9093//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)
9094// op(reg_vec2,mem_vec,imm)
Elena Demikhovsky9e380862015-06-03 10:56:40 +00009095// op(reg_vec2,broadcast(eltVt),imm)
9096multiclass avx512_3Op_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
Igor Breger2ae0fe32015-08-31 11:14:02 +00009097 X86VectorVTInfo _>:
9098 avx512_3Op_rm_imm8<opc, OpcodeStr, OpNode, _, _>{
9099
Craig Topper05948fb2016-08-02 05:11:15 +00009100 let ExeDomain = _.ExeDomain in
Craig Toppere1cac152016-06-07 07:27:54 +00009101 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
9102 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),
9103 OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1",
9104 "$src1, ${src2}"##_.BroadcastStr##", $src3",
9105 (OpNode (_.VT _.RC:$src1),
9106 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
9107 (i8 imm:$src3))>, EVEX_B;
Elena Demikhovsky9e380862015-06-03 10:56:40 +00009108}
9109
Elena Demikhovsky3425c932015-06-02 08:28:57 +00009110//handle scalar instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)
9111// op(reg_vec2,mem_scalar,imm)
9112//all instruction created with FROUND_CURRENT
9113multiclass avx512_fp_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper05948fb2016-08-02 05:11:15 +00009114 X86VectorVTInfo _> {
9115 let ExeDomain = _.ExeDomain in {
Elena Demikhovsky3425c932015-06-02 08:28:57 +00009116 defm rri : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009117 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
Elena Demikhovsky3425c932015-06-02 08:28:57 +00009118 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
9119 (OpNode (_.VT _.RC:$src1),
9120 (_.VT _.RC:$src2),
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009121 (i32 imm:$src3),
Elena Demikhovsky3425c932015-06-02 08:28:57 +00009122 (i32 FROUND_CURRENT))>;
Craig Toppere1cac152016-06-07 07:27:54 +00009123 defm rmi : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
Igor Bregere73ef852016-09-11 12:38:46 +00009124 (ins _.RC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3),
Craig Toppere1cac152016-06-07 07:27:54 +00009125 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
9126 (OpNode (_.VT _.RC:$src1),
9127 (_.VT (scalar_to_vector
9128 (_.ScalarLdFrag addr:$src2))),
9129 (i32 imm:$src3),
9130 (i32 FROUND_CURRENT))>;
Craig Topper05948fb2016-08-02 05:11:15 +00009131 }
Elena Demikhovsky3425c932015-06-02 08:28:57 +00009132}
9133
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00009134//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}
9135multiclass avx512_fp_sae_packed_imm<bits<8> opc, string OpcodeStr,
9136 SDNode OpNode, X86VectorVTInfo _>{
Craig Topper05948fb2016-08-02 05:11:15 +00009137 let ExeDomain = _.ExeDomain in
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00009138 defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009139 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
Craig Topperbfe13ff2016-01-11 00:44:52 +00009140 OpcodeStr, "$src3, {sae}, $src2, $src1",
9141 "$src1, $src2, {sae}, $src3",
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00009142 (OpNode (_.VT _.RC:$src1),
9143 (_.VT _.RC:$src2),
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009144 (i32 imm:$src3),
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00009145 (i32 FROUND_NO_EXC))>, EVEX_B;
9146}
Elena Demikhovsky3425c932015-06-02 08:28:57 +00009147//handle scalar instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}
9148multiclass avx512_fp_sae_scalar_imm<bits<8> opc, string OpcodeStr,
9149 SDNode OpNode, X86VectorVTInfo _> {
Craig Toppercac5d692017-02-26 06:45:37 +00009150 let ExeDomain = _.ExeDomain in
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009151 defm NAME#rrib : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
9152 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
Craig Topperbfe13ff2016-01-11 00:44:52 +00009153 OpcodeStr, "$src3, {sae}, $src2, $src1",
9154 "$src1, $src2, {sae}, $src3",
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009155 (OpNode (_.VT _.RC:$src1),
9156 (_.VT _.RC:$src2),
9157 (i32 imm:$src3),
9158 (i32 FROUND_NO_EXC))>, EVEX_B;
Elena Demikhovsky3425c932015-06-02 08:28:57 +00009159}
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00009160
Elena Demikhovsky3582eb32015-06-01 11:05:34 +00009161multiclass avx512_common_fp_sae_packed_imm<string OpcodeStr,
9162 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
Elena Demikhovsky9e380862015-06-03 10:56:40 +00009163 let Predicates = [prd] in {
9164 defm Z : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
Igor Breger00d9f842015-06-08 14:03:17 +00009165 avx512_fp_sae_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00009166 EVEX_V512;
9167
Elena Demikhovsky9e380862015-06-03 10:56:40 +00009168 }
9169 let Predicates = [prd, HasVLX] in {
9170 defm Z128 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info128>,
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00009171 EVEX_V128;
Elena Demikhovsky9e380862015-06-03 10:56:40 +00009172 defm Z256 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info256>,
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00009173 EVEX_V256;
Elena Demikhovsky9e380862015-06-03 10:56:40 +00009174 }
Elena Demikhovsky42c96d92015-06-01 06:50:49 +00009175}
9176
Igor Breger2ae0fe32015-08-31 11:14:02 +00009177multiclass avx512_common_3Op_rm_imm8<bits<8> opc, SDNode OpNode, string OpStr,
9178 AVX512VLVectorVTInfo DestInfo, AVX512VLVectorVTInfo SrcInfo>{
9179 let Predicates = [HasBWI] in {
9180 defm Z : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info512,
9181 SrcInfo.info512>, EVEX_V512, AVX512AIi8Base, EVEX_4V;
9182 }
9183 let Predicates = [HasBWI, HasVLX] in {
9184 defm Z128 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info128,
9185 SrcInfo.info128>, EVEX_V128, AVX512AIi8Base, EVEX_4V;
9186 defm Z256 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info256,
9187 SrcInfo.info256>, EVEX_V256, AVX512AIi8Base, EVEX_4V;
9188 }
9189}
9190
Igor Breger00d9f842015-06-08 14:03:17 +00009191multiclass avx512_common_3Op_imm8<string OpcodeStr, AVX512VLVectorVTInfo _,
9192 bits<8> opc, SDNode OpNode>{
9193 let Predicates = [HasAVX512] in {
9194 defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
9195 }
9196 let Predicates = [HasAVX512, HasVLX] in {
9197 defm Z128 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128;
9198 defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
9199 }
9200}
9201
Elena Demikhovsky3425c932015-06-02 08:28:57 +00009202multiclass avx512_common_fp_sae_scalar_imm<string OpcodeStr,
9203 X86VectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
9204 let Predicates = [prd] in {
9205 defm Z128 : avx512_fp_scalar_imm<opc, OpcodeStr, OpNode, _>,
9206 avx512_fp_sae_scalar_imm<opc, OpcodeStr, OpNode, _>;
Elena Demikhovsky9e380862015-06-03 10:56:40 +00009207 }
Elena Demikhovsky3425c932015-06-02 08:28:57 +00009208}
9209
Igor Breger1e58e8a2015-09-02 11:18:55 +00009210multiclass avx512_common_unary_fp_sae_packed_imm_all<string OpcodeStr,
9211 bits<8> opcPs, bits<8> opcPd, SDNode OpNode, Predicate prd>{
9212 defm PS : avx512_common_unary_fp_sae_packed_imm<OpcodeStr, avx512vl_f32_info,
9213 opcPs, OpNode, prd>, EVEX_CD8<32, CD8VF>;
9214 defm PD : avx512_common_unary_fp_sae_packed_imm<OpcodeStr, avx512vl_f64_info,
9215 opcPd, OpNode, prd>, EVEX_CD8<64, CD8VF>, VEX_W;
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009216}
9217
Elena Demikhovsky3582eb32015-06-01 11:05:34 +00009218
Igor Breger1e58e8a2015-09-02 11:18:55 +00009219defm VREDUCE : avx512_common_unary_fp_sae_packed_imm_all<"vreduce", 0x56, 0x56,
9220 X86VReduce, HasDQI>, AVX512AIi8Base, EVEX;
9221defm VRNDSCALE : avx512_common_unary_fp_sae_packed_imm_all<"vrndscale", 0x08, 0x09,
9222 X86VRndScale, HasAVX512>, AVX512AIi8Base, EVEX;
9223defm VGETMANT : avx512_common_unary_fp_sae_packed_imm_all<"vgetmant", 0x26, 0x26,
9224 X86VGetMant, HasAVX512>, AVX512AIi8Base, EVEX;
9225
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009226
Elena Demikhovsky3582eb32015-06-01 11:05:34 +00009227defm VRANGEPD : avx512_common_fp_sae_packed_imm<"vrangepd", avx512vl_f64_info,
9228 0x50, X86VRange, HasDQI>,
9229 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
9230defm VRANGEPS : avx512_common_fp_sae_packed_imm<"vrangeps", avx512vl_f32_info,
9231 0x50, X86VRange, HasDQI>,
9232 AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
9233
Elena Demikhovsky8938f5a2015-06-02 14:12:54 +00009234defm VRANGESD: avx512_common_fp_sae_scalar_imm<"vrangesd", f64x_info,
9235 0x51, X86VRange, HasDQI>,
9236 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
9237defm VRANGESS: avx512_common_fp_sae_scalar_imm<"vrangess", f32x_info,
9238 0x51, X86VRange, HasDQI>,
9239 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
9240
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009241defm VREDUCESD: avx512_common_fp_sae_scalar_imm<"vreducesd", f64x_info,
9242 0x57, X86Reduces, HasDQI>,
9243 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
9244defm VREDUCESS: avx512_common_fp_sae_scalar_imm<"vreducess", f32x_info,
9245 0x57, X86Reduces, HasDQI>,
9246 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
Elena Demikhovsky9e380862015-06-03 10:56:40 +00009247
Igor Breger1e58e8a2015-09-02 11:18:55 +00009248defm VGETMANTSD: avx512_common_fp_sae_scalar_imm<"vgetmantsd", f64x_info,
9249 0x27, X86GetMants, HasAVX512>,
9250 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
9251defm VGETMANTSS: avx512_common_fp_sae_scalar_imm<"vgetmantss", f32x_info,
9252 0x27, X86GetMants, HasAVX512>,
9253 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
9254
Elena Demikhovsky9e380862015-06-03 10:56:40 +00009255multiclass avx512_shuff_packed_128<string OpcodeStr, AVX512VLVectorVTInfo _,
9256 bits<8> opc, SDNode OpNode = X86Shuf128>{
9257 let Predicates = [HasAVX512] in {
9258 defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
9259
9260 }
9261 let Predicates = [HasAVX512, HasVLX] in {
9262 defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
9263 }
9264}
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009265let Predicates = [HasAVX512] in {
9266def : Pat<(v16f32 (ffloor VR512:$src)),
Ahmed Bougacha58a19742017-06-26 16:00:24 +00009267 (VRNDSCALEPSZrri VR512:$src, (i32 0x9))>;
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009268def : Pat<(v16f32 (fnearbyint VR512:$src)),
9269 (VRNDSCALEPSZrri VR512:$src, (i32 0xC))>;
9270def : Pat<(v16f32 (fceil VR512:$src)),
Ahmed Bougacha58a19742017-06-26 16:00:24 +00009271 (VRNDSCALEPSZrri VR512:$src, (i32 0xA))>;
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009272def : Pat<(v16f32 (frint VR512:$src)),
9273 (VRNDSCALEPSZrri VR512:$src, (i32 0x4))>;
9274def : Pat<(v16f32 (ftrunc VR512:$src)),
Ahmed Bougacha58a19742017-06-26 16:00:24 +00009275 (VRNDSCALEPSZrri VR512:$src, (i32 0xB))>;
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009276
9277def : Pat<(v8f64 (ffloor VR512:$src)),
Ahmed Bougacha58a19742017-06-26 16:00:24 +00009278 (VRNDSCALEPDZrri VR512:$src, (i32 0x9))>;
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009279def : Pat<(v8f64 (fnearbyint VR512:$src)),
9280 (VRNDSCALEPDZrri VR512:$src, (i32 0xC))>;
9281def : Pat<(v8f64 (fceil VR512:$src)),
Ahmed Bougacha58a19742017-06-26 16:00:24 +00009282 (VRNDSCALEPDZrri VR512:$src, (i32 0xA))>;
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009283def : Pat<(v8f64 (frint VR512:$src)),
9284 (VRNDSCALEPDZrri VR512:$src, (i32 0x4))>;
9285def : Pat<(v8f64 (ftrunc VR512:$src)),
Ahmed Bougacha58a19742017-06-26 16:00:24 +00009286 (VRNDSCALEPDZrri VR512:$src, (i32 0xB))>;
Asaf Badouha5b2e5e2015-07-22 12:00:43 +00009287}
Elena Demikhovsky9e380862015-06-03 10:56:40 +00009288
9289defm VSHUFF32X4 : avx512_shuff_packed_128<"vshuff32x4",avx512vl_f32_info, 0x23>,
9290 AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
9291defm VSHUFF64X2 : avx512_shuff_packed_128<"vshuff64x2",avx512vl_f64_info, 0x23>,
9292 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
9293defm VSHUFI32X4 : avx512_shuff_packed_128<"vshufi32x4",avx512vl_i32_info, 0x43>,
9294 AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
9295defm VSHUFI64X2 : avx512_shuff_packed_128<"vshufi64x2",avx512vl_i64_info, 0x43>,
9296 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
Igor Breger00d9f842015-06-08 14:03:17 +00009297
Craig Topperb561e662017-01-19 02:34:29 +00009298let Predicates = [HasAVX512] in {
9299// Provide fallback in case the load node that is used in the broadcast
9300// patterns above is used by additional users, which prevents the pattern
9301// selection.
9302def : Pat<(v8f64 (X86SubVBroadcast (v2f64 VR128X:$src))),
9303 (VSHUFF64X2Zrri (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
9304 (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
9305 0)>;
9306def : Pat<(v8i64 (X86SubVBroadcast (v2i64 VR128X:$src))),
9307 (VSHUFI64X2Zrri (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
9308 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
9309 0)>;
9310
9311def : Pat<(v16f32 (X86SubVBroadcast (v4f32 VR128X:$src))),
9312 (VSHUFF32X4Zrri (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
9313 (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
9314 0)>;
9315def : Pat<(v16i32 (X86SubVBroadcast (v4i32 VR128X:$src))),
9316 (VSHUFI32X4Zrri (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
9317 (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
9318 0)>;
9319
9320def : Pat<(v32i16 (X86SubVBroadcast (v8i16 VR128X:$src))),
9321 (VSHUFI32X4Zrri (INSERT_SUBREG (v32i16 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
9322 (INSERT_SUBREG (v32i16 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
9323 0)>;
9324
9325def : Pat<(v64i8 (X86SubVBroadcast (v16i8 VR128X:$src))),
9326 (VSHUFI32X4Zrri (INSERT_SUBREG (v64i8 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
9327 (INSERT_SUBREG (v64i8 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
9328 0)>;
9329}
9330
Craig Topperc48fa892015-12-27 19:45:21 +00009331multiclass avx512_valign<string OpcodeStr, AVX512VLVectorVTInfo VTInfo_I> {
Igor Breger00d9f842015-06-08 14:03:17 +00009332 defm NAME: avx512_common_3Op_imm8<OpcodeStr, VTInfo_I, 0x03, X86VAlign>,
9333 AVX512AIi8Base, EVEX_4V;
Igor Breger00d9f842015-06-08 14:03:17 +00009334}
9335
Craig Topperc48fa892015-12-27 19:45:21 +00009336defm VALIGND: avx512_valign<"valignd", avx512vl_i32_info>,
Igor Breger00d9f842015-06-08 14:03:17 +00009337 EVEX_CD8<32, CD8VF>;
Craig Topperc48fa892015-12-27 19:45:21 +00009338defm VALIGNQ: avx512_valign<"valignq", avx512vl_i64_info>,
Igor Breger00d9f842015-06-08 14:03:17 +00009339 EVEX_CD8<64, CD8VF>, VEX_W;
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00009340
Craig Topper7a299302016-06-09 07:06:38 +00009341multiclass avx512_vpalignr_lowering<X86VectorVTInfo _ , list<Predicate> p>{
Igor Breger2ae0fe32015-08-31 11:14:02 +00009342 let Predicates = p in
9343 def NAME#_.VTName#rri:
9344 Pat<(_.VT (X86PAlignr _.RC:$src1, _.RC:$src2, (i8 imm:$imm))),
9345 (!cast<Instruction>(NAME#_.ZSuffix#rri)
9346 _.RC:$src1, _.RC:$src2, imm:$imm)>;
9347}
9348
Craig Topper7a299302016-06-09 07:06:38 +00009349multiclass avx512_vpalignr_lowering_common<AVX512VLVectorVTInfo _>:
9350 avx512_vpalignr_lowering<_.info512, [HasBWI]>,
9351 avx512_vpalignr_lowering<_.info128, [HasBWI, HasVLX]>,
9352 avx512_vpalignr_lowering<_.info256, [HasBWI, HasVLX]>;
Igor Breger2ae0fe32015-08-31 11:14:02 +00009353
Craig Topper7a299302016-06-09 07:06:38 +00009354defm VPALIGNR: avx512_common_3Op_rm_imm8<0x0F, X86PAlignr, "vpalignr" ,
Igor Breger2ae0fe32015-08-31 11:14:02 +00009355 avx512vl_i8_info, avx512vl_i8_info>,
Craig Topper7a299302016-06-09 07:06:38 +00009356 avx512_vpalignr_lowering_common<avx512vl_i16_info>,
9357 avx512_vpalignr_lowering_common<avx512vl_i32_info>,
9358 avx512_vpalignr_lowering_common<avx512vl_f32_info>,
9359 avx512_vpalignr_lowering_common<avx512vl_i64_info>,
9360 avx512_vpalignr_lowering_common<avx512vl_f64_info>,
Igor Breger2ae0fe32015-08-31 11:14:02 +00009361 EVEX_CD8<8, CD8VF>;
9362
Igor Bregerf3ded812015-08-31 13:09:30 +00009363defm VDBPSADBW: avx512_common_3Op_rm_imm8<0x42, X86dbpsadbw, "vdbpsadbw" ,
9364 avx512vl_i16_info, avx512vl_i8_info>, EVEX_CD8<8, CD8VF>;
9365
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00009366multiclass avx512_unary_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
9367 X86VectorVTInfo _> {
Craig Toppere9e84c82017-01-31 05:18:24 +00009368 let ExeDomain = _.ExeDomain in {
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00009369 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
Igor Breger24cab0f2015-11-16 07:22:00 +00009370 (ins _.RC:$src1), OpcodeStr,
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00009371 "$src1", "$src1",
9372 (_.VT (OpNode _.RC:$src1))>, EVEX, AVX5128IBase;
9373
Craig Toppere1cac152016-06-07 07:27:54 +00009374 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
9375 (ins _.MemOp:$src1), OpcodeStr,
9376 "$src1", "$src1",
9377 (_.VT (OpNode (bitconvert (_.LdFrag addr:$src1))))>,
9378 EVEX, AVX5128IBase, EVEX_CD8<_.EltSize, CD8VF>;
Craig Toppere9e84c82017-01-31 05:18:24 +00009379 }
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00009380}
9381
9382multiclass avx512_unary_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
9383 X86VectorVTInfo _> :
9384 avx512_unary_rm<opc, OpcodeStr, OpNode, _> {
Craig Toppere1cac152016-06-07 07:27:54 +00009385 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
9386 (ins _.ScalarMemOp:$src1), OpcodeStr,
9387 "${src1}"##_.BroadcastStr,
9388 "${src1}"##_.BroadcastStr,
9389 (_.VT (OpNode (X86VBroadcast
9390 (_.ScalarLdFrag addr:$src1))))>,
9391 EVEX, AVX5128IBase, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>;
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00009392}
9393
9394multiclass avx512_unary_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
9395 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
9396 let Predicates = [prd] in
9397 defm Z : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info512>, EVEX_V512;
9398
9399 let Predicates = [prd, HasVLX] in {
9400 defm Z256 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info256>,
9401 EVEX_V256;
9402 defm Z128 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info128>,
9403 EVEX_V128;
9404 }
9405}
9406
9407multiclass avx512_unary_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
9408 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
9409 let Predicates = [prd] in
9410 defm Z : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info512>,
9411 EVEX_V512;
9412
9413 let Predicates = [prd, HasVLX] in {
9414 defm Z256 : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info256>,
9415 EVEX_V256;
9416 defm Z128 : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info128>,
9417 EVEX_V128;
9418 }
9419}
9420
9421multiclass avx512_unary_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr,
9422 SDNode OpNode, Predicate prd> {
Igor Breger24cab0f2015-11-16 07:22:00 +00009423 defm Q : avx512_unary_rmb_vl<opc_q, OpcodeStr#"q", OpNode, avx512vl_i64_info,
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00009424 prd>, VEX_W;
Igor Breger24cab0f2015-11-16 07:22:00 +00009425 defm D : avx512_unary_rmb_vl<opc_d, OpcodeStr#"d", OpNode, avx512vl_i32_info,
9426 prd>;
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00009427}
9428
9429multiclass avx512_unary_rm_vl_bw<bits<8> opc_b, bits<8> opc_w, string OpcodeStr,
9430 SDNode OpNode, Predicate prd> {
Igor Breger24cab0f2015-11-16 07:22:00 +00009431 defm W : avx512_unary_rm_vl<opc_w, OpcodeStr#"w", OpNode, avx512vl_i16_info, prd>;
9432 defm B : avx512_unary_rm_vl<opc_b, OpcodeStr#"b", OpNode, avx512vl_i8_info, prd>;
Elena Demikhovsky5e2f8c42015-06-23 08:19:46 +00009433}
9434
9435multiclass avx512_unary_rm_vl_all<bits<8> opc_b, bits<8> opc_w,
9436 bits<8> opc_d, bits<8> opc_q,
9437 string OpcodeStr, SDNode OpNode> {
9438 defm NAME : avx512_unary_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode,
9439 HasAVX512>,
9440 avx512_unary_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode,
9441 HasBWI>;
9442}
9443
Simon Pilgrimcf2da962017-03-14 21:26:58 +00009444defm VPABS : avx512_unary_rm_vl_all<0x1C, 0x1D, 0x1E, 0x1F, "vpabs", abs>;
Igor Bregerf2460112015-07-26 14:41:44 +00009445
Simon Pilgrimfea153f2017-05-06 19:11:59 +00009446// VPABS: Use 512bit version to implement 128/256 bit in case NoVLX.
9447let Predicates = [HasAVX512, NoVLX] in {
9448 def : Pat<(v4i64 (abs VR256X:$src)),
9449 (EXTRACT_SUBREG
9450 (VPABSQZrr
9451 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)),
9452 sub_ymm)>;
9453 def : Pat<(v2i64 (abs VR128X:$src)),
9454 (EXTRACT_SUBREG
9455 (VPABSQZrr
9456 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)),
9457 sub_xmm)>;
9458}
9459
Igor Breger0dcd8bc2015-09-03 09:05:31 +00009460multiclass avx512_ctlz<bits<8> opc, string OpcodeStr, Predicate prd>{
9461
9462 defm NAME : avx512_unary_rm_vl_dq<opc, opc, OpcodeStr, ctlz, prd>;
Igor Breger0dcd8bc2015-09-03 09:05:31 +00009463}
9464
9465defm VPLZCNT : avx512_ctlz<0x44, "vplzcnt", HasCDI>;
9466defm VPCONFLICT : avx512_unary_rm_vl_dq<0xC4, 0xC4, "vpconflict", X86Conflict, HasCDI>;
9467
Simon Pilgrimc89aa0b2017-05-05 12:20:34 +00009468// VPLZCNT: Use 512bit version to implement 128/256 bit in case NoVLX.
9469let Predicates = [HasCDI, NoVLX] in {
9470 def : Pat<(v4i64 (ctlz VR256X:$src)),
9471 (EXTRACT_SUBREG
9472 (VPLZCNTQZrr
9473 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)),
9474 sub_ymm)>;
9475 def : Pat<(v2i64 (ctlz VR128X:$src)),
9476 (EXTRACT_SUBREG
9477 (VPLZCNTQZrr
9478 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)),
9479 sub_xmm)>;
9480
9481 def : Pat<(v8i32 (ctlz VR256X:$src)),
9482 (EXTRACT_SUBREG
9483 (VPLZCNTDZrr
9484 (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)),
9485 sub_ymm)>;
9486 def : Pat<(v4i32 (ctlz VR128X:$src)),
9487 (EXTRACT_SUBREG
9488 (VPLZCNTDZrr
9489 (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)),
9490 sub_xmm)>;
9491}
9492
Igor Breger24cab0f2015-11-16 07:22:00 +00009493//===---------------------------------------------------------------------===//
Oren Ben Simhon7bf27f02017-05-25 13:45:23 +00009494// Counts number of ones - VPOPCNTD and VPOPCNTQ
9495//===---------------------------------------------------------------------===//
9496
9497multiclass avx512_unary_rmb_popcnt<bits<8> opc, string OpcodeStr, X86VectorVTInfo VTInfo> {
9498 let Predicates = [HasVPOPCNTDQ] in
9499 defm Z : avx512_unary_rmb<opc, OpcodeStr, ctpop, VTInfo>, EVEX_V512;
9500}
9501
9502// Use 512bit version to implement 128/256 bit.
9503multiclass avx512_unary_lowering<SDNode OpNode, AVX512VLVectorVTInfo _, Predicate prd> {
9504 let Predicates = [prd] in {
9505 def Z256_Alt : Pat<(_.info256.VT(OpNode _.info256.RC:$src1)),
9506 (EXTRACT_SUBREG
9507 (!cast<Instruction>(NAME # "Zrr")
9508 (INSERT_SUBREG(_.info512.VT(IMPLICIT_DEF)),
9509 _.info256.RC:$src1,
9510 _.info256.SubRegIdx)),
9511 _.info256.SubRegIdx)>;
9512
9513 def Z128_Alt : Pat<(_.info128.VT(OpNode _.info128.RC:$src1)),
9514 (EXTRACT_SUBREG
9515 (!cast<Instruction>(NAME # "Zrr")
9516 (INSERT_SUBREG(_.info512.VT(IMPLICIT_DEF)),
9517 _.info128.RC:$src1,
9518 _.info128.SubRegIdx)),
9519 _.info128.SubRegIdx)>;
9520 }
9521}
9522
9523defm VPOPCNTD : avx512_unary_rmb_popcnt<0x55, "vpopcntd", v16i32_info>,
9524 avx512_unary_lowering<ctpop, avx512vl_i32_info, HasVPOPCNTDQ>;
9525defm VPOPCNTQ : avx512_unary_rmb_popcnt<0x55, "vpopcntq", v8i64_info>,
9526 avx512_unary_lowering<ctpop, avx512vl_i64_info, HasVPOPCNTDQ>, VEX_W;
9527
9528//===---------------------------------------------------------------------===//
Igor Breger24cab0f2015-11-16 07:22:00 +00009529// Replicate Single FP - MOVSHDUP and MOVSLDUP
9530//===---------------------------------------------------------------------===//
9531multiclass avx512_replicate<bits<8> opc, string OpcodeStr, SDNode OpNode>{
9532 defm NAME: avx512_unary_rm_vl<opc, OpcodeStr, OpNode, avx512vl_f32_info,
9533 HasAVX512>, XS;
Igor Breger24cab0f2015-11-16 07:22:00 +00009534}
9535
9536defm VMOVSHDUP : avx512_replicate<0x16, "vmovshdup", X86Movshdup>;
9537defm VMOVSLDUP : avx512_replicate<0x12, "vmovsldup", X86Movsldup>;
Igor Breger1f782962015-11-19 08:26:56 +00009538
9539//===----------------------------------------------------------------------===//
9540// AVX-512 - MOVDDUP
9541//===----------------------------------------------------------------------===//
9542
9543multiclass avx512_movddup_128<bits<8> opc, string OpcodeStr, SDNode OpNode,
9544 X86VectorVTInfo _> {
Craig Toppere9e84c82017-01-31 05:18:24 +00009545 let ExeDomain = _.ExeDomain in {
Igor Breger1f782962015-11-19 08:26:56 +00009546 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
9547 (ins _.RC:$src), OpcodeStr, "$src", "$src",
9548 (_.VT (OpNode (_.VT _.RC:$src)))>, EVEX;
Craig Toppere1cac152016-06-07 07:27:54 +00009549 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
9550 (ins _.ScalarMemOp:$src), OpcodeStr, "$src", "$src",
9551 (_.VT (OpNode (_.VT (scalar_to_vector
9552 (_.ScalarLdFrag addr:$src)))))>,
9553 EVEX, EVEX_CD8<_.EltSize, CD8VH>;
Craig Toppere9e84c82017-01-31 05:18:24 +00009554 }
Igor Breger1f782962015-11-19 08:26:56 +00009555}
9556
9557multiclass avx512_movddup_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
9558 AVX512VLVectorVTInfo VTInfo> {
9559
9560 defm Z : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info512>, EVEX_V512;
9561
9562 let Predicates = [HasAVX512, HasVLX] in {
9563 defm Z256 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info256>,
9564 EVEX_V256;
9565 defm Z128 : avx512_movddup_128<opc, OpcodeStr, OpNode, VTInfo.info128>,
9566 EVEX_V128;
9567 }
9568}
9569
9570multiclass avx512_movddup<bits<8> opc, string OpcodeStr, SDNode OpNode>{
9571 defm NAME: avx512_movddup_common<opc, OpcodeStr, OpNode,
9572 avx512vl_f64_info>, XD, VEX_W;
Igor Breger1f782962015-11-19 08:26:56 +00009573}
9574
9575defm VMOVDDUP : avx512_movddup<0x12, "vmovddup", X86Movddup>;
9576
Craig Topper7eb0e7c2016-09-29 05:54:43 +00009577let Predicates = [HasVLX] in {
Igor Breger1f782962015-11-19 08:26:56 +00009578def : Pat<(X86Movddup (loadv2f64 addr:$src)),
Craig Topper7eb0e7c2016-09-29 05:54:43 +00009579 (VMOVDDUPZ128rm addr:$src)>;
Igor Breger1f782962015-11-19 08:26:56 +00009580def : Pat<(v2f64 (X86VBroadcast (loadf64 addr:$src))),
Craig Topper7eb0e7c2016-09-29 05:54:43 +00009581 (VMOVDDUPZ128rm addr:$src)>;
9582def : Pat<(v2f64 (X86VBroadcast f64:$src)),
9583 (VMOVDDUPZ128rr (COPY_TO_REGCLASS FR64X:$src, VR128X))>;
Craig Topperda84ff32017-01-07 22:20:23 +00009584
9585def : Pat<(vselect (v2i1 VK2WM:$mask), (X86Movddup (loadv2f64 addr:$src)),
9586 (v2f64 VR128X:$src0)),
9587 (VMOVDDUPZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
9588def : Pat<(vselect (v2i1 VK2WM:$mask), (X86Movddup (loadv2f64 addr:$src)),
9589 (bitconvert (v4i32 immAllZerosV))),
9590 (VMOVDDUPZ128rmkz VK2WM:$mask, addr:$src)>;
9591
9592def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast f64:$src)),
9593 (v2f64 VR128X:$src0)),
9594 (VMOVDDUPZ128rrk VR128X:$src0, VK2WM:$mask,
9595 (COPY_TO_REGCLASS FR64X:$src, VR128X))>;
9596def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast f64:$src)),
9597 (bitconvert (v4i32 immAllZerosV))),
9598 (VMOVDDUPZ128rrkz VK2WM:$mask, (COPY_TO_REGCLASS FR64X:$src, VR128X))>;
9599
9600def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast (loadf64 addr:$src))),
9601 (v2f64 VR128X:$src0)),
9602 (VMOVDDUPZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
9603def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast (loadf64 addr:$src))),
9604 (bitconvert (v4i32 immAllZerosV))),
9605 (VMOVDDUPZ128rmkz VK2WM:$mask, addr:$src)>;
Craig Topper7eb0e7c2016-09-29 05:54:43 +00009606}
Igor Breger1f782962015-11-19 08:26:56 +00009607
Igor Bregerf2460112015-07-26 14:41:44 +00009608//===----------------------------------------------------------------------===//
9609// AVX-512 - Unpack Instructions
9610//===----------------------------------------------------------------------===//
Craig Topper9433f972016-08-02 06:16:53 +00009611defm VUNPCKH : avx512_fp_binop_p<0x15, "vunpckh", X86Unpckh, HasAVX512,
9612 SSE_ALU_ITINS_S>;
9613defm VUNPCKL : avx512_fp_binop_p<0x14, "vunpckl", X86Unpckl, HasAVX512,
9614 SSE_ALU_ITINS_S>;
Igor Bregerf2460112015-07-26 14:41:44 +00009615
9616defm VPUNPCKLBW : avx512_binop_rm_vl_b<0x60, "vpunpcklbw", X86Unpckl,
9617 SSE_INTALU_ITINS_P, HasBWI>;
9618defm VPUNPCKHBW : avx512_binop_rm_vl_b<0x68, "vpunpckhbw", X86Unpckh,
9619 SSE_INTALU_ITINS_P, HasBWI>;
9620defm VPUNPCKLWD : avx512_binop_rm_vl_w<0x61, "vpunpcklwd", X86Unpckl,
9621 SSE_INTALU_ITINS_P, HasBWI>;
9622defm VPUNPCKHWD : avx512_binop_rm_vl_w<0x69, "vpunpckhwd", X86Unpckh,
9623 SSE_INTALU_ITINS_P, HasBWI>;
9624
9625defm VPUNPCKLDQ : avx512_binop_rm_vl_d<0x62, "vpunpckldq", X86Unpckl,
9626 SSE_INTALU_ITINS_P, HasAVX512>;
9627defm VPUNPCKHDQ : avx512_binop_rm_vl_d<0x6A, "vpunpckhdq", X86Unpckh,
9628 SSE_INTALU_ITINS_P, HasAVX512>;
9629defm VPUNPCKLQDQ : avx512_binop_rm_vl_q<0x6C, "vpunpcklqdq", X86Unpckl,
9630 SSE_INTALU_ITINS_P, HasAVX512>;
9631defm VPUNPCKHQDQ : avx512_binop_rm_vl_q<0x6D, "vpunpckhqdq", X86Unpckh,
9632 SSE_INTALU_ITINS_P, HasAVX512>;
Igor Bregerdefab3c2015-10-08 12:55:01 +00009633
9634//===----------------------------------------------------------------------===//
9635// AVX-512 - Extract & Insert Integer Instructions
9636//===----------------------------------------------------------------------===//
9637
9638multiclass avx512_extract_elt_bw_m<bits<8> opc, string OpcodeStr, SDNode OpNode,
9639 X86VectorVTInfo _> {
Craig Toppere1cac152016-06-07 07:27:54 +00009640 def mr : AVX512Ii8<opc, MRMDestMem, (outs),
9641 (ins _.ScalarMemOp:$dst, _.RC:$src1, u8imm:$src2),
9642 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
9643 [(store (_.EltVT (trunc (assertzext (OpNode (_.VT _.RC:$src1),
9644 imm:$src2)))),
9645 addr:$dst)]>,
9646 EVEX, EVEX_CD8<_.EltSize, CD8VT1>;
Igor Bregerdefab3c2015-10-08 12:55:01 +00009647}
9648
9649multiclass avx512_extract_elt_b<string OpcodeStr, X86VectorVTInfo _> {
9650 let Predicates = [HasBWI] in {
9651 def rr : AVX512Ii8<0x14, MRMDestReg, (outs GR32orGR64:$dst),
9652 (ins _.RC:$src1, u8imm:$src2),
9653 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
9654 [(set GR32orGR64:$dst,
9655 (X86pextrb (_.VT _.RC:$src1), imm:$src2))]>,
9656 EVEX, TAPD;
9657
9658 defm NAME : avx512_extract_elt_bw_m<0x14, OpcodeStr, X86pextrb, _>, TAPD;
9659 }
9660}
9661
9662multiclass avx512_extract_elt_w<string OpcodeStr, X86VectorVTInfo _> {
9663 let Predicates = [HasBWI] in {
9664 def rr : AVX512Ii8<0xC5, MRMSrcReg, (outs GR32orGR64:$dst),
9665 (ins _.RC:$src1, u8imm:$src2),
9666 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
9667 [(set GR32orGR64:$dst,
9668 (X86pextrw (_.VT _.RC:$src1), imm:$src2))]>,
9669 EVEX, PD;
9670
Craig Topper99f6b622016-05-01 01:03:56 +00009671 let hasSideEffects = 0 in
Igor Breger55747302015-11-18 08:46:16 +00009672 def rr_REV : AVX512Ii8<0x15, MRMDestReg, (outs GR32orGR64:$dst),
9673 (ins _.RC:$src1, u8imm:$src2),
9674 OpcodeStr#".s\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
Ayman Musa0b4f97d2017-05-28 12:39:37 +00009675 EVEX, TAPD, FoldGenData<NAME#rr>;
Igor Breger55747302015-11-18 08:46:16 +00009676
Igor Bregerdefab3c2015-10-08 12:55:01 +00009677 defm NAME : avx512_extract_elt_bw_m<0x15, OpcodeStr, X86pextrw, _>, TAPD;
9678 }
9679}
9680
9681multiclass avx512_extract_elt_dq<string OpcodeStr, X86VectorVTInfo _,
9682 RegisterClass GRC> {
9683 let Predicates = [HasDQI] in {
9684 def rr : AVX512Ii8<0x16, MRMDestReg, (outs GRC:$dst),
9685 (ins _.RC:$src1, u8imm:$src2),
9686 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
9687 [(set GRC:$dst,
9688 (extractelt (_.VT _.RC:$src1), imm:$src2))]>,
9689 EVEX, TAPD;
9690
Craig Toppere1cac152016-06-07 07:27:54 +00009691 def mr : AVX512Ii8<0x16, MRMDestMem, (outs),
9692 (ins _.ScalarMemOp:$dst, _.RC:$src1, u8imm:$src2),
9693 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
9694 [(store (extractelt (_.VT _.RC:$src1),
9695 imm:$src2),addr:$dst)]>,
9696 EVEX, EVEX_CD8<_.EltSize, CD8VT1>, TAPD;
Igor Bregerdefab3c2015-10-08 12:55:01 +00009697 }
9698}
9699
9700defm VPEXTRBZ : avx512_extract_elt_b<"vpextrb", v16i8x_info>;
9701defm VPEXTRWZ : avx512_extract_elt_w<"vpextrw", v8i16x_info>;
9702defm VPEXTRDZ : avx512_extract_elt_dq<"vpextrd", v4i32x_info, GR32>;
9703defm VPEXTRQZ : avx512_extract_elt_dq<"vpextrq", v2i64x_info, GR64>, VEX_W;
9704
9705multiclass avx512_insert_elt_m<bits<8> opc, string OpcodeStr, SDNode OpNode,
9706 X86VectorVTInfo _, PatFrag LdFrag> {
9707 def rm : AVX512Ii8<opc, MRMSrcMem, (outs _.RC:$dst),
9708 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),
9709 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
9710 [(set _.RC:$dst,
9711 (_.VT (OpNode _.RC:$src1, (LdFrag addr:$src2), imm:$src3)))]>,
9712 EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>;
9713}
9714
9715multiclass avx512_insert_elt_bw<bits<8> opc, string OpcodeStr, SDNode OpNode,
9716 X86VectorVTInfo _, PatFrag LdFrag> {
9717 let Predicates = [HasBWI] in {
9718 def rr : AVX512Ii8<opc, MRMSrcReg, (outs _.RC:$dst),
9719 (ins _.RC:$src1, GR32orGR64:$src2, u8imm:$src3),
9720 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
9721 [(set _.RC:$dst,
9722 (OpNode _.RC:$src1, GR32orGR64:$src2, imm:$src3))]>, EVEX_4V;
9723
9724 defm NAME : avx512_insert_elt_m<opc, OpcodeStr, OpNode, _, LdFrag>;
9725 }
9726}
9727
9728multiclass avx512_insert_elt_dq<bits<8> opc, string OpcodeStr,
9729 X86VectorVTInfo _, RegisterClass GRC> {
9730 let Predicates = [HasDQI] in {
9731 def rr : AVX512Ii8<opc, MRMSrcReg, (outs _.RC:$dst),
9732 (ins _.RC:$src1, GRC:$src2, u8imm:$src3),
9733 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
9734 [(set _.RC:$dst,
9735 (_.VT (insertelt _.RC:$src1, GRC:$src2, imm:$src3)))]>,
9736 EVEX_4V, TAPD;
9737
9738 defm NAME : avx512_insert_elt_m<opc, OpcodeStr, insertelt, _,
9739 _.ScalarLdFrag>, TAPD;
9740 }
9741}
9742
9743defm VPINSRBZ : avx512_insert_elt_bw<0x20, "vpinsrb", X86pinsrb, v16i8x_info,
9744 extloadi8>, TAPD;
9745defm VPINSRWZ : avx512_insert_elt_bw<0xC4, "vpinsrw", X86pinsrw, v8i16x_info,
9746 extloadi16>, PD;
9747defm VPINSRDZ : avx512_insert_elt_dq<0x22, "vpinsrd", v4i32x_info, GR32>;
9748defm VPINSRQZ : avx512_insert_elt_dq<0x22, "vpinsrq", v2i64x_info, GR64>, VEX_W;
Igor Bregera6297c72015-09-02 10:50:58 +00009749//===----------------------------------------------------------------------===//
9750// VSHUFPS - VSHUFPD Operations
9751//===----------------------------------------------------------------------===//
9752multiclass avx512_shufp<string OpcodeStr, AVX512VLVectorVTInfo VTInfo_I,
9753 AVX512VLVectorVTInfo VTInfo_FP>{
9754 defm NAME: avx512_common_3Op_imm8<OpcodeStr, VTInfo_FP, 0xC6, X86Shufp>,
9755 EVEX_CD8<VTInfo_FP.info512.EltSize, CD8VF>,
9756 AVX512AIi8Base, EVEX_4V;
Igor Bregera6297c72015-09-02 10:50:58 +00009757}
9758
9759defm VSHUFPS: avx512_shufp<"vshufps", avx512vl_i32_info, avx512vl_f32_info>, PS;
9760defm VSHUFPD: avx512_shufp<"vshufpd", avx512vl_i64_info, avx512vl_f64_info>, PD, VEX_W;
Asaf Badouhd2c35992015-09-02 14:21:54 +00009761//===----------------------------------------------------------------------===//
9762// AVX-512 - Byte shift Left/Right
9763//===----------------------------------------------------------------------===//
9764
9765multiclass avx512_shift_packed<bits<8> opc, SDNode OpNode, Format MRMr,
9766 Format MRMm, string OpcodeStr, X86VectorVTInfo _>{
9767 def rr : AVX512<opc, MRMr,
9768 (outs _.RC:$dst), (ins _.RC:$src1, u8imm:$src2),
9769 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
9770 [(set _.RC:$dst,(_.VT (OpNode _.RC:$src1, (i8 imm:$src2))))]>;
Craig Toppere1cac152016-06-07 07:27:54 +00009771 def rm : AVX512<opc, MRMm,
9772 (outs _.RC:$dst), (ins _.MemOp:$src1, u8imm:$src2),
9773 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
9774 [(set _.RC:$dst,(_.VT (OpNode
Simon Pilgrim255fdd02016-06-11 12:54:37 +00009775 (_.VT (bitconvert (_.LdFrag addr:$src1))),
9776 (i8 imm:$src2))))]>;
Asaf Badouhd2c35992015-09-02 14:21:54 +00009777}
9778
Simon Pilgrim18bcf932016-02-03 09:41:59 +00009779multiclass avx512_shift_packed_all<bits<8> opc, SDNode OpNode, Format MRMr,
Asaf Badouhd2c35992015-09-02 14:21:54 +00009780 Format MRMm, string OpcodeStr, Predicate prd>{
9781 let Predicates = [prd] in
Simon Pilgrim18bcf932016-02-03 09:41:59 +00009782 defm Z512 : avx512_shift_packed<opc, OpNode, MRMr, MRMm,
Simon Pilgrim255fdd02016-06-11 12:54:37 +00009783 OpcodeStr, v64i8_info>, EVEX_V512;
Asaf Badouhd2c35992015-09-02 14:21:54 +00009784 let Predicates = [prd, HasVLX] in {
Simon Pilgrim18bcf932016-02-03 09:41:59 +00009785 defm Z256 : avx512_shift_packed<opc, OpNode, MRMr, MRMm,
Simon Pilgrim255fdd02016-06-11 12:54:37 +00009786 OpcodeStr, v32i8x_info>, EVEX_V256;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00009787 defm Z128 : avx512_shift_packed<opc, OpNode, MRMr, MRMm,
Simon Pilgrim255fdd02016-06-11 12:54:37 +00009788 OpcodeStr, v16i8x_info>, EVEX_V128;
Asaf Badouhd2c35992015-09-02 14:21:54 +00009789 }
9790}
Simon Pilgrim18bcf932016-02-03 09:41:59 +00009791defm VPSLLDQ : avx512_shift_packed_all<0x73, X86vshldq, MRM7r, MRM7m, "vpslldq",
Asaf Badouhd2c35992015-09-02 14:21:54 +00009792 HasBWI>, AVX512PDIi8Base, EVEX_4V;
Simon Pilgrim18bcf932016-02-03 09:41:59 +00009793defm VPSRLDQ : avx512_shift_packed_all<0x73, X86vshrdq, MRM3r, MRM3m, "vpsrldq",
Asaf Badouhd2c35992015-09-02 14:21:54 +00009794 HasBWI>, AVX512PDIi8Base, EVEX_4V;
9795
9796
Simon Pilgrim18bcf932016-02-03 09:41:59 +00009797multiclass avx512_psadbw_packed<bits<8> opc, SDNode OpNode,
Cong Houdb6220f2015-11-24 19:51:26 +00009798 string OpcodeStr, X86VectorVTInfo _dst,
9799 X86VectorVTInfo _src>{
Asaf Badouhd2c35992015-09-02 14:21:54 +00009800 def rr : AVX512BI<opc, MRMSrcReg,
Cong Houdb6220f2015-11-24 19:51:26 +00009801 (outs _dst.RC:$dst), (ins _src.RC:$src1, _src.RC:$src2),
Asaf Badouhd2c35992015-09-02 14:21:54 +00009802 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
Cong Houdb6220f2015-11-24 19:51:26 +00009803 [(set _dst.RC:$dst,(_dst.VT
9804 (OpNode (_src.VT _src.RC:$src1),
9805 (_src.VT _src.RC:$src2))))]>;
Craig Toppere1cac152016-06-07 07:27:54 +00009806 def rm : AVX512BI<opc, MRMSrcMem,
9807 (outs _dst.RC:$dst), (ins _src.RC:$src1, _src.MemOp:$src2),
9808 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
9809 [(set _dst.RC:$dst,(_dst.VT
9810 (OpNode (_src.VT _src.RC:$src1),
9811 (_src.VT (bitconvert
9812 (_src.LdFrag addr:$src2))))))]>;
Asaf Badouhd2c35992015-09-02 14:21:54 +00009813}
9814
Simon Pilgrim18bcf932016-02-03 09:41:59 +00009815multiclass avx512_psadbw_packed_all<bits<8> opc, SDNode OpNode,
Asaf Badouhd2c35992015-09-02 14:21:54 +00009816 string OpcodeStr, Predicate prd> {
9817 let Predicates = [prd] in
Cong Houdb6220f2015-11-24 19:51:26 +00009818 defm Z512 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v8i64_info,
9819 v64i8_info>, EVEX_V512;
Asaf Badouhd2c35992015-09-02 14:21:54 +00009820 let Predicates = [prd, HasVLX] in {
Cong Houdb6220f2015-11-24 19:51:26 +00009821 defm Z256 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v4i64x_info,
9822 v32i8x_info>, EVEX_V256;
9823 defm Z128 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v2i64x_info,
9824 v16i8x_info>, EVEX_V128;
Asaf Badouhd2c35992015-09-02 14:21:54 +00009825 }
9826}
9827
Simon Pilgrim18bcf932016-02-03 09:41:59 +00009828defm VPSADBW : avx512_psadbw_packed_all<0xf6, X86psadbw, "vpsadbw",
Asaf Badouhd2c35992015-09-02 14:21:54 +00009829 HasBWI>, EVEX_4V;
Igor Bregerb4bb1902015-10-15 12:33:24 +00009830
Craig Topper4e794c72017-02-19 19:36:58 +00009831// Transforms to swizzle an immediate to enable better matching when
9832// memory operand isn't in the right place.
9833def VPTERNLOG321_imm8 : SDNodeXForm<imm, [{
9834 // Convert a VPTERNLOG immediate by swapping operand 0 and operand 2.
9835 uint8_t Imm = N->getZExtValue();
9836 // Swap bits 1/4 and 3/6.
9837 uint8_t NewImm = Imm & 0xa5;
9838 if (Imm & 0x02) NewImm |= 0x10;
9839 if (Imm & 0x10) NewImm |= 0x02;
9840 if (Imm & 0x08) NewImm |= 0x40;
9841 if (Imm & 0x40) NewImm |= 0x08;
9842 return getI8Imm(NewImm, SDLoc(N));
9843}]>;
9844def VPTERNLOG213_imm8 : SDNodeXForm<imm, [{
9845 // Convert a VPTERNLOG immediate by swapping operand 1 and operand 2.
9846 uint8_t Imm = N->getZExtValue();
9847 // Swap bits 2/4 and 3/5.
9848 uint8_t NewImm = Imm & 0xc3;
Craig Toppera5fa2e42017-02-20 07:00:34 +00009849 if (Imm & 0x04) NewImm |= 0x10;
9850 if (Imm & 0x10) NewImm |= 0x04;
Craig Topper4e794c72017-02-19 19:36:58 +00009851 if (Imm & 0x08) NewImm |= 0x20;
9852 if (Imm & 0x20) NewImm |= 0x08;
9853 return getI8Imm(NewImm, SDLoc(N));
9854}]>;
Craig Topper48905772017-02-19 21:32:15 +00009855def VPTERNLOG132_imm8 : SDNodeXForm<imm, [{
9856 // Convert a VPTERNLOG immediate by swapping operand 1 and operand 2.
9857 uint8_t Imm = N->getZExtValue();
9858 // Swap bits 1/2 and 5/6.
9859 uint8_t NewImm = Imm & 0x99;
9860 if (Imm & 0x02) NewImm |= 0x04;
9861 if (Imm & 0x04) NewImm |= 0x02;
9862 if (Imm & 0x20) NewImm |= 0x40;
9863 if (Imm & 0x40) NewImm |= 0x20;
9864 return getI8Imm(NewImm, SDLoc(N));
9865}]>;
Craig Topperc6c68f52017-02-20 07:00:40 +00009866def VPTERNLOG231_imm8 : SDNodeXForm<imm, [{
9867 // Convert a VPTERNLOG immediate by moving operand 1 to the end.
9868 uint8_t Imm = N->getZExtValue();
9869 // Move bits 1->2, 2->4, 3->6, 4->1, 5->3, 6->5
9870 uint8_t NewImm = Imm & 0x81;
9871 if (Imm & 0x02) NewImm |= 0x04;
9872 if (Imm & 0x04) NewImm |= 0x10;
9873 if (Imm & 0x08) NewImm |= 0x40;
9874 if (Imm & 0x10) NewImm |= 0x02;
9875 if (Imm & 0x20) NewImm |= 0x08;
9876 if (Imm & 0x40) NewImm |= 0x20;
9877 return getI8Imm(NewImm, SDLoc(N));
9878}]>;
9879def VPTERNLOG312_imm8 : SDNodeXForm<imm, [{
9880 // Convert a VPTERNLOG immediate by moving operand 2 to the beginning.
9881 uint8_t Imm = N->getZExtValue();
9882 // Move bits 1->4, 2->1, 3->5, 4->2, 5->6, 6->3
9883 uint8_t NewImm = Imm & 0x81;
9884 if (Imm & 0x02) NewImm |= 0x10;
9885 if (Imm & 0x04) NewImm |= 0x02;
9886 if (Imm & 0x08) NewImm |= 0x20;
9887 if (Imm & 0x10) NewImm |= 0x04;
9888 if (Imm & 0x20) NewImm |= 0x40;
9889 if (Imm & 0x40) NewImm |= 0x08;
9890 return getI8Imm(NewImm, SDLoc(N));
9891}]>;
Craig Topper4e794c72017-02-19 19:36:58 +00009892
Igor Bregerb4bb1902015-10-15 12:33:24 +00009893multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper05948fb2016-08-02 05:11:15 +00009894 X86VectorVTInfo _>{
9895 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {
Igor Bregerb4bb1902015-10-15 12:33:24 +00009896 defm rri : AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
9897 (ins _.RC:$src2, _.RC:$src3, u8imm:$src4),
Igor Breger252c2d92016-02-22 12:37:41 +00009898 OpcodeStr, "$src4, $src3, $src2", "$src2, $src3, $src4",
Igor Bregerb4bb1902015-10-15 12:33:24 +00009899 (OpNode (_.VT _.RC:$src1),
9900 (_.VT _.RC:$src2),
9901 (_.VT _.RC:$src3),
Craig Topper202b4532016-09-22 03:00:50 +00009902 (i8 imm:$src4)), 1, 1>, AVX512AIi8Base, EVEX_4V;
Craig Toppere1cac152016-06-07 07:27:54 +00009903 defm rmi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
9904 (ins _.RC:$src2, _.MemOp:$src3, u8imm:$src4),
9905 OpcodeStr, "$src4, $src3, $src2", "$src2, $src3, $src4",
9906 (OpNode (_.VT _.RC:$src1),
9907 (_.VT _.RC:$src2),
9908 (_.VT (bitconvert (_.LdFrag addr:$src3))),
Craig Topper202b4532016-09-22 03:00:50 +00009909 (i8 imm:$src4)), 1, 0>,
Craig Toppere1cac152016-06-07 07:27:54 +00009910 AVX512AIi8Base, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>;
9911 defm rmbi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
9912 (ins _.RC:$src2, _.ScalarMemOp:$src3, u8imm:$src4),
9913 OpcodeStr, "$src4, ${src3}"##_.BroadcastStr##", $src2",
9914 "$src2, ${src3}"##_.BroadcastStr##", $src4",
9915 (OpNode (_.VT _.RC:$src1),
9916 (_.VT _.RC:$src2),
9917 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
Craig Topper202b4532016-09-22 03:00:50 +00009918 (i8 imm:$src4)), 1, 0>, EVEX_B,
Craig Toppere1cac152016-06-07 07:27:54 +00009919 AVX512AIi8Base, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>;
Igor Bregerb4bb1902015-10-15 12:33:24 +00009920 }// Constraints = "$src1 = $dst"
Craig Topper4e794c72017-02-19 19:36:58 +00009921
9922 // Additional patterns for matching passthru operand in other positions.
Craig Topper4e794c72017-02-19 19:36:58 +00009923 def : Pat<(_.VT (vselect _.KRCWM:$mask,
9924 (OpNode _.RC:$src3, _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
9925 _.RC:$src1)),
9926 (!cast<Instruction>(NAME#_.ZSuffix#rrik) _.RC:$src1, _.KRCWM:$mask,
9927 _.RC:$src2, _.RC:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
9928 def : Pat<(_.VT (vselect _.KRCWM:$mask,
9929 (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3, (i8 imm:$src4)),
9930 _.RC:$src1)),
9931 (!cast<Instruction>(NAME#_.ZSuffix#rrik) _.RC:$src1, _.KRCWM:$mask,
9932 _.RC:$src2, _.RC:$src3, (VPTERNLOG213_imm8 imm:$src4))>;
Craig Topper48905772017-02-19 21:32:15 +00009933
9934 // Additional patterns for matching loads in other positions.
9935 def : Pat<(_.VT (OpNode (bitconvert (_.LdFrag addr:$src3)),
9936 _.RC:$src2, _.RC:$src1, (i8 imm:$src4))),
9937 (!cast<Instruction>(NAME#_.ZSuffix#rmi) _.RC:$src1, _.RC:$src2,
9938 addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
9939 def : Pat<(_.VT (OpNode _.RC:$src1,
9940 (bitconvert (_.LdFrag addr:$src3)),
9941 _.RC:$src2, (i8 imm:$src4))),
9942 (!cast<Instruction>(NAME#_.ZSuffix#rmi) _.RC:$src1, _.RC:$src2,
9943 addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
9944
9945 // Additional patterns for matching zero masking with loads in other
9946 // positions.
Craig Topper48905772017-02-19 21:32:15 +00009947 def : Pat<(_.VT (vselect _.KRCWM:$mask,
9948 (OpNode (bitconvert (_.LdFrag addr:$src3)),
9949 _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
9950 _.ImmAllZerosV)),
9951 (!cast<Instruction>(NAME#_.ZSuffix#rmikz) _.RC:$src1, _.KRCWM:$mask,
9952 _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
9953 def : Pat<(_.VT (vselect _.KRCWM:$mask,
9954 (OpNode _.RC:$src1, (bitconvert (_.LdFrag addr:$src3)),
9955 _.RC:$src2, (i8 imm:$src4)),
9956 _.ImmAllZerosV)),
9957 (!cast<Instruction>(NAME#_.ZSuffix#rmikz) _.RC:$src1, _.KRCWM:$mask,
9958 _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
Craig Topper48905772017-02-19 21:32:15 +00009959
9960 // Additional patterns for matching masked loads with different
9961 // operand orders.
Craig Topper48905772017-02-19 21:32:15 +00009962 def : Pat<(_.VT (vselect _.KRCWM:$mask,
9963 (OpNode _.RC:$src1, (bitconvert (_.LdFrag addr:$src3)),
9964 _.RC:$src2, (i8 imm:$src4)),
9965 _.RC:$src1)),
9966 (!cast<Instruction>(NAME#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
9967 _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
Craig Topperc6c68f52017-02-20 07:00:40 +00009968 def : Pat<(_.VT (vselect _.KRCWM:$mask,
9969 (OpNode (bitconvert (_.LdFrag addr:$src3)),
9970 _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
9971 _.RC:$src1)),
9972 (!cast<Instruction>(NAME#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
9973 _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
9974 def : Pat<(_.VT (vselect _.KRCWM:$mask,
9975 (OpNode _.RC:$src2, _.RC:$src1,
9976 (bitconvert (_.LdFrag addr:$src3)), (i8 imm:$src4)),
9977 _.RC:$src1)),
9978 (!cast<Instruction>(NAME#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
9979 _.RC:$src2, addr:$src3, (VPTERNLOG213_imm8 imm:$src4))>;
9980 def : Pat<(_.VT (vselect _.KRCWM:$mask,
9981 (OpNode _.RC:$src2, (bitconvert (_.LdFrag addr:$src3)),
9982 _.RC:$src1, (i8 imm:$src4)),
9983 _.RC:$src1)),
9984 (!cast<Instruction>(NAME#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
9985 _.RC:$src2, addr:$src3, (VPTERNLOG231_imm8 imm:$src4))>;
9986 def : Pat<(_.VT (vselect _.KRCWM:$mask,
9987 (OpNode (bitconvert (_.LdFrag addr:$src3)),
9988 _.RC:$src1, _.RC:$src2, (i8 imm:$src4)),
9989 _.RC:$src1)),
9990 (!cast<Instruction>(NAME#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
9991 _.RC:$src2, addr:$src3, (VPTERNLOG312_imm8 imm:$src4))>;
Craig Topper5b4e36a2017-02-20 02:47:42 +00009992
9993 // Additional patterns for matching broadcasts in other positions.
9994 def : Pat<(_.VT (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
9995 _.RC:$src2, _.RC:$src1, (i8 imm:$src4))),
9996 (!cast<Instruction>(NAME#_.ZSuffix#rmbi) _.RC:$src1, _.RC:$src2,
9997 addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
9998 def : Pat<(_.VT (OpNode _.RC:$src1,
9999 (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
10000 _.RC:$src2, (i8 imm:$src4))),
10001 (!cast<Instruction>(NAME#_.ZSuffix#rmbi) _.RC:$src1, _.RC:$src2,
10002 addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
10003
10004 // Additional patterns for matching zero masking with broadcasts in other
10005 // positions.
10006 def : Pat<(_.VT (vselect _.KRCWM:$mask,
10007 (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
10008 _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
10009 _.ImmAllZerosV)),
10010 (!cast<Instruction>(NAME#_.ZSuffix#rmbikz) _.RC:$src1,
10011 _.KRCWM:$mask, _.RC:$src2, addr:$src3,
10012 (VPTERNLOG321_imm8 imm:$src4))>;
10013 def : Pat<(_.VT (vselect _.KRCWM:$mask,
10014 (OpNode _.RC:$src1,
10015 (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
10016 _.RC:$src2, (i8 imm:$src4)),
10017 _.ImmAllZerosV)),
10018 (!cast<Instruction>(NAME#_.ZSuffix#rmbikz) _.RC:$src1,
10019 _.KRCWM:$mask, _.RC:$src2, addr:$src3,
10020 (VPTERNLOG132_imm8 imm:$src4))>;
10021
10022 // Additional patterns for matching masked broadcasts with different
10023 // operand orders.
10024 def : Pat<(_.VT (vselect _.KRCWM:$mask,
10025 (OpNode _.RC:$src1,
10026 (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
10027 _.RC:$src2, (i8 imm:$src4)),
10028 _.RC:$src1)),
10029 (!cast<Instruction>(NAME#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
10030 _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
Craig Topper2012dda2017-02-20 17:44:09 +000010031 def : Pat<(_.VT (vselect _.KRCWM:$mask,
10032 (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
10033 _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
10034 _.RC:$src1)),
10035 (!cast<Instruction>(NAME#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
10036 _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
10037 def : Pat<(_.VT (vselect _.KRCWM:$mask,
10038 (OpNode _.RC:$src2, _.RC:$src1,
10039 (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
10040 (i8 imm:$src4)), _.RC:$src1)),
10041 (!cast<Instruction>(NAME#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
10042 _.RC:$src2, addr:$src3, (VPTERNLOG213_imm8 imm:$src4))>;
10043 def : Pat<(_.VT (vselect _.KRCWM:$mask,
10044 (OpNode _.RC:$src2,
10045 (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
10046 _.RC:$src1, (i8 imm:$src4)),
10047 _.RC:$src1)),
10048 (!cast<Instruction>(NAME#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
10049 _.RC:$src2, addr:$src3, (VPTERNLOG231_imm8 imm:$src4))>;
10050 def : Pat<(_.VT (vselect _.KRCWM:$mask,
10051 (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
10052 _.RC:$src1, _.RC:$src2, (i8 imm:$src4)),
10053 _.RC:$src1)),
10054 (!cast<Instruction>(NAME#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
10055 _.RC:$src2, addr:$src3, (VPTERNLOG312_imm8 imm:$src4))>;
Igor Bregerb4bb1902015-10-15 12:33:24 +000010056}
10057
10058multiclass avx512_common_ternlog<string OpcodeStr, AVX512VLVectorVTInfo _>{
10059 let Predicates = [HasAVX512] in
10060 defm Z : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, _.info512>, EVEX_V512;
10061 let Predicates = [HasAVX512, HasVLX] in {
10062 defm Z128 : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, _.info128>, EVEX_V128;
10063 defm Z256 : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, _.info256>, EVEX_V256;
10064 }
10065}
10066
10067defm VPTERNLOGD : avx512_common_ternlog<"vpternlogd", avx512vl_i32_info>;
10068defm VPTERNLOGQ : avx512_common_ternlog<"vpternlogq", avx512vl_i64_info>, VEX_W;
10069
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +000010070//===----------------------------------------------------------------------===//
10071// AVX-512 - FixupImm
10072//===----------------------------------------------------------------------===//
10073
10074multiclass avx512_fixupimm_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
Craig Topper05948fb2016-08-02 05:11:15 +000010075 X86VectorVTInfo _>{
10076 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +000010077 defm rri : AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
10078 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),
10079 OpcodeStr##_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",
10080 (OpNode (_.VT _.RC:$src1),
10081 (_.VT _.RC:$src2),
10082 (_.IntVT _.RC:$src3),
10083 (i32 imm:$src4),
10084 (i32 FROUND_CURRENT))>;
Craig Toppere1cac152016-06-07 07:27:54 +000010085 defm rmi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
10086 (ins _.RC:$src2, _.MemOp:$src3, i32u8imm:$src4),
10087 OpcodeStr##_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",
10088 (OpNode (_.VT _.RC:$src1),
10089 (_.VT _.RC:$src2),
10090 (_.IntVT (bitconvert (_.LdFrag addr:$src3))),
10091 (i32 imm:$src4),
10092 (i32 FROUND_CURRENT))>;
10093 defm rmbi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
10094 (ins _.RC:$src2, _.ScalarMemOp:$src3, i32u8imm:$src4),
10095 OpcodeStr##_.Suffix, "$src4, ${src3}"##_.BroadcastStr##", $src2",
10096 "$src2, ${src3}"##_.BroadcastStr##", $src4",
10097 (OpNode (_.VT _.RC:$src1),
10098 (_.VT _.RC:$src2),
10099 (_.IntVT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
10100 (i32 imm:$src4),
10101 (i32 FROUND_CURRENT))>, EVEX_B;
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +000010102 } // Constraints = "$src1 = $dst"
10103}
10104
10105multiclass avx512_fixupimm_packed_sae<bits<8> opc, string OpcodeStr,
Craig Topper05948fb2016-08-02 05:11:15 +000010106 SDNode OpNode, X86VectorVTInfo _>{
10107let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +000010108 defm rrib : AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
10109 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),
Simon Pilgrim18bcf932016-02-03 09:41:59 +000010110 OpcodeStr##_.Suffix, "$src4, {sae}, $src3, $src2",
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +000010111 "$src2, $src3, {sae}, $src4",
10112 (OpNode (_.VT _.RC:$src1),
10113 (_.VT _.RC:$src2),
10114 (_.IntVT _.RC:$src3),
10115 (i32 imm:$src4),
10116 (i32 FROUND_NO_EXC))>, EVEX_B;
10117 }
10118}
10119
10120multiclass avx512_fixupimm_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode,
10121 X86VectorVTInfo _, X86VectorVTInfo _src3VT> {
Craig Topper05948fb2016-08-02 05:11:15 +000010122 let Constraints = "$src1 = $dst" , Predicates = [HasAVX512],
10123 ExeDomain = _.ExeDomain in {
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +000010124 defm rri : AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
10125 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),
10126 OpcodeStr##_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",
10127 (OpNode (_.VT _.RC:$src1),
10128 (_.VT _.RC:$src2),
10129 (_src3VT.VT _src3VT.RC:$src3),
10130 (i32 imm:$src4),
10131 (i32 FROUND_CURRENT))>;
10132
10133 defm rrib : AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
10134 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),
10135 OpcodeStr##_.Suffix, "$src4, {sae}, $src3, $src2",
10136 "$src2, $src3, {sae}, $src4",
10137 (OpNode (_.VT _.RC:$src1),
10138 (_.VT _.RC:$src2),
10139 (_src3VT.VT _src3VT.RC:$src3),
10140 (i32 imm:$src4),
10141 (i32 FROUND_NO_EXC))>, EVEX_B;
Craig Toppere1cac152016-06-07 07:27:54 +000010142 defm rmi : AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
10143 (ins _.RC:$src2, _.ScalarMemOp:$src3, i32u8imm:$src4),
10144 OpcodeStr##_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",
10145 (OpNode (_.VT _.RC:$src1),
10146 (_.VT _.RC:$src2),
10147 (_src3VT.VT (scalar_to_vector
10148 (_src3VT.ScalarLdFrag addr:$src3))),
10149 (i32 imm:$src4),
10150 (i32 FROUND_CURRENT))>;
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +000010151 }
10152}
10153
10154multiclass avx512_fixupimm_packed_all<AVX512VLVectorVTInfo _Vec>{
10155 let Predicates = [HasAVX512] in
10156 defm Z : avx512_fixupimm_packed<0x54, "vfixupimm", X86VFixupimm, _Vec.info512>,
10157 avx512_fixupimm_packed_sae<0x54, "vfixupimm", X86VFixupimm, _Vec.info512>,
10158 AVX512AIi8Base, EVEX_4V, EVEX_V512;
10159 let Predicates = [HasAVX512, HasVLX] in {
10160 defm Z128 : avx512_fixupimm_packed<0x54, "vfixupimm", X86VFixupimm, _Vec.info128>,
10161 AVX512AIi8Base, EVEX_4V, EVEX_V128;
10162 defm Z256 : avx512_fixupimm_packed<0x54, "vfixupimm", X86VFixupimm, _Vec.info256>,
10163 AVX512AIi8Base, EVEX_4V, EVEX_V256;
10164 }
10165}
10166
Simon Pilgrim18bcf932016-02-03 09:41:59 +000010167defm VFIXUPIMMSS : avx512_fixupimm_scalar<0x55, "vfixupimm", X86VFixupimmScalar,
10168 f32x_info, v4i32x_info>,
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +000010169 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +000010170defm VFIXUPIMMSD : avx512_fixupimm_scalar<0x55, "vfixupimm", X86VFixupimmScalar,
10171 f64x_info, v2i64x_info>,
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +000010172 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
Simon Pilgrim18bcf932016-02-03 09:41:59 +000010173defm VFIXUPIMMPS : avx512_fixupimm_packed_all<avx512vl_f32_info>,
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +000010174 EVEX_CD8<32, CD8VF>;
Simon Pilgrim18bcf932016-02-03 09:41:59 +000010175defm VFIXUPIMMPD : avx512_fixupimm_packed_all<avx512vl_f64_info>,
Asaf Badouhd4a0d9a2016-01-19 14:21:39 +000010176 EVEX_CD8<64, CD8VF>, VEX_W;
Craig Topper5625d242016-07-29 06:06:00 +000010177
10178
10179
10180// Patterns used to select SSE scalar fp arithmetic instructions from
10181// either:
10182//
10183// (1) a scalar fp operation followed by a blend
10184//
10185// The effect is that the backend no longer emits unnecessary vector
10186// insert instructions immediately after SSE scalar fp instructions
10187// like addss or mulss.
10188//
10189// For example, given the following code:
10190// __m128 foo(__m128 A, __m128 B) {
10191// A[0] += B[0];
10192// return A;
10193// }
10194//
10195// Previously we generated:
10196// addss %xmm0, %xmm1
10197// movss %xmm1, %xmm0
10198//
10199// We now generate:
10200// addss %xmm1, %xmm0
10201//
10202// (2) a vector packed single/double fp operation followed by a vector insert
10203//
10204// The effect is that the backend converts the packed fp instruction
10205// followed by a vector insert into a single SSE scalar fp instruction.
10206//
10207// For example, given the following code:
10208// __m128 foo(__m128 A, __m128 B) {
10209// __m128 C = A + B;
10210// return (__m128) {c[0], a[1], a[2], a[3]};
10211// }
10212//
10213// Previously we generated:
10214// addps %xmm0, %xmm1
10215// movss %xmm1, %xmm0
10216//
10217// We now generate:
10218// addss %xmm1, %xmm0
10219
10220// TODO: Some canonicalization in lowering would simplify the number of
10221// patterns we have to try to match.
10222multiclass AVX512_scalar_math_f32_patterns<SDNode Op, string OpcPrefix> {
10223 let Predicates = [HasAVX512] in {
Simon Pilgrimae17cf22016-10-01 15:33:01 +000010224 // extracted scalar math op with insert via movss
Craig Topper5ef13ba2016-12-26 07:26:07 +000010225 def : Pat<(v4f32 (X86Movss (v4f32 VR128X:$dst), (v4f32 (scalar_to_vector
10226 (Op (f32 (extractelt (v4f32 VR128X:$dst), (iPTR 0))),
10227 FR32X:$src))))),
Simon Pilgrimae17cf22016-10-01 15:33:01 +000010228 (!cast<I>("V"#OpcPrefix#SSZrr_Int) v4f32:$dst,
Craig Topper5ef13ba2016-12-26 07:26:07 +000010229 (COPY_TO_REGCLASS FR32X:$src, VR128X))>;
Simon Pilgrimae17cf22016-10-01 15:33:01 +000010230
Craig Topper5625d242016-07-29 06:06:00 +000010231 // extracted scalar math op with insert via blend
Craig Topper5ef13ba2016-12-26 07:26:07 +000010232 def : Pat<(v4f32 (X86Blendi (v4f32 VR128X:$dst), (v4f32 (scalar_to_vector
10233 (Op (f32 (extractelt (v4f32 VR128X:$dst), (iPTR 0))),
10234 FR32X:$src))), (i8 1))),
Craig Topper5625d242016-07-29 06:06:00 +000010235 (!cast<I>("V"#OpcPrefix#SSZrr_Int) v4f32:$dst,
Craig Topper5ef13ba2016-12-26 07:26:07 +000010236 (COPY_TO_REGCLASS FR32X:$src, VR128X))>;
Craig Topper5625d242016-07-29 06:06:00 +000010237
10238 // vector math op with insert via movss
Craig Topper5ef13ba2016-12-26 07:26:07 +000010239 def : Pat<(v4f32 (X86Movss (v4f32 VR128X:$dst),
10240 (Op (v4f32 VR128X:$dst), (v4f32 VR128X:$src)))),
Craig Topper5625d242016-07-29 06:06:00 +000010241 (!cast<I>("V"#OpcPrefix#SSZrr_Int) v4f32:$dst, v4f32:$src)>;
10242
10243 // vector math op with insert via blend
Craig Topper5ef13ba2016-12-26 07:26:07 +000010244 def : Pat<(v4f32 (X86Blendi (v4f32 VR128X:$dst),
10245 (Op (v4f32 VR128X:$dst), (v4f32 VR128X:$src)), (i8 1))),
Craig Topper5625d242016-07-29 06:06:00 +000010246 (!cast<I>("V"#OpcPrefix#SSZrr_Int) v4f32:$dst, v4f32:$src)>;
Craig Topper83f21452016-12-27 01:56:24 +000010247
10248 // extracted masked scalar math op with insert via movss
10249 def : Pat<(X86Movss (v4f32 VR128X:$src1),
10250 (scalar_to_vector
10251 (X86selects VK1WM:$mask,
10252 (Op (f32 (extractelt (v4f32 VR128X:$src1), (iPTR 0))),
10253 FR32X:$src2),
10254 FR32X:$src0))),
10255 (!cast<I>("V"#OpcPrefix#SSZrr_Intk) (COPY_TO_REGCLASS FR32X:$src0, VR128X),
10256 VK1WM:$mask, v4f32:$src1,
10257 (COPY_TO_REGCLASS FR32X:$src2, VR128X))>;
Craig Topper5625d242016-07-29 06:06:00 +000010258 }
10259}
10260
10261defm : AVX512_scalar_math_f32_patterns<fadd, "ADD">;
10262defm : AVX512_scalar_math_f32_patterns<fsub, "SUB">;
10263defm : AVX512_scalar_math_f32_patterns<fmul, "MUL">;
10264defm : AVX512_scalar_math_f32_patterns<fdiv, "DIV">;
10265
10266multiclass AVX512_scalar_math_f64_patterns<SDNode Op, string OpcPrefix> {
10267 let Predicates = [HasAVX512] in {
10268 // extracted scalar math op with insert via movsd
Craig Topper5ef13ba2016-12-26 07:26:07 +000010269 def : Pat<(v2f64 (X86Movsd (v2f64 VR128X:$dst), (v2f64 (scalar_to_vector
10270 (Op (f64 (extractelt (v2f64 VR128X:$dst), (iPTR 0))),
10271 FR64X:$src))))),
Craig Topper5625d242016-07-29 06:06:00 +000010272 (!cast<I>("V"#OpcPrefix#SDZrr_Int) v2f64:$dst,
Craig Topper5ef13ba2016-12-26 07:26:07 +000010273 (COPY_TO_REGCLASS FR64X:$src, VR128X))>;
Craig Topper5625d242016-07-29 06:06:00 +000010274
10275 // extracted scalar math op with insert via blend
Craig Topper5ef13ba2016-12-26 07:26:07 +000010276 def : Pat<(v2f64 (X86Blendi (v2f64 VR128X:$dst), (v2f64 (scalar_to_vector
10277 (Op (f64 (extractelt (v2f64 VR128X:$dst), (iPTR 0))),
10278 FR64X:$src))), (i8 1))),
Craig Topper5625d242016-07-29 06:06:00 +000010279 (!cast<I>("V"#OpcPrefix#SDZrr_Int) v2f64:$dst,
Craig Topper5ef13ba2016-12-26 07:26:07 +000010280 (COPY_TO_REGCLASS FR64X:$src, VR128X))>;
Craig Topper5625d242016-07-29 06:06:00 +000010281
10282 // vector math op with insert via movsd
Craig Topper5ef13ba2016-12-26 07:26:07 +000010283 def : Pat<(v2f64 (X86Movsd (v2f64 VR128X:$dst),
10284 (Op (v2f64 VR128X:$dst), (v2f64 VR128X:$src)))),
Craig Topper5625d242016-07-29 06:06:00 +000010285 (!cast<I>("V"#OpcPrefix#SDZrr_Int) v2f64:$dst, v2f64:$src)>;
10286
10287 // vector math op with insert via blend
Craig Topper5ef13ba2016-12-26 07:26:07 +000010288 def : Pat<(v2f64 (X86Blendi (v2f64 VR128X:$dst),
10289 (Op (v2f64 VR128X:$dst), (v2f64 VR128X:$src)), (i8 1))),
Craig Topper5625d242016-07-29 06:06:00 +000010290 (!cast<I>("V"#OpcPrefix#SDZrr_Int) v2f64:$dst, v2f64:$src)>;
Craig Topper83f21452016-12-27 01:56:24 +000010291
10292 // extracted masked scalar math op with insert via movss
10293 def : Pat<(X86Movsd (v2f64 VR128X:$src1),
10294 (scalar_to_vector
10295 (X86selects VK1WM:$mask,
10296 (Op (f64 (extractelt (v2f64 VR128X:$src1), (iPTR 0))),
10297 FR64X:$src2),
10298 FR64X:$src0))),
10299 (!cast<I>("V"#OpcPrefix#SDZrr_Intk) (COPY_TO_REGCLASS FR64X:$src0, VR128X),
10300 VK1WM:$mask, v2f64:$src1,
10301 (COPY_TO_REGCLASS FR64X:$src2, VR128X))>;
Craig Topper5625d242016-07-29 06:06:00 +000010302 }
10303}
10304
10305defm : AVX512_scalar_math_f64_patterns<fadd, "ADD">;
10306defm : AVX512_scalar_math_f64_patterns<fsub, "SUB">;
10307defm : AVX512_scalar_math_f64_patterns<fmul, "MUL">;
10308defm : AVX512_scalar_math_f64_patterns<fdiv, "DIV">;