Eric Christopher | 06b32cd | 2015-02-20 00:36:53 +0000 | [diff] [blame] | 1 | //===-- X86InstrAVX512.td - AVX512 Instruction Set ---------*- tablegen -*-===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file describes the X86 AVX512 instruction set, defining the |
| 11 | // instructions, and properties of the instructions which are needed for code |
| 12 | // generation, machine code emission, and analysis. |
| 13 | // |
| 14 | //===----------------------------------------------------------------------===// |
| 15 | |
Adam Nemet | 5ed17da | 2014-08-21 19:50:07 +0000 | [diff] [blame] | 16 | // Group template arguments that can be derived from the vector type (EltNum x |
| 17 | // EltVT). These are things like the register class for the writemask, etc. |
| 18 | // The idea is to pass one of these as the template argument rather than the |
| 19 | // individual arguments. |
Elena Demikhovsky | fa4a6c1 | 2014-12-09 07:06:32 +0000 | [diff] [blame] | 20 | // The template is also used for scalar types, in this case numelts is 1. |
Robert Khasanov | 4204c1a | 2014-12-12 14:21:30 +0000 | [diff] [blame] | 21 | class X86VectorVTInfo<int numelts, ValueType eltvt, RegisterClass rc, |
Adam Nemet | 5ed17da | 2014-08-21 19:50:07 +0000 | [diff] [blame] | 22 | string suffix = ""> { |
| 23 | RegisterClass RC = rc; |
Robert Khasanov | 4204c1a | 2014-12-12 14:21:30 +0000 | [diff] [blame] | 24 | ValueType EltVT = eltvt; |
Adam Nemet | 449b3f0 | 2014-10-15 23:42:09 +0000 | [diff] [blame] | 25 | int NumElts = numelts; |
Adam Nemet | 5ed17da | 2014-08-21 19:50:07 +0000 | [diff] [blame] | 26 | |
| 27 | // Corresponding mask register class. |
| 28 | RegisterClass KRC = !cast<RegisterClass>("VK" # NumElts); |
| 29 | |
| 30 | // Corresponding write-mask register class. |
| 31 | RegisterClass KRCWM = !cast<RegisterClass>("VK" # NumElts # "WM"); |
| 32 | |
| 33 | // The GPR register class that can hold the write mask. Use GR8 for fewer |
| 34 | // than 8 elements. Use shift-right and equal to work around the lack of |
| 35 | // !lt in tablegen. |
| 36 | RegisterClass MRC = |
| 37 | !cast<RegisterClass>("GR" # |
| 38 | !if (!eq (!srl(NumElts, 3), 0), 8, NumElts)); |
| 39 | |
| 40 | // Suffix used in the instruction mnemonic. |
| 41 | string Suffix = suffix; |
| 42 | |
Elena Demikhovsky | fa4a6c1 | 2014-12-09 07:06:32 +0000 | [diff] [blame] | 43 | // VTName is a string name for vector VT. For vector types it will be |
| 44 | // v # NumElts # EltVT, so for vector of 8 elements of i32 it will be v8i32 |
| 45 | // It is a little bit complex for scalar types, where NumElts = 1. |
| 46 | // In this case we build v4f32 or v2f64 |
| 47 | string VTName = "v" # !if (!eq (NumElts, 1), |
| 48 | !if (!eq (EltVT.Size, 32), 4, |
| 49 | !if (!eq (EltVT.Size, 64), 2, NumElts)), NumElts) # EltVT; |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 50 | |
Adam Nemet | 5ed17da | 2014-08-21 19:50:07 +0000 | [diff] [blame] | 51 | // The vector VT. |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 52 | ValueType VT = !cast<ValueType>(VTName); |
Adam Nemet | 5ed17da | 2014-08-21 19:50:07 +0000 | [diff] [blame] | 53 | |
| 54 | string EltTypeName = !cast<string>(EltVT); |
| 55 | // Size of the element type in bits, e.g. 32 for v16i32. |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 56 | string EltSizeName = !subst("i", "", !subst("f", "", EltTypeName)); |
| 57 | int EltSize = EltVT.Size; |
Adam Nemet | 5ed17da | 2014-08-21 19:50:07 +0000 | [diff] [blame] | 58 | |
| 59 | // "i" for integer types and "f" for floating-point types |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 60 | string TypeVariantName = !subst(EltSizeName, "", EltTypeName); |
Adam Nemet | 5ed17da | 2014-08-21 19:50:07 +0000 | [diff] [blame] | 61 | |
| 62 | // Size of RC in bits, e.g. 512 for VR512. |
| 63 | int Size = VT.Size; |
| 64 | |
| 65 | // The corresponding memory operand, e.g. i512mem for VR512. |
| 66 | X86MemOperand MemOp = !cast<X86MemOperand>(TypeVariantName # Size # "mem"); |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 67 | X86MemOperand ScalarMemOp = !cast<X86MemOperand>(EltVT # "mem"); |
| 68 | |
| 69 | // Load patterns |
| 70 | // Note: For 128/256-bit integer VT we choose loadv2i64/loadv4i64 |
| 71 | // due to load promotion during legalization |
| 72 | PatFrag LdFrag = !cast<PatFrag>("load" # |
| 73 | !if (!eq (TypeVariantName, "i"), |
| 74 | !if (!eq (Size, 128), "v2i64", |
| 75 | !if (!eq (Size, 256), "v4i64", |
| 76 | VTName)), VTName)); |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 77 | |
| 78 | PatFrag AlignedLdFrag = !cast<PatFrag>("alignedload" # |
| 79 | !if (!eq (TypeVariantName, "i"), |
| 80 | !if (!eq (Size, 128), "v2i64", |
| 81 | !if (!eq (Size, 256), "v4i64", |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 82 | !if (!eq (Size, 512), |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 83 | !if (!eq (EltSize, 64), "v8i64", "v16i32"), |
| 84 | VTName))), VTName)); |
| 85 | |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 86 | PatFrag ScalarLdFrag = !cast<PatFrag>("load" # EltVT); |
Adam Nemet | 5ed17da | 2014-08-21 19:50:07 +0000 | [diff] [blame] | 87 | |
| 88 | // The corresponding float type, e.g. v16f32 for v16i32 |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 89 | // Note: For EltSize < 32, FloatVT is illegal and TableGen |
| 90 | // fails to compile, so we choose FloatVT = VT |
| 91 | ValueType FloatVT = !cast<ValueType>( |
| 92 | !if (!eq (!srl(EltSize,5),0), |
| 93 | VTName, |
| 94 | !if (!eq(TypeVariantName, "i"), |
| 95 | "v" # NumElts # "f" # EltSize, |
| 96 | VTName))); |
Adam Nemet | 5ed17da | 2014-08-21 19:50:07 +0000 | [diff] [blame] | 97 | |
| 98 | // The string to specify embedded broadcast in assembly. |
| 99 | string BroadcastStr = "{1to" # NumElts # "}"; |
Adam Nemet | 55536c6 | 2014-09-25 23:48:45 +0000 | [diff] [blame] | 100 | |
Adam Nemet | 449b3f0 | 2014-10-15 23:42:09 +0000 | [diff] [blame] | 101 | // 8-bit compressed displacement tuple/subvector format. This is only |
| 102 | // defined for NumElts <= 8. |
| 103 | CD8VForm CD8TupleForm = !if (!eq (!srl(NumElts, 4), 0), |
| 104 | !cast<CD8VForm>("CD8VT" # NumElts), ?); |
| 105 | |
Adam Nemet | 55536c6 | 2014-09-25 23:48:45 +0000 | [diff] [blame] | 106 | SubRegIndex SubRegIdx = !if (!eq (Size, 128), sub_xmm, |
| 107 | !if (!eq (Size, 256), sub_ymm, ?)); |
| 108 | |
| 109 | Domain ExeDomain = !if (!eq (EltTypeName, "f32"), SSEPackedSingle, |
| 110 | !if (!eq (EltTypeName, "f64"), SSEPackedDouble, |
| 111 | SSEPackedInt)); |
Adam Nemet | 0937723 | 2014-10-08 23:25:31 +0000 | [diff] [blame] | 112 | |
Elena Demikhovsky | 52e81bc | 2015-02-23 15:12:31 +0000 | [diff] [blame] | 113 | RegisterClass FRC = !if (!eq (EltTypeName, "f32"), FR32X, FR64X); |
| 114 | |
Adam Nemet | 0937723 | 2014-10-08 23:25:31 +0000 | [diff] [blame] | 115 | // A vector type of the same width with element type i32. This is used to |
| 116 | // create the canonical constant zero node ImmAllZerosV. |
| 117 | ValueType i32VT = !cast<ValueType>("v" # !srl(Size, 5) # "i32"); |
| 118 | dag ImmAllZerosV = (VT (bitconvert (i32VT immAllZerosV))); |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 119 | |
| 120 | string ZSuffix = !if (!eq (Size, 128), "Z128", |
| 121 | !if (!eq (Size, 256), "Z256", "Z")); |
Adam Nemet | 5ed17da | 2014-08-21 19:50:07 +0000 | [diff] [blame] | 122 | } |
| 123 | |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 124 | def v64i8_info : X86VectorVTInfo<64, i8, VR512, "b">; |
| 125 | def v32i16_info : X86VectorVTInfo<32, i16, VR512, "w">; |
Adam Nemet | 5ed17da | 2014-08-21 19:50:07 +0000 | [diff] [blame] | 126 | def v16i32_info : X86VectorVTInfo<16, i32, VR512, "d">; |
| 127 | def v8i64_info : X86VectorVTInfo<8, i64, VR512, "q">; |
Adam Nemet | 6bddb8c | 2014-09-29 22:54:41 +0000 | [diff] [blame] | 128 | def v16f32_info : X86VectorVTInfo<16, f32, VR512, "ps">; |
| 129 | def v8f64_info : X86VectorVTInfo<8, f64, VR512, "pd">; |
Adam Nemet | 5ed17da | 2014-08-21 19:50:07 +0000 | [diff] [blame] | 130 | |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 131 | // "x" in v32i8x_info means RC = VR256X |
| 132 | def v32i8x_info : X86VectorVTInfo<32, i8, VR256X, "b">; |
| 133 | def v16i16x_info : X86VectorVTInfo<16, i16, VR256X, "w">; |
| 134 | def v8i32x_info : X86VectorVTInfo<8, i32, VR256X, "d">; |
| 135 | def v4i64x_info : X86VectorVTInfo<4, i64, VR256X, "q">; |
Robert Khasanov | 3e534c9 | 2014-10-28 16:37:13 +0000 | [diff] [blame] | 136 | def v8f32x_info : X86VectorVTInfo<8, f32, VR256X, "ps">; |
| 137 | def v4f64x_info : X86VectorVTInfo<4, f64, VR256X, "pd">; |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 138 | |
| 139 | def v16i8x_info : X86VectorVTInfo<16, i8, VR128X, "b">; |
| 140 | def v8i16x_info : X86VectorVTInfo<8, i16, VR128X, "w">; |
| 141 | def v4i32x_info : X86VectorVTInfo<4, i32, VR128X, "d">; |
| 142 | def v2i64x_info : X86VectorVTInfo<2, i64, VR128X, "q">; |
Robert Khasanov | 3e534c9 | 2014-10-28 16:37:13 +0000 | [diff] [blame] | 143 | def v4f32x_info : X86VectorVTInfo<4, f32, VR128X, "ps">; |
| 144 | def v2f64x_info : X86VectorVTInfo<2, f64, VR128X, "pd">; |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 145 | |
Elena Demikhovsky | fa4a6c1 | 2014-12-09 07:06:32 +0000 | [diff] [blame] | 146 | // We map scalar types to the smallest (128-bit) vector type |
| 147 | // with the appropriate element type. This allows to use the same masking logic. |
Elena Demikhovsky | 905a5a6 | 2014-11-26 10:46:49 +0000 | [diff] [blame] | 148 | def f32x_info : X86VectorVTInfo<1, f32, VR128X, "ss">; |
| 149 | def f64x_info : X86VectorVTInfo<1, f64, VR128X, "sd">; |
| 150 | |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 151 | class AVX512VLVectorVTInfo<X86VectorVTInfo i512, X86VectorVTInfo i256, |
| 152 | X86VectorVTInfo i128> { |
| 153 | X86VectorVTInfo info512 = i512; |
| 154 | X86VectorVTInfo info256 = i256; |
| 155 | X86VectorVTInfo info128 = i128; |
| 156 | } |
| 157 | |
| 158 | def avx512vl_i8_info : AVX512VLVectorVTInfo<v64i8_info, v32i8x_info, |
| 159 | v16i8x_info>; |
| 160 | def avx512vl_i16_info : AVX512VLVectorVTInfo<v32i16_info, v16i16x_info, |
| 161 | v8i16x_info>; |
| 162 | def avx512vl_i32_info : AVX512VLVectorVTInfo<v16i32_info, v8i32x_info, |
| 163 | v4i32x_info>; |
| 164 | def avx512vl_i64_info : AVX512VLVectorVTInfo<v8i64_info, v4i64x_info, |
| 165 | v2i64x_info>; |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 166 | def avx512vl_f32_info : AVX512VLVectorVTInfo<v16f32_info, v8f32x_info, |
| 167 | v4f32x_info>; |
| 168 | def avx512vl_f64_info : AVX512VLVectorVTInfo<v8f64_info, v4f64x_info, |
| 169 | v2f64x_info>; |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 170 | |
Adam Nemet | 52bb6cf | 2014-10-08 23:25:23 +0000 | [diff] [blame] | 171 | // This multiclass generates the masking variants from the non-masking |
| 172 | // variant. It only provides the assembly pieces for the masking variants. |
| 173 | // It assumes custom ISel patterns for masking which can be provided as |
| 174 | // template arguments. |
Adam Nemet | 3480142 | 2014-10-08 23:25:39 +0000 | [diff] [blame] | 175 | multiclass AVX512_maskable_custom<bits<8> O, Format F, |
| 176 | dag Outs, |
| 177 | dag Ins, dag MaskingIns, dag ZeroMaskingIns, |
| 178 | string OpcodeStr, |
| 179 | string AttSrcAsm, string IntelSrcAsm, |
| 180 | list<dag> Pattern, |
| 181 | list<dag> MaskingPattern, |
| 182 | list<dag> ZeroMaskingPattern, |
| 183 | string MaskingConstraint = "", |
| 184 | InstrItinClass itin = NoItinerary, |
| 185 | bit IsCommutable = 0> { |
Adam Nemet | 52bb6cf | 2014-10-08 23:25:23 +0000 | [diff] [blame] | 186 | let isCommutable = IsCommutable in |
| 187 | def NAME: AVX512<O, F, Outs, Ins, |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 188 | OpcodeStr#"\t{"#AttSrcAsm#", $dst|"# |
| 189 | "$dst , "#IntelSrcAsm#"}", |
Adam Nemet | 52bb6cf | 2014-10-08 23:25:23 +0000 | [diff] [blame] | 190 | Pattern, itin>; |
| 191 | |
| 192 | // Prefer over VMOV*rrk Pat<> |
| 193 | let AddedComplexity = 20 in |
| 194 | def NAME#k: AVX512<O, F, Outs, MaskingIns, |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 195 | OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"# |
| 196 | "$dst {${mask}}, "#IntelSrcAsm#"}", |
Adam Nemet | 52bb6cf | 2014-10-08 23:25:23 +0000 | [diff] [blame] | 197 | MaskingPattern, itin>, |
| 198 | EVEX_K { |
| 199 | // In case of the 3src subclass this is overridden with a let. |
| 200 | string Constraints = MaskingConstraint; |
| 201 | } |
| 202 | let AddedComplexity = 30 in // Prefer over VMOV*rrkz Pat<> |
| 203 | def NAME#kz: AVX512<O, F, Outs, ZeroMaskingIns, |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 204 | OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}} {z}|"# |
| 205 | "$dst {${mask}} {z}, "#IntelSrcAsm#"}", |
Adam Nemet | 52bb6cf | 2014-10-08 23:25:23 +0000 | [diff] [blame] | 206 | ZeroMaskingPattern, |
| 207 | itin>, |
| 208 | EVEX_KZ; |
| 209 | } |
| 210 | |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 211 | |
Adam Nemet | 3480142 | 2014-10-08 23:25:39 +0000 | [diff] [blame] | 212 | // Common base class of AVX512_maskable and AVX512_maskable_3src. |
| 213 | multiclass AVX512_maskable_common<bits<8> O, Format F, X86VectorVTInfo _, |
| 214 | dag Outs, |
| 215 | dag Ins, dag MaskingIns, dag ZeroMaskingIns, |
| 216 | string OpcodeStr, |
| 217 | string AttSrcAsm, string IntelSrcAsm, |
| 218 | dag RHS, dag MaskingRHS, |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 219 | SDNode Select = vselect, |
Adam Nemet | 3480142 | 2014-10-08 23:25:39 +0000 | [diff] [blame] | 220 | string MaskingConstraint = "", |
| 221 | InstrItinClass itin = NoItinerary, |
| 222 | bit IsCommutable = 0> : |
| 223 | AVX512_maskable_custom<O, F, Outs, Ins, MaskingIns, ZeroMaskingIns, OpcodeStr, |
| 224 | AttSrcAsm, IntelSrcAsm, |
| 225 | [(set _.RC:$dst, RHS)], |
| 226 | [(set _.RC:$dst, MaskingRHS)], |
| 227 | [(set _.RC:$dst, |
Elena Demikhovsky | 905a5a6 | 2014-11-26 10:46:49 +0000 | [diff] [blame] | 228 | (Select _.KRCWM:$mask, RHS, _.ImmAllZerosV))], |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 229 | MaskingConstraint, NoItinerary, IsCommutable>; |
Adam Nemet | 2e2537f | 2014-08-07 17:53:55 +0000 | [diff] [blame] | 230 | |
Adam Nemet | 2e91ee5 | 2014-08-14 17:13:19 +0000 | [diff] [blame] | 231 | // This multiclass generates the unconditional/non-masking, the masking and |
Elena Demikhovsky | 905a5a6 | 2014-11-26 10:46:49 +0000 | [diff] [blame] | 232 | // the zero-masking variant of the vector instruction. In the masking case, the |
Adam Nemet | 2e91ee5 | 2014-08-14 17:13:19 +0000 | [diff] [blame] | 233 | // perserved vector elements come from a new dummy input operand tied to $dst. |
Adam Nemet | 3480142 | 2014-10-08 23:25:39 +0000 | [diff] [blame] | 234 | multiclass AVX512_maskable<bits<8> O, Format F, X86VectorVTInfo _, |
| 235 | dag Outs, dag Ins, string OpcodeStr, |
| 236 | string AttSrcAsm, string IntelSrcAsm, |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 237 | dag RHS, |
Elena Demikhovsky | be8808d | 2014-11-12 07:31:03 +0000 | [diff] [blame] | 238 | InstrItinClass itin = NoItinerary, |
Adam Nemet | 3480142 | 2014-10-08 23:25:39 +0000 | [diff] [blame] | 239 | bit IsCommutable = 0> : |
| 240 | AVX512_maskable_common<O, F, _, Outs, Ins, |
| 241 | !con((ins _.RC:$src0, _.KRCWM:$mask), Ins), |
| 242 | !con((ins _.KRCWM:$mask), Ins), |
| 243 | OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS, |
Elena Demikhovsky | 905a5a6 | 2014-11-26 10:46:49 +0000 | [diff] [blame] | 244 | (vselect _.KRCWM:$mask, RHS, _.RC:$src0), vselect, |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 245 | "$src0 = $dst", itin, IsCommutable>; |
Elena Demikhovsky | 905a5a6 | 2014-11-26 10:46:49 +0000 | [diff] [blame] | 246 | |
| 247 | // This multiclass generates the unconditional/non-masking, the masking and |
| 248 | // the zero-masking variant of the scalar instruction. |
| 249 | multiclass AVX512_maskable_scalar<bits<8> O, Format F, X86VectorVTInfo _, |
| 250 | dag Outs, dag Ins, string OpcodeStr, |
| 251 | string AttSrcAsm, string IntelSrcAsm, |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 252 | dag RHS, |
Elena Demikhovsky | 905a5a6 | 2014-11-26 10:46:49 +0000 | [diff] [blame] | 253 | InstrItinClass itin = NoItinerary, |
| 254 | bit IsCommutable = 0> : |
| 255 | AVX512_maskable_common<O, F, _, Outs, Ins, |
| 256 | !con((ins _.RC:$src0, _.KRCWM:$mask), Ins), |
| 257 | !con((ins _.KRCWM:$mask), Ins), |
| 258 | OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS, |
| 259 | (X86select _.KRCWM:$mask, RHS, _.RC:$src0), X86select, |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 260 | "$src0 = $dst", itin, IsCommutable>; |
Adam Nemet | 2e91ee5 | 2014-08-14 17:13:19 +0000 | [diff] [blame] | 261 | |
Adam Nemet | 3480142 | 2014-10-08 23:25:39 +0000 | [diff] [blame] | 262 | // Similar to AVX512_maskable but in this case one of the source operands |
Adam Nemet | 2e91ee5 | 2014-08-14 17:13:19 +0000 | [diff] [blame] | 263 | // ($src1) is already tied to $dst so we just use that for the preserved |
| 264 | // vector elements. NOTE that the NonTiedIns (the ins dag) should exclude |
| 265 | // $src1. |
Adam Nemet | 3480142 | 2014-10-08 23:25:39 +0000 | [diff] [blame] | 266 | multiclass AVX512_maskable_3src<bits<8> O, Format F, X86VectorVTInfo _, |
| 267 | dag Outs, dag NonTiedIns, string OpcodeStr, |
| 268 | string AttSrcAsm, string IntelSrcAsm, |
| 269 | dag RHS> : |
| 270 | AVX512_maskable_common<O, F, _, Outs, |
| 271 | !con((ins _.RC:$src1), NonTiedIns), |
| 272 | !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns), |
| 273 | !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns), |
| 274 | OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS, |
| 275 | (vselect _.KRCWM:$mask, RHS, _.RC:$src1)>; |
Adam Nemet | 2e91ee5 | 2014-08-14 17:13:19 +0000 | [diff] [blame] | 276 | |
Igor Breger | 15820b0 | 2015-07-01 13:24:28 +0000 | [diff] [blame] | 277 | multiclass AVX512_maskable_3src_scalar<bits<8> O, Format F, X86VectorVTInfo _, |
| 278 | dag Outs, dag NonTiedIns, string OpcodeStr, |
| 279 | string AttSrcAsm, string IntelSrcAsm, |
| 280 | dag RHS> : |
| 281 | AVX512_maskable_common<O, F, _, Outs, |
| 282 | !con((ins _.RC:$src1), NonTiedIns), |
| 283 | !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns), |
| 284 | !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns), |
| 285 | OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS, |
| 286 | (X86select _.KRCWM:$mask, RHS, _.RC:$src1)>; |
Adam Nemet | 2b5cdbb | 2014-10-08 23:25:33 +0000 | [diff] [blame] | 287 | |
Adam Nemet | 3480142 | 2014-10-08 23:25:39 +0000 | [diff] [blame] | 288 | multiclass AVX512_maskable_in_asm<bits<8> O, Format F, X86VectorVTInfo _, |
| 289 | dag Outs, dag Ins, |
| 290 | string OpcodeStr, |
| 291 | string AttSrcAsm, string IntelSrcAsm, |
| 292 | list<dag> Pattern> : |
| 293 | AVX512_maskable_custom<O, F, Outs, Ins, |
| 294 | !con((ins _.RC:$src0, _.KRCWM:$mask), Ins), |
| 295 | !con((ins _.KRCWM:$mask), Ins), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 296 | OpcodeStr, AttSrcAsm, IntelSrcAsm, Pattern, [], [], |
Adam Nemet | 3480142 | 2014-10-08 23:25:39 +0000 | [diff] [blame] | 297 | "$src0 = $dst">; |
Adam Nemet | 2b5cdbb | 2014-10-08 23:25:33 +0000 | [diff] [blame] | 298 | |
Elena Demikhovsky | 431b81e | 2015-04-21 13:13:46 +0000 | [diff] [blame] | 299 | |
| 300 | // Instruction with mask that puts result in mask register, |
| 301 | // like "compare" and "vptest" |
| 302 | multiclass AVX512_maskable_custom_cmp<bits<8> O, Format F, |
| 303 | dag Outs, |
| 304 | dag Ins, dag MaskingIns, |
| 305 | string OpcodeStr, |
| 306 | string AttSrcAsm, string IntelSrcAsm, |
| 307 | list<dag> Pattern, |
| 308 | list<dag> MaskingPattern, |
| 309 | string Round = "", |
| 310 | InstrItinClass itin = NoItinerary> { |
| 311 | def NAME: AVX512<O, F, Outs, Ins, |
| 312 | OpcodeStr#"\t{"#AttSrcAsm#", $dst "#Round#"|"# |
| 313 | "$dst "#Round#", "#IntelSrcAsm#"}", |
| 314 | Pattern, itin>; |
| 315 | |
| 316 | def NAME#k: AVX512<O, F, Outs, MaskingIns, |
Elena Demikhovsky | 29792e9 | 2015-05-07 11:24:42 +0000 | [diff] [blame] | 317 | OpcodeStr#"\t{"#Round#AttSrcAsm#", $dst {${mask}}|"# |
| 318 | "$dst {${mask}}, "#IntelSrcAsm#Round#"}", |
Elena Demikhovsky | 431b81e | 2015-04-21 13:13:46 +0000 | [diff] [blame] | 319 | MaskingPattern, itin>, EVEX_K; |
| 320 | } |
| 321 | |
| 322 | multiclass AVX512_maskable_common_cmp<bits<8> O, Format F, X86VectorVTInfo _, |
| 323 | dag Outs, |
| 324 | dag Ins, dag MaskingIns, |
| 325 | string OpcodeStr, |
| 326 | string AttSrcAsm, string IntelSrcAsm, |
| 327 | dag RHS, dag MaskingRHS, |
| 328 | string Round = "", |
| 329 | InstrItinClass itin = NoItinerary> : |
| 330 | AVX512_maskable_custom_cmp<O, F, Outs, Ins, MaskingIns, OpcodeStr, |
| 331 | AttSrcAsm, IntelSrcAsm, |
| 332 | [(set _.KRC:$dst, RHS)], |
| 333 | [(set _.KRC:$dst, MaskingRHS)], |
| 334 | Round, NoItinerary>; |
| 335 | |
| 336 | multiclass AVX512_maskable_cmp<bits<8> O, Format F, X86VectorVTInfo _, |
| 337 | dag Outs, dag Ins, string OpcodeStr, |
| 338 | string AttSrcAsm, string IntelSrcAsm, |
| 339 | dag RHS, string Round = "", |
| 340 | InstrItinClass itin = NoItinerary> : |
| 341 | AVX512_maskable_common_cmp<O, F, _, Outs, Ins, |
| 342 | !con((ins _.KRCWM:$mask), Ins), |
| 343 | OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS, |
| 344 | (and _.KRCWM:$mask, RHS), |
| 345 | Round, itin>; |
| 346 | |
Elena Demikhovsky | 29792e9 | 2015-05-07 11:24:42 +0000 | [diff] [blame] | 347 | multiclass AVX512_maskable_cmp_alt<bits<8> O, Format F, X86VectorVTInfo _, |
| 348 | dag Outs, dag Ins, string OpcodeStr, |
| 349 | string AttSrcAsm, string IntelSrcAsm> : |
| 350 | AVX512_maskable_custom_cmp<O, F, Outs, |
| 351 | Ins, !con((ins _.KRCWM:$mask),Ins), OpcodeStr, |
| 352 | AttSrcAsm, IntelSrcAsm, |
| 353 | [],[],"", NoItinerary>; |
| 354 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 355 | // Bitcasts between 512-bit vector types. Return the original type since |
| 356 | // no instruction is needed for the conversion |
| 357 | let Predicates = [HasAVX512] in { |
Robert Khasanov | bfa0131 | 2014-07-21 14:54:21 +0000 | [diff] [blame] | 358 | def : Pat<(v8f64 (bitconvert (v8i64 VR512:$src))), (v8f64 VR512:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 359 | def : Pat<(v8f64 (bitconvert (v16i32 VR512:$src))), (v8f64 VR512:$src)>; |
Robert Khasanov | bfa0131 | 2014-07-21 14:54:21 +0000 | [diff] [blame] | 360 | def : Pat<(v8f64 (bitconvert (v32i16 VR512:$src))), (v8f64 VR512:$src)>; |
| 361 | def : Pat<(v8f64 (bitconvert (v64i8 VR512:$src))), (v8f64 VR512:$src)>; |
| 362 | def : Pat<(v8f64 (bitconvert (v16f32 VR512:$src))), (v8f64 VR512:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 363 | def : Pat<(v16f32 (bitconvert (v8i64 VR512:$src))), (v16f32 VR512:$src)>; |
Robert Khasanov | bfa0131 | 2014-07-21 14:54:21 +0000 | [diff] [blame] | 364 | def : Pat<(v16f32 (bitconvert (v16i32 VR512:$src))), (v16f32 VR512:$src)>; |
| 365 | def : Pat<(v16f32 (bitconvert (v32i16 VR512:$src))), (v16f32 VR512:$src)>; |
| 366 | def : Pat<(v16f32 (bitconvert (v64i8 VR512:$src))), (v16f32 VR512:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 367 | def : Pat<(v16f32 (bitconvert (v8f64 VR512:$src))), (v16f32 VR512:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 368 | def : Pat<(v8i64 (bitconvert (v16i32 VR512:$src))), (v8i64 VR512:$src)>; |
Robert Khasanov | bfa0131 | 2014-07-21 14:54:21 +0000 | [diff] [blame] | 369 | def : Pat<(v8i64 (bitconvert (v32i16 VR512:$src))), (v8i64 VR512:$src)>; |
| 370 | def : Pat<(v8i64 (bitconvert (v64i8 VR512:$src))), (v8i64 VR512:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 371 | def : Pat<(v8i64 (bitconvert (v8f64 VR512:$src))), (v8i64 VR512:$src)>; |
Robert Khasanov | bfa0131 | 2014-07-21 14:54:21 +0000 | [diff] [blame] | 372 | def : Pat<(v8i64 (bitconvert (v16f32 VR512:$src))), (v8i64 VR512:$src)>; |
| 373 | def : Pat<(v16i32 (bitconvert (v8i64 VR512:$src))), (v16i32 VR512:$src)>; |
Elena Demikhovsky | 40a7714 | 2014-08-11 09:59:08 +0000 | [diff] [blame] | 374 | def : Pat<(v16i32 (bitconvert (v16f32 VR512:$src))), (v16i32 VR512:$src)>; |
Robert Khasanov | bfa0131 | 2014-07-21 14:54:21 +0000 | [diff] [blame] | 375 | def : Pat<(v16i32 (bitconvert (v32i16 VR512:$src))), (v16i32 VR512:$src)>; |
| 376 | def : Pat<(v16i32 (bitconvert (v64i8 VR512:$src))), (v16i32 VR512:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 377 | def : Pat<(v16i32 (bitconvert (v8f64 VR512:$src))), (v16i32 VR512:$src)>; |
Robert Khasanov | bfa0131 | 2014-07-21 14:54:21 +0000 | [diff] [blame] | 378 | def : Pat<(v32i16 (bitconvert (v8i64 VR512:$src))), (v32i16 VR512:$src)>; |
| 379 | def : Pat<(v32i16 (bitconvert (v16i32 VR512:$src))), (v32i16 VR512:$src)>; |
| 380 | def : Pat<(v32i16 (bitconvert (v64i8 VR512:$src))), (v32i16 VR512:$src)>; |
| 381 | def : Pat<(v32i16 (bitconvert (v8f64 VR512:$src))), (v32i16 VR512:$src)>; |
| 382 | def : Pat<(v32i16 (bitconvert (v16f32 VR512:$src))), (v32i16 VR512:$src)>; |
| 383 | def : Pat<(v32i16 (bitconvert (v16f32 VR512:$src))), (v32i16 VR512:$src)>; |
| 384 | def : Pat<(v64i8 (bitconvert (v8i64 VR512:$src))), (v64i8 VR512:$src)>; |
| 385 | def : Pat<(v64i8 (bitconvert (v16i32 VR512:$src))), (v64i8 VR512:$src)>; |
| 386 | def : Pat<(v64i8 (bitconvert (v32i16 VR512:$src))), (v64i8 VR512:$src)>; |
| 387 | def : Pat<(v64i8 (bitconvert (v8f64 VR512:$src))), (v64i8 VR512:$src)>; |
| 388 | def : Pat<(v64i8 (bitconvert (v16f32 VR512:$src))), (v64i8 VR512:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 389 | |
| 390 | def : Pat<(v2i64 (bitconvert (v4i32 VR128X:$src))), (v2i64 VR128X:$src)>; |
| 391 | def : Pat<(v2i64 (bitconvert (v8i16 VR128X:$src))), (v2i64 VR128X:$src)>; |
| 392 | def : Pat<(v2i64 (bitconvert (v16i8 VR128X:$src))), (v2i64 VR128X:$src)>; |
| 393 | def : Pat<(v2i64 (bitconvert (v2f64 VR128X:$src))), (v2i64 VR128X:$src)>; |
| 394 | def : Pat<(v2i64 (bitconvert (v4f32 VR128X:$src))), (v2i64 VR128X:$src)>; |
| 395 | def : Pat<(v4i32 (bitconvert (v2i64 VR128X:$src))), (v4i32 VR128X:$src)>; |
| 396 | def : Pat<(v4i32 (bitconvert (v8i16 VR128X:$src))), (v4i32 VR128X:$src)>; |
| 397 | def : Pat<(v4i32 (bitconvert (v16i8 VR128X:$src))), (v4i32 VR128X:$src)>; |
| 398 | def : Pat<(v4i32 (bitconvert (v2f64 VR128X:$src))), (v4i32 VR128X:$src)>; |
| 399 | def : Pat<(v4i32 (bitconvert (v4f32 VR128X:$src))), (v4i32 VR128X:$src)>; |
| 400 | def : Pat<(v8i16 (bitconvert (v2i64 VR128X:$src))), (v8i16 VR128X:$src)>; |
| 401 | def : Pat<(v8i16 (bitconvert (v4i32 VR128X:$src))), (v8i16 VR128X:$src)>; |
| 402 | def : Pat<(v8i16 (bitconvert (v16i8 VR128X:$src))), (v8i16 VR128X:$src)>; |
| 403 | def : Pat<(v8i16 (bitconvert (v2f64 VR128X:$src))), (v8i16 VR128X:$src)>; |
| 404 | def : Pat<(v8i16 (bitconvert (v4f32 VR128X:$src))), (v8i16 VR128X:$src)>; |
| 405 | def : Pat<(v16i8 (bitconvert (v2i64 VR128X:$src))), (v16i8 VR128X:$src)>; |
| 406 | def : Pat<(v16i8 (bitconvert (v4i32 VR128X:$src))), (v16i8 VR128X:$src)>; |
| 407 | def : Pat<(v16i8 (bitconvert (v8i16 VR128X:$src))), (v16i8 VR128X:$src)>; |
| 408 | def : Pat<(v16i8 (bitconvert (v2f64 VR128X:$src))), (v16i8 VR128X:$src)>; |
| 409 | def : Pat<(v16i8 (bitconvert (v4f32 VR128X:$src))), (v16i8 VR128X:$src)>; |
| 410 | def : Pat<(v4f32 (bitconvert (v2i64 VR128X:$src))), (v4f32 VR128X:$src)>; |
| 411 | def : Pat<(v4f32 (bitconvert (v4i32 VR128X:$src))), (v4f32 VR128X:$src)>; |
| 412 | def : Pat<(v4f32 (bitconvert (v8i16 VR128X:$src))), (v4f32 VR128X:$src)>; |
| 413 | def : Pat<(v4f32 (bitconvert (v16i8 VR128X:$src))), (v4f32 VR128X:$src)>; |
| 414 | def : Pat<(v4f32 (bitconvert (v2f64 VR128X:$src))), (v4f32 VR128X:$src)>; |
| 415 | def : Pat<(v2f64 (bitconvert (v2i64 VR128X:$src))), (v2f64 VR128X:$src)>; |
| 416 | def : Pat<(v2f64 (bitconvert (v4i32 VR128X:$src))), (v2f64 VR128X:$src)>; |
| 417 | def : Pat<(v2f64 (bitconvert (v8i16 VR128X:$src))), (v2f64 VR128X:$src)>; |
| 418 | def : Pat<(v2f64 (bitconvert (v16i8 VR128X:$src))), (v2f64 VR128X:$src)>; |
| 419 | def : Pat<(v2f64 (bitconvert (v4f32 VR128X:$src))), (v2f64 VR128X:$src)>; |
| 420 | |
| 421 | // Bitcasts between 256-bit vector types. Return the original type since |
| 422 | // no instruction is needed for the conversion |
| 423 | def : Pat<(v4f64 (bitconvert (v8f32 VR256X:$src))), (v4f64 VR256X:$src)>; |
| 424 | def : Pat<(v4f64 (bitconvert (v8i32 VR256X:$src))), (v4f64 VR256X:$src)>; |
| 425 | def : Pat<(v4f64 (bitconvert (v4i64 VR256X:$src))), (v4f64 VR256X:$src)>; |
| 426 | def : Pat<(v4f64 (bitconvert (v16i16 VR256X:$src))), (v4f64 VR256X:$src)>; |
| 427 | def : Pat<(v4f64 (bitconvert (v32i8 VR256X:$src))), (v4f64 VR256X:$src)>; |
| 428 | def : Pat<(v8f32 (bitconvert (v8i32 VR256X:$src))), (v8f32 VR256X:$src)>; |
| 429 | def : Pat<(v8f32 (bitconvert (v4i64 VR256X:$src))), (v8f32 VR256X:$src)>; |
| 430 | def : Pat<(v8f32 (bitconvert (v4f64 VR256X:$src))), (v8f32 VR256X:$src)>; |
| 431 | def : Pat<(v8f32 (bitconvert (v32i8 VR256X:$src))), (v8f32 VR256X:$src)>; |
| 432 | def : Pat<(v8f32 (bitconvert (v16i16 VR256X:$src))), (v8f32 VR256X:$src)>; |
| 433 | def : Pat<(v4i64 (bitconvert (v8f32 VR256X:$src))), (v4i64 VR256X:$src)>; |
| 434 | def : Pat<(v4i64 (bitconvert (v8i32 VR256X:$src))), (v4i64 VR256X:$src)>; |
| 435 | def : Pat<(v4i64 (bitconvert (v4f64 VR256X:$src))), (v4i64 VR256X:$src)>; |
| 436 | def : Pat<(v4i64 (bitconvert (v32i8 VR256X:$src))), (v4i64 VR256X:$src)>; |
| 437 | def : Pat<(v4i64 (bitconvert (v16i16 VR256X:$src))), (v4i64 VR256X:$src)>; |
| 438 | def : Pat<(v32i8 (bitconvert (v4f64 VR256X:$src))), (v32i8 VR256X:$src)>; |
| 439 | def : Pat<(v32i8 (bitconvert (v4i64 VR256X:$src))), (v32i8 VR256X:$src)>; |
| 440 | def : Pat<(v32i8 (bitconvert (v8f32 VR256X:$src))), (v32i8 VR256X:$src)>; |
| 441 | def : Pat<(v32i8 (bitconvert (v8i32 VR256X:$src))), (v32i8 VR256X:$src)>; |
| 442 | def : Pat<(v32i8 (bitconvert (v16i16 VR256X:$src))), (v32i8 VR256X:$src)>; |
| 443 | def : Pat<(v8i32 (bitconvert (v32i8 VR256X:$src))), (v8i32 VR256X:$src)>; |
| 444 | def : Pat<(v8i32 (bitconvert (v16i16 VR256X:$src))), (v8i32 VR256X:$src)>; |
| 445 | def : Pat<(v8i32 (bitconvert (v8f32 VR256X:$src))), (v8i32 VR256X:$src)>; |
| 446 | def : Pat<(v8i32 (bitconvert (v4i64 VR256X:$src))), (v8i32 VR256X:$src)>; |
| 447 | def : Pat<(v8i32 (bitconvert (v4f64 VR256X:$src))), (v8i32 VR256X:$src)>; |
| 448 | def : Pat<(v16i16 (bitconvert (v8f32 VR256X:$src))), (v16i16 VR256X:$src)>; |
| 449 | def : Pat<(v16i16 (bitconvert (v8i32 VR256X:$src))), (v16i16 VR256X:$src)>; |
| 450 | def : Pat<(v16i16 (bitconvert (v4i64 VR256X:$src))), (v16i16 VR256X:$src)>; |
| 451 | def : Pat<(v16i16 (bitconvert (v4f64 VR256X:$src))), (v16i16 VR256X:$src)>; |
| 452 | def : Pat<(v16i16 (bitconvert (v32i8 VR256X:$src))), (v16i16 VR256X:$src)>; |
| 453 | } |
| 454 | |
| 455 | // |
| 456 | // AVX-512: VPXOR instruction writes zero to its upper part, it's safe build zeros. |
| 457 | // |
| 458 | |
| 459 | let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1, |
| 460 | isPseudo = 1, Predicates = [HasAVX512] in { |
| 461 | def AVX512_512_SET0 : I<0, Pseudo, (outs VR512:$dst), (ins), "", |
| 462 | [(set VR512:$dst, (v16f32 immAllZerosV))]>; |
| 463 | } |
| 464 | |
Craig Topper | fb1746b | 2014-01-30 06:03:19 +0000 | [diff] [blame] | 465 | let Predicates = [HasAVX512] in { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 466 | def : Pat<(v8i64 immAllZerosV), (AVX512_512_SET0)>; |
| 467 | def : Pat<(v16i32 immAllZerosV), (AVX512_512_SET0)>; |
| 468 | def : Pat<(v8f64 immAllZerosV), (AVX512_512_SET0)>; |
Craig Topper | fb1746b | 2014-01-30 06:03:19 +0000 | [diff] [blame] | 469 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 470 | |
| 471 | //===----------------------------------------------------------------------===// |
| 472 | // AVX-512 - VECTOR INSERT |
| 473 | // |
Adam Nemet | 4e2ef47 | 2014-10-02 23:18:28 +0000 | [diff] [blame] | 474 | |
Adam Nemet | 4285c1f | 2014-10-15 23:42:17 +0000 | [diff] [blame] | 475 | multiclass vinsert_for_size_no_alt<int Opcode, |
| 476 | X86VectorVTInfo From, X86VectorVTInfo To, |
| 477 | PatFrag vinsert_insert, |
| 478 | SDNodeXForm INSERT_get_vinsert_imm> { |
Adam Nemet | 4e2ef47 | 2014-10-02 23:18:28 +0000 | [diff] [blame] | 479 | let hasSideEffects = 0, ExeDomain = To.ExeDomain in { |
| 480 | def rr : AVX512AIi8<Opcode, MRMSrcReg, (outs VR512:$dst), |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 481 | (ins VR512:$src1, From.RC:$src2, u8imm:$src3), |
Adam Nemet | 449b3f0 | 2014-10-15 23:42:09 +0000 | [diff] [blame] | 482 | "vinsert" # From.EltTypeName # "x" # From.NumElts # |
| 483 | "\t{$src3, $src2, $src1, $dst|" |
Adam Nemet | 4e2ef47 | 2014-10-02 23:18:28 +0000 | [diff] [blame] | 484 | "$dst, $src1, $src2, $src3}", |
Adam Nemet | 4dca3ce | 2014-10-02 23:18:30 +0000 | [diff] [blame] | 485 | [(set To.RC:$dst, (vinsert_insert:$src3 (To.VT VR512:$src1), |
| 486 | (From.VT From.RC:$src2), |
| 487 | (iPTR imm)))]>, |
| 488 | EVEX_4V, EVEX_V512; |
Adam Nemet | 4e2ef47 | 2014-10-02 23:18:28 +0000 | [diff] [blame] | 489 | |
| 490 | let mayLoad = 1 in |
| 491 | def rm : AVX512AIi8<Opcode, MRMSrcMem, (outs VR512:$dst), |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 492 | (ins VR512:$src1, From.MemOp:$src2, u8imm:$src3), |
Adam Nemet | 449b3f0 | 2014-10-15 23:42:09 +0000 | [diff] [blame] | 493 | "vinsert" # From.EltTypeName # "x" # From.NumElts # |
| 494 | "\t{$src3, $src2, $src1, $dst|" |
Adam Nemet | 4e2ef47 | 2014-10-02 23:18:28 +0000 | [diff] [blame] | 495 | "$dst, $src1, $src2, $src3}", |
Adam Nemet | 449b3f0 | 2014-10-15 23:42:09 +0000 | [diff] [blame] | 496 | []>, |
| 497 | EVEX_4V, EVEX_V512, EVEX_CD8<From.EltSize, From.CD8TupleForm>; |
Adam Nemet | 4e2ef47 | 2014-10-02 23:18:28 +0000 | [diff] [blame] | 498 | } |
Adam Nemet | 4285c1f | 2014-10-15 23:42:17 +0000 | [diff] [blame] | 499 | } |
Adam Nemet | 4e2ef47 | 2014-10-02 23:18:28 +0000 | [diff] [blame] | 500 | |
Adam Nemet | 4285c1f | 2014-10-15 23:42:17 +0000 | [diff] [blame] | 501 | multiclass vinsert_for_size<int Opcode, |
| 502 | X86VectorVTInfo From, X86VectorVTInfo To, |
| 503 | X86VectorVTInfo AltFrom, X86VectorVTInfo AltTo, |
| 504 | PatFrag vinsert_insert, |
| 505 | SDNodeXForm INSERT_get_vinsert_imm> : |
| 506 | vinsert_for_size_no_alt<Opcode, From, To, |
| 507 | vinsert_insert, INSERT_get_vinsert_imm> { |
Adam Nemet | 4e2ef47 | 2014-10-02 23:18:28 +0000 | [diff] [blame] | 508 | // Codegen pattern with the alternative types, e.g. v2i64 -> v8i64 for |
Adam Nemet | 4285c1f | 2014-10-15 23:42:17 +0000 | [diff] [blame] | 509 | // vinserti32x4. Only add this if 64x2 and friends are not supported |
| 510 | // natively via AVX512DQ. |
| 511 | let Predicates = [NoDQI] in |
| 512 | def : Pat<(vinsert_insert:$ins |
| 513 | (AltTo.VT VR512:$src1), (AltFrom.VT From.RC:$src2), (iPTR imm)), |
| 514 | (AltTo.VT (!cast<Instruction>(NAME # From.EltSize # "x4rr") |
| 515 | VR512:$src1, From.RC:$src2, |
| 516 | (INSERT_get_vinsert_imm VR512:$ins)))>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 517 | } |
| 518 | |
Adam Nemet | b1c3ef4 | 2014-10-15 23:42:04 +0000 | [diff] [blame] | 519 | multiclass vinsert_for_type<ValueType EltVT32, int Opcode128, |
| 520 | ValueType EltVT64, int Opcode256> { |
| 521 | defm NAME # "32x4" : vinsert_for_size<Opcode128, |
Adam Nemet | 4e2ef47 | 2014-10-02 23:18:28 +0000 | [diff] [blame] | 522 | X86VectorVTInfo< 4, EltVT32, VR128X>, |
| 523 | X86VectorVTInfo<16, EltVT32, VR512>, |
| 524 | X86VectorVTInfo< 2, EltVT64, VR128X>, |
| 525 | X86VectorVTInfo< 8, EltVT64, VR512>, |
| 526 | vinsert128_insert, |
| 527 | INSERT_get_vinsert128_imm>; |
Adam Nemet | 4285c1f | 2014-10-15 23:42:17 +0000 | [diff] [blame] | 528 | let Predicates = [HasDQI] in |
| 529 | defm NAME # "64x2" : vinsert_for_size_no_alt<Opcode128, |
| 530 | X86VectorVTInfo< 2, EltVT64, VR128X>, |
| 531 | X86VectorVTInfo< 8, EltVT64, VR512>, |
| 532 | vinsert128_insert, |
| 533 | INSERT_get_vinsert128_imm>, VEX_W; |
Adam Nemet | b1c3ef4 | 2014-10-15 23:42:04 +0000 | [diff] [blame] | 534 | defm NAME # "64x4" : vinsert_for_size<Opcode256, |
Adam Nemet | 4e2ef47 | 2014-10-02 23:18:28 +0000 | [diff] [blame] | 535 | X86VectorVTInfo< 4, EltVT64, VR256X>, |
| 536 | X86VectorVTInfo< 8, EltVT64, VR512>, |
| 537 | X86VectorVTInfo< 8, EltVT32, VR256>, |
| 538 | X86VectorVTInfo<16, EltVT32, VR512>, |
| 539 | vinsert256_insert, |
| 540 | INSERT_get_vinsert256_imm>, VEX_W; |
Adam Nemet | 4285c1f | 2014-10-15 23:42:17 +0000 | [diff] [blame] | 541 | let Predicates = [HasDQI] in |
| 542 | defm NAME # "32x8" : vinsert_for_size_no_alt<Opcode256, |
| 543 | X86VectorVTInfo< 8, EltVT32, VR256X>, |
| 544 | X86VectorVTInfo<16, EltVT32, VR512>, |
| 545 | vinsert256_insert, |
| 546 | INSERT_get_vinsert256_imm>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 547 | } |
| 548 | |
Adam Nemet | 4e2ef47 | 2014-10-02 23:18:28 +0000 | [diff] [blame] | 549 | defm VINSERTF : vinsert_for_type<f32, 0x18, f64, 0x1a>; |
| 550 | defm VINSERTI : vinsert_for_type<i32, 0x38, i64, 0x3a>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 551 | |
| 552 | // vinsertps - insert f32 to XMM |
| 553 | def VINSERTPSzrr : AVX512AIi8<0x21, MRMSrcReg, (outs VR128X:$dst), |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 554 | (ins VR128X:$src1, VR128X:$src2, u8imm:$src3), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 555 | "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", |
Filipe Cabecinhas | 2035221 | 2014-04-21 20:07:29 +0000 | [diff] [blame] | 556 | [(set VR128X:$dst, (X86insertps VR128X:$src1, VR128X:$src2, imm:$src3))]>, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 557 | EVEX_4V; |
| 558 | def VINSERTPSzrm: AVX512AIi8<0x21, MRMSrcMem, (outs VR128X:$dst), |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 559 | (ins VR128X:$src1, f32mem:$src2, u8imm:$src3), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 560 | "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", |
Filipe Cabecinhas | 2035221 | 2014-04-21 20:07:29 +0000 | [diff] [blame] | 561 | [(set VR128X:$dst, (X86insertps VR128X:$src1, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 562 | (v4f32 (scalar_to_vector (loadf32 addr:$src2))), |
| 563 | imm:$src3))]>, EVEX_4V, EVEX_CD8<32, CD8VT1>; |
| 564 | |
| 565 | //===----------------------------------------------------------------------===// |
| 566 | // AVX-512 VECTOR EXTRACT |
| 567 | //--- |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 568 | |
Adam Nemet | 55536c6 | 2014-09-25 23:48:45 +0000 | [diff] [blame] | 569 | multiclass vextract_for_size<int Opcode, |
| 570 | X86VectorVTInfo From, X86VectorVTInfo To, |
| 571 | X86VectorVTInfo AltFrom, X86VectorVTInfo AltTo, |
| 572 | PatFrag vextract_extract, |
| 573 | SDNodeXForm EXTRACT_get_vextract_imm> { |
| 574 | let hasSideEffects = 0, ExeDomain = To.ExeDomain in { |
Adam Nemet | 3480142 | 2014-10-08 23:25:39 +0000 | [diff] [blame] | 575 | defm rr : AVX512_maskable_in_asm<Opcode, MRMDestReg, To, (outs To.RC:$dst), |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 576 | (ins VR512:$src1, u8imm:$idx), |
Adam Nemet | 2b5cdbb | 2014-10-08 23:25:33 +0000 | [diff] [blame] | 577 | "vextract" # To.EltTypeName # "x4", |
| 578 | "$idx, $src1", "$src1, $idx", |
| 579 | [(set To.RC:$dst, (vextract_extract:$idx (From.VT VR512:$src1), |
| 580 | (iPTR imm)))]>, |
| 581 | AVX512AIi8Base, EVEX, EVEX_V512; |
Adam Nemet | 55536c6 | 2014-09-25 23:48:45 +0000 | [diff] [blame] | 582 | let mayStore = 1 in |
| 583 | def rm : AVX512AIi8<Opcode, MRMDestMem, (outs), |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 584 | (ins To.MemOp:$dst, VR512:$src1, u8imm:$src2), |
Adam Nemet | 55536c6 | 2014-09-25 23:48:45 +0000 | [diff] [blame] | 585 | "vextract" # To.EltTypeName # "x4\t{$src2, $src1, $dst|" |
| 586 | "$dst, $src1, $src2}", |
| 587 | []>, EVEX, EVEX_V512, EVEX_CD8<To.EltSize, CD8VT4>; |
| 588 | } |
| 589 | |
Adam Nemet | 55536c6 | 2014-09-25 23:48:45 +0000 | [diff] [blame] | 590 | // Codegen pattern with the alternative types, e.g. v8i64 -> v2i64 for |
| 591 | // vextracti32x4 |
| 592 | def : Pat<(vextract_extract:$ext (AltFrom.VT VR512:$src1), (iPTR imm)), |
| 593 | (AltTo.VT (!cast<Instruction>(NAME # To.EltSize # "x4rr") |
| 594 | VR512:$src1, |
| 595 | (EXTRACT_get_vextract_imm To.RC:$ext)))>; |
| 596 | |
| 597 | // A 128/256-bit subvector extract from the first 512-bit vector position is |
| 598 | // a subregister copy that needs no instruction. |
| 599 | def : Pat<(To.VT (extract_subvector (From.VT VR512:$src), (iPTR 0))), |
| 600 | (To.VT |
| 601 | (EXTRACT_SUBREG (From.VT VR512:$src), To.SubRegIdx))>; |
| 602 | |
| 603 | // And for the alternative types. |
| 604 | def : Pat<(AltTo.VT (extract_subvector (AltFrom.VT VR512:$src), (iPTR 0))), |
| 605 | (AltTo.VT |
| 606 | (EXTRACT_SUBREG (AltFrom.VT VR512:$src), AltTo.SubRegIdx))>; |
Adam Nemet | 47b2d5f | 2014-10-08 23:25:37 +0000 | [diff] [blame] | 607 | |
| 608 | // Intrinsic call with masking. |
| 609 | def : Pat<(!cast<Intrinsic>("int_x86_avx512_mask_vextract" # To.EltTypeName # |
| 610 | "x4_512") |
| 611 | VR512:$src1, (iPTR imm:$idx), To.RC:$src0, GR8:$mask), |
| 612 | (!cast<Instruction>(NAME # To.EltSize # "x4rrk") To.RC:$src0, |
| 613 | (v4i1 (COPY_TO_REGCLASS GR8:$mask, VK4WM)), |
| 614 | VR512:$src1, imm:$idx)>; |
| 615 | |
| 616 | // Intrinsic call with zero-masking. |
| 617 | def : Pat<(!cast<Intrinsic>("int_x86_avx512_mask_vextract" # To.EltTypeName # |
| 618 | "x4_512") |
| 619 | VR512:$src1, (iPTR imm:$idx), To.ImmAllZerosV, GR8:$mask), |
| 620 | (!cast<Instruction>(NAME # To.EltSize # "x4rrkz") |
| 621 | (v4i1 (COPY_TO_REGCLASS GR8:$mask, VK4WM)), |
| 622 | VR512:$src1, imm:$idx)>; |
| 623 | |
| 624 | // Intrinsic call without masking. |
| 625 | def : Pat<(!cast<Intrinsic>("int_x86_avx512_mask_vextract" # To.EltTypeName # |
| 626 | "x4_512") |
| 627 | VR512:$src1, (iPTR imm:$idx), To.ImmAllZerosV, (i8 -1)), |
| 628 | (!cast<Instruction>(NAME # To.EltSize # "x4rr") |
| 629 | VR512:$src1, imm:$idx)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 630 | } |
| 631 | |
Adam Nemet | 55536c6 | 2014-09-25 23:48:45 +0000 | [diff] [blame] | 632 | multiclass vextract_for_type<ValueType EltVT32, int Opcode32, |
| 633 | ValueType EltVT64, int Opcode64> { |
| 634 | defm NAME # "32x4" : vextract_for_size<Opcode32, |
| 635 | X86VectorVTInfo<16, EltVT32, VR512>, |
| 636 | X86VectorVTInfo< 4, EltVT32, VR128X>, |
| 637 | X86VectorVTInfo< 8, EltVT64, VR512>, |
| 638 | X86VectorVTInfo< 2, EltVT64, VR128X>, |
| 639 | vextract128_extract, |
| 640 | EXTRACT_get_vextract128_imm>; |
| 641 | defm NAME # "64x4" : vextract_for_size<Opcode64, |
| 642 | X86VectorVTInfo< 8, EltVT64, VR512>, |
| 643 | X86VectorVTInfo< 4, EltVT64, VR256X>, |
| 644 | X86VectorVTInfo<16, EltVT32, VR512>, |
| 645 | X86VectorVTInfo< 8, EltVT32, VR256>, |
| 646 | vextract256_extract, |
| 647 | EXTRACT_get_vextract256_imm>, VEX_W; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 648 | } |
| 649 | |
Adam Nemet | 55536c6 | 2014-09-25 23:48:45 +0000 | [diff] [blame] | 650 | defm VEXTRACTF : vextract_for_type<f32, 0x19, f64, 0x1b>; |
| 651 | defm VEXTRACTI : vextract_for_type<i32, 0x39, i64, 0x3b>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 652 | |
| 653 | // A 128-bit subvector insert to the first 512-bit vector position |
| 654 | // is a subregister copy that needs no instruction. |
| 655 | def : Pat<(insert_subvector undef, (v2i64 VR128X:$src), (iPTR 0)), |
| 656 | (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), |
| 657 | (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm), |
| 658 | sub_ymm)>; |
| 659 | def : Pat<(insert_subvector undef, (v2f64 VR128X:$src), (iPTR 0)), |
| 660 | (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), |
| 661 | (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm), |
| 662 | sub_ymm)>; |
| 663 | def : Pat<(insert_subvector undef, (v4i32 VR128X:$src), (iPTR 0)), |
| 664 | (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), |
| 665 | (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm), |
| 666 | sub_ymm)>; |
| 667 | def : Pat<(insert_subvector undef, (v4f32 VR128X:$src), (iPTR 0)), |
| 668 | (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), |
| 669 | (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm), |
| 670 | sub_ymm)>; |
| 671 | |
| 672 | def : Pat<(insert_subvector undef, (v4i64 VR256X:$src), (iPTR 0)), |
| 673 | (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>; |
| 674 | def : Pat<(insert_subvector undef, (v4f64 VR256X:$src), (iPTR 0)), |
| 675 | (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>; |
| 676 | def : Pat<(insert_subvector undef, (v8i32 VR256X:$src), (iPTR 0)), |
| 677 | (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>; |
| 678 | def : Pat<(insert_subvector undef, (v8f32 VR256X:$src), (iPTR 0)), |
| 679 | (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>; |
| 680 | |
| 681 | // vextractps - extract 32 bits from XMM |
| 682 | def VEXTRACTPSzrr : AVX512AIi8<0x17, MRMDestReg, (outs GR32:$dst), |
Craig Topper | fc946a0 | 2015-01-25 02:21:13 +0000 | [diff] [blame] | 683 | (ins VR128X:$src1, u8imm:$src2), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 684 | "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 685 | [(set GR32:$dst, (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2))]>, |
| 686 | EVEX; |
| 687 | |
| 688 | def VEXTRACTPSzmr : AVX512AIi8<0x17, MRMDestMem, (outs), |
Craig Topper | fc946a0 | 2015-01-25 02:21:13 +0000 | [diff] [blame] | 689 | (ins f32mem:$dst, VR128X:$src1, u8imm:$src2), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 690 | "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 691 | [(store (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2), |
Elena Demikhovsky | 2aafc22 | 2014-02-11 07:25:59 +0000 | [diff] [blame] | 692 | addr:$dst)]>, EVEX, EVEX_CD8<32, CD8VT1>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 693 | |
| 694 | //===---------------------------------------------------------------------===// |
| 695 | // AVX-512 BROADCAST |
| 696 | //--- |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 697 | multiclass avx512_fp_broadcast<bits<8> opc, SDNode OpNode, RegisterClass SrcRC, |
| 698 | ValueType svt, X86VectorVTInfo _> { |
| 699 | defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 700 | (ins SrcRC:$src), "vbroadcast"## !subst("p", "s", _.Suffix), |
| 701 | "$src", "$src", (_.VT (OpNode (svt SrcRC:$src)))>, |
| 702 | T8PD, EVEX; |
| 703 | |
| 704 | let mayLoad = 1 in { |
| 705 | defm m : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 706 | (ins _.ScalarMemOp:$src), |
| 707 | "vbroadcast"##!subst("p", "s", _.Suffix), "$src", "$src", |
| 708 | (_.VT (OpNode (_.ScalarLdFrag addr:$src)))>, |
| 709 | T8PD, EVEX; |
| 710 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 711 | } |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 712 | |
| 713 | multiclass avx512_fp_broadcast_vl<bits<8> opc, SDNode OpNode, |
| 714 | AVX512VLVectorVTInfo _> { |
| 715 | defm Z : avx512_fp_broadcast<opc, OpNode, VR128X, _.info128.VT, _.info512>, |
| 716 | EVEX_V512; |
| 717 | |
| 718 | let Predicates = [HasVLX] in { |
| 719 | defm Z256 : avx512_fp_broadcast<opc, OpNode, VR128X, _.info128.VT, _.info256>, |
| 720 | EVEX_V256; |
| 721 | } |
| 722 | } |
| 723 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 724 | let ExeDomain = SSEPackedSingle in { |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 725 | defm VBROADCASTSS : avx512_fp_broadcast_vl<0x18, X86VBroadcast, |
| 726 | avx512vl_f32_info>, EVEX_CD8<32, CD8VT1>; |
| 727 | let Predicates = [HasVLX] in { |
| 728 | defm VBROADCASTSSZ128 : avx512_fp_broadcast<0x18, X86VBroadcast, VR128X, |
| 729 | v4f32, v4f32x_info>, EVEX_V128, |
| 730 | EVEX_CD8<32, CD8VT1>; |
| 731 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 732 | } |
| 733 | |
| 734 | let ExeDomain = SSEPackedDouble in { |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 735 | defm VBROADCASTSD : avx512_fp_broadcast_vl<0x19, X86VBroadcast, |
| 736 | avx512vl_f64_info>, VEX_W, EVEX_CD8<64, CD8VT1>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 737 | } |
| 738 | |
Robert Khasanov | 8d9b93e | 2014-12-16 16:12:11 +0000 | [diff] [blame] | 739 | // avx512_broadcast_pat introduces patterns for broadcast with a scalar argument. |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 740 | // Later, we can canonize broadcast instructions before ISel phase and |
Robert Khasanov | 8d9b93e | 2014-12-16 16:12:11 +0000 | [diff] [blame] | 741 | // eliminate additional patterns on ISel. |
Robert Khasanov | 8e8c399 | 2014-12-09 18:45:30 +0000 | [diff] [blame] | 742 | // SrcRC_v and SrcRC_s are RegisterClasses for vector and scalar |
| 743 | // representations of source |
| 744 | multiclass avx512_broadcast_pat<string InstName, SDNode OpNode, |
| 745 | X86VectorVTInfo _, RegisterClass SrcRC_v, |
| 746 | RegisterClass SrcRC_s> { |
Robert Khasanov | 4204c1a | 2014-12-12 14:21:30 +0000 | [diff] [blame] | 747 | def : Pat<(_.VT (OpNode (_.EltVT SrcRC_s:$src))), |
Robert Khasanov | 8e8c399 | 2014-12-09 18:45:30 +0000 | [diff] [blame] | 748 | (!cast<Instruction>(InstName##"r") |
| 749 | (COPY_TO_REGCLASS SrcRC_s:$src, SrcRC_v))>; |
| 750 | |
| 751 | let AddedComplexity = 30 in { |
| 752 | def : Pat<(_.VT (vselect _.KRCWM:$mask, |
Robert Khasanov | 4204c1a | 2014-12-12 14:21:30 +0000 | [diff] [blame] | 753 | (OpNode (_.EltVT SrcRC_s:$src)), _.RC:$src0)), |
Robert Khasanov | 8e8c399 | 2014-12-09 18:45:30 +0000 | [diff] [blame] | 754 | (!cast<Instruction>(InstName##"rk") _.RC:$src0, _.KRCWM:$mask, |
| 755 | (COPY_TO_REGCLASS SrcRC_s:$src, SrcRC_v))>; |
| 756 | |
| 757 | def : Pat<(_.VT(vselect _.KRCWM:$mask, |
Robert Khasanov | 4204c1a | 2014-12-12 14:21:30 +0000 | [diff] [blame] | 758 | (OpNode (_.EltVT SrcRC_s:$src)), _.ImmAllZerosV)), |
Robert Khasanov | 8e8c399 | 2014-12-09 18:45:30 +0000 | [diff] [blame] | 759 | (!cast<Instruction>(InstName##"rkz") _.KRCWM:$mask, |
| 760 | (COPY_TO_REGCLASS SrcRC_s:$src, SrcRC_v))>; |
| 761 | } |
| 762 | } |
| 763 | |
| 764 | defm : avx512_broadcast_pat<"VBROADCASTSSZ", X86VBroadcast, v16f32_info, |
| 765 | VR128X, FR32X>; |
| 766 | defm : avx512_broadcast_pat<"VBROADCASTSDZ", X86VBroadcast, v8f64_info, |
| 767 | VR128X, FR64X>; |
| 768 | |
| 769 | let Predicates = [HasVLX] in { |
| 770 | defm : avx512_broadcast_pat<"VBROADCASTSSZ256", X86VBroadcast, |
| 771 | v8f32x_info, VR128X, FR32X>; |
| 772 | defm : avx512_broadcast_pat<"VBROADCASTSSZ128", X86VBroadcast, |
| 773 | v4f32x_info, VR128X, FR32X>; |
| 774 | defm : avx512_broadcast_pat<"VBROADCASTSDZ256", X86VBroadcast, |
| 775 | v4f64x_info, VR128X, FR64X>; |
| 776 | } |
| 777 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 778 | def : Pat<(v16f32 (X86VBroadcast (loadf32 addr:$src))), |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 779 | (VBROADCASTSSZm addr:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 780 | def : Pat<(v8f64 (X86VBroadcast (loadf64 addr:$src))), |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 781 | (VBROADCASTSDZm addr:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 782 | |
Quentin Colombet | 4bf1c28 | 2013-10-25 17:47:18 +0000 | [diff] [blame] | 783 | def : Pat<(int_x86_avx512_vbroadcast_ss_512 addr:$src), |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 784 | (VBROADCASTSSZm addr:$src)>; |
Quentin Colombet | 4bf1c28 | 2013-10-25 17:47:18 +0000 | [diff] [blame] | 785 | def : Pat<(int_x86_avx512_vbroadcast_sd_512 addr:$src), |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 786 | (VBROADCASTSDZm addr:$src)>; |
Quentin Colombet | 4bf1c28 | 2013-10-25 17:47:18 +0000 | [diff] [blame] | 787 | |
Robert Khasanov | cbc5703 | 2014-12-09 16:38:41 +0000 | [diff] [blame] | 788 | multiclass avx512_int_broadcast_reg<bits<8> opc, X86VectorVTInfo _, |
| 789 | RegisterClass SrcRC> { |
| 790 | defm r : AVX512_maskable_in_asm<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 791 | (ins SrcRC:$src), "vpbroadcast"##_.Suffix, |
| 792 | "$src", "$src", []>, T8PD, EVEX; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 793 | } |
| 794 | |
Robert Khasanov | cbc5703 | 2014-12-09 16:38:41 +0000 | [diff] [blame] | 795 | multiclass avx512_int_broadcast_reg_vl<bits<8> opc, AVX512VLVectorVTInfo _, |
| 796 | RegisterClass SrcRC, Predicate prd> { |
| 797 | let Predicates = [prd] in |
| 798 | defm Z : avx512_int_broadcast_reg<opc, _.info512, SrcRC>, EVEX_V512; |
| 799 | let Predicates = [prd, HasVLX] in { |
| 800 | defm Z256 : avx512_int_broadcast_reg<opc, _.info256, SrcRC>, EVEX_V256; |
| 801 | defm Z128 : avx512_int_broadcast_reg<opc, _.info128, SrcRC>, EVEX_V128; |
| 802 | } |
| 803 | } |
| 804 | |
| 805 | defm VPBROADCASTBr : avx512_int_broadcast_reg_vl<0x7A, avx512vl_i8_info, GR32, |
| 806 | HasBWI>; |
| 807 | defm VPBROADCASTWr : avx512_int_broadcast_reg_vl<0x7B, avx512vl_i16_info, GR32, |
| 808 | HasBWI>; |
| 809 | defm VPBROADCASTDr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i32_info, GR32, |
| 810 | HasAVX512>; |
| 811 | defm VPBROADCASTQr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i64_info, GR64, |
| 812 | HasAVX512>, VEX_W; |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 813 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 814 | def : Pat <(v16i32 (X86vzext VK16WM:$mask)), |
Robert Khasanov | cbc5703 | 2014-12-09 16:38:41 +0000 | [diff] [blame] | 815 | (VPBROADCASTDrZrkz VK16WM:$mask, (i32 (MOV32ri 0x1)))>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 816 | |
| 817 | def : Pat <(v8i64 (X86vzext VK8WM:$mask)), |
Robert Khasanov | cbc5703 | 2014-12-09 16:38:41 +0000 | [diff] [blame] | 818 | (VPBROADCASTQrZrkz VK8WM:$mask, (i64 (MOV64ri 0x1)))>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 819 | |
| 820 | def : Pat<(v16i32 (X86VBroadcast (i32 GR32:$src))), |
Robert Khasanov | cbc5703 | 2014-12-09 16:38:41 +0000 | [diff] [blame] | 821 | (VPBROADCASTDrZr GR32:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 822 | def : Pat<(v8i64 (X86VBroadcast (i64 GR64:$src))), |
Robert Khasanov | cbc5703 | 2014-12-09 16:38:41 +0000 | [diff] [blame] | 823 | (VPBROADCASTQrZr GR64:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 824 | |
Cameron McInally | 394d557 | 2013-10-31 13:56:31 +0000 | [diff] [blame] | 825 | def : Pat<(v16i32 (int_x86_avx512_pbroadcastd_i32_512 (i32 GR32:$src))), |
Robert Khasanov | cbc5703 | 2014-12-09 16:38:41 +0000 | [diff] [blame] | 826 | (VPBROADCASTDrZr GR32:$src)>; |
Cameron McInally | 394d557 | 2013-10-31 13:56:31 +0000 | [diff] [blame] | 827 | def : Pat<(v8i64 (int_x86_avx512_pbroadcastq_i64_512 (i64 GR64:$src))), |
Robert Khasanov | cbc5703 | 2014-12-09 16:38:41 +0000 | [diff] [blame] | 828 | (VPBROADCASTQrZr GR64:$src)>; |
Cameron McInally | 394d557 | 2013-10-31 13:56:31 +0000 | [diff] [blame] | 829 | |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 830 | def : Pat<(v16i32 (int_x86_avx512_mask_pbroadcast_d_gpr_512 (i32 GR32:$src), |
| 831 | (v16i32 immAllZerosV), (i16 GR16:$mask))), |
Robert Khasanov | cbc5703 | 2014-12-09 16:38:41 +0000 | [diff] [blame] | 832 | (VPBROADCASTDrZrkz (COPY_TO_REGCLASS GR16:$mask, VK16WM), GR32:$src)>; |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 833 | def : Pat<(v8i64 (int_x86_avx512_mask_pbroadcast_q_gpr_512 (i64 GR64:$src), |
| 834 | (bc_v8i64 (v16i32 immAllZerosV)), (i8 GR8:$mask))), |
Robert Khasanov | cbc5703 | 2014-12-09 16:38:41 +0000 | [diff] [blame] | 835 | (VPBROADCASTQrZrkz (COPY_TO_REGCLASS GR8:$mask, VK8WM), GR64:$src)>; |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 836 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 837 | multiclass avx512_int_broadcast_rm<bits<8> opc, string OpcodeStr, |
| 838 | X86MemOperand x86memop, PatFrag ld_frag, |
| 839 | RegisterClass DstRC, ValueType OpVT, ValueType SrcVT, |
| 840 | RegisterClass KRC> { |
| 841 | def rr : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst), (ins VR128X:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 842 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 843 | [(set DstRC:$dst, |
| 844 | (OpVT (X86VBroadcast (SrcVT VR128X:$src))))]>, EVEX; |
Elena Demikhovsky | 60eb9db | 2015-05-04 12:40:50 +0000 | [diff] [blame] | 845 | def rrk : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst), (ins KRC:$mask, |
| 846 | VR128X:$src), |
| 847 | !strconcat(OpcodeStr, |
| 848 | "\t{$src, ${dst} {${mask}} |${dst} {${mask}}, $src}"), |
| 849 | []>, EVEX, EVEX_K; |
| 850 | def rrkz : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst), (ins KRC:$mask, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 851 | VR128X:$src), |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 852 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 853 | "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), |
Elena Demikhovsky | 60eb9db | 2015-05-04 12:40:50 +0000 | [diff] [blame] | 854 | []>, EVEX, EVEX_KZ; |
Elena Demikhovsky | dd0794e | 2013-10-24 07:16:35 +0000 | [diff] [blame] | 855 | let mayLoad = 1 in { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 856 | def rm : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 857 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 858 | [(set DstRC:$dst, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 859 | (OpVT (X86VBroadcast (ld_frag addr:$src))))]>, EVEX; |
Elena Demikhovsky | 60eb9db | 2015-05-04 12:40:50 +0000 | [diff] [blame] | 860 | def rmk : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins KRC:$mask, |
| 861 | x86memop:$src), |
| 862 | !strconcat(OpcodeStr, |
| 863 | "\t{$src, ${dst} {${mask}}|${dst} {${mask}} , $src}"), |
| 864 | []>, EVEX, EVEX_K; |
| 865 | def rmkz : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins KRC:$mask, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 866 | x86memop:$src), |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 867 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 868 | "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), |
Elena Demikhovsky | 60eb9db | 2015-05-04 12:40:50 +0000 | [diff] [blame] | 869 | [(set DstRC:$dst, (OpVT (vselect KRC:$mask, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 870 | (X86VBroadcast (ld_frag addr:$src)), |
Elena Demikhovsky | 60eb9db | 2015-05-04 12:40:50 +0000 | [diff] [blame] | 871 | (OpVT (bitconvert (v16i32 immAllZerosV))))))]>, EVEX, EVEX_KZ; |
Elena Demikhovsky | dd0794e | 2013-10-24 07:16:35 +0000 | [diff] [blame] | 872 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 873 | } |
| 874 | |
| 875 | defm VPBROADCASTDZ : avx512_int_broadcast_rm<0x58, "vpbroadcastd", i32mem, |
| 876 | loadi32, VR512, v16i32, v4i32, VK16WM>, |
| 877 | EVEX_V512, EVEX_CD8<32, CD8VT1>; |
| 878 | defm VPBROADCASTQZ : avx512_int_broadcast_rm<0x59, "vpbroadcastq", i64mem, |
| 879 | loadi64, VR512, v8i64, v2i64, VK8WM>, EVEX_V512, VEX_W, |
| 880 | EVEX_CD8<64, CD8VT1>; |
| 881 | |
Elena Demikhovsky | ad9c396 | 2015-05-18 06:42:57 +0000 | [diff] [blame] | 882 | multiclass avx512_subvec_broadcast_rm<bits<8> opc, string OpcodeStr, |
| 883 | X86VectorVTInfo _Dst, X86VectorVTInfo _Src> { |
Adam Nemet | 73f72e1 | 2014-06-27 00:43:38 +0000 | [diff] [blame] | 884 | let mayLoad = 1 in { |
Elena Demikhovsky | ad9c396 | 2015-05-18 06:42:57 +0000 | [diff] [blame] | 885 | def rm : AVX5128I<opc, MRMSrcMem, (outs _Dst.RC:$dst), (ins _Src.MemOp:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 886 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 887 | [(set _Dst.RC:$dst, |
| 888 | (_Dst.VT (X86SubVBroadcast |
Elena Demikhovsky | ad9c396 | 2015-05-18 06:42:57 +0000 | [diff] [blame] | 889 | (_Src.VT (bitconvert (_Src.LdFrag addr:$src))))))]>, EVEX; |
| 890 | def rmk : AVX5128I<opc, MRMSrcMem, (outs _Dst.RC:$dst), (ins _Dst.KRCWM:$mask, |
| 891 | _Src.MemOp:$src), |
Adam Nemet | 73f72e1 | 2014-06-27 00:43:38 +0000 | [diff] [blame] | 892 | !strconcat(OpcodeStr, |
Elena Demikhovsky | ad9c396 | 2015-05-18 06:42:57 +0000 | [diff] [blame] | 893 | "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"), |
| 894 | []>, EVEX, EVEX_K; |
| 895 | def rmkz : AVX5128I<opc, MRMSrcMem, (outs _Dst.RC:$dst), (ins _Dst.KRCWM:$mask, |
| 896 | _Src.MemOp:$src), |
| 897 | !strconcat(OpcodeStr, |
| 898 | "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), |
Adam Nemet | 73f72e1 | 2014-06-27 00:43:38 +0000 | [diff] [blame] | 899 | []>, EVEX, EVEX_KZ; |
| 900 | } |
| 901 | } |
| 902 | |
Elena Demikhovsky | ad9c396 | 2015-05-18 06:42:57 +0000 | [diff] [blame] | 903 | defm VBROADCASTI32X4 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4", |
| 904 | v16i32_info, v4i32x_info>, |
Adam Nemet | 73f72e1 | 2014-06-27 00:43:38 +0000 | [diff] [blame] | 905 | EVEX_V512, EVEX_CD8<32, CD8VT4>; |
Elena Demikhovsky | ad9c396 | 2015-05-18 06:42:57 +0000 | [diff] [blame] | 906 | defm VBROADCASTF32X4 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4", |
| 907 | v16f32_info, v4f32x_info>, |
| 908 | EVEX_V512, EVEX_CD8<32, CD8VT4>; |
| 909 | defm VBROADCASTI64X4 : avx512_subvec_broadcast_rm<0x5b, "vbroadcasti64x4", |
| 910 | v8i64_info, v4i64x_info>, VEX_W, |
Adam Nemet | 73f72e1 | 2014-06-27 00:43:38 +0000 | [diff] [blame] | 911 | EVEX_V512, EVEX_CD8<64, CD8VT4>; |
Elena Demikhovsky | ad9c396 | 2015-05-18 06:42:57 +0000 | [diff] [blame] | 912 | defm VBROADCASTF64X4 : avx512_subvec_broadcast_rm<0x1b, "vbroadcastf64x4", |
| 913 | v8f64_info, v4f64x_info>, VEX_W, |
| 914 | EVEX_V512, EVEX_CD8<64, CD8VT4>; |
| 915 | |
| 916 | let Predicates = [HasVLX] in { |
| 917 | defm VBROADCASTI32X4Z256 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4", |
| 918 | v8i32x_info, v4i32x_info>, |
| 919 | EVEX_V256, EVEX_CD8<32, CD8VT4>; |
| 920 | defm VBROADCASTF32X4Z256 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4", |
| 921 | v8f32x_info, v4f32x_info>, |
| 922 | EVEX_V256, EVEX_CD8<32, CD8VT4>; |
| 923 | } |
| 924 | let Predicates = [HasVLX, HasDQI] in { |
| 925 | defm VBROADCASTI64X2Z128 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti64x2", |
| 926 | v4i64x_info, v2i64x_info>, VEX_W, |
| 927 | EVEX_V256, EVEX_CD8<64, CD8VT2>; |
| 928 | defm VBROADCASTF64X2Z128 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf64x2", |
| 929 | v4f64x_info, v2f64x_info>, VEX_W, |
| 930 | EVEX_V256, EVEX_CD8<64, CD8VT2>; |
| 931 | } |
| 932 | let Predicates = [HasDQI] in { |
| 933 | defm VBROADCASTI64X2 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti64x2", |
| 934 | v8i64_info, v2i64x_info>, VEX_W, |
| 935 | EVEX_V512, EVEX_CD8<64, CD8VT2>; |
| 936 | defm VBROADCASTI32X8 : avx512_subvec_broadcast_rm<0x5b, "vbroadcasti32x8", |
| 937 | v16i32_info, v8i32x_info>, |
| 938 | EVEX_V512, EVEX_CD8<32, CD8VT8>; |
| 939 | defm VBROADCASTF64X2 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf64x2", |
| 940 | v8f64_info, v2f64x_info>, VEX_W, |
| 941 | EVEX_V512, EVEX_CD8<64, CD8VT2>; |
| 942 | defm VBROADCASTF32X8 : avx512_subvec_broadcast_rm<0x1b, "vbroadcastf32x8", |
| 943 | v16f32_info, v8f32x_info>, |
| 944 | EVEX_V512, EVEX_CD8<32, CD8VT8>; |
| 945 | } |
Adam Nemet | 73f72e1 | 2014-06-27 00:43:38 +0000 | [diff] [blame] | 946 | |
Cameron McInally | 394d557 | 2013-10-31 13:56:31 +0000 | [diff] [blame] | 947 | def : Pat<(v16i32 (int_x86_avx512_pbroadcastd_512 (v4i32 VR128X:$src))), |
| 948 | (VPBROADCASTDZrr VR128X:$src)>; |
| 949 | def : Pat<(v8i64 (int_x86_avx512_pbroadcastq_512 (v2i64 VR128X:$src))), |
| 950 | (VPBROADCASTQZrr VR128X:$src)>; |
| 951 | |
Robert Khasanov | dd09a8f | 2014-10-28 12:28:51 +0000 | [diff] [blame] | 952 | def : Pat<(v16f32 (X86VBroadcast (v16f32 VR512:$src))), |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 953 | (VBROADCASTSSZr (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm))>; |
Elena Demikhovsky | 08ce53c | 2015-05-18 07:06:23 +0000 | [diff] [blame] | 954 | def : Pat<(v16f32 (X86VBroadcast (v8f32 VR256X:$src))), |
| 955 | (VBROADCASTSSZr (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm))>; |
| 956 | |
Robert Khasanov | dd09a8f | 2014-10-28 12:28:51 +0000 | [diff] [blame] | 957 | def : Pat<(v8f64 (X86VBroadcast (v8f64 VR512:$src))), |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 958 | (VBROADCASTSDZr (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm))>; |
Elena Demikhovsky | 08ce53c | 2015-05-18 07:06:23 +0000 | [diff] [blame] | 959 | def : Pat<(v8f64 (X86VBroadcast (v4f64 VR256X:$src))), |
| 960 | (VBROADCASTSDZr (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm))>; |
Robert Khasanov | dd09a8f | 2014-10-28 12:28:51 +0000 | [diff] [blame] | 961 | |
| 962 | def : Pat<(v16i32 (X86VBroadcast (v16i32 VR512:$src))), |
| 963 | (VPBROADCASTDZrr (EXTRACT_SUBREG (v16i32 VR512:$src), sub_xmm))>; |
Elena Demikhovsky | 08ce53c | 2015-05-18 07:06:23 +0000 | [diff] [blame] | 964 | def : Pat<(v16i32 (X86VBroadcast (v8i32 VR256X:$src))), |
| 965 | (VPBROADCASTDZrr (EXTRACT_SUBREG (v8i32 VR256X:$src), sub_xmm))>; |
| 966 | |
Robert Khasanov | dd09a8f | 2014-10-28 12:28:51 +0000 | [diff] [blame] | 967 | def : Pat<(v8i64 (X86VBroadcast (v8i64 VR512:$src))), |
| 968 | (VPBROADCASTQZrr (EXTRACT_SUBREG (v8i64 VR512:$src), sub_xmm))>; |
Elena Demikhovsky | 08ce53c | 2015-05-18 07:06:23 +0000 | [diff] [blame] | 969 | def : Pat<(v8i64 (X86VBroadcast (v4i64 VR256X:$src))), |
| 970 | (VPBROADCASTQZrr (EXTRACT_SUBREG (v4i64 VR256X:$src), sub_xmm))>; |
Robert Khasanov | dd09a8f | 2014-10-28 12:28:51 +0000 | [diff] [blame] | 971 | |
Quentin Colombet | 8761a8f | 2013-10-25 18:04:12 +0000 | [diff] [blame] | 972 | def : Pat<(v16f32 (int_x86_avx512_vbroadcast_ss_ps_512 (v4f32 VR128X:$src))), |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 973 | (VBROADCASTSSZr VR128X:$src)>; |
Quentin Colombet | 8761a8f | 2013-10-25 18:04:12 +0000 | [diff] [blame] | 974 | def : Pat<(v8f64 (int_x86_avx512_vbroadcast_sd_pd_512 (v2f64 VR128X:$src))), |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 975 | (VBROADCASTSDZr VR128X:$src)>; |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 976 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 977 | // Provide fallback in case the load node that is used in the patterns above |
| 978 | // is used by additional users, which prevents the pattern selection. |
| 979 | def : Pat<(v16f32 (X86VBroadcast FR32X:$src)), |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 980 | (VBROADCASTSSZr (COPY_TO_REGCLASS FR32X:$src, VR128X))>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 981 | def : Pat<(v8f64 (X86VBroadcast FR64X:$src)), |
Robert Khasanov | af318f7 | 2014-10-30 14:21:47 +0000 | [diff] [blame] | 982 | (VBROADCASTSDZr (COPY_TO_REGCLASS FR64X:$src, VR128X))>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 983 | |
| 984 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 985 | //===----------------------------------------------------------------------===// |
| 986 | // AVX-512 BROADCAST MASK TO VECTOR REGISTER |
| 987 | //--- |
| 988 | |
| 989 | multiclass avx512_mask_broadcast<bits<8> opc, string OpcodeStr, |
Elena Demikhovsky | 4b01b73 | 2014-10-26 09:52:24 +0000 | [diff] [blame] | 990 | RegisterClass KRC> { |
| 991 | let Predicates = [HasCDI] in |
| 992 | def Zrr : AVX512XS8I<opc, MRMSrcReg, (outs VR512:$dst), (ins KRC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 993 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | 4b01b73 | 2014-10-26 09:52:24 +0000 | [diff] [blame] | 994 | []>, EVEX, EVEX_V512; |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 995 | |
Elena Demikhovsky | 4b01b73 | 2014-10-26 09:52:24 +0000 | [diff] [blame] | 996 | let Predicates = [HasCDI, HasVLX] in { |
| 997 | def Z128rr : AVX512XS8I<opc, MRMSrcReg, (outs VR128:$dst), (ins KRC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 998 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | 4b01b73 | 2014-10-26 09:52:24 +0000 | [diff] [blame] | 999 | []>, EVEX, EVEX_V128; |
| 1000 | def Z256rr : AVX512XS8I<opc, MRMSrcReg, (outs VR256:$dst), (ins KRC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 1001 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | 4b01b73 | 2014-10-26 09:52:24 +0000 | [diff] [blame] | 1002 | []>, EVEX, EVEX_V256; |
| 1003 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1004 | } |
| 1005 | |
Cameron McInally | c43c8f9 | 2014-06-13 11:40:31 +0000 | [diff] [blame] | 1006 | let Predicates = [HasCDI] in { |
Elena Demikhovsky | 4b01b73 | 2014-10-26 09:52:24 +0000 | [diff] [blame] | 1007 | defm VPBROADCASTMW2D : avx512_mask_broadcast<0x3A, "vpbroadcastmw2d", |
| 1008 | VK16>; |
| 1009 | defm VPBROADCASTMB2Q : avx512_mask_broadcast<0x2A, "vpbroadcastmb2q", |
| 1010 | VK8>, VEX_W; |
Cameron McInally | c43c8f9 | 2014-06-13 11:40:31 +0000 | [diff] [blame] | 1011 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1012 | |
| 1013 | //===----------------------------------------------------------------------===// |
| 1014 | // AVX-512 - VPERM |
| 1015 | // |
| 1016 | // -- immediate form -- |
Adam Nemet | 8d85b0c | 2014-10-27 23:08:37 +0000 | [diff] [blame] | 1017 | multiclass avx512_perm_imm<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 1018 | X86VectorVTInfo _> { |
| 1019 | let ExeDomain = _.ExeDomain in { |
| 1020 | def ri : AVX512AIi8<opc, MRMSrcReg, (outs _.RC:$dst), |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 1021 | (ins _.RC:$src1, u8imm:$src2), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1022 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 1023 | "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Adam Nemet | 8d85b0c | 2014-10-27 23:08:37 +0000 | [diff] [blame] | 1024 | [(set _.RC:$dst, |
| 1025 | (_.VT (OpNode _.RC:$src1, (i8 imm:$src2))))]>, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1026 | EVEX; |
Adam Nemet | 8d85b0c | 2014-10-27 23:08:37 +0000 | [diff] [blame] | 1027 | def mi : AVX512AIi8<opc, MRMSrcMem, (outs _.RC:$dst), |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 1028 | (ins _.MemOp:$src1, u8imm:$src2), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1029 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 1030 | "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Adam Nemet | 8d85b0c | 2014-10-27 23:08:37 +0000 | [diff] [blame] | 1031 | [(set _.RC:$dst, |
Craig Topper | 820d492 | 2015-02-09 04:04:50 +0000 | [diff] [blame] | 1032 | (_.VT (OpNode (_.LdFrag addr:$src1), |
Adam Nemet | 8d85b0c | 2014-10-27 23:08:37 +0000 | [diff] [blame] | 1033 | (i8 imm:$src2))))]>, |
| 1034 | EVEX, EVEX_CD8<_.EltSize, CD8VF>; |
| 1035 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1036 | } |
| 1037 | |
Adam Nemet | cf7a4a2 | 2014-10-27 23:08:40 +0000 | [diff] [blame] | 1038 | multiclass avx512_permil<bits<8> OpcImm, bits<8> OpcVar, X86VectorVTInfo _, |
| 1039 | X86VectorVTInfo Ctrl> : |
| 1040 | avx512_perm_imm<OpcImm, "vpermil" # _.Suffix, X86VPermilpi, _> { |
| 1041 | let ExeDomain = _.ExeDomain in { |
| 1042 | def rr : AVX5128I<OpcVar, MRMSrcReg, (outs _.RC:$dst), |
| 1043 | (ins _.RC:$src1, _.RC:$src2), |
| 1044 | !strconcat("vpermil" # _.Suffix, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 1045 | "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Adam Nemet | cf7a4a2 | 2014-10-27 23:08:40 +0000 | [diff] [blame] | 1046 | [(set _.RC:$dst, |
| 1047 | (_.VT (X86VPermilpv _.RC:$src1, |
| 1048 | (Ctrl.VT Ctrl.RC:$src2))))]>, |
| 1049 | EVEX_4V; |
| 1050 | def rm : AVX5128I<OpcVar, MRMSrcMem, (outs _.RC:$dst), |
| 1051 | (ins _.RC:$src1, Ctrl.MemOp:$src2), |
| 1052 | !strconcat("vpermil" # _.Suffix, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 1053 | "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Adam Nemet | cf7a4a2 | 2014-10-27 23:08:40 +0000 | [diff] [blame] | 1054 | [(set _.RC:$dst, |
| 1055 | (_.VT (X86VPermilpv _.RC:$src1, |
Craig Topper | 820d492 | 2015-02-09 04:04:50 +0000 | [diff] [blame] | 1056 | (Ctrl.VT (Ctrl.LdFrag addr:$src2)))))]>, |
Adam Nemet | cf7a4a2 | 2014-10-27 23:08:40 +0000 | [diff] [blame] | 1057 | EVEX_4V; |
| 1058 | } |
| 1059 | } |
Adam Nemet | cf7a4a2 | 2014-10-27 23:08:40 +0000 | [diff] [blame] | 1060 | defm VPERMILPSZ : avx512_permil<0x04, 0x0C, v16f32_info, v16i32_info>, |
Adam Nemet | 8d85b0c | 2014-10-27 23:08:37 +0000 | [diff] [blame] | 1061 | EVEX_V512; |
Adam Nemet | cf7a4a2 | 2014-10-27 23:08:40 +0000 | [diff] [blame] | 1062 | defm VPERMILPDZ : avx512_permil<0x05, 0x0D, v8f64_info, v8i64_info>, |
Adam Nemet | 8d85b0c | 2014-10-27 23:08:37 +0000 | [diff] [blame] | 1063 | EVEX_V512, VEX_W; |
Adam Nemet | 9aad131 | 2014-10-27 23:08:34 +0000 | [diff] [blame] | 1064 | |
| 1065 | def : Pat<(v16i32 (X86VPermilpi VR512:$src1, (i8 imm:$imm))), |
| 1066 | (VPERMILPSZri VR512:$src1, imm:$imm)>; |
| 1067 | def : Pat<(v8i64 (X86VPermilpi VR512:$src1, (i8 imm:$imm))), |
| 1068 | (VPERMILPDZri VR512:$src1, imm:$imm)>; |
| 1069 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1070 | // -- VPERM2I - 3 source operands form -- |
Elena Demikhovsky | d3057e5 | 2015-06-18 08:56:19 +0000 | [diff] [blame] | 1071 | multiclass avx512_perm_3src<bits<8> opc, string OpcodeStr, |
| 1072 | SDNode OpNode, X86VectorVTInfo _> { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1073 | let Constraints = "$src1 = $dst" in { |
Elena Demikhovsky | d3057e5 | 2015-06-18 08:56:19 +0000 | [diff] [blame] | 1074 | defm rr: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 1075 | (ins _.RC:$src2, _.RC:$src3), |
| 1076 | OpcodeStr, "$src3, $src2", "$src2, $src3", |
| 1077 | (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>, EVEX_4V, |
| 1078 | AVX5128IBase; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1079 | |
Elena Demikhovsky | d3057e5 | 2015-06-18 08:56:19 +0000 | [diff] [blame] | 1080 | let mayLoad = 1 in |
| 1081 | defm rm: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 1082 | (ins _.RC:$src2, _.MemOp:$src3), |
| 1083 | OpcodeStr, "$src3, $src2", "$src2, $src3", |
| 1084 | (_.VT (OpNode _.RC:$src1, _.RC:$src2, |
| 1085 | (_.VT (bitconvert (_.LdFrag addr:$src3)))))>, |
| 1086 | EVEX_4V, AVX5128IBase; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1087 | } |
| 1088 | } |
Elena Demikhovsky | d3057e5 | 2015-06-18 08:56:19 +0000 | [diff] [blame] | 1089 | multiclass avx512_perm_3src_mb<bits<8> opc, string OpcodeStr, |
| 1090 | SDNode OpNode, X86VectorVTInfo _> { |
| 1091 | let mayLoad = 1, Constraints = "$src1 = $dst" in |
| 1092 | defm rmb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 1093 | (ins _.RC:$src2, _.ScalarMemOp:$src3), |
| 1094 | OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"), |
| 1095 | !strconcat("$src2, ${src3}", _.BroadcastStr ), |
| 1096 | (_.VT (OpNode _.RC:$src1, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 1097 | _.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3)))))>, |
Elena Demikhovsky | d3057e5 | 2015-06-18 08:56:19 +0000 | [diff] [blame] | 1098 | AVX5128IBase, EVEX_4V, EVEX_B; |
Adam Nemet | efe9c98 | 2014-07-02 21:25:58 +0000 | [diff] [blame] | 1099 | } |
| 1100 | |
Elena Demikhovsky | d3057e5 | 2015-06-18 08:56:19 +0000 | [diff] [blame] | 1101 | multiclass avx512_perm_3src_sizes<bits<8> opc, string OpcodeStr, |
| 1102 | SDNode OpNode, AVX512VLVectorVTInfo VTInfo> { |
| 1103 | let Predicates = [HasAVX512] in |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 1104 | defm NAME: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info512>, |
Elena Demikhovsky | d3057e5 | 2015-06-18 08:56:19 +0000 | [diff] [blame] | 1105 | avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info512>, EVEX_V512; |
| 1106 | let Predicates = [HasVLX] in { |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 1107 | defm NAME#128: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info128>, |
Elena Demikhovsky | d3057e5 | 2015-06-18 08:56:19 +0000 | [diff] [blame] | 1108 | avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info128>, |
| 1109 | EVEX_V128; |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 1110 | defm NAME#256: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info256>, |
Elena Demikhovsky | d3057e5 | 2015-06-18 08:56:19 +0000 | [diff] [blame] | 1111 | avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info256>, |
| 1112 | EVEX_V256; |
| 1113 | } |
| 1114 | } |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 1115 | multiclass avx512_perm_3src_sizes_w<bits<8> opc, string OpcodeStr, |
Elena Demikhovsky | d3057e5 | 2015-06-18 08:56:19 +0000 | [diff] [blame] | 1116 | SDNode OpNode, AVX512VLVectorVTInfo VTInfo> { |
| 1117 | let Predicates = [HasBWI] in |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 1118 | defm NAME: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info512>, |
Elena Demikhovsky | d3057e5 | 2015-06-18 08:56:19 +0000 | [diff] [blame] | 1119 | avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info512>, |
| 1120 | EVEX_V512; |
| 1121 | let Predicates = [HasBWI, HasVLX] in { |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 1122 | defm NAME#128: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info128>, |
Elena Demikhovsky | d3057e5 | 2015-06-18 08:56:19 +0000 | [diff] [blame] | 1123 | avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info128>, |
| 1124 | EVEX_V128; |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 1125 | defm NAME#256: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info256>, |
Elena Demikhovsky | d3057e5 | 2015-06-18 08:56:19 +0000 | [diff] [blame] | 1126 | avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info256>, |
| 1127 | EVEX_V256; |
| 1128 | } |
| 1129 | } |
| 1130 | defm VPERMI2D : avx512_perm_3src_sizes<0x76, "vpermi2d", X86VPermiv3, |
| 1131 | avx512vl_i32_info>, EVEX_CD8<32, CD8VF>; |
| 1132 | defm VPERMI2Q : avx512_perm_3src_sizes<0x76, "vpermi2q", X86VPermiv3, |
| 1133 | avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>; |
| 1134 | defm VPERMI2PS : avx512_perm_3src_sizes<0x77, "vpermi2ps", X86VPermiv3, |
| 1135 | avx512vl_f32_info>, EVEX_CD8<32, CD8VF>; |
| 1136 | defm VPERMI2PD : avx512_perm_3src_sizes<0x77, "vpermi2pd", X86VPermiv3, |
| 1137 | avx512vl_f64_info>, VEX_W, EVEX_CD8<64, CD8VF>; |
| 1138 | |
| 1139 | defm VPERMT2D : avx512_perm_3src_sizes<0x7E, "vpermt2d", X86VPermv3, |
| 1140 | avx512vl_i32_info>, EVEX_CD8<32, CD8VF>; |
| 1141 | defm VPERMT2Q : avx512_perm_3src_sizes<0x7E, "vpermt2q", X86VPermv3, |
| 1142 | avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>; |
| 1143 | defm VPERMT2PS : avx512_perm_3src_sizes<0x7F, "vpermt2ps", X86VPermv3, |
| 1144 | avx512vl_f32_info>, EVEX_CD8<32, CD8VF>; |
| 1145 | defm VPERMT2PD : avx512_perm_3src_sizes<0x7F, "vpermt2pd", X86VPermv3, |
| 1146 | avx512vl_f64_info>, VEX_W, EVEX_CD8<64, CD8VF>; |
| 1147 | |
| 1148 | defm VPERMT2W : avx512_perm_3src_sizes_w<0x7D, "vpermt2w", X86VPermv3, |
| 1149 | avx512vl_i16_info>, VEX_W, EVEX_CD8<16, CD8VF>; |
| 1150 | defm VPERMI2W : avx512_perm_3src_sizes_w<0x75, "vpermi2w", X86VPermiv3, |
| 1151 | avx512vl_i16_info>, VEX_W, EVEX_CD8<16, CD8VF>; |
Elena Demikhovsky | 299cf511 | 2014-04-29 09:09:15 +0000 | [diff] [blame] | 1152 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1153 | //===----------------------------------------------------------------------===// |
| 1154 | // AVX-512 - BLEND using mask |
| 1155 | // |
Elena Demikhovsky | 949b0d4 | 2014-12-22 13:52:48 +0000 | [diff] [blame] | 1156 | multiclass avx512_blendmask<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> { |
| 1157 | let ExeDomain = _.ExeDomain in { |
| 1158 | def rr : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst), |
| 1159 | (ins _.RC:$src1, _.RC:$src2), |
| 1160 | !strconcat(OpcodeStr, |
| 1161 | "\t{$src2, $src1, ${dst} |${dst}, $src1, $src2}"), |
| 1162 | []>, EVEX_4V; |
| 1163 | def rrk : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst), |
| 1164 | (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2), |
Elena Demikhovsky | 172a27c | 2014-01-08 10:54:22 +0000 | [diff] [blame] | 1165 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 1166 | "\t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"), |
Elena Demikhovsky | 949b0d4 | 2014-12-22 13:52:48 +0000 | [diff] [blame] | 1167 | [(set _.RC:$dst, (X86select _.KRCWM:$mask, (_.VT _.RC:$src1), |
| 1168 | (_.VT _.RC:$src2)))]>, EVEX_4V, EVEX_K; |
| 1169 | def rrkz : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst), |
| 1170 | (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2), |
| 1171 | !strconcat(OpcodeStr, |
| 1172 | "\t{$src2, $src1, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src1, $src2}"), |
| 1173 | []>, EVEX_4V, EVEX_KZ; |
| 1174 | let mayLoad = 1 in { |
| 1175 | def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst), |
| 1176 | (ins _.RC:$src1, _.MemOp:$src2), |
| 1177 | !strconcat(OpcodeStr, |
| 1178 | "\t{$src2, $src1, ${dst} |${dst}, $src1, $src2}"), |
| 1179 | []>, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>; |
| 1180 | def rmk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst), |
| 1181 | (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2), |
Elena Demikhovsky | 172a27c | 2014-01-08 10:54:22 +0000 | [diff] [blame] | 1182 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 1183 | "\t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"), |
Elena Demikhovsky | 949b0d4 | 2014-12-22 13:52:48 +0000 | [diff] [blame] | 1184 | [(set _.RC:$dst, (X86select _.KRCWM:$mask, (_.VT _.RC:$src1), |
| 1185 | (_.VT (bitconvert (_.LdFrag addr:$src2)))))]>, |
| 1186 | EVEX_4V, EVEX_K, EVEX_CD8<_.EltSize, CD8VF>; |
| 1187 | def rmkz : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst), |
| 1188 | (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2), |
| 1189 | !strconcat(OpcodeStr, |
| 1190 | "\t{$src2, $src1, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src1, $src2}"), |
| 1191 | []>, EVEX_4V, EVEX_KZ, EVEX_CD8<_.EltSize, CD8VF>; |
| 1192 | } |
| 1193 | } |
| 1194 | } |
| 1195 | multiclass avx512_blendmask_rmb<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> { |
| 1196 | |
| 1197 | def rmbk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst), |
| 1198 | (ins _.KRCWM:$mask, _.RC:$src1, _.ScalarMemOp:$src2), |
| 1199 | !strconcat(OpcodeStr, |
| 1200 | "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|", |
| 1201 | "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"), |
| 1202 | [(set _.RC:$dst,(X86select _.KRCWM:$mask, (_.VT _.RC:$src1), |
| 1203 | (X86VBroadcast (_.ScalarLdFrag addr:$src2))))]>, |
Elena Demikhovsky | 3121449 | 2014-12-23 09:36:28 +0000 | [diff] [blame] | 1204 | EVEX_4V, EVEX_K, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>; |
Elena Demikhovsky | 949b0d4 | 2014-12-22 13:52:48 +0000 | [diff] [blame] | 1205 | |
| 1206 | def rmb : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst), |
| 1207 | (ins _.RC:$src1, _.ScalarMemOp:$src2), |
| 1208 | !strconcat(OpcodeStr, |
| 1209 | "\t{${src2}", _.BroadcastStr, ", $src1, $dst|", |
| 1210 | "$dst, $src1, ${src2}", _.BroadcastStr, "}"), |
Elena Demikhovsky | 3121449 | 2014-12-23 09:36:28 +0000 | [diff] [blame] | 1211 | []>, EVEX_4V, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>; |
Elena Demikhovsky | 949b0d4 | 2014-12-22 13:52:48 +0000 | [diff] [blame] | 1212 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1213 | } |
| 1214 | |
Elena Demikhovsky | 949b0d4 | 2014-12-22 13:52:48 +0000 | [diff] [blame] | 1215 | multiclass blendmask_dq <bits<8> opc, string OpcodeStr, |
| 1216 | AVX512VLVectorVTInfo VTInfo> { |
| 1217 | defm Z : avx512_blendmask <opc, OpcodeStr, VTInfo.info512>, |
| 1218 | avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info512>, EVEX_V512; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1219 | |
Elena Demikhovsky | 949b0d4 | 2014-12-22 13:52:48 +0000 | [diff] [blame] | 1220 | let Predicates = [HasVLX] in { |
| 1221 | defm Z256 : avx512_blendmask<opc, OpcodeStr, VTInfo.info256>, |
| 1222 | avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info256>, EVEX_V256; |
| 1223 | defm Z128 : avx512_blendmask<opc, OpcodeStr, VTInfo.info128>, |
| 1224 | avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info128>, EVEX_V128; |
| 1225 | } |
| 1226 | } |
Elena Demikhovsky | 172a27c | 2014-01-08 10:54:22 +0000 | [diff] [blame] | 1227 | |
Elena Demikhovsky | 949b0d4 | 2014-12-22 13:52:48 +0000 | [diff] [blame] | 1228 | multiclass blendmask_bw <bits<8> opc, string OpcodeStr, |
| 1229 | AVX512VLVectorVTInfo VTInfo> { |
| 1230 | let Predicates = [HasBWI] in |
| 1231 | defm Z : avx512_blendmask <opc, OpcodeStr, VTInfo.info512>, EVEX_V512; |
Elena Demikhovsky | 172a27c | 2014-01-08 10:54:22 +0000 | [diff] [blame] | 1232 | |
Elena Demikhovsky | 949b0d4 | 2014-12-22 13:52:48 +0000 | [diff] [blame] | 1233 | let Predicates = [HasBWI, HasVLX] in { |
| 1234 | defm Z256 : avx512_blendmask <opc, OpcodeStr, VTInfo.info256>, EVEX_V256; |
| 1235 | defm Z128 : avx512_blendmask <opc, OpcodeStr, VTInfo.info128>, EVEX_V128; |
| 1236 | } |
| 1237 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1238 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1239 | |
Elena Demikhovsky | 949b0d4 | 2014-12-22 13:52:48 +0000 | [diff] [blame] | 1240 | defm VBLENDMPS : blendmask_dq <0x65, "vblendmps", avx512vl_f32_info>; |
| 1241 | defm VBLENDMPD : blendmask_dq <0x65, "vblendmpd", avx512vl_f64_info>, VEX_W; |
| 1242 | defm VPBLENDMD : blendmask_dq <0x64, "vpblendmd", avx512vl_i32_info>; |
| 1243 | defm VPBLENDMQ : blendmask_dq <0x64, "vpblendmq", avx512vl_i64_info>, VEX_W; |
| 1244 | defm VPBLENDMB : blendmask_bw <0x66, "vpblendmb", avx512vl_i8_info>; |
| 1245 | defm VPBLENDMW : blendmask_bw <0x66, "vpblendmw", avx512vl_i16_info>, VEX_W; |
Elena Demikhovsky | 172a27c | 2014-01-08 10:54:22 +0000 | [diff] [blame] | 1246 | |
Elena Demikhovsky | 172a27c | 2014-01-08 10:54:22 +0000 | [diff] [blame] | 1247 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1248 | let Predicates = [HasAVX512] in { |
| 1249 | def : Pat<(v8f32 (vselect (v8i1 VK8WM:$mask), (v8f32 VR256X:$src1), |
| 1250 | (v8f32 VR256X:$src2))), |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 1251 | (EXTRACT_SUBREG |
Elena Demikhovsky | 949b0d4 | 2014-12-22 13:52:48 +0000 | [diff] [blame] | 1252 | (v16f32 (VBLENDMPSZrrk (COPY_TO_REGCLASS VK8WM:$mask, VK16WM), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1253 | (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), |
| 1254 | (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>; |
| 1255 | |
| 1256 | def : Pat<(v8i32 (vselect (v8i1 VK8WM:$mask), (v8i32 VR256X:$src1), |
| 1257 | (v8i32 VR256X:$src2))), |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 1258 | (EXTRACT_SUBREG |
Elena Demikhovsky | 949b0d4 | 2014-12-22 13:52:48 +0000 | [diff] [blame] | 1259 | (v16i32 (VPBLENDMDZrrk (COPY_TO_REGCLASS VK8WM:$mask, VK16WM), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1260 | (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), |
| 1261 | (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>; |
| 1262 | } |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 1263 | //===----------------------------------------------------------------------===// |
| 1264 | // Compare Instructions |
| 1265 | //===----------------------------------------------------------------------===// |
| 1266 | |
| 1267 | // avx512_cmp_scalar - AVX512 CMPSS and CMPSD |
| 1268 | multiclass avx512_cmp_scalar<RegisterClass RC, X86MemOperand x86memop, |
Craig Topper | 1d60952 | 2015-01-25 08:49:19 +0000 | [diff] [blame] | 1269 | SDNode OpNode, ValueType VT, |
| 1270 | PatFrag ld_frag, string Suffix> { |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 1271 | def rr : AVX512Ii8<0xC2, MRMSrcReg, |
Craig Topper | 1d60952 | 2015-01-25 08:49:19 +0000 | [diff] [blame] | 1272 | (outs VK1:$dst), (ins RC:$src1, RC:$src2, AVXCC:$cc), |
| 1273 | !strconcat("vcmp${cc}", Suffix, |
| 1274 | "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Craig Topper | 6e3a582 | 2014-12-27 20:08:45 +0000 | [diff] [blame] | 1275 | [(set VK1:$dst, (OpNode (VT RC:$src1), RC:$src2, imm:$cc))], |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 1276 | IIC_SSE_ALU_F32S_RR>, EVEX_4V; |
| 1277 | def rm : AVX512Ii8<0xC2, MRMSrcMem, |
Craig Topper | 1d60952 | 2015-01-25 08:49:19 +0000 | [diff] [blame] | 1278 | (outs VK1:$dst), (ins RC:$src1, x86memop:$src2, AVXCC:$cc), |
| 1279 | !strconcat("vcmp${cc}", Suffix, |
| 1280 | "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Craig Topper | 6e3a582 | 2014-12-27 20:08:45 +0000 | [diff] [blame] | 1281 | [(set VK1:$dst, (OpNode (VT RC:$src1), |
| 1282 | (ld_frag addr:$src2), imm:$cc))], IIC_SSE_ALU_F32P_RM>, EVEX_4V; |
Craig Topper | 0550ce7 | 2014-01-05 04:55:55 +0000 | [diff] [blame] | 1283 | let isAsmParserOnly = 1, hasSideEffects = 0 in { |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 1284 | def rri_alt : AVX512Ii8<0xC2, MRMSrcReg, |
Craig Topper | f38dea1 | 2015-01-21 06:07:53 +0000 | [diff] [blame] | 1285 | (outs VK1:$dst), (ins RC:$src1, RC:$src2, u8imm:$cc), |
Craig Topper | 1d60952 | 2015-01-25 08:49:19 +0000 | [diff] [blame] | 1286 | !strconcat("vcmp", Suffix, |
| 1287 | "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"), |
| 1288 | [], IIC_SSE_ALU_F32S_RR>, EVEX_4V; |
Craig Topper | 9f4d485 | 2015-01-20 12:15:30 +0000 | [diff] [blame] | 1289 | let mayLoad = 1 in |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 1290 | def rmi_alt : AVX512Ii8<0xC2, MRMSrcMem, |
Craig Topper | f38dea1 | 2015-01-21 06:07:53 +0000 | [diff] [blame] | 1291 | (outs VK1:$dst), (ins RC:$src1, x86memop:$src2, u8imm:$cc), |
Craig Topper | 1d60952 | 2015-01-25 08:49:19 +0000 | [diff] [blame] | 1292 | !strconcat("vcmp", Suffix, |
| 1293 | "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"), |
| 1294 | [], IIC_SSE_ALU_F32P_RM>, EVEX_4V; |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 1295 | } |
| 1296 | } |
| 1297 | |
| 1298 | let Predicates = [HasAVX512] in { |
Craig Topper | 1d60952 | 2015-01-25 08:49:19 +0000 | [diff] [blame] | 1299 | defm VCMPSSZ : avx512_cmp_scalar<FR32X, f32mem, X86cmpms, f32, loadf32, "ss">, |
| 1300 | XS; |
| 1301 | defm VCMPSDZ : avx512_cmp_scalar<FR64X, f64mem, X86cmpms, f64, loadf64, "sd">, |
| 1302 | XD, VEX_W; |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 1303 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1304 | |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 1305 | multiclass avx512_icmp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 1306 | X86VectorVTInfo _> { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1307 | def rr : AVX512BI<opc, MRMSrcReg, |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 1308 | (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2), |
| 1309 | !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
| 1310 | [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2)))], |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1311 | IIC_SSE_ALU_F32P_RR>, EVEX_4V; |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 1312 | let mayLoad = 1 in |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1313 | def rm : AVX512BI<opc, MRMSrcMem, |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 1314 | (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2), |
| 1315 | !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
| 1316 | [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), |
| 1317 | (_.VT (bitconvert (_.LdFrag addr:$src2)))))], |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1318 | IIC_SSE_ALU_F32P_RM>, EVEX_4V; |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 1319 | def rrk : AVX512BI<opc, MRMSrcReg, |
| 1320 | (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2), |
| 1321 | !strconcat(OpcodeStr, "\t{$src2, $src1, $dst {${mask}}|", |
| 1322 | "$dst {${mask}}, $src1, $src2}"), |
| 1323 | [(set _.KRC:$dst, (and _.KRCWM:$mask, |
| 1324 | (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))))], |
| 1325 | IIC_SSE_ALU_F32P_RR>, EVEX_4V, EVEX_K; |
| 1326 | let mayLoad = 1 in |
| 1327 | def rmk : AVX512BI<opc, MRMSrcMem, |
| 1328 | (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2), |
| 1329 | !strconcat(OpcodeStr, "\t{$src2, $src1, $dst {${mask}}|", |
| 1330 | "$dst {${mask}}, $src1, $src2}"), |
| 1331 | [(set _.KRC:$dst, (and _.KRCWM:$mask, |
| 1332 | (OpNode (_.VT _.RC:$src1), |
| 1333 | (_.VT (bitconvert |
| 1334 | (_.LdFrag addr:$src2))))))], |
| 1335 | IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1336 | } |
| 1337 | |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 1338 | multiclass avx512_icmp_packed_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode, |
Robert Khasanov | f70f798 | 2014-09-18 14:06:55 +0000 | [diff] [blame] | 1339 | X86VectorVTInfo _> : |
| 1340 | avx512_icmp_packed<opc, OpcodeStr, OpNode, _> { |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 1341 | let mayLoad = 1 in { |
| 1342 | def rmb : AVX512BI<opc, MRMSrcMem, |
| 1343 | (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2), |
| 1344 | !strconcat(OpcodeStr, "\t{${src2}", _.BroadcastStr, ", $src1, $dst", |
| 1345 | "|$dst, $src1, ${src2}", _.BroadcastStr, "}"), |
| 1346 | [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), |
| 1347 | (X86VBroadcast (_.ScalarLdFrag addr:$src2))))], |
| 1348 | IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_B; |
| 1349 | def rmbk : AVX512BI<opc, MRMSrcMem, |
| 1350 | (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, |
| 1351 | _.ScalarMemOp:$src2), |
| 1352 | !strconcat(OpcodeStr, |
| 1353 | "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|", |
| 1354 | "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"), |
| 1355 | [(set _.KRC:$dst, (and _.KRCWM:$mask, |
| 1356 | (OpNode (_.VT _.RC:$src1), |
| 1357 | (X86VBroadcast |
| 1358 | (_.ScalarLdFrag addr:$src2)))))], |
| 1359 | IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K, EVEX_B; |
| 1360 | } |
| 1361 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1362 | |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 1363 | multiclass avx512_icmp_packed_vl<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 1364 | AVX512VLVectorVTInfo VTInfo, Predicate prd> { |
| 1365 | let Predicates = [prd] in |
| 1366 | defm Z : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info512>, |
| 1367 | EVEX_V512; |
| 1368 | |
| 1369 | let Predicates = [prd, HasVLX] in { |
| 1370 | defm Z256 : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info256>, |
| 1371 | EVEX_V256; |
| 1372 | defm Z128 : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info128>, |
| 1373 | EVEX_V128; |
| 1374 | } |
| 1375 | } |
| 1376 | |
| 1377 | multiclass avx512_icmp_packed_rmb_vl<bits<8> opc, string OpcodeStr, |
| 1378 | SDNode OpNode, AVX512VLVectorVTInfo VTInfo, |
| 1379 | Predicate prd> { |
| 1380 | let Predicates = [prd] in |
| 1381 | defm Z : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info512>, |
| 1382 | EVEX_V512; |
| 1383 | |
| 1384 | let Predicates = [prd, HasVLX] in { |
| 1385 | defm Z256 : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info256>, |
| 1386 | EVEX_V256; |
| 1387 | defm Z128 : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info128>, |
| 1388 | EVEX_V128; |
| 1389 | } |
| 1390 | } |
| 1391 | |
| 1392 | defm VPCMPEQB : avx512_icmp_packed_vl<0x74, "vpcmpeqb", X86pcmpeqm, |
| 1393 | avx512vl_i8_info, HasBWI>, |
| 1394 | EVEX_CD8<8, CD8VF>; |
| 1395 | |
| 1396 | defm VPCMPEQW : avx512_icmp_packed_vl<0x75, "vpcmpeqw", X86pcmpeqm, |
| 1397 | avx512vl_i16_info, HasBWI>, |
| 1398 | EVEX_CD8<16, CD8VF>; |
| 1399 | |
Robert Khasanov | f70f798 | 2014-09-18 14:06:55 +0000 | [diff] [blame] | 1400 | defm VPCMPEQD : avx512_icmp_packed_rmb_vl<0x76, "vpcmpeqd", X86pcmpeqm, |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 1401 | avx512vl_i32_info, HasAVX512>, |
| 1402 | EVEX_CD8<32, CD8VF>; |
| 1403 | |
Robert Khasanov | f70f798 | 2014-09-18 14:06:55 +0000 | [diff] [blame] | 1404 | defm VPCMPEQQ : avx512_icmp_packed_rmb_vl<0x29, "vpcmpeqq", X86pcmpeqm, |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 1405 | avx512vl_i64_info, HasAVX512>, |
| 1406 | T8PD, VEX_W, EVEX_CD8<64, CD8VF>; |
| 1407 | |
| 1408 | defm VPCMPGTB : avx512_icmp_packed_vl<0x64, "vpcmpgtb", X86pcmpgtm, |
| 1409 | avx512vl_i8_info, HasBWI>, |
| 1410 | EVEX_CD8<8, CD8VF>; |
| 1411 | |
| 1412 | defm VPCMPGTW : avx512_icmp_packed_vl<0x65, "vpcmpgtw", X86pcmpgtm, |
| 1413 | avx512vl_i16_info, HasBWI>, |
| 1414 | EVEX_CD8<16, CD8VF>; |
| 1415 | |
Robert Khasanov | f70f798 | 2014-09-18 14:06:55 +0000 | [diff] [blame] | 1416 | defm VPCMPGTD : avx512_icmp_packed_rmb_vl<0x66, "vpcmpgtd", X86pcmpgtm, |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 1417 | avx512vl_i32_info, HasAVX512>, |
| 1418 | EVEX_CD8<32, CD8VF>; |
| 1419 | |
Robert Khasanov | f70f798 | 2014-09-18 14:06:55 +0000 | [diff] [blame] | 1420 | defm VPCMPGTQ : avx512_icmp_packed_rmb_vl<0x37, "vpcmpgtq", X86pcmpgtm, |
Robert Khasanov | 2ea081d | 2014-08-25 14:49:34 +0000 | [diff] [blame] | 1421 | avx512vl_i64_info, HasAVX512>, |
| 1422 | T8PD, VEX_W, EVEX_CD8<64, CD8VF>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1423 | |
| 1424 | def : Pat<(v8i1 (X86pcmpgtm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))), |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1425 | (COPY_TO_REGCLASS (VPCMPGTDZrr |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1426 | (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), |
| 1427 | (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm))), VK8)>; |
| 1428 | |
| 1429 | def : Pat<(v8i1 (X86pcmpeqm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))), |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1430 | (COPY_TO_REGCLASS (VPCMPEQDZrr |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1431 | (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), |
| 1432 | (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm))), VK8)>; |
| 1433 | |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1434 | multiclass avx512_icmp_cc<bits<8> opc, string Suffix, SDNode OpNode, |
| 1435 | X86VectorVTInfo _> { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1436 | def rri : AVX512AIi8<opc, MRMSrcReg, |
Craig Topper | 7d3c6d3 | 2015-01-28 10:09:56 +0000 | [diff] [blame] | 1437 | (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, AVX512ICC:$cc), |
Adam Nemet | 1efcb90 | 2014-07-01 18:03:43 +0000 | [diff] [blame] | 1438 | !strconcat("vpcmp${cc}", Suffix, |
| 1439 | "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1440 | [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), |
| 1441 | imm:$cc))], |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1442 | IIC_SSE_ALU_F32P_RR>, EVEX_4V; |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1443 | let mayLoad = 1 in |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1444 | def rmi : AVX512AIi8<opc, MRMSrcMem, |
Craig Topper | 7d3c6d3 | 2015-01-28 10:09:56 +0000 | [diff] [blame] | 1445 | (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2, AVX512ICC:$cc), |
Adam Nemet | 1efcb90 | 2014-07-01 18:03:43 +0000 | [diff] [blame] | 1446 | !strconcat("vpcmp${cc}", Suffix, |
| 1447 | "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1448 | [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), |
| 1449 | (_.VT (bitconvert (_.LdFrag addr:$src2))), |
Craig Topper | 6e3a582 | 2014-12-27 20:08:45 +0000 | [diff] [blame] | 1450 | imm:$cc))], |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1451 | IIC_SSE_ALU_F32P_RM>, EVEX_4V; |
| 1452 | def rrik : AVX512AIi8<opc, MRMSrcReg, |
| 1453 | (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2, |
Craig Topper | 7d3c6d3 | 2015-01-28 10:09:56 +0000 | [diff] [blame] | 1454 | AVX512ICC:$cc), |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1455 | !strconcat("vpcmp${cc}", Suffix, |
| 1456 | "\t{$src2, $src1, $dst {${mask}}|", |
| 1457 | "$dst {${mask}}, $src1, $src2}"), |
| 1458 | [(set _.KRC:$dst, (and _.KRCWM:$mask, |
| 1459 | (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), |
Craig Topper | 6e3a582 | 2014-12-27 20:08:45 +0000 | [diff] [blame] | 1460 | imm:$cc)))], |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1461 | IIC_SSE_ALU_F32P_RR>, EVEX_4V, EVEX_K; |
| 1462 | let mayLoad = 1 in |
| 1463 | def rmik : AVX512AIi8<opc, MRMSrcMem, |
| 1464 | (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2, |
Craig Topper | 7d3c6d3 | 2015-01-28 10:09:56 +0000 | [diff] [blame] | 1465 | AVX512ICC:$cc), |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1466 | !strconcat("vpcmp${cc}", Suffix, |
| 1467 | "\t{$src2, $src1, $dst {${mask}}|", |
| 1468 | "$dst {${mask}}, $src1, $src2}"), |
| 1469 | [(set _.KRC:$dst, (and _.KRCWM:$mask, |
| 1470 | (OpNode (_.VT _.RC:$src1), |
| 1471 | (_.VT (bitconvert (_.LdFrag addr:$src2))), |
Craig Topper | 6e3a582 | 2014-12-27 20:08:45 +0000 | [diff] [blame] | 1472 | imm:$cc)))], |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1473 | IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K; |
| 1474 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1475 | // Accept explicit immediate argument form instead of comparison code. |
Craig Topper | 0550ce7 | 2014-01-05 04:55:55 +0000 | [diff] [blame] | 1476 | let isAsmParserOnly = 1, hasSideEffects = 0 in { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1477 | def rri_alt : AVX512AIi8<opc, MRMSrcReg, |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 1478 | (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, u8imm:$cc), |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1479 | !strconcat("vpcmp", Suffix, "\t{$cc, $src2, $src1, $dst|", |
| 1480 | "$dst, $src1, $src2, $cc}"), |
Adam Nemet | 1efcb90 | 2014-07-01 18:03:43 +0000 | [diff] [blame] | 1481 | [], IIC_SSE_ALU_F32P_RR>, EVEX_4V; |
Craig Topper | 9f4d485 | 2015-01-20 12:15:30 +0000 | [diff] [blame] | 1482 | let mayLoad = 1 in |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1483 | def rmi_alt : AVX512AIi8<opc, MRMSrcMem, |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 1484 | (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc), |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1485 | !strconcat("vpcmp", Suffix, "\t{$cc, $src2, $src1, $dst|", |
| 1486 | "$dst, $src1, $src2, $cc}"), |
Adam Nemet | 1efcb90 | 2014-07-01 18:03:43 +0000 | [diff] [blame] | 1487 | [], IIC_SSE_ALU_F32P_RM>, EVEX_4V; |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1488 | def rrik_alt : AVX512AIi8<opc, MRMSrcReg, |
| 1489 | (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2, |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 1490 | u8imm:$cc), |
Adam Nemet | 16de248 | 2014-07-01 18:03:45 +0000 | [diff] [blame] | 1491 | !strconcat("vpcmp", Suffix, |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1492 | "\t{$cc, $src2, $src1, $dst {${mask}}|", |
| 1493 | "$dst {${mask}}, $src1, $src2, $cc}"), |
| 1494 | [], IIC_SSE_ALU_F32P_RR>, EVEX_4V, EVEX_K; |
Craig Topper | 9f4d485 | 2015-01-20 12:15:30 +0000 | [diff] [blame] | 1495 | let mayLoad = 1 in |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1496 | def rmik_alt : AVX512AIi8<opc, MRMSrcMem, |
| 1497 | (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2, |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 1498 | u8imm:$cc), |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1499 | !strconcat("vpcmp", Suffix, |
| 1500 | "\t{$cc, $src2, $src1, $dst {${mask}}|", |
| 1501 | "$dst {${mask}}, $src1, $src2, $cc}"), |
Adam Nemet | 16de248 | 2014-07-01 18:03:45 +0000 | [diff] [blame] | 1502 | [], IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1503 | } |
| 1504 | } |
| 1505 | |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1506 | multiclass avx512_icmp_cc_rmb<bits<8> opc, string Suffix, SDNode OpNode, |
Robert Khasanov | f70f798 | 2014-09-18 14:06:55 +0000 | [diff] [blame] | 1507 | X86VectorVTInfo _> : |
| 1508 | avx512_icmp_cc<opc, Suffix, OpNode, _> { |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1509 | def rmib : AVX512AIi8<opc, MRMSrcMem, |
| 1510 | (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2, |
Craig Topper | 7d3c6d3 | 2015-01-28 10:09:56 +0000 | [diff] [blame] | 1511 | AVX512ICC:$cc), |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1512 | !strconcat("vpcmp${cc}", Suffix, |
| 1513 | "\t{${src2}", _.BroadcastStr, ", $src1, $dst|", |
| 1514 | "$dst, $src1, ${src2}", _.BroadcastStr, "}"), |
| 1515 | [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), |
| 1516 | (X86VBroadcast (_.ScalarLdFrag addr:$src2)), |
Craig Topper | 6e3a582 | 2014-12-27 20:08:45 +0000 | [diff] [blame] | 1517 | imm:$cc))], |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1518 | IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_B; |
| 1519 | def rmibk : AVX512AIi8<opc, MRMSrcMem, |
| 1520 | (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, |
Craig Topper | 7d3c6d3 | 2015-01-28 10:09:56 +0000 | [diff] [blame] | 1521 | _.ScalarMemOp:$src2, AVX512ICC:$cc), |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1522 | !strconcat("vpcmp${cc}", Suffix, |
| 1523 | "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|", |
| 1524 | "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"), |
| 1525 | [(set _.KRC:$dst, (and _.KRCWM:$mask, |
| 1526 | (OpNode (_.VT _.RC:$src1), |
| 1527 | (X86VBroadcast (_.ScalarLdFrag addr:$src2)), |
Craig Topper | 6e3a582 | 2014-12-27 20:08:45 +0000 | [diff] [blame] | 1528 | imm:$cc)))], |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1529 | IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K, EVEX_B; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1530 | |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1531 | // Accept explicit immediate argument form instead of comparison code. |
Craig Topper | 9f4d485 | 2015-01-20 12:15:30 +0000 | [diff] [blame] | 1532 | let isAsmParserOnly = 1, hasSideEffects = 0, mayLoad = 1 in { |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1533 | def rmib_alt : AVX512AIi8<opc, MRMSrcMem, |
| 1534 | (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2, |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 1535 | u8imm:$cc), |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1536 | !strconcat("vpcmp", Suffix, |
| 1537 | "\t{$cc, ${src2}", _.BroadcastStr, ", $src1, $dst|", |
| 1538 | "$dst, $src1, ${src2}", _.BroadcastStr, ", $cc}"), |
| 1539 | [], IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_B; |
| 1540 | def rmibk_alt : AVX512AIi8<opc, MRMSrcMem, |
| 1541 | (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 1542 | _.ScalarMemOp:$src2, u8imm:$cc), |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1543 | !strconcat("vpcmp", Suffix, |
| 1544 | "\t{$cc, ${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|", |
| 1545 | "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, ", $cc}"), |
| 1546 | [], IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K, EVEX_B; |
| 1547 | } |
| 1548 | } |
| 1549 | |
| 1550 | multiclass avx512_icmp_cc_vl<bits<8> opc, string Suffix, SDNode OpNode, |
| 1551 | AVX512VLVectorVTInfo VTInfo, Predicate prd> { |
| 1552 | let Predicates = [prd] in |
| 1553 | defm Z : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info512>, EVEX_V512; |
| 1554 | |
| 1555 | let Predicates = [prd, HasVLX] in { |
| 1556 | defm Z256 : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info256>, EVEX_V256; |
| 1557 | defm Z128 : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info128>, EVEX_V128; |
| 1558 | } |
| 1559 | } |
| 1560 | |
| 1561 | multiclass avx512_icmp_cc_rmb_vl<bits<8> opc, string Suffix, SDNode OpNode, |
| 1562 | AVX512VLVectorVTInfo VTInfo, Predicate prd> { |
| 1563 | let Predicates = [prd] in |
| 1564 | defm Z : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info512>, |
| 1565 | EVEX_V512; |
| 1566 | |
| 1567 | let Predicates = [prd, HasVLX] in { |
| 1568 | defm Z256 : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info256>, |
| 1569 | EVEX_V256; |
| 1570 | defm Z128 : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info128>, |
| 1571 | EVEX_V128; |
| 1572 | } |
| 1573 | } |
| 1574 | |
| 1575 | defm VPCMPB : avx512_icmp_cc_vl<0x3F, "b", X86cmpm, avx512vl_i8_info, |
| 1576 | HasBWI>, EVEX_CD8<8, CD8VF>; |
| 1577 | defm VPCMPUB : avx512_icmp_cc_vl<0x3E, "ub", X86cmpmu, avx512vl_i8_info, |
| 1578 | HasBWI>, EVEX_CD8<8, CD8VF>; |
| 1579 | |
| 1580 | defm VPCMPW : avx512_icmp_cc_vl<0x3F, "w", X86cmpm, avx512vl_i16_info, |
| 1581 | HasBWI>, VEX_W, EVEX_CD8<16, CD8VF>; |
| 1582 | defm VPCMPUW : avx512_icmp_cc_vl<0x3E, "uw", X86cmpmu, avx512vl_i16_info, |
| 1583 | HasBWI>, VEX_W, EVEX_CD8<16, CD8VF>; |
| 1584 | |
Robert Khasanov | f70f798 | 2014-09-18 14:06:55 +0000 | [diff] [blame] | 1585 | defm VPCMPD : avx512_icmp_cc_rmb_vl<0x1F, "d", X86cmpm, avx512vl_i32_info, |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1586 | HasAVX512>, EVEX_CD8<32, CD8VF>; |
Robert Khasanov | f70f798 | 2014-09-18 14:06:55 +0000 | [diff] [blame] | 1587 | defm VPCMPUD : avx512_icmp_cc_rmb_vl<0x1E, "ud", X86cmpmu, avx512vl_i32_info, |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1588 | HasAVX512>, EVEX_CD8<32, CD8VF>; |
| 1589 | |
Robert Khasanov | f70f798 | 2014-09-18 14:06:55 +0000 | [diff] [blame] | 1590 | defm VPCMPQ : avx512_icmp_cc_rmb_vl<0x1F, "q", X86cmpm, avx512vl_i64_info, |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1591 | HasAVX512>, VEX_W, EVEX_CD8<64, CD8VF>; |
Robert Khasanov | f70f798 | 2014-09-18 14:06:55 +0000 | [diff] [blame] | 1592 | defm VPCMPUQ : avx512_icmp_cc_rmb_vl<0x1E, "uq", X86cmpmu, avx512vl_i64_info, |
Robert Khasanov | 29e3b96 | 2014-08-27 09:34:37 +0000 | [diff] [blame] | 1593 | HasAVX512>, VEX_W, EVEX_CD8<64, CD8VF>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1594 | |
Elena Demikhovsky | 29792e9 | 2015-05-07 11:24:42 +0000 | [diff] [blame] | 1595 | multiclass avx512_vcmp_common<X86VectorVTInfo _> { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1596 | |
Elena Demikhovsky | 29792e9 | 2015-05-07 11:24:42 +0000 | [diff] [blame] | 1597 | defm rri : AVX512_maskable_cmp<0xC2, MRMSrcReg, _, |
| 1598 | (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2,AVXCC:$cc), |
| 1599 | "vcmp${cc}"#_.Suffix, |
| 1600 | "$src2, $src1", "$src1, $src2", |
| 1601 | (X86cmpm (_.VT _.RC:$src1), |
| 1602 | (_.VT _.RC:$src2), |
| 1603 | imm:$cc)>; |
| 1604 | |
| 1605 | let mayLoad = 1 in { |
| 1606 | defm rmi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _, |
| 1607 | (outs _.KRC:$dst),(ins _.RC:$src1, _.MemOp:$src2, AVXCC:$cc), |
| 1608 | "vcmp${cc}"#_.Suffix, |
| 1609 | "$src2, $src1", "$src1, $src2", |
| 1610 | (X86cmpm (_.VT _.RC:$src1), |
| 1611 | (_.VT (bitconvert (_.LdFrag addr:$src2))), |
| 1612 | imm:$cc)>; |
| 1613 | |
| 1614 | defm rmbi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _, |
| 1615 | (outs _.KRC:$dst), |
| 1616 | (ins _.RC:$src1, _.ScalarMemOp:$src2, AVXCC:$cc), |
| 1617 | "vcmp${cc}"#_.Suffix, |
| 1618 | "${src2}"##_.BroadcastStr##", $src1", |
| 1619 | "$src1, ${src2}"##_.BroadcastStr, |
| 1620 | (X86cmpm (_.VT _.RC:$src1), |
| 1621 | (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))), |
| 1622 | imm:$cc)>,EVEX_B; |
| 1623 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1624 | // Accept explicit immediate argument form instead of comparison code. |
Craig Topper | 0550ce7 | 2014-01-05 04:55:55 +0000 | [diff] [blame] | 1625 | let isAsmParserOnly = 1, hasSideEffects = 0 in { |
Elena Demikhovsky | 29792e9 | 2015-05-07 11:24:42 +0000 | [diff] [blame] | 1626 | defm rri_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _, |
| 1627 | (outs _.KRC:$dst), |
| 1628 | (ins _.RC:$src1, _.RC:$src2, u8imm:$cc), |
| 1629 | "vcmp"#_.Suffix, |
| 1630 | "$cc, $src2, $src1", "$src1, $src2, $cc">; |
| 1631 | |
| 1632 | let mayLoad = 1 in { |
| 1633 | defm rmi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _, |
| 1634 | (outs _.KRC:$dst), |
| 1635 | (ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc), |
| 1636 | "vcmp"#_.Suffix, |
| 1637 | "$cc, $src2, $src1", "$src1, $src2, $cc">; |
| 1638 | |
| 1639 | defm rmbi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _, |
| 1640 | (outs _.KRC:$dst), |
| 1641 | (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$cc), |
| 1642 | "vcmp"#_.Suffix, |
| 1643 | "$cc, ${src2}"##_.BroadcastStr##", $src1", |
| 1644 | "$src1, ${src2}"##_.BroadcastStr##", $cc">,EVEX_B; |
| 1645 | } |
| 1646 | } |
| 1647 | } |
| 1648 | |
| 1649 | multiclass avx512_vcmp_sae<X86VectorVTInfo _> { |
| 1650 | // comparison code form (VCMP[EQ/LT/LE/...] |
| 1651 | defm rrib : AVX512_maskable_cmp<0xC2, MRMSrcReg, _, |
| 1652 | (outs _.KRC:$dst),(ins _.RC:$src1, _.RC:$src2, AVXCC:$cc), |
| 1653 | "vcmp${cc}"#_.Suffix, |
| 1654 | "{sae}, $src2, $src1", "$src1, $src2,{sae}", |
| 1655 | (X86cmpmRnd (_.VT _.RC:$src1), |
| 1656 | (_.VT _.RC:$src2), |
| 1657 | imm:$cc, |
| 1658 | (i32 FROUND_NO_EXC))>, EVEX_B; |
| 1659 | |
| 1660 | let isAsmParserOnly = 1, hasSideEffects = 0 in { |
| 1661 | defm rrib_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _, |
| 1662 | (outs _.KRC:$dst), |
| 1663 | (ins _.RC:$src1, _.RC:$src2, u8imm:$cc), |
| 1664 | "vcmp"#_.Suffix, |
| 1665 | "$cc,{sae}, $src2, $src1", |
| 1666 | "$src1, $src2,{sae}, $cc">, EVEX_B; |
| 1667 | } |
| 1668 | } |
| 1669 | |
| 1670 | multiclass avx512_vcmp<AVX512VLVectorVTInfo _> { |
| 1671 | let Predicates = [HasAVX512] in { |
| 1672 | defm Z : avx512_vcmp_common<_.info512>, |
| 1673 | avx512_vcmp_sae<_.info512>, EVEX_V512; |
| 1674 | |
| 1675 | } |
| 1676 | let Predicates = [HasAVX512,HasVLX] in { |
| 1677 | defm Z128 : avx512_vcmp_common<_.info128>, EVEX_V128; |
| 1678 | defm Z256 : avx512_vcmp_common<_.info256>, EVEX_V256; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1679 | } |
| 1680 | } |
| 1681 | |
Elena Demikhovsky | 29792e9 | 2015-05-07 11:24:42 +0000 | [diff] [blame] | 1682 | defm VCMPPD : avx512_vcmp<avx512vl_f64_info>, |
| 1683 | AVX512PDIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; |
| 1684 | defm VCMPPS : avx512_vcmp<avx512vl_f32_info>, |
| 1685 | AVX512PSIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1686 | |
| 1687 | def : Pat<(v8i1 (X86cmpm (v8f32 VR256X:$src1), (v8f32 VR256X:$src2), imm:$cc)), |
| 1688 | (COPY_TO_REGCLASS (VCMPPSZrri |
| 1689 | (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), |
| 1690 | (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), |
| 1691 | imm:$cc), VK8)>; |
| 1692 | def : Pat<(v8i1 (X86cmpm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)), |
| 1693 | (COPY_TO_REGCLASS (VPCMPDZrri |
| 1694 | (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), |
| 1695 | (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), |
| 1696 | imm:$cc), VK8)>; |
| 1697 | def : Pat<(v8i1 (X86cmpmu (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)), |
| 1698 | (COPY_TO_REGCLASS (VPCMPUDZrri |
| 1699 | (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), |
| 1700 | (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), |
| 1701 | imm:$cc), VK8)>; |
Elena Demikhovsky | de3f751 | 2014-01-01 15:12:34 +0000 | [diff] [blame] | 1702 | |
Elena Demikhovsky | 29792e9 | 2015-05-07 11:24:42 +0000 | [diff] [blame] | 1703 | //----------------------------------------------------------------- |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1704 | // Mask register copy, including |
| 1705 | // - copy between mask registers |
| 1706 | // - load/store mask registers |
| 1707 | // - copy from GPR to mask register and vice versa |
| 1708 | // |
| 1709 | multiclass avx512_mask_mov<bits<8> opc_kk, bits<8> opc_km, bits<8> opc_mk, |
| 1710 | string OpcodeStr, RegisterClass KRC, |
Elena Demikhovsky | ba84672 | 2015-02-17 09:20:12 +0000 | [diff] [blame] | 1711 | ValueType vvt, X86MemOperand x86memop> { |
Elena Demikhovsky | f404e05 | 2014-01-05 14:21:07 +0000 | [diff] [blame] | 1712 | let hasSideEffects = 0 in { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1713 | def kk : I<opc_kk, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 1714 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1715 | let mayLoad = 1 in |
| 1716 | def km : I<opc_km, MRMSrcMem, (outs KRC:$dst), (ins x86memop:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 1717 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | ba84672 | 2015-02-17 09:20:12 +0000 | [diff] [blame] | 1718 | [(set KRC:$dst, (vvt (load addr:$src)))]>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1719 | let mayStore = 1 in |
| 1720 | def mk : I<opc_mk, MRMDestMem, (outs), (ins x86memop:$dst, KRC:$src), |
Elena Demikhovsky | 1a603b3 | 2015-01-25 12:47:15 +0000 | [diff] [blame] | 1721 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
| 1722 | [(store KRC:$src, addr:$dst)]>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1723 | } |
| 1724 | } |
| 1725 | |
| 1726 | multiclass avx512_mask_mov_gpr<bits<8> opc_kr, bits<8> opc_rk, |
| 1727 | string OpcodeStr, |
| 1728 | RegisterClass KRC, RegisterClass GRC> { |
Elena Demikhovsky | f404e05 | 2014-01-05 14:21:07 +0000 | [diff] [blame] | 1729 | let hasSideEffects = 0 in { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1730 | def kr : I<opc_kr, MRMSrcReg, (outs KRC:$dst), (ins GRC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 1731 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1732 | def rk : I<opc_rk, MRMSrcReg, (outs GRC:$dst), (ins KRC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 1733 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1734 | } |
| 1735 | } |
| 1736 | |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1737 | let Predicates = [HasDQI] in |
Elena Demikhovsky | ba84672 | 2015-02-17 09:20:12 +0000 | [diff] [blame] | 1738 | defm KMOVB : avx512_mask_mov<0x90, 0x90, 0x91, "kmovb", VK8, v8i1, i8mem>, |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1739 | avx512_mask_mov_gpr<0x92, 0x93, "kmovb", VK8, GR32>, |
| 1740 | VEX, PD; |
| 1741 | |
| 1742 | let Predicates = [HasAVX512] in |
Elena Demikhovsky | ba84672 | 2015-02-17 09:20:12 +0000 | [diff] [blame] | 1743 | defm KMOVW : avx512_mask_mov<0x90, 0x90, 0x91, "kmovw", VK16, v16i1, i16mem>, |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1744 | avx512_mask_mov_gpr<0x92, 0x93, "kmovw", VK16, GR32>, |
Craig Topper | 5ccb617 | 2014-02-18 00:21:49 +0000 | [diff] [blame] | 1745 | VEX, PS; |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1746 | |
| 1747 | let Predicates = [HasBWI] in { |
Elena Demikhovsky | ba84672 | 2015-02-17 09:20:12 +0000 | [diff] [blame] | 1748 | defm KMOVD : avx512_mask_mov<0x90, 0x90, 0x91, "kmovd", VK32, v32i1,i32mem>, |
| 1749 | VEX, PD, VEX_W; |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1750 | defm KMOVD : avx512_mask_mov_gpr<0x92, 0x93, "kmovd", VK32, GR32>, |
| 1751 | VEX, XD; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1752 | } |
| 1753 | |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1754 | let Predicates = [HasBWI] in { |
Elena Demikhovsky | ba84672 | 2015-02-17 09:20:12 +0000 | [diff] [blame] | 1755 | defm KMOVQ : avx512_mask_mov<0x90, 0x90, 0x91, "kmovq", VK64, v64i1, i64mem>, |
| 1756 | VEX, PS, VEX_W; |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1757 | defm KMOVQ : avx512_mask_mov_gpr<0x92, 0x93, "kmovq", VK64, GR64>, |
| 1758 | VEX, XD, VEX_W; |
| 1759 | } |
| 1760 | |
| 1761 | // GR from/to mask register |
| 1762 | let Predicates = [HasDQI] in { |
| 1763 | def : Pat<(v8i1 (bitconvert (i8 GR8:$src))), |
| 1764 | (KMOVBkr (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit))>; |
| 1765 | def : Pat<(i8 (bitconvert (v8i1 VK8:$src))), |
| 1766 | (EXTRACT_SUBREG (KMOVBrk VK8:$src), sub_8bit)>; |
| 1767 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1768 | let Predicates = [HasAVX512] in { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1769 | def : Pat<(v16i1 (bitconvert (i16 GR16:$src))), |
| 1770 | (KMOVWkr (SUBREG_TO_REG (i32 0), GR16:$src, sub_16bit))>; |
| 1771 | def : Pat<(i16 (bitconvert (v16i1 VK16:$src))), |
| 1772 | (EXTRACT_SUBREG (KMOVWrk VK16:$src), sub_16bit)>; |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1773 | } |
| 1774 | let Predicates = [HasBWI] in { |
| 1775 | def : Pat<(v32i1 (bitconvert (i32 GR32:$src))), (KMOVDkr GR32:$src)>; |
| 1776 | def : Pat<(i32 (bitconvert (v32i1 VK32:$src))), (KMOVDrk VK32:$src)>; |
| 1777 | } |
| 1778 | let Predicates = [HasBWI] in { |
| 1779 | def : Pat<(v64i1 (bitconvert (i64 GR64:$src))), (KMOVQkr GR64:$src)>; |
| 1780 | def : Pat<(i64 (bitconvert (v64i1 VK64:$src))), (KMOVQrk VK64:$src)>; |
| 1781 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1782 | |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1783 | // Load/store kreg |
| 1784 | let Predicates = [HasDQI] in { |
| 1785 | def : Pat<(store (i8 (bitconvert (v8i1 VK8:$src))), addr:$dst), |
| 1786 | (KMOVBmk addr:$dst, VK8:$src)>; |
Elena Demikhovsky | ba84672 | 2015-02-17 09:20:12 +0000 | [diff] [blame] | 1787 | def : Pat<(v8i1 (bitconvert (i8 (load addr:$src)))), |
| 1788 | (KMOVBkm addr:$src)>; |
Elena Demikhovsky | 9f83c73 | 2015-09-02 09:20:58 +0000 | [diff] [blame] | 1789 | |
| 1790 | def : Pat<(store VK4:$src, addr:$dst), |
| 1791 | (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK4:$src, VK8))>; |
| 1792 | def : Pat<(store VK2:$src, addr:$dst), |
| 1793 | (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK2:$src, VK8))>; |
Elena Demikhovsky | ba84672 | 2015-02-17 09:20:12 +0000 | [diff] [blame] | 1794 | } |
| 1795 | let Predicates = [HasAVX512, NoDQI] in { |
| 1796 | def : Pat<(store (i8 (bitconvert (v8i1 VK8:$src))), addr:$dst), |
| 1797 | (KMOVWmk addr:$dst, (COPY_TO_REGCLASS VK8:$src, VK16))>; |
| 1798 | def : Pat<(v8i1 (bitconvert (i8 (load addr:$src)))), |
| 1799 | (COPY_TO_REGCLASS (KMOVWkm addr:$src), VK8)>; |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1800 | } |
| 1801 | let Predicates = [HasAVX512] in { |
| 1802 | def : Pat<(store (i16 (bitconvert (v16i1 VK16:$src))), addr:$dst), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1803 | (KMOVWmk addr:$dst, VK16:$src)>; |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 1804 | def : Pat<(i1 (load addr:$src)), |
Elena Demikhovsky | f61727d | 2015-05-20 14:32:03 +0000 | [diff] [blame] | 1805 | (COPY_TO_REGCLASS (AND16ri (i16 (SUBREG_TO_REG (i32 0), |
| 1806 | (MOV8rm addr:$src), sub_8bit)), |
| 1807 | (i16 1)), VK1)>; |
Elena Demikhovsky | ba84672 | 2015-02-17 09:20:12 +0000 | [diff] [blame] | 1808 | def : Pat<(v16i1 (bitconvert (i16 (load addr:$src)))), |
| 1809 | (KMOVWkm addr:$src)>; |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1810 | } |
| 1811 | let Predicates = [HasBWI] in { |
| 1812 | def : Pat<(store (i32 (bitconvert (v32i1 VK32:$src))), addr:$dst), |
| 1813 | (KMOVDmk addr:$dst, VK32:$src)>; |
Elena Demikhovsky | ba84672 | 2015-02-17 09:20:12 +0000 | [diff] [blame] | 1814 | def : Pat<(v32i1 (bitconvert (i32 (load addr:$src)))), |
| 1815 | (KMOVDkm addr:$src)>; |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1816 | } |
| 1817 | let Predicates = [HasBWI] in { |
| 1818 | def : Pat<(store (i64 (bitconvert (v64i1 VK64:$src))), addr:$dst), |
| 1819 | (KMOVQmk addr:$dst, VK64:$src)>; |
Elena Demikhovsky | ba84672 | 2015-02-17 09:20:12 +0000 | [diff] [blame] | 1820 | def : Pat<(v64i1 (bitconvert (i64 (load addr:$src)))), |
| 1821 | (KMOVQkm addr:$src)>; |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1822 | } |
Elena Demikhovsky | c5f6726 | 2013-12-17 08:33:15 +0000 | [diff] [blame] | 1823 | |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1824 | let Predicates = [HasAVX512] in { |
Elena Demikhovsky | 34d2d76 | 2014-08-18 11:59:06 +0000 | [diff] [blame] | 1825 | def : Pat<(i1 (trunc (i64 GR64:$src))), |
| 1826 | (COPY_TO_REGCLASS (KMOVWkr (AND32ri (EXTRACT_SUBREG $src, sub_32bit), |
| 1827 | (i32 1))), VK1)>; |
| 1828 | |
Elena Demikhovsky | 64c9548 | 2013-12-24 14:24:07 +0000 | [diff] [blame] | 1829 | def : Pat<(i1 (trunc (i32 GR32:$src))), |
Elena Demikhovsky | c965701 | 2014-02-20 06:34:39 +0000 | [diff] [blame] | 1830 | (COPY_TO_REGCLASS (KMOVWkr (AND32ri $src, (i32 1))), VK1)>; |
Elena Demikhovsky | 64c9548 | 2013-12-24 14:24:07 +0000 | [diff] [blame] | 1831 | |
| 1832 | def : Pat<(i1 (trunc (i8 GR8:$src))), |
Elena Demikhovsky | c965701 | 2014-02-20 06:34:39 +0000 | [diff] [blame] | 1833 | (COPY_TO_REGCLASS |
| 1834 | (KMOVWkr (AND32ri (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit), (i32 1))), |
| 1835 | VK1)>; |
| 1836 | def : Pat<(i1 (trunc (i16 GR16:$src))), |
| 1837 | (COPY_TO_REGCLASS |
| 1838 | (KMOVWkr (AND32ri (SUBREG_TO_REG (i32 0), $src, sub_16bit), (i32 1))), |
| 1839 | VK1)>; |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1840 | |
Elena Demikhovsky | 3ebfe11 | 2014-02-23 14:28:35 +0000 | [diff] [blame] | 1841 | def : Pat<(i32 (zext VK1:$src)), |
| 1842 | (AND32ri (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1))>; |
Elena Demikhovsky | 86c7b46 | 2015-05-27 14:09:33 +0000 | [diff] [blame] | 1843 | def : Pat<(i32 (anyext VK1:$src)), |
| 1844 | (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16))>; |
Elena Demikhovsky | 64c9548 | 2013-12-24 14:24:07 +0000 | [diff] [blame] | 1845 | def : Pat<(i8 (zext VK1:$src)), |
| 1846 | (EXTRACT_SUBREG |
Elena Demikhovsky | 3ebfe11 | 2014-02-23 14:28:35 +0000 | [diff] [blame] | 1847 | (AND32ri (KMOVWrk |
| 1848 | (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1)), sub_8bit)>; |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 1849 | def : Pat<(i64 (zext VK1:$src)), |
Elena Demikhovsky | 3ebfe11 | 2014-02-23 14:28:35 +0000 | [diff] [blame] | 1850 | (AND64ri8 (SUBREG_TO_REG (i64 0), |
| 1851 | (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), sub_32bit), (i64 1))>; |
Elena Demikhovsky | 750498c | 2014-02-17 07:29:33 +0000 | [diff] [blame] | 1852 | def : Pat<(i16 (zext VK1:$src)), |
| 1853 | (EXTRACT_SUBREG |
Elena Demikhovsky | 3ebfe11 | 2014-02-23 14:28:35 +0000 | [diff] [blame] | 1854 | (AND32ri (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1)), |
| 1855 | sub_16bit)>; |
Elena Demikhovsky | cf0b9ba | 2014-04-09 12:37:50 +0000 | [diff] [blame] | 1856 | def : Pat<(v16i1 (scalar_to_vector VK1:$src)), |
| 1857 | (COPY_TO_REGCLASS VK1:$src, VK16)>; |
| 1858 | def : Pat<(v8i1 (scalar_to_vector VK1:$src)), |
| 1859 | (COPY_TO_REGCLASS VK1:$src, VK8)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1860 | } |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1861 | let Predicates = [HasBWI] in { |
| 1862 | def : Pat<(v32i1 (scalar_to_vector VK1:$src)), |
| 1863 | (COPY_TO_REGCLASS VK1:$src, VK32)>; |
| 1864 | def : Pat<(v64i1 (scalar_to_vector VK1:$src)), |
| 1865 | (COPY_TO_REGCLASS VK1:$src, VK64)>; |
| 1866 | } |
| 1867 | |
| 1868 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1869 | // With AVX-512 only, 8-bit mask is promoted to 16-bit mask. |
Elena Demikhovsky | 75d1489 | 2015-05-10 10:33:32 +0000 | [diff] [blame] | 1870 | let Predicates = [HasAVX512, NoDQI] in { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1871 | // GR from/to 8-bit mask without native support |
| 1872 | def : Pat<(v8i1 (bitconvert (i8 GR8:$src))), |
| 1873 | (COPY_TO_REGCLASS |
Elena Demikhovsky | f61727d | 2015-05-20 14:32:03 +0000 | [diff] [blame] | 1874 | (KMOVWkr (MOVZX32rr8 GR8 :$src)), VK8)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1875 | def : Pat<(i8 (bitconvert (v8i1 VK8:$src))), |
| 1876 | (EXTRACT_SUBREG |
| 1877 | (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16)), |
| 1878 | sub_8bit)>; |
Elena Demikhovsky | 75d1489 | 2015-05-10 10:33:32 +0000 | [diff] [blame] | 1879 | } |
Elena Demikhovsky | f61727d | 2015-05-20 14:32:03 +0000 | [diff] [blame] | 1880 | |
Elena Demikhovsky | 75d1489 | 2015-05-10 10:33:32 +0000 | [diff] [blame] | 1881 | let Predicates = [HasAVX512] in { |
Elena Demikhovsky | 9f423d6 | 2014-02-10 07:02:39 +0000 | [diff] [blame] | 1882 | def : Pat<(i1 (X86Vextract VK16:$src, (iPTR 0))), |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 1883 | (COPY_TO_REGCLASS VK16:$src, VK1)>; |
Elena Demikhovsky | 9f423d6 | 2014-02-10 07:02:39 +0000 | [diff] [blame] | 1884 | def : Pat<(i1 (X86Vextract VK8:$src, (iPTR 0))), |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 1885 | (COPY_TO_REGCLASS VK8:$src, VK1)>; |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1886 | } |
| 1887 | let Predicates = [HasBWI] in { |
| 1888 | def : Pat<(i1 (X86Vextract VK32:$src, (iPTR 0))), |
| 1889 | (COPY_TO_REGCLASS VK32:$src, VK1)>; |
| 1890 | def : Pat<(i1 (X86Vextract VK64:$src, (iPTR 0))), |
| 1891 | (COPY_TO_REGCLASS VK64:$src, VK1)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1892 | } |
| 1893 | |
| 1894 | // Mask unary operation |
| 1895 | // - KNOT |
| 1896 | multiclass avx512_mask_unop<bits<8> opc, string OpcodeStr, |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1897 | RegisterClass KRC, SDPatternOperator OpNode, |
| 1898 | Predicate prd> { |
| 1899 | let Predicates = [prd] in |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1900 | def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 1901 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1902 | [(set KRC:$dst, (OpNode KRC:$src))]>; |
| 1903 | } |
| 1904 | |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1905 | multiclass avx512_mask_unop_all<bits<8> opc, string OpcodeStr, |
| 1906 | SDPatternOperator OpNode> { |
| 1907 | defm B : avx512_mask_unop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode, |
| 1908 | HasDQI>, VEX, PD; |
| 1909 | defm W : avx512_mask_unop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode, |
| 1910 | HasAVX512>, VEX, PS; |
| 1911 | defm D : avx512_mask_unop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode, |
| 1912 | HasBWI>, VEX, PD, VEX_W; |
| 1913 | defm Q : avx512_mask_unop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode, |
| 1914 | HasBWI>, VEX, PS, VEX_W; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1915 | } |
| 1916 | |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1917 | defm KNOT : avx512_mask_unop_all<0x44, "knot", not>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1918 | |
Elena Demikhovsky | e382c3f | 2013-12-10 13:53:10 +0000 | [diff] [blame] | 1919 | multiclass avx512_mask_unop_int<string IntName, string InstName> { |
| 1920 | let Predicates = [HasAVX512] in |
| 1921 | def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_w") |
| 1922 | (i16 GR16:$src)), |
| 1923 | (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"Wrr") |
| 1924 | (v16i1 (COPY_TO_REGCLASS GR16:$src, VK16))), GR16)>; |
| 1925 | } |
| 1926 | defm : avx512_mask_unop_int<"knot", "KNOT">; |
| 1927 | |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1928 | let Predicates = [HasDQI] in |
| 1929 | def : Pat<(xor VK8:$src1, (v8i1 immAllOnesV)), (KNOTBrr VK8:$src1)>; |
| 1930 | let Predicates = [HasAVX512] in |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1931 | def : Pat<(xor VK16:$src1, (v16i1 immAllOnesV)), (KNOTWrr VK16:$src1)>; |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1932 | let Predicates = [HasBWI] in |
| 1933 | def : Pat<(xor VK32:$src1, (v32i1 immAllOnesV)), (KNOTDrr VK32:$src1)>; |
| 1934 | let Predicates = [HasBWI] in |
| 1935 | def : Pat<(xor VK64:$src1, (v64i1 immAllOnesV)), (KNOTQrr VK64:$src1)>; |
| 1936 | |
| 1937 | // KNL does not support KMOVB, 8-bit mask is promoted to 16-bit |
Elena Demikhovsky | d2cb3c8 | 2015-02-12 08:40:34 +0000 | [diff] [blame] | 1938 | let Predicates = [HasAVX512, NoDQI] in { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1939 | def : Pat<(xor VK8:$src1, (v8i1 immAllOnesV)), |
| 1940 | (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$src1, VK16)), VK8)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1941 | def : Pat<(not VK8:$src), |
| 1942 | (COPY_TO_REGCLASS |
| 1943 | (KNOTWrr (COPY_TO_REGCLASS VK8:$src, VK16)), VK8)>; |
Robert Khasanov | 74acbb7 | 2014-07-23 14:49:42 +0000 | [diff] [blame] | 1944 | } |
Elena Demikhovsky | d1084c5 | 2015-04-27 12:57:59 +0000 | [diff] [blame] | 1945 | def : Pat<(xor VK4:$src1, (v4i1 immAllOnesV)), |
| 1946 | (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK4:$src1, VK16)), VK4)>; |
| 1947 | def : Pat<(xor VK2:$src1, (v2i1 immAllOnesV)), |
| 1948 | (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK2:$src1, VK16)), VK2)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1949 | |
| 1950 | // Mask binary operation |
Elena Demikhovsky | e382c3f | 2013-12-10 13:53:10 +0000 | [diff] [blame] | 1951 | // - KAND, KANDN, KOR, KXNOR, KXOR |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1952 | multiclass avx512_mask_binop<bits<8> opc, string OpcodeStr, |
Robert Khasanov | 595683d | 2014-07-28 13:46:45 +0000 | [diff] [blame] | 1953 | RegisterClass KRC, SDPatternOperator OpNode, |
Elena Demikhovsky | d1084c5 | 2015-04-27 12:57:59 +0000 | [diff] [blame] | 1954 | Predicate prd, bit IsCommutable> { |
| 1955 | let Predicates = [prd], isCommutable = IsCommutable in |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1956 | def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2), |
| 1957 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 1958 | "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1959 | [(set KRC:$dst, (OpNode KRC:$src1, KRC:$src2))]>; |
| 1960 | } |
| 1961 | |
Robert Khasanov | 595683d | 2014-07-28 13:46:45 +0000 | [diff] [blame] | 1962 | multiclass avx512_mask_binop_all<bits<8> opc, string OpcodeStr, |
Igor Breger | 59ac339 | 2015-08-31 11:50:23 +0000 | [diff] [blame] | 1963 | SDPatternOperator OpNode, bit IsCommutable, |
| 1964 | Predicate prdW = HasAVX512> { |
Robert Khasanov | 595683d | 2014-07-28 13:46:45 +0000 | [diff] [blame] | 1965 | defm B : avx512_mask_binop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode, |
Elena Demikhovsky | d1084c5 | 2015-04-27 12:57:59 +0000 | [diff] [blame] | 1966 | HasDQI, IsCommutable>, VEX_4V, VEX_L, PD; |
Robert Khasanov | 595683d | 2014-07-28 13:46:45 +0000 | [diff] [blame] | 1967 | defm W : avx512_mask_binop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode, |
Igor Breger | 59ac339 | 2015-08-31 11:50:23 +0000 | [diff] [blame] | 1968 | prdW, IsCommutable>, VEX_4V, VEX_L, PS; |
Robert Khasanov | 595683d | 2014-07-28 13:46:45 +0000 | [diff] [blame] | 1969 | defm D : avx512_mask_binop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode, |
Elena Demikhovsky | d1084c5 | 2015-04-27 12:57:59 +0000 | [diff] [blame] | 1970 | HasBWI, IsCommutable>, VEX_4V, VEX_L, VEX_W, PD; |
Robert Khasanov | 595683d | 2014-07-28 13:46:45 +0000 | [diff] [blame] | 1971 | defm Q : avx512_mask_binop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode, |
Elena Demikhovsky | d1084c5 | 2015-04-27 12:57:59 +0000 | [diff] [blame] | 1972 | HasBWI, IsCommutable>, VEX_4V, VEX_L, VEX_W, PS; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1973 | } |
| 1974 | |
| 1975 | def andn : PatFrag<(ops node:$i0, node:$i1), (and (not node:$i0), node:$i1)>; |
| 1976 | def xnor : PatFrag<(ops node:$i0, node:$i1), (not (xor node:$i0, node:$i1))>; |
| 1977 | |
Elena Demikhovsky | d1084c5 | 2015-04-27 12:57:59 +0000 | [diff] [blame] | 1978 | defm KAND : avx512_mask_binop_all<0x41, "kand", and, 1>; |
| 1979 | defm KOR : avx512_mask_binop_all<0x45, "kor", or, 1>; |
| 1980 | defm KXNOR : avx512_mask_binop_all<0x46, "kxnor", xnor, 1>; |
| 1981 | defm KXOR : avx512_mask_binop_all<0x47, "kxor", xor, 1>; |
| 1982 | defm KANDN : avx512_mask_binop_all<0x42, "kandn", andn, 0>; |
Igor Breger | 59ac339 | 2015-08-31 11:50:23 +0000 | [diff] [blame] | 1983 | defm KADD : avx512_mask_binop_all<0x4A, "kadd", add, 1, HasDQI>; |
Elena Demikhovsky | b64d7e8 | 2013-12-25 10:06:40 +0000 | [diff] [blame] | 1984 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1985 | multiclass avx512_mask_binop_int<string IntName, string InstName> { |
| 1986 | let Predicates = [HasAVX512] in |
Elena Demikhovsky | e382c3f | 2013-12-10 13:53:10 +0000 | [diff] [blame] | 1987 | def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_w") |
| 1988 | (i16 GR16:$src1), (i16 GR16:$src2)), |
| 1989 | (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"Wrr") |
| 1990 | (v16i1 (COPY_TO_REGCLASS GR16:$src1, VK16)), |
| 1991 | (v16i1 (COPY_TO_REGCLASS GR16:$src2, VK16))), GR16)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1992 | } |
| 1993 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 1994 | defm : avx512_mask_binop_int<"kand", "KAND">; |
| 1995 | defm : avx512_mask_binop_int<"kandn", "KANDN">; |
| 1996 | defm : avx512_mask_binop_int<"kor", "KOR">; |
| 1997 | defm : avx512_mask_binop_int<"kxnor", "KXNOR">; |
| 1998 | defm : avx512_mask_binop_int<"kxor", "KXOR">; |
Elena Demikhovsky | e382c3f | 2013-12-10 13:53:10 +0000 | [diff] [blame] | 1999 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2000 | multiclass avx512_binop_pat<SDPatternOperator OpNode, Instruction Inst> { |
Elena Demikhovsky | d1084c5 | 2015-04-27 12:57:59 +0000 | [diff] [blame] | 2001 | // With AVX512F, 8-bit mask is promoted to 16-bit mask, |
| 2002 | // for the DQI set, this type is legal and KxxxB instruction is used |
| 2003 | let Predicates = [NoDQI] in |
| 2004 | def : Pat<(OpNode VK8:$src1, VK8:$src2), |
| 2005 | (COPY_TO_REGCLASS |
| 2006 | (Inst (COPY_TO_REGCLASS VK8:$src1, VK16), |
| 2007 | (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>; |
| 2008 | |
| 2009 | // All types smaller than 8 bits require conversion anyway |
| 2010 | def : Pat<(OpNode VK1:$src1, VK1:$src2), |
| 2011 | (COPY_TO_REGCLASS (Inst |
| 2012 | (COPY_TO_REGCLASS VK1:$src1, VK16), |
| 2013 | (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>; |
| 2014 | def : Pat<(OpNode VK2:$src1, VK2:$src2), |
| 2015 | (COPY_TO_REGCLASS (Inst |
| 2016 | (COPY_TO_REGCLASS VK2:$src1, VK16), |
| 2017 | (COPY_TO_REGCLASS VK2:$src2, VK16)), VK1)>; |
| 2018 | def : Pat<(OpNode VK4:$src1, VK4:$src2), |
| 2019 | (COPY_TO_REGCLASS (Inst |
| 2020 | (COPY_TO_REGCLASS VK4:$src1, VK16), |
| 2021 | (COPY_TO_REGCLASS VK4:$src2, VK16)), VK1)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2022 | } |
| 2023 | |
| 2024 | defm : avx512_binop_pat<and, KANDWrr>; |
| 2025 | defm : avx512_binop_pat<andn, KANDNWrr>; |
| 2026 | defm : avx512_binop_pat<or, KORWrr>; |
| 2027 | defm : avx512_binop_pat<xnor, KXNORWrr>; |
| 2028 | defm : avx512_binop_pat<xor, KXORWrr>; |
| 2029 | |
Elena Demikhovsky | d1084c5 | 2015-04-27 12:57:59 +0000 | [diff] [blame] | 2030 | def : Pat<(xor (xor VK16:$src1, VK16:$src2), (v16i1 immAllOnesV)), |
| 2031 | (KXNORWrr VK16:$src1, VK16:$src2)>; |
| 2032 | def : Pat<(xor (xor VK8:$src1, VK8:$src2), (v8i1 immAllOnesV)), |
Elena Demikhovsky | 00c9ad5 | 2015-06-10 06:49:28 +0000 | [diff] [blame] | 2033 | (KXNORBrr VK8:$src1, VK8:$src2)>, Requires<[HasDQI]>; |
Elena Demikhovsky | d1084c5 | 2015-04-27 12:57:59 +0000 | [diff] [blame] | 2034 | def : Pat<(xor (xor VK32:$src1, VK32:$src2), (v32i1 immAllOnesV)), |
Elena Demikhovsky | 00c9ad5 | 2015-06-10 06:49:28 +0000 | [diff] [blame] | 2035 | (KXNORDrr VK32:$src1, VK32:$src2)>, Requires<[HasBWI]>; |
Elena Demikhovsky | d1084c5 | 2015-04-27 12:57:59 +0000 | [diff] [blame] | 2036 | def : Pat<(xor (xor VK64:$src1, VK64:$src2), (v64i1 immAllOnesV)), |
Elena Demikhovsky | 00c9ad5 | 2015-06-10 06:49:28 +0000 | [diff] [blame] | 2037 | (KXNORQrr VK64:$src1, VK64:$src2)>, Requires<[HasBWI]>; |
Elena Demikhovsky | d1084c5 | 2015-04-27 12:57:59 +0000 | [diff] [blame] | 2038 | |
| 2039 | let Predicates = [NoDQI] in |
| 2040 | def : Pat<(xor (xor VK8:$src1, VK8:$src2), (v8i1 immAllOnesV)), |
| 2041 | (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK8:$src1, VK16), |
| 2042 | (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>; |
| 2043 | |
| 2044 | def : Pat<(xor (xor VK4:$src1, VK4:$src2), (v4i1 immAllOnesV)), |
| 2045 | (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK4:$src1, VK16), |
| 2046 | (COPY_TO_REGCLASS VK4:$src2, VK16)), VK4)>; |
| 2047 | |
| 2048 | def : Pat<(xor (xor VK2:$src1, VK2:$src2), (v2i1 immAllOnesV)), |
| 2049 | (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK2:$src1, VK16), |
| 2050 | (COPY_TO_REGCLASS VK2:$src2, VK16)), VK2)>; |
| 2051 | |
| 2052 | def : Pat<(xor (xor VK1:$src1, VK1:$src2), (i1 1)), |
| 2053 | (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK1:$src1, VK16), |
| 2054 | (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>; |
| 2055 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2056 | // Mask unpacking |
| 2057 | multiclass avx512_mask_unpck<bits<8> opc, string OpcodeStr, |
Elena Demikhovsky | e382c3f | 2013-12-10 13:53:10 +0000 | [diff] [blame] | 2058 | RegisterClass KRC> { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2059 | let Predicates = [HasAVX512] in |
Elena Demikhovsky | e382c3f | 2013-12-10 13:53:10 +0000 | [diff] [blame] | 2060 | def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2061 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 2062 | "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2063 | } |
| 2064 | |
| 2065 | multiclass avx512_mask_unpck_bw<bits<8> opc, string OpcodeStr> { |
Elena Demikhovsky | e382c3f | 2013-12-10 13:53:10 +0000 | [diff] [blame] | 2066 | defm BW : avx512_mask_unpck<opc, !strconcat(OpcodeStr, "bw"), VK16>, |
Craig Topper | ae11aed | 2014-01-14 07:41:20 +0000 | [diff] [blame] | 2067 | VEX_4V, VEX_L, PD; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2068 | } |
| 2069 | |
| 2070 | defm KUNPCK : avx512_mask_unpck_bw<0x4b, "kunpck">; |
Elena Demikhovsky | c5f6726 | 2013-12-17 08:33:15 +0000 | [diff] [blame] | 2071 | def : Pat<(v16i1 (concat_vectors (v8i1 VK8:$src1), (v8i1 VK8:$src2))), |
| 2072 | (KUNPCKBWrr (COPY_TO_REGCLASS VK8:$src2, VK16), |
| 2073 | (COPY_TO_REGCLASS VK8:$src1, VK16))>; |
| 2074 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2075 | |
| 2076 | multiclass avx512_mask_unpck_int<string IntName, string InstName> { |
| 2077 | let Predicates = [HasAVX512] in |
Elena Demikhovsky | e382c3f | 2013-12-10 13:53:10 +0000 | [diff] [blame] | 2078 | def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_bw") |
| 2079 | (i16 GR16:$src1), (i16 GR16:$src2)), |
| 2080 | (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"BWrr") |
| 2081 | (v16i1 (COPY_TO_REGCLASS GR16:$src1, VK16)), |
| 2082 | (v16i1 (COPY_TO_REGCLASS GR16:$src2, VK16))), GR16)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2083 | } |
Elena Demikhovsky | e382c3f | 2013-12-10 13:53:10 +0000 | [diff] [blame] | 2084 | defm : avx512_mask_unpck_int<"kunpck", "KUNPCK">; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2085 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2086 | // Mask bit testing |
| 2087 | multiclass avx512_mask_testop<bits<8> opc, string OpcodeStr, RegisterClass KRC, |
Igor Breger | 5ea0a681 | 2015-08-31 13:30:19 +0000 | [diff] [blame] | 2088 | SDNode OpNode, Predicate prd> { |
| 2089 | let Predicates = [prd], Defs = [EFLAGS] in |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2090 | def rr : I<opc, MRMSrcReg, (outs), (ins KRC:$src1, KRC:$src2), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 2091 | !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2092 | [(set EFLAGS, (OpNode KRC:$src1, KRC:$src2))]>; |
| 2093 | } |
| 2094 | |
Igor Breger | 5ea0a681 | 2015-08-31 13:30:19 +0000 | [diff] [blame] | 2095 | multiclass avx512_mask_testop_w<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 2096 | Predicate prdW = HasAVX512> { |
| 2097 | defm B : avx512_mask_testop<opc, OpcodeStr#"b", VK8, OpNode, HasDQI>, |
| 2098 | VEX, PD; |
| 2099 | defm W : avx512_mask_testop<opc, OpcodeStr#"w", VK16, OpNode, prdW>, |
| 2100 | VEX, PS; |
| 2101 | defm Q : avx512_mask_testop<opc, OpcodeStr#"q", VK64, OpNode, HasBWI>, |
| 2102 | VEX, PS, VEX_W; |
| 2103 | defm D : avx512_mask_testop<opc, OpcodeStr#"d", VK32, OpNode, HasBWI>, |
| 2104 | VEX, PD, VEX_W; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2105 | } |
| 2106 | |
| 2107 | defm KORTEST : avx512_mask_testop_w<0x98, "kortest", X86kortest>; |
Igor Breger | 5ea0a681 | 2015-08-31 13:30:19 +0000 | [diff] [blame] | 2108 | defm KTEST : avx512_mask_testop_w<0x99, "ktest", X86ktest, HasDQI>; |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 2109 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2110 | // Mask shift |
| 2111 | multiclass avx512_mask_shiftop<bits<8> opc, string OpcodeStr, RegisterClass KRC, |
| 2112 | SDNode OpNode> { |
| 2113 | let Predicates = [HasAVX512] in |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 2114 | def ri : Ii8<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src, u8imm:$imm), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2115 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 2116 | "\t{$imm, $src, $dst|$dst, $src, $imm}"), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2117 | [(set KRC:$dst, (OpNode KRC:$src, (i8 imm:$imm)))]>; |
| 2118 | } |
| 2119 | |
| 2120 | multiclass avx512_mask_shiftop_w<bits<8> opc1, bits<8> opc2, string OpcodeStr, |
| 2121 | SDNode OpNode> { |
| 2122 | defm W : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "w"), VK16, OpNode>, |
Elena Demikhovsky | 1a603b3 | 2015-01-25 12:47:15 +0000 | [diff] [blame] | 2123 | VEX, TAPD, VEX_W; |
| 2124 | let Predicates = [HasDQI] in |
| 2125 | defm B : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "b"), VK8, OpNode>, |
| 2126 | VEX, TAPD; |
| 2127 | let Predicates = [HasBWI] in { |
| 2128 | defm Q : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "q"), VK64, OpNode>, |
| 2129 | VEX, TAPD, VEX_W; |
| 2130 | let Predicates = [HasDQI] in |
| 2131 | defm D : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "d"), VK32, OpNode>, |
| 2132 | VEX, TAPD; |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 2133 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2134 | } |
| 2135 | |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 2136 | defm KSHIFTL : avx512_mask_shiftop_w<0x32, 0x33, "kshiftl", X86vshli>; |
| 2137 | defm KSHIFTR : avx512_mask_shiftop_w<0x30, 0x31, "kshiftr", X86vsrli>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2138 | |
| 2139 | // Mask setting all 0s or 1s |
| 2140 | multiclass avx512_mask_setop<RegisterClass KRC, ValueType VT, PatFrag Val> { |
| 2141 | let Predicates = [HasAVX512] in |
| 2142 | let isReMaterializable = 1, isAsCheapAsAMove = 1, isPseudo = 1 in |
| 2143 | def #NAME# : I<0, Pseudo, (outs KRC:$dst), (ins), "", |
| 2144 | [(set KRC:$dst, (VT Val))]>; |
| 2145 | } |
| 2146 | |
| 2147 | multiclass avx512_mask_setop_w<PatFrag Val> { |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 2148 | defm B : avx512_mask_setop<VK8, v8i1, Val>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2149 | defm W : avx512_mask_setop<VK16, v16i1, Val>; |
Elena Demikhovsky | d1084c5 | 2015-04-27 12:57:59 +0000 | [diff] [blame] | 2150 | defm D : avx512_mask_setop<VK32, v32i1, Val>; |
| 2151 | defm Q : avx512_mask_setop<VK64, v64i1, Val>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2152 | } |
| 2153 | |
| 2154 | defm KSET0 : avx512_mask_setop_w<immAllZerosV>; |
| 2155 | defm KSET1 : avx512_mask_setop_w<immAllOnesV>; |
| 2156 | |
| 2157 | // With AVX-512 only, 8-bit mask is promoted to 16-bit mask. |
| 2158 | let Predicates = [HasAVX512] in { |
| 2159 | def : Pat<(v8i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK8)>; |
| 2160 | def : Pat<(v8i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK8)>; |
Elena Demikhovsky | d1084c5 | 2015-04-27 12:57:59 +0000 | [diff] [blame] | 2161 | def : Pat<(v4i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK4)>; |
| 2162 | def : Pat<(v2i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK2)>; |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 2163 | def : Pat<(i1 0), (COPY_TO_REGCLASS (KSET0W), VK1)>; |
Elena Demikhovsky | 1d6a495 | 2015-05-17 07:28:51 +0000 | [diff] [blame] | 2164 | def : Pat<(i1 1), (COPY_TO_REGCLASS (KSHIFTRWri (KSET1W), (i8 15)), VK1)>; |
| 2165 | def : Pat<(i1 -1), (COPY_TO_REGCLASS (KSHIFTRWri (KSET1W), (i8 15)), VK1)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2166 | } |
| 2167 | def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 0))), |
| 2168 | (v8i1 (COPY_TO_REGCLASS VK16:$src, VK8))>; |
| 2169 | |
| 2170 | def : Pat<(v16i1 (insert_subvector undef, (v8i1 VK8:$src), (iPTR 0))), |
| 2171 | (v16i1 (COPY_TO_REGCLASS VK8:$src, VK16))>; |
| 2172 | |
| 2173 | def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 8))), |
| 2174 | (v8i1 (COPY_TO_REGCLASS (KSHIFTRWri VK16:$src, (i8 8)), VK8))>; |
| 2175 | |
Elena Demikhovsky | 86c7b46 | 2015-05-27 14:09:33 +0000 | [diff] [blame] | 2176 | def : Pat<(v32i1 (extract_subvector (v64i1 VK64:$src), (iPTR 0))), |
| 2177 | (v32i1 (COPY_TO_REGCLASS VK64:$src, VK32))>; |
| 2178 | |
| 2179 | def : Pat<(v32i1 (extract_subvector (v64i1 VK64:$src), (iPTR 32))), |
| 2180 | (v32i1 (COPY_TO_REGCLASS (KSHIFTRQri VK64:$src, (i8 32)), VK32))>; |
| 2181 | |
Robert Khasanov | 5aa4445 | 2014-09-30 11:41:54 +0000 | [diff] [blame] | 2182 | let Predicates = [HasVLX] in { |
| 2183 | def : Pat<(v8i1 (insert_subvector undef, (v4i1 VK4:$src), (iPTR 0))), |
| 2184 | (v8i1 (COPY_TO_REGCLASS VK4:$src, VK8))>; |
| 2185 | def : Pat<(v8i1 (insert_subvector undef, (v2i1 VK2:$src), (iPTR 0))), |
| 2186 | (v8i1 (COPY_TO_REGCLASS VK2:$src, VK8))>; |
Elena Demikhovsky | de05f10 | 2015-03-05 15:11:35 +0000 | [diff] [blame] | 2187 | def : Pat<(v4i1 (insert_subvector undef, (v2i1 VK2:$src), (iPTR 0))), |
| 2188 | (v4i1 (COPY_TO_REGCLASS VK2:$src, VK4))>; |
Robert Khasanov | 5aa4445 | 2014-09-30 11:41:54 +0000 | [diff] [blame] | 2189 | def : Pat<(v4i1 (extract_subvector (v8i1 VK8:$src), (iPTR 0))), |
| 2190 | (v4i1 (COPY_TO_REGCLASS VK8:$src, VK4))>; |
| 2191 | def : Pat<(v2i1 (extract_subvector (v8i1 VK8:$src), (iPTR 0))), |
| 2192 | (v2i1 (COPY_TO_REGCLASS VK8:$src, VK2))>; |
| 2193 | } |
| 2194 | |
Elena Demikhovsky | 9737e38 | 2014-03-02 09:19:44 +0000 | [diff] [blame] | 2195 | def : Pat<(v8i1 (X86vshli VK8:$src, (i8 imm:$imm))), |
Elena Demikhovsky | 1a603b3 | 2015-01-25 12:47:15 +0000 | [diff] [blame] | 2196 | (v8i1 (COPY_TO_REGCLASS |
| 2197 | (KSHIFTLWri (COPY_TO_REGCLASS VK8:$src, VK16), |
| 2198 | (I8Imm $imm)), VK8))>, Requires<[HasAVX512, NoDQI]>; |
Elena Demikhovsky | 9737e38 | 2014-03-02 09:19:44 +0000 | [diff] [blame] | 2199 | |
| 2200 | def : Pat<(v8i1 (X86vsrli VK8:$src, (i8 imm:$imm))), |
Elena Demikhovsky | 1a603b3 | 2015-01-25 12:47:15 +0000 | [diff] [blame] | 2201 | (v8i1 (COPY_TO_REGCLASS |
| 2202 | (KSHIFTRWri (COPY_TO_REGCLASS VK8:$src, VK16), |
| 2203 | (I8Imm $imm)), VK8))>, Requires<[HasAVX512, NoDQI]>; |
Elena Demikhovsky | de05f10 | 2015-03-05 15:11:35 +0000 | [diff] [blame] | 2204 | |
| 2205 | def : Pat<(v4i1 (X86vshli VK4:$src, (i8 imm:$imm))), |
| 2206 | (v4i1 (COPY_TO_REGCLASS |
| 2207 | (KSHIFTLWri (COPY_TO_REGCLASS VK4:$src, VK16), |
| 2208 | (I8Imm $imm)), VK4))>, Requires<[HasAVX512]>; |
| 2209 | |
| 2210 | def : Pat<(v4i1 (X86vsrli VK4:$src, (i8 imm:$imm))), |
| 2211 | (v4i1 (COPY_TO_REGCLASS |
| 2212 | (KSHIFTRWri (COPY_TO_REGCLASS VK4:$src, VK16), |
| 2213 | (I8Imm $imm)), VK4))>, Requires<[HasAVX512]>; |
| 2214 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2215 | //===----------------------------------------------------------------------===// |
| 2216 | // AVX-512 - Aligned and unaligned load and store |
| 2217 | // |
| 2218 | |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2219 | |
| 2220 | multiclass avx512_load<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2221 | PatFrag ld_frag, PatFrag mload, |
| 2222 | bit IsReMaterializable = 1> { |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2223 | let hasSideEffects = 0 in { |
| 2224 | def rr : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst), (ins _.RC:$src), |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2225 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [], |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2226 | _.ExeDomain>, EVEX; |
| 2227 | def rrkz : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst), |
| 2228 | (ins _.KRCWM:$mask, _.RC:$src), |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2229 | !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}} {z}|", |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2230 | "${dst} {${mask}} {z}, $src}"), [], _.ExeDomain>, |
| 2231 | EVEX, EVEX_KZ; |
| 2232 | |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2233 | let canFoldAsLoad = 1, isReMaterializable = IsReMaterializable, |
| 2234 | SchedRW = [WriteLoad] in |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2235 | def rm : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst), (ins _.MemOp:$src), |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2236 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2237 | [(set _.RC:$dst, (_.VT (bitconvert (ld_frag addr:$src))))], |
| 2238 | _.ExeDomain>, EVEX; |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2239 | |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2240 | let Constraints = "$src0 = $dst" in { |
| 2241 | def rrk : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst), |
| 2242 | (ins _.RC:$src0, _.KRCWM:$mask, _.RC:$src1), |
| 2243 | !strconcat(OpcodeStr, "\t{$src1, ${dst} {${mask}}|", |
| 2244 | "${dst} {${mask}}, $src1}"), |
| 2245 | [(set _.RC:$dst, (_.VT (vselect _.KRCWM:$mask, |
| 2246 | (_.VT _.RC:$src1), |
| 2247 | (_.VT _.RC:$src0))))], _.ExeDomain>, |
| 2248 | EVEX, EVEX_K; |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2249 | let mayLoad = 1, SchedRW = [WriteLoad] in |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2250 | def rmk : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst), |
| 2251 | (ins _.RC:$src0, _.KRCWM:$mask, _.MemOp:$src1), |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2252 | !strconcat(OpcodeStr, "\t{$src1, ${dst} {${mask}}|", |
| 2253 | "${dst} {${mask}}, $src1}"), |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2254 | [(set _.RC:$dst, (_.VT |
| 2255 | (vselect _.KRCWM:$mask, |
| 2256 | (_.VT (bitconvert (ld_frag addr:$src1))), |
| 2257 | (_.VT _.RC:$src0))))], _.ExeDomain>, EVEX, EVEX_K; |
Elena Demikhovsky | fd05667 | 2014-03-13 12:05:52 +0000 | [diff] [blame] | 2258 | } |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2259 | let mayLoad = 1, SchedRW = [WriteLoad] in |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2260 | def rmkz : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst), |
| 2261 | (ins _.KRCWM:$mask, _.MemOp:$src), |
| 2262 | OpcodeStr #"\t{$src, ${dst} {${mask}} {z}|"# |
| 2263 | "${dst} {${mask}} {z}, $src}", |
| 2264 | [(set _.RC:$dst, (_.VT (vselect _.KRCWM:$mask, |
| 2265 | (_.VT (bitconvert (ld_frag addr:$src))), _.ImmAllZerosV)))], |
| 2266 | _.ExeDomain>, EVEX, EVEX_KZ; |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2267 | } |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2268 | def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, undef)), |
| 2269 | (!cast<Instruction>(NAME#_.ZSuffix##rmkz) _.KRCWM:$mask, addr:$ptr)>; |
| 2270 | |
| 2271 | def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, _.ImmAllZerosV)), |
| 2272 | (!cast<Instruction>(NAME#_.ZSuffix##rmkz) _.KRCWM:$mask, addr:$ptr)>; |
| 2273 | |
| 2274 | def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src0))), |
| 2275 | (!cast<Instruction>(NAME#_.ZSuffix##rmk) _.RC:$src0, |
| 2276 | _.KRCWM:$mask, addr:$ptr)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2277 | } |
| 2278 | |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2279 | multiclass avx512_alignedload_vl<bits<8> opc, string OpcodeStr, |
| 2280 | AVX512VLVectorVTInfo _, |
| 2281 | Predicate prd, |
| 2282 | bit IsReMaterializable = 1> { |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2283 | let Predicates = [prd] in |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2284 | defm Z : avx512_load<opc, OpcodeStr, _.info512, _.info512.AlignedLdFrag, |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2285 | masked_load_aligned512, IsReMaterializable>, EVEX_V512; |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2286 | |
| 2287 | let Predicates = [prd, HasVLX] in { |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2288 | defm Z256 : avx512_load<opc, OpcodeStr, _.info256, _.info256.AlignedLdFrag, |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2289 | masked_load_aligned256, IsReMaterializable>, EVEX_V256; |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2290 | defm Z128 : avx512_load<opc, OpcodeStr, _.info128, _.info128.AlignedLdFrag, |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2291 | masked_load_aligned128, IsReMaterializable>, EVEX_V128; |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2292 | } |
| 2293 | } |
| 2294 | |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2295 | multiclass avx512_load_vl<bits<8> opc, string OpcodeStr, |
| 2296 | AVX512VLVectorVTInfo _, |
| 2297 | Predicate prd, |
| 2298 | bit IsReMaterializable = 1> { |
| 2299 | let Predicates = [prd] in |
| 2300 | defm Z : avx512_load<opc, OpcodeStr, _.info512, _.info512.LdFrag, |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2301 | masked_load_unaligned, IsReMaterializable>, EVEX_V512; |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2302 | |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2303 | let Predicates = [prd, HasVLX] in { |
| 2304 | defm Z256 : avx512_load<opc, OpcodeStr, _.info256, _.info256.LdFrag, |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2305 | masked_load_unaligned, IsReMaterializable>, EVEX_V256; |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2306 | defm Z128 : avx512_load<opc, OpcodeStr, _.info128, _.info128.LdFrag, |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2307 | masked_load_unaligned, IsReMaterializable>, EVEX_V128; |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2308 | } |
| 2309 | } |
| 2310 | |
| 2311 | multiclass avx512_store<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2312 | PatFrag st_frag, PatFrag mstore> { |
Craig Topper | 9fdd078 | 2015-01-15 09:37:15 +0000 | [diff] [blame] | 2313 | let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in { |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2314 | def rr_alt : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst), (ins _.RC:$src), |
| 2315 | OpcodeStr # "\t{$src, $dst|$dst, $src}", [], |
| 2316 | _.ExeDomain>, EVEX; |
Elena Demikhovsky | fd05667 | 2014-03-13 12:05:52 +0000 | [diff] [blame] | 2317 | let Constraints = "$src1 = $dst" in |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2318 | def rrk_alt : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst), |
| 2319 | (ins _.RC:$src1, _.KRCWM:$mask, _.RC:$src2), |
| 2320 | OpcodeStr # |
| 2321 | "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}", |
| 2322 | [], _.ExeDomain>, EVEX, EVEX_K; |
| 2323 | def rrkz_alt : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst), |
| 2324 | (ins _.KRCWM:$mask, _.RC:$src), |
| 2325 | OpcodeStr # |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 2326 | "\t{$src, ${dst} {${mask}} {z}|" # |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2327 | "${dst} {${mask}} {z}, $src}", |
| 2328 | [], _.ExeDomain>, EVEX, EVEX_KZ; |
Elena Demikhovsky | fd05667 | 2014-03-13 12:05:52 +0000 | [diff] [blame] | 2329 | } |
| 2330 | let mayStore = 1 in { |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2331 | def mr : AVX512PI<opc, MRMDestMem, (outs), (ins _.MemOp:$dst, _.RC:$src), |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2332 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2333 | [(st_frag (_.VT _.RC:$src), addr:$dst)], _.ExeDomain>, EVEX; |
Elena Demikhovsky | fd05667 | 2014-03-13 12:05:52 +0000 | [diff] [blame] | 2334 | def mrk : AVX512PI<opc, MRMDestMem, (outs), |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2335 | (ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src), |
| 2336 | OpcodeStr # "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}", |
| 2337 | [], _.ExeDomain>, EVEX, EVEX_K; |
Elena Demikhovsky | fd05667 | 2014-03-13 12:05:52 +0000 | [diff] [blame] | 2338 | } |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2339 | |
| 2340 | def: Pat<(mstore addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src)), |
| 2341 | (!cast<Instruction>(NAME#_.ZSuffix##mrk) addr:$ptr, |
| 2342 | _.KRCWM:$mask, _.RC:$src)>; |
Elena Demikhovsky | fd05667 | 2014-03-13 12:05:52 +0000 | [diff] [blame] | 2343 | } |
| 2344 | |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2345 | |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2346 | multiclass avx512_store_vl< bits<8> opc, string OpcodeStr, |
| 2347 | AVX512VLVectorVTInfo _, Predicate prd> { |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2348 | let Predicates = [prd] in |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2349 | defm Z : avx512_store<opc, OpcodeStr, _.info512, store, |
| 2350 | masked_store_unaligned>, EVEX_V512; |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2351 | |
| 2352 | let Predicates = [prd, HasVLX] in { |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2353 | defm Z256 : avx512_store<opc, OpcodeStr, _.info256, store, |
| 2354 | masked_store_unaligned>, EVEX_V256; |
| 2355 | defm Z128 : avx512_store<opc, OpcodeStr, _.info128, store, |
| 2356 | masked_store_unaligned>, EVEX_V128; |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2357 | } |
| 2358 | } |
| 2359 | |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2360 | multiclass avx512_alignedstore_vl<bits<8> opc, string OpcodeStr, |
| 2361 | AVX512VLVectorVTInfo _, Predicate prd> { |
| 2362 | let Predicates = [prd] in |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2363 | defm Z : avx512_store<opc, OpcodeStr, _.info512, alignedstore512, |
| 2364 | masked_store_aligned512>, EVEX_V512; |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2365 | |
| 2366 | let Predicates = [prd, HasVLX] in { |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2367 | defm Z256 : avx512_store<opc, OpcodeStr, _.info256, alignedstore256, |
| 2368 | masked_store_aligned256>, EVEX_V256; |
| 2369 | defm Z128 : avx512_store<opc, OpcodeStr, _.info128, alignedstore, |
| 2370 | masked_store_aligned128>, EVEX_V128; |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2371 | } |
| 2372 | } |
| 2373 | |
| 2374 | defm VMOVAPS : avx512_alignedload_vl<0x28, "vmovaps", avx512vl_f32_info, |
| 2375 | HasAVX512>, |
| 2376 | avx512_alignedstore_vl<0x29, "vmovaps", avx512vl_f32_info, |
| 2377 | HasAVX512>, PS, EVEX_CD8<32, CD8VF>; |
| 2378 | |
| 2379 | defm VMOVAPD : avx512_alignedload_vl<0x28, "vmovapd", avx512vl_f64_info, |
| 2380 | HasAVX512>, |
| 2381 | avx512_alignedstore_vl<0x29, "vmovapd", avx512vl_f64_info, |
| 2382 | HasAVX512>, PD, VEX_W, EVEX_CD8<64, CD8VF>; |
| 2383 | |
| 2384 | defm VMOVUPS : avx512_load_vl<0x10, "vmovups", avx512vl_f32_info, HasAVX512>, |
| 2385 | avx512_store_vl<0x11, "vmovups", avx512vl_f32_info, HasAVX512>, |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2386 | PS, EVEX_CD8<32, CD8VF>; |
| 2387 | |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2388 | defm VMOVUPD : avx512_load_vl<0x10, "vmovupd", avx512vl_f64_info, HasAVX512, 0>, |
| 2389 | avx512_store_vl<0x11, "vmovupd", avx512vl_f64_info, HasAVX512>, |
| 2390 | PD, VEX_W, EVEX_CD8<64, CD8VF>; |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2391 | |
Elena Demikhovsky | fd05667 | 2014-03-13 12:05:52 +0000 | [diff] [blame] | 2392 | def: Pat<(v8f64 (int_x86_avx512_mask_loadu_pd_512 addr:$ptr, |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2393 | (bc_v8f64 (v16i32 immAllZerosV)), GR8:$mask)), |
Elena Demikhovsky | fd05667 | 2014-03-13 12:05:52 +0000 | [diff] [blame] | 2394 | (VMOVUPDZrmkz (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), addr:$ptr)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2395 | |
Elena Demikhovsky | fd05667 | 2014-03-13 12:05:52 +0000 | [diff] [blame] | 2396 | def: Pat<(v16f32 (int_x86_avx512_mask_loadu_ps_512 addr:$ptr, |
| 2397 | (bc_v16f32 (v16i32 immAllZerosV)), GR16:$mask)), |
| 2398 | (VMOVUPSZrmkz (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), addr:$ptr)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2399 | |
Adam Nemet | 3e8b22b | 2015-01-16 18:50:09 +0000 | [diff] [blame] | 2400 | def: Pat<(v8f64 (int_x86_avx512_mask_load_pd_512 addr:$ptr, |
| 2401 | (bc_v8f64 (v16i32 immAllZerosV)), GR8:$mask)), |
| 2402 | (VMOVAPDZrmkz (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), addr:$ptr)>; |
| 2403 | |
| 2404 | def: Pat<(v16f32 (int_x86_avx512_mask_load_ps_512 addr:$ptr, |
| 2405 | (bc_v16f32 (v16i32 immAllZerosV)), GR16:$mask)), |
| 2406 | (VMOVAPSZrmkz (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), addr:$ptr)>; |
| 2407 | |
| 2408 | def: Pat<(v8f64 (int_x86_avx512_mask_load_pd_512 addr:$ptr, |
| 2409 | (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))), |
| 2410 | (VMOVAPDZrm addr:$ptr)>; |
| 2411 | |
| 2412 | def: Pat<(v16f32 (int_x86_avx512_mask_load_ps_512 addr:$ptr, |
| 2413 | (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1))), |
| 2414 | (VMOVAPSZrm addr:$ptr)>; |
| 2415 | |
Elena Demikhovsky | fd05667 | 2014-03-13 12:05:52 +0000 | [diff] [blame] | 2416 | def: Pat<(int_x86_avx512_mask_storeu_ps_512 addr:$ptr, (v16f32 VR512:$src), |
| 2417 | GR16:$mask), |
| 2418 | (VMOVUPSZmrk addr:$ptr, (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), |
| 2419 | VR512:$src)>; |
| 2420 | def: Pat<(int_x86_avx512_mask_storeu_pd_512 addr:$ptr, (v8f64 VR512:$src), |
| 2421 | GR8:$mask), |
| 2422 | (VMOVUPDZmrk addr:$ptr, (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), |
| 2423 | VR512:$src)>; |
Elena Demikhovsky | 1f3ed41 | 2013-10-22 09:19:28 +0000 | [diff] [blame] | 2424 | |
Adam Nemet | 3e8b22b | 2015-01-16 18:50:09 +0000 | [diff] [blame] | 2425 | def: Pat<(int_x86_avx512_mask_store_ps_512 addr:$ptr, (v16f32 VR512:$src), |
| 2426 | GR16:$mask), |
| 2427 | (VMOVAPSZmrk addr:$ptr, (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), |
| 2428 | VR512:$src)>; |
| 2429 | def: Pat<(int_x86_avx512_mask_store_pd_512 addr:$ptr, (v8f64 VR512:$src), |
| 2430 | GR8:$mask), |
| 2431 | (VMOVAPDZmrk addr:$ptr, (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), |
| 2432 | VR512:$src)>; |
| 2433 | |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2434 | let Predicates = [HasAVX512, NoVLX] in { |
Igor Breger | 074a64e | 2015-07-24 17:24:15 +0000 | [diff] [blame] | 2435 | def: Pat<(X86mstore addr:$ptr, VK8WM:$mask, (v8f32 VR256:$src)), |
Elena Demikhovsky | f1de34b | 2014-12-04 09:40:44 +0000 | [diff] [blame] | 2436 | (VMOVUPSZmrk addr:$ptr, |
| 2437 | (v16i1 (COPY_TO_REGCLASS VK8WM:$mask, VK16WM)), |
| 2438 | (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR256:$src, sub_ymm))>; |
| 2439 | |
| 2440 | def: Pat<(v8f32 (masked_load addr:$ptr, VK8WM:$mask, undef)), |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 2441 | (v8f32 (EXTRACT_SUBREG (v16f32 (VMOVUPSZrmkz |
Elena Demikhovsky | f1de34b | 2014-12-04 09:40:44 +0000 | [diff] [blame] | 2442 | (v16i1 (COPY_TO_REGCLASS VK8WM:$mask, VK16WM)), addr:$ptr)), sub_ymm))>; |
| 2443 | |
Elena Demikhovsky | fb73ca5 | 2014-12-19 23:27:57 +0000 | [diff] [blame] | 2444 | def: Pat<(v8f32 (masked_load addr:$ptr, VK8WM:$mask, (v8f32 VR256:$src0))), |
| 2445 | (v8f32 (EXTRACT_SUBREG (v16f32 (VMOVUPSZrmk |
| 2446 | (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR256:$src0, sub_ymm), |
| 2447 | (v16i1 (COPY_TO_REGCLASS VK8WM:$mask, VK16WM)), addr:$ptr)), sub_ymm))>; |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2448 | } |
Elena Demikhovsky | fb73ca5 | 2014-12-19 23:27:57 +0000 | [diff] [blame] | 2449 | |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2450 | defm VMOVDQA32 : avx512_alignedload_vl<0x6F, "vmovdqa32", avx512vl_i32_info, |
| 2451 | HasAVX512>, |
| 2452 | avx512_alignedstore_vl<0x7F, "vmovdqa32", avx512vl_i32_info, |
| 2453 | HasAVX512>, PD, EVEX_CD8<32, CD8VF>; |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2454 | |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2455 | defm VMOVDQA64 : avx512_alignedload_vl<0x6F, "vmovdqa64", avx512vl_i64_info, |
| 2456 | HasAVX512>, |
| 2457 | avx512_alignedstore_vl<0x7F, "vmovdqa64", avx512vl_i64_info, |
| 2458 | HasAVX512>, PD, VEX_W, EVEX_CD8<64, CD8VF>; |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2459 | |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2460 | defm VMOVDQU8 : avx512_load_vl<0x6F, "vmovdqu8", avx512vl_i8_info, HasBWI>, |
| 2461 | avx512_store_vl<0x7F, "vmovdqu8", avx512vl_i8_info, |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2462 | HasBWI>, XD, EVEX_CD8<8, CD8VF>; |
| 2463 | |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2464 | defm VMOVDQU16 : avx512_load_vl<0x6F, "vmovdqu16", avx512vl_i16_info, HasBWI>, |
| 2465 | avx512_store_vl<0x7F, "vmovdqu16", avx512vl_i16_info, |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2466 | HasBWI>, XD, VEX_W, EVEX_CD8<16, CD8VF>; |
| 2467 | |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2468 | defm VMOVDQU32 : avx512_load_vl<0x6F, "vmovdqu32", avx512vl_i32_info, HasAVX512>, |
| 2469 | avx512_store_vl<0x7F, "vmovdqu32", avx512vl_i32_info, |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2470 | HasAVX512>, XS, EVEX_CD8<32, CD8VF>; |
| 2471 | |
Elena Demikhovsky | 2689d78 | 2015-03-02 12:46:21 +0000 | [diff] [blame] | 2472 | defm VMOVDQU64 : avx512_load_vl<0x6F, "vmovdqu64", avx512vl_i64_info, HasAVX512>, |
| 2473 | avx512_store_vl<0x7F, "vmovdqu64", avx512vl_i64_info, |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2474 | HasAVX512>, XS, VEX_W, EVEX_CD8<64, CD8VF>; |
Elena Demikhovsky | 1f3ed41 | 2013-10-22 09:19:28 +0000 | [diff] [blame] | 2475 | |
Elena Demikhovsky | cf0b9ba | 2014-04-09 12:37:50 +0000 | [diff] [blame] | 2476 | def: Pat<(v16i32 (int_x86_avx512_mask_loadu_d_512 addr:$ptr, |
| 2477 | (v16i32 immAllZerosV), GR16:$mask)), |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2478 | (VMOVDQU32Zrmkz (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), addr:$ptr)>; |
Elena Demikhovsky | cf0b9ba | 2014-04-09 12:37:50 +0000 | [diff] [blame] | 2479 | |
| 2480 | def: Pat<(v8i64 (int_x86_avx512_mask_loadu_q_512 addr:$ptr, |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2481 | (bc_v8i64 (v16i32 immAllZerosV)), GR8:$mask)), |
| 2482 | (VMOVDQU64Zrmkz (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), addr:$ptr)>; |
Elena Demikhovsky | cf0b9ba | 2014-04-09 12:37:50 +0000 | [diff] [blame] | 2483 | |
Elena Demikhovsky | e73333a | 2014-05-04 13:35:37 +0000 | [diff] [blame] | 2484 | def: Pat<(int_x86_avx512_mask_storeu_d_512 addr:$ptr, (v16i32 VR512:$src), |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2485 | GR16:$mask), |
| 2486 | (VMOVDQU32Zmrk addr:$ptr, (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), |
Elena Demikhovsky | e73333a | 2014-05-04 13:35:37 +0000 | [diff] [blame] | 2487 | VR512:$src)>; |
| 2488 | def: Pat<(int_x86_avx512_mask_storeu_q_512 addr:$ptr, (v8i64 VR512:$src), |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2489 | GR8:$mask), |
| 2490 | (VMOVDQU64Zmrk addr:$ptr, (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), |
Elena Demikhovsky | e73333a | 2014-05-04 13:35:37 +0000 | [diff] [blame] | 2491 | VR512:$src)>; |
| 2492 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2493 | let AddedComplexity = 20 in { |
Elena Demikhovsky | a30e437 | 2014-02-05 07:05:03 +0000 | [diff] [blame] | 2494 | def : Pat<(v8i64 (vselect VK8WM:$mask, (v8i64 VR512:$src), |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2495 | (bc_v8i64 (v16i32 immAllZerosV)))), |
| 2496 | (VMOVDQU64Zrrkz VK8WM:$mask, VR512:$src)>; |
Elena Demikhovsky | a30e437 | 2014-02-05 07:05:03 +0000 | [diff] [blame] | 2497 | |
| 2498 | def : Pat<(v8i64 (vselect VK8WM:$mask, (bc_v8i64 (v16i32 immAllZerosV)), |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2499 | (v8i64 VR512:$src))), |
| 2500 | (VMOVDQU64Zrrkz (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$mask, VK16)), |
Elena Demikhovsky | a30e437 | 2014-02-05 07:05:03 +0000 | [diff] [blame] | 2501 | VK8), VR512:$src)>; |
| 2502 | |
| 2503 | def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 VR512:$src), |
| 2504 | (v16i32 immAllZerosV))), |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2505 | (VMOVDQU32Zrrkz VK16WM:$mask, VR512:$src)>; |
Elena Demikhovsky | a30e437 | 2014-02-05 07:05:03 +0000 | [diff] [blame] | 2506 | |
| 2507 | def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 immAllZerosV), |
Robert Khasanov | 7ca7df0 | 2014-08-04 14:35:15 +0000 | [diff] [blame] | 2508 | (v16i32 VR512:$src))), |
| 2509 | (VMOVDQU32Zrrkz (KNOTWrr VK16WM:$mask), VR512:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2510 | } |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2511 | // NoVLX patterns |
| 2512 | let Predicates = [HasAVX512, NoVLX] in { |
Igor Breger | 074a64e | 2015-07-24 17:24:15 +0000 | [diff] [blame] | 2513 | def: Pat<(X86mstore addr:$ptr, VK8WM:$mask, (v8i32 VR256:$src)), |
Elena Demikhovsky | f1de34b | 2014-12-04 09:40:44 +0000 | [diff] [blame] | 2514 | (VMOVDQU32Zmrk addr:$ptr, |
| 2515 | (v16i1 (COPY_TO_REGCLASS VK8WM:$mask, VK16WM)), |
| 2516 | (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR256:$src, sub_ymm))>; |
| 2517 | |
| 2518 | def: Pat<(v8i32 (masked_load addr:$ptr, VK8WM:$mask, undef)), |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 2519 | (v8i32 (EXTRACT_SUBREG (v16i32 (VMOVDQU32Zrmkz |
Elena Demikhovsky | f1de34b | 2014-12-04 09:40:44 +0000 | [diff] [blame] | 2520 | (v16i1 (COPY_TO_REGCLASS VK8WM:$mask, VK16WM)), addr:$ptr)), sub_ymm))>; |
Elena Demikhovsky | d207f17 | 2015-03-03 15:03:35 +0000 | [diff] [blame] | 2521 | } |
Elena Demikhovsky | f1de34b | 2014-12-04 09:40:44 +0000 | [diff] [blame] | 2522 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2523 | // Move Int Doubleword to Packed Double Int |
| 2524 | // |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2525 | def VMOVDI2PDIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR32:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2526 | "vmovd\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2527 | [(set VR128X:$dst, |
| 2528 | (v4i32 (scalar_to_vector GR32:$src)))], IIC_SSE_MOVDQ>, |
| 2529 | EVEX, VEX_LIG; |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2530 | def VMOVDI2PDIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst), (ins i32mem:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2531 | "vmovd\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2532 | [(set VR128X:$dst, |
| 2533 | (v4i32 (scalar_to_vector (loadi32 addr:$src))))], |
| 2534 | IIC_SSE_MOVDQ>, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2535 | def VMOV64toPQIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR64:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2536 | "vmovq\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2537 | [(set VR128X:$dst, |
| 2538 | (v2i64 (scalar_to_vector GR64:$src)))], |
| 2539 | IIC_SSE_MOVDQ>, EVEX, VEX_W, VEX_LIG; |
Craig Topper | 88adf2a | 2013-10-12 05:41:08 +0000 | [diff] [blame] | 2540 | let isCodeGenOnly = 1 in { |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2541 | def VMOV64toSDZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR64:$dst), (ins GR64:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2542 | "vmovq\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2543 | [(set FR64:$dst, (bitconvert GR64:$src))], |
| 2544 | IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteMove]>; |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2545 | def VMOVSDto64Zrr : AVX512BI<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2546 | "vmovq\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2547 | [(set GR64:$dst, (bitconvert FR64:$src))], |
| 2548 | IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteMove]>; |
Craig Topper | 88adf2a | 2013-10-12 05:41:08 +0000 | [diff] [blame] | 2549 | } |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2550 | def VMOVSDto64Zmr : AVX512BI<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2551 | "vmovq\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2552 | [(store (i64 (bitconvert FR64:$src)), addr:$dst)], |
| 2553 | IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteStore]>, |
| 2554 | EVEX_CD8<64, CD8VT1>; |
| 2555 | |
| 2556 | // Move Int Doubleword to Single Scalar |
| 2557 | // |
Craig Topper | 88adf2a | 2013-10-12 05:41:08 +0000 | [diff] [blame] | 2558 | let isCodeGenOnly = 1 in { |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2559 | def VMOVDI2SSZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR32X:$dst), (ins GR32:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2560 | "vmovd\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2561 | [(set FR32X:$dst, (bitconvert GR32:$src))], |
| 2562 | IIC_SSE_MOVDQ>, EVEX, VEX_LIG; |
| 2563 | |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2564 | def VMOVDI2SSZrm : AVX512BI<0x6E, MRMSrcMem, (outs FR32X:$dst), (ins i32mem:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2565 | "vmovd\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2566 | [(set FR32X:$dst, (bitconvert (loadi32 addr:$src)))], |
| 2567 | IIC_SSE_MOVDQ>, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; |
Craig Topper | 88adf2a | 2013-10-12 05:41:08 +0000 | [diff] [blame] | 2568 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2569 | |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2570 | // Move doubleword from xmm register to r/m32 |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2571 | // |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2572 | def VMOVPDI2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128X:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2573 | "vmovd\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2574 | [(set GR32:$dst, (vector_extract (v4i32 VR128X:$src), |
| 2575 | (iPTR 0)))], IIC_SSE_MOVD_ToGP>, |
| 2576 | EVEX, VEX_LIG; |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2577 | def VMOVPDI2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2578 | (ins i32mem:$dst, VR128X:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2579 | "vmovd\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2580 | [(store (i32 (vector_extract (v4i32 VR128X:$src), |
| 2581 | (iPTR 0))), addr:$dst)], IIC_SSE_MOVDQ>, |
| 2582 | EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; |
| 2583 | |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2584 | // Move quadword from xmm1 register to r/m64 |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2585 | // |
| 2586 | def VMOVPQIto64Zrr : I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128X:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2587 | "vmovq\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2588 | [(set GR64:$dst, (extractelt (v2i64 VR128X:$src), |
| 2589 | (iPTR 0)))], |
Craig Topper | ae11aed | 2014-01-14 07:41:20 +0000 | [diff] [blame] | 2590 | IIC_SSE_MOVD_ToGP>, PD, EVEX, VEX_LIG, VEX_W, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2591 | Requires<[HasAVX512, In64BitMode]>; |
| 2592 | |
Elena Demikhovsky | 85aeffa | 2013-10-03 12:03:26 +0000 | [diff] [blame] | 2593 | def VMOVPQIto64Zmr : I<0xD6, MRMDestMem, (outs), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2594 | (ins i64mem:$dst, VR128X:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2595 | "vmovq\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2596 | [(store (extractelt (v2i64 VR128X:$src), (iPTR 0)), |
| 2597 | addr:$dst)], IIC_SSE_MOVDQ>, |
Craig Topper | ae11aed | 2014-01-14 07:41:20 +0000 | [diff] [blame] | 2598 | EVEX, PD, VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2599 | Sched<[WriteStore]>, Requires<[HasAVX512, In64BitMode]>; |
| 2600 | |
| 2601 | // Move Scalar Single to Double Int |
| 2602 | // |
Craig Topper | 88adf2a | 2013-10-12 05:41:08 +0000 | [diff] [blame] | 2603 | let isCodeGenOnly = 1 in { |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2604 | def VMOVSS2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2605 | (ins FR32X:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2606 | "vmovd\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2607 | [(set GR32:$dst, (bitconvert FR32X:$src))], |
| 2608 | IIC_SSE_MOVD_ToGP>, EVEX, VEX_LIG; |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2609 | def VMOVSS2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2610 | (ins i32mem:$dst, FR32X:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2611 | "vmovd\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2612 | [(store (i32 (bitconvert FR32X:$src)), addr:$dst)], |
| 2613 | IIC_SSE_MOVDQ>, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; |
Craig Topper | 88adf2a | 2013-10-12 05:41:08 +0000 | [diff] [blame] | 2614 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2615 | |
| 2616 | // Move Quadword Int to Packed Quadword Int |
| 2617 | // |
Elena Demikhovsky | 767fc96 | 2014-01-14 15:10:08 +0000 | [diff] [blame] | 2618 | def VMOVQI2PQIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2619 | (ins i64mem:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2620 | "vmovq\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2621 | [(set VR128X:$dst, |
| 2622 | (v2i64 (scalar_to_vector (loadi64 addr:$src))))]>, |
| 2623 | EVEX, VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; |
| 2624 | |
| 2625 | //===----------------------------------------------------------------------===// |
| 2626 | // AVX-512 MOVSS, MOVSD |
| 2627 | //===----------------------------------------------------------------------===// |
| 2628 | |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 2629 | multiclass avx512_move_scalar <string asm, RegisterClass RC, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2630 | SDNode OpNode, ValueType vt, |
| 2631 | X86MemOperand x86memop, PatFrag mem_pat> { |
Elena Demikhovsky | f404e05 | 2014-01-05 14:21:07 +0000 | [diff] [blame] | 2632 | let hasSideEffects = 0 in { |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 2633 | def rr : SI<0x10, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src1, RC:$src2), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 2634 | !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2635 | [(set VR128X:$dst, (vt (OpNode VR128X:$src1, |
| 2636 | (scalar_to_vector RC:$src2))))], |
| 2637 | IIC_SSE_MOV_S_RR>, EVEX_4V, VEX_LIG; |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 2638 | let Constraints = "$src1 = $dst" in |
| 2639 | def rrk : SI<0x10, MRMSrcReg, (outs VR128X:$dst), |
| 2640 | (ins VR128X:$src1, VK1WM:$mask, RC:$src2, RC:$src3), |
| 2641 | !strconcat(asm, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 2642 | "\t{$src3, $src2, $dst {${mask}}|$dst {${mask}}, $src2, $src3}"), |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 2643 | [], IIC_SSE_MOV_S_RR>, EVEX_4V, VEX_LIG, EVEX_K; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2644 | def rm : SI<0x10, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 2645 | !strconcat(asm, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2646 | [(set RC:$dst, (mem_pat addr:$src))], IIC_SSE_MOV_S_RM>, |
| 2647 | EVEX, VEX_LIG; |
Elena Demikhovsky | ff620ed | 2014-08-27 07:38:43 +0000 | [diff] [blame] | 2648 | let mayStore = 1 in { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2649 | def mr: SI<0x11, MRMDestMem, (outs), (ins x86memop:$dst, RC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 2650 | !strconcat(asm, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2651 | [(store RC:$src, addr:$dst)], IIC_SSE_MOV_S_MR>, |
| 2652 | EVEX, VEX_LIG; |
Elena Demikhovsky | ff620ed | 2014-08-27 07:38:43 +0000 | [diff] [blame] | 2653 | def mrk: SI<0x11, MRMDestMem, (outs), (ins x86memop:$dst, VK1WM:$mask, RC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 2654 | !strconcat(asm, "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}"), |
Elena Demikhovsky | ff620ed | 2014-08-27 07:38:43 +0000 | [diff] [blame] | 2655 | [], IIC_SSE_MOV_S_MR>, |
| 2656 | EVEX, VEX_LIG, EVEX_K; |
| 2657 | } // mayStore |
Elena Demikhovsky | f404e05 | 2014-01-05 14:21:07 +0000 | [diff] [blame] | 2658 | } //hasSideEffects = 0 |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2659 | } |
| 2660 | |
| 2661 | let ExeDomain = SSEPackedSingle in |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2662 | defm VMOVSSZ : avx512_move_scalar<"movss", FR32X, X86Movss, v4f32, f32mem, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2663 | loadf32>, XS, EVEX_CD8<32, CD8VT1>; |
| 2664 | |
| 2665 | let ExeDomain = SSEPackedDouble in |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2666 | defm VMOVSDZ : avx512_move_scalar<"movsd", FR64X, X86Movsd, v2f64, f64mem, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2667 | loadf64>, XD, VEX_W, EVEX_CD8<64, CD8VT1>; |
| 2668 | |
Elena Demikhovsky | 47fc44e | 2013-12-16 13:52:35 +0000 | [diff] [blame] | 2669 | def : Pat<(f32 (X86select VK1WM:$mask, (f32 FR32X:$src1), (f32 FR32X:$src2))), |
| 2670 | (COPY_TO_REGCLASS (VMOVSSZrrk (COPY_TO_REGCLASS FR32X:$src2, VR128X), |
| 2671 | VK1WM:$mask, (f32 (IMPLICIT_DEF)), FR32X:$src1), FR32X)>; |
| 2672 | |
| 2673 | def : Pat<(f64 (X86select VK1WM:$mask, (f64 FR64X:$src1), (f64 FR64X:$src2))), |
| 2674 | (COPY_TO_REGCLASS (VMOVSDZrrk (COPY_TO_REGCLASS FR64X:$src2, VR128X), |
| 2675 | VK1WM:$mask, (f64 (IMPLICIT_DEF)), FR64X:$src1), FR64X)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2676 | |
Elena Demikhovsky | ff620ed | 2014-08-27 07:38:43 +0000 | [diff] [blame] | 2677 | def : Pat<(int_x86_avx512_mask_store_ss addr:$dst, VR128X:$src, GR8:$mask), |
| 2678 | (VMOVSSZmrk addr:$dst, (i1 (COPY_TO_REGCLASS GR8:$mask, VK1WM)), |
| 2679 | (COPY_TO_REGCLASS VR128X:$src, FR32X))>; |
| 2680 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2681 | // For the disassembler |
Craig Topper | 3484fc2 | 2014-01-05 04:17:28 +0000 | [diff] [blame] | 2682 | let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2683 | def VMOVSSZrr_REV : SI<0x11, MRMDestReg, (outs VR128X:$dst), |
| 2684 | (ins VR128X:$src1, FR32X:$src2), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2685 | "movss\t{$src2, $src1, $dst|$dst, $src1, $src2}", [], |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2686 | IIC_SSE_MOV_S_RR>, |
| 2687 | XS, EVEX_4V, VEX_LIG; |
| 2688 | def VMOVSDZrr_REV : SI<0x11, MRMDestReg, (outs VR128X:$dst), |
| 2689 | (ins VR128X:$src1, FR64X:$src2), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2690 | "movsd\t{$src2, $src1, $dst|$dst, $src1, $src2}", [], |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2691 | IIC_SSE_MOV_S_RR>, |
| 2692 | XD, EVEX_4V, VEX_LIG, VEX_W; |
| 2693 | } |
| 2694 | |
| 2695 | let Predicates = [HasAVX512] in { |
| 2696 | let AddedComplexity = 15 in { |
| 2697 | // Move scalar to XMM zero-extended, zeroing a VR128X then do a |
| 2698 | // MOVS{S,D} to the lower bits. |
| 2699 | def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector FR32X:$src)))), |
| 2700 | (VMOVSSZrr (v4f32 (V_SET0)), FR32X:$src)>; |
| 2701 | def : Pat<(v4f32 (X86vzmovl (v4f32 VR128X:$src))), |
| 2702 | (VMOVSSZrr (v4f32 (V_SET0)), (COPY_TO_REGCLASS VR128X:$src, FR32X))>; |
| 2703 | def : Pat<(v4i32 (X86vzmovl (v4i32 VR128X:$src))), |
| 2704 | (VMOVSSZrr (v4i32 (V_SET0)), (COPY_TO_REGCLASS VR128X:$src, FR32X))>; |
| 2705 | def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64X:$src)))), |
| 2706 | (VMOVSDZrr (v2f64 (V_SET0)), FR64X:$src)>; |
| 2707 | |
| 2708 | // Move low f32 and clear high bits. |
| 2709 | def : Pat<(v8f32 (X86vzmovl (v8f32 VR256X:$src))), |
| 2710 | (SUBREG_TO_REG (i32 0), |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 2711 | (VMOVSSZrr (v4f32 (V_SET0)), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2712 | (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm)), sub_xmm)>; |
| 2713 | def : Pat<(v8i32 (X86vzmovl (v8i32 VR256X:$src))), |
| 2714 | (SUBREG_TO_REG (i32 0), |
| 2715 | (VMOVSSZrr (v4i32 (V_SET0)), |
| 2716 | (EXTRACT_SUBREG (v8i32 VR256X:$src), sub_xmm)), sub_xmm)>; |
| 2717 | } |
| 2718 | |
| 2719 | let AddedComplexity = 20 in { |
| 2720 | // MOVSSrm zeros the high parts of the register; represent this |
| 2721 | // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0 |
| 2722 | def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))), |
| 2723 | (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>; |
| 2724 | def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))), |
| 2725 | (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>; |
| 2726 | def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))), |
| 2727 | (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>; |
| 2728 | |
| 2729 | // MOVSDrm zeros the high parts of the register; represent this |
| 2730 | // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0 |
| 2731 | def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))), |
| 2732 | (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; |
| 2733 | def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))), |
| 2734 | (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; |
| 2735 | def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))), |
| 2736 | (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; |
| 2737 | def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))), |
| 2738 | (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; |
| 2739 | def : Pat<(v2f64 (X86vzload addr:$src)), |
| 2740 | (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; |
| 2741 | |
| 2742 | // Represent the same patterns above but in the form they appear for |
| 2743 | // 256-bit types |
| 2744 | def : Pat<(v8i32 (X86vzmovl (insert_subvector undef, |
| 2745 | (v4i32 (scalar_to_vector (loadi32 addr:$src))), (iPTR 0)))), |
Elena Demikhovsky | 34586e7 | 2013-10-02 12:20:42 +0000 | [diff] [blame] | 2746 | (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrm addr:$src), sub_xmm)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2747 | def : Pat<(v8f32 (X86vzmovl (insert_subvector undef, |
| 2748 | (v4f32 (scalar_to_vector (loadf32 addr:$src))), (iPTR 0)))), |
| 2749 | (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>; |
| 2750 | def : Pat<(v4f64 (X86vzmovl (insert_subvector undef, |
| 2751 | (v2f64 (scalar_to_vector (loadf64 addr:$src))), (iPTR 0)))), |
| 2752 | (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>; |
| 2753 | } |
| 2754 | def : Pat<(v8f32 (X86vzmovl (insert_subvector undef, |
| 2755 | (v4f32 (scalar_to_vector FR32X:$src)), (iPTR 0)))), |
| 2756 | (SUBREG_TO_REG (i32 0), (v4f32 (VMOVSSZrr (v4f32 (V_SET0)), |
| 2757 | FR32X:$src)), sub_xmm)>; |
| 2758 | def : Pat<(v4f64 (X86vzmovl (insert_subvector undef, |
| 2759 | (v2f64 (scalar_to_vector FR64X:$src)), (iPTR 0)))), |
| 2760 | (SUBREG_TO_REG (i64 0), (v2f64 (VMOVSDZrr (v2f64 (V_SET0)), |
| 2761 | FR64X:$src)), sub_xmm)>; |
| 2762 | def : Pat<(v4i64 (X86vzmovl (insert_subvector undef, |
| 2763 | (v2i64 (scalar_to_vector (loadi64 addr:$src))), (iPTR 0)))), |
Elena Demikhovsky | 34586e7 | 2013-10-02 12:20:42 +0000 | [diff] [blame] | 2764 | (SUBREG_TO_REG (i64 0), (VMOVQI2PQIZrm addr:$src), sub_xmm)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2765 | |
| 2766 | // Move low f64 and clear high bits. |
| 2767 | def : Pat<(v4f64 (X86vzmovl (v4f64 VR256X:$src))), |
| 2768 | (SUBREG_TO_REG (i32 0), |
| 2769 | (VMOVSDZrr (v2f64 (V_SET0)), |
| 2770 | (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm)), sub_xmm)>; |
| 2771 | |
| 2772 | def : Pat<(v4i64 (X86vzmovl (v4i64 VR256X:$src))), |
| 2773 | (SUBREG_TO_REG (i32 0), (VMOVSDZrr (v2i64 (V_SET0)), |
| 2774 | (EXTRACT_SUBREG (v4i64 VR256X:$src), sub_xmm)), sub_xmm)>; |
| 2775 | |
| 2776 | // Extract and store. |
| 2777 | def : Pat<(store (f32 (vector_extract (v4f32 VR128X:$src), (iPTR 0))), |
| 2778 | addr:$dst), |
| 2779 | (VMOVSSZmr addr:$dst, (COPY_TO_REGCLASS (v4f32 VR128X:$src), FR32X))>; |
| 2780 | def : Pat<(store (f64 (vector_extract (v2f64 VR128X:$src), (iPTR 0))), |
| 2781 | addr:$dst), |
| 2782 | (VMOVSDZmr addr:$dst, (COPY_TO_REGCLASS (v2f64 VR128X:$src), FR64X))>; |
| 2783 | |
| 2784 | // Shuffle with VMOVSS |
| 2785 | def : Pat<(v4i32 (X86Movss VR128X:$src1, VR128X:$src2)), |
| 2786 | (VMOVSSZrr (v4i32 VR128X:$src1), |
| 2787 | (COPY_TO_REGCLASS (v4i32 VR128X:$src2), FR32X))>; |
| 2788 | def : Pat<(v4f32 (X86Movss VR128X:$src1, VR128X:$src2)), |
| 2789 | (VMOVSSZrr (v4f32 VR128X:$src1), |
| 2790 | (COPY_TO_REGCLASS (v4f32 VR128X:$src2), FR32X))>; |
| 2791 | |
| 2792 | // 256-bit variants |
| 2793 | def : Pat<(v8i32 (X86Movss VR256X:$src1, VR256X:$src2)), |
| 2794 | (SUBREG_TO_REG (i32 0), |
| 2795 | (VMOVSSZrr (EXTRACT_SUBREG (v8i32 VR256X:$src1), sub_xmm), |
| 2796 | (EXTRACT_SUBREG (v8i32 VR256X:$src2), sub_xmm)), |
| 2797 | sub_xmm)>; |
| 2798 | def : Pat<(v8f32 (X86Movss VR256X:$src1, VR256X:$src2)), |
| 2799 | (SUBREG_TO_REG (i32 0), |
| 2800 | (VMOVSSZrr (EXTRACT_SUBREG (v8f32 VR256X:$src1), sub_xmm), |
| 2801 | (EXTRACT_SUBREG (v8f32 VR256X:$src2), sub_xmm)), |
| 2802 | sub_xmm)>; |
| 2803 | |
| 2804 | // Shuffle with VMOVSD |
| 2805 | def : Pat<(v2i64 (X86Movsd VR128X:$src1, VR128X:$src2)), |
| 2806 | (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; |
| 2807 | def : Pat<(v2f64 (X86Movsd VR128X:$src1, VR128X:$src2)), |
| 2808 | (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; |
| 2809 | def : Pat<(v4f32 (X86Movsd VR128X:$src1, VR128X:$src2)), |
| 2810 | (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; |
| 2811 | def : Pat<(v4i32 (X86Movsd VR128X:$src1, VR128X:$src2)), |
| 2812 | (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; |
| 2813 | |
| 2814 | // 256-bit variants |
| 2815 | def : Pat<(v4i64 (X86Movsd VR256X:$src1, VR256X:$src2)), |
| 2816 | (SUBREG_TO_REG (i32 0), |
| 2817 | (VMOVSDZrr (EXTRACT_SUBREG (v4i64 VR256X:$src1), sub_xmm), |
| 2818 | (EXTRACT_SUBREG (v4i64 VR256X:$src2), sub_xmm)), |
| 2819 | sub_xmm)>; |
| 2820 | def : Pat<(v4f64 (X86Movsd VR256X:$src1, VR256X:$src2)), |
| 2821 | (SUBREG_TO_REG (i32 0), |
| 2822 | (VMOVSDZrr (EXTRACT_SUBREG (v4f64 VR256X:$src1), sub_xmm), |
| 2823 | (EXTRACT_SUBREG (v4f64 VR256X:$src2), sub_xmm)), |
| 2824 | sub_xmm)>; |
| 2825 | |
| 2826 | def : Pat<(v2f64 (X86Movlpd VR128X:$src1, VR128X:$src2)), |
| 2827 | (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; |
| 2828 | def : Pat<(v2i64 (X86Movlpd VR128X:$src1, VR128X:$src2)), |
| 2829 | (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; |
| 2830 | def : Pat<(v4f32 (X86Movlps VR128X:$src1, VR128X:$src2)), |
| 2831 | (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; |
| 2832 | def : Pat<(v4i32 (X86Movlps VR128X:$src1, VR128X:$src2)), |
| 2833 | (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; |
| 2834 | } |
| 2835 | |
| 2836 | let AddedComplexity = 15 in |
| 2837 | def VMOVZPQILo2PQIZrr : AVX512XSI<0x7E, MRMSrcReg, (outs VR128X:$dst), |
| 2838 | (ins VR128X:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2839 | "vmovq\t{$src, $dst|$dst, $src}", |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 2840 | [(set VR128X:$dst, (v2i64 (X86vzmovl |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2841 | (v2i64 VR128X:$src))))], |
| 2842 | IIC_SSE_MOVQ_RR>, EVEX, VEX_W; |
| 2843 | |
| 2844 | let AddedComplexity = 20 in |
| 2845 | def VMOVZPQILo2PQIZrm : AVX512XSI<0x7E, MRMSrcMem, (outs VR128X:$dst), |
| 2846 | (ins i128mem:$src), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 2847 | "vmovq\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2848 | [(set VR128X:$dst, (v2i64 (X86vzmovl |
| 2849 | (loadv2i64 addr:$src))))], |
| 2850 | IIC_SSE_MOVDQ>, EVEX, VEX_W, |
| 2851 | EVEX_CD8<8, CD8VT8>; |
| 2852 | |
| 2853 | let Predicates = [HasAVX512] in { |
| 2854 | // AVX 128-bit movd/movq instruction write zeros in the high 128-bit part. |
| 2855 | let AddedComplexity = 20 in { |
| 2856 | def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector (loadi32 addr:$src))))), |
| 2857 | (VMOVDI2PDIZrm addr:$src)>; |
Elena Demikhovsky | 3b75f5d | 2013-10-01 08:38:02 +0000 | [diff] [blame] | 2858 | def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector GR64:$src)))), |
| 2859 | (VMOV64toPQIZrr GR64:$src)>; |
| 2860 | def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))), |
| 2861 | (VMOVDI2PDIZrr GR32:$src)>; |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 2862 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2863 | def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))), |
| 2864 | (VMOVDI2PDIZrm addr:$src)>; |
| 2865 | def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))), |
| 2866 | (VMOVDI2PDIZrm addr:$src)>; |
| 2867 | def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))), |
| 2868 | (VMOVZPQILo2PQIZrm addr:$src)>; |
| 2869 | def : Pat<(v2f64 (X86vzmovl (v2f64 VR128X:$src))), |
| 2870 | (VMOVZPQILo2PQIZrr VR128X:$src)>; |
Cameron McInally | 30bbb21 | 2013-12-05 00:11:25 +0000 | [diff] [blame] | 2871 | def : Pat<(v2i64 (X86vzload addr:$src)), |
| 2872 | (VMOVZPQILo2PQIZrm addr:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2873 | } |
Elena Demikhovsky | 3b75f5d | 2013-10-01 08:38:02 +0000 | [diff] [blame] | 2874 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2875 | // Use regular 128-bit instructions to match 256-bit scalar_to_vec+zext. |
| 2876 | def : Pat<(v8i32 (X86vzmovl (insert_subvector undef, |
| 2877 | (v4i32 (scalar_to_vector GR32:$src)),(iPTR 0)))), |
| 2878 | (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src), sub_xmm)>; |
| 2879 | def : Pat<(v4i64 (X86vzmovl (insert_subvector undef, |
| 2880 | (v2i64 (scalar_to_vector GR64:$src)),(iPTR 0)))), |
| 2881 | (SUBREG_TO_REG (i64 0), (VMOV64toPQIZrr GR64:$src), sub_xmm)>; |
| 2882 | } |
| 2883 | |
| 2884 | def : Pat<(v16i32 (X86Vinsert (v16i32 immAllZerosV), GR32:$src2, (iPTR 0))), |
| 2885 | (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src2), sub_xmm)>; |
| 2886 | |
| 2887 | def : Pat<(v8i64 (X86Vinsert (bc_v8i64 (v16i32 immAllZerosV)), GR64:$src2, (iPTR 0))), |
| 2888 | (SUBREG_TO_REG (i32 0), (VMOV64toPQIZrr GR64:$src2), sub_xmm)>; |
| 2889 | |
| 2890 | def : Pat<(v16i32 (X86Vinsert undef, GR32:$src2, (iPTR 0))), |
| 2891 | (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src2), sub_xmm)>; |
| 2892 | |
| 2893 | def : Pat<(v8i64 (X86Vinsert undef, GR64:$src2, (iPTR 0))), |
| 2894 | (SUBREG_TO_REG (i32 0), (VMOV64toPQIZrr GR64:$src2), sub_xmm)>; |
| 2895 | |
| 2896 | //===----------------------------------------------------------------------===// |
Adam Nemet | 7f62b23 | 2014-06-10 16:39:53 +0000 | [diff] [blame] | 2897 | // AVX-512 - Non-temporals |
| 2898 | //===----------------------------------------------------------------------===// |
Robert Khasanov | ed88297 | 2014-08-13 10:46:00 +0000 | [diff] [blame] | 2899 | let SchedRW = [WriteLoad] in { |
| 2900 | def VMOVNTDQAZrm : AVX512PI<0x2A, MRMSrcMem, (outs VR512:$dst), |
| 2901 | (ins i512mem:$src), "vmovntdqa\t{$src, $dst|$dst, $src}", |
| 2902 | [(set VR512:$dst, (int_x86_avx512_movntdqa addr:$src))], |
| 2903 | SSEPackedInt>, EVEX, T8PD, EVEX_V512, |
| 2904 | EVEX_CD8<64, CD8VF>; |
Adam Nemet | 7f62b23 | 2014-06-10 16:39:53 +0000 | [diff] [blame] | 2905 | |
Robert Khasanov | ed88297 | 2014-08-13 10:46:00 +0000 | [diff] [blame] | 2906 | let Predicates = [HasAVX512, HasVLX] in { |
| 2907 | def VMOVNTDQAZ256rm : AVX512PI<0x2A, MRMSrcMem, (outs VR256X:$dst), |
| 2908 | (ins i256mem:$src), |
| 2909 | "vmovntdqa\t{$src, $dst|$dst, $src}", [], |
| 2910 | SSEPackedInt>, EVEX, T8PD, EVEX_V256, |
| 2911 | EVEX_CD8<64, CD8VF>; |
Adam Nemet | 7f62b23 | 2014-06-10 16:39:53 +0000 | [diff] [blame] | 2912 | |
Robert Khasanov | ed88297 | 2014-08-13 10:46:00 +0000 | [diff] [blame] | 2913 | def VMOVNTDQAZ128rm : AVX512PI<0x2A, MRMSrcMem, (outs VR128X:$dst), |
| 2914 | (ins i128mem:$src), |
| 2915 | "vmovntdqa\t{$src, $dst|$dst, $src}", [], |
| 2916 | SSEPackedInt>, EVEX, T8PD, EVEX_V128, |
| 2917 | EVEX_CD8<64, CD8VF>; |
| 2918 | } |
Adam Nemet | efd0785 | 2014-06-18 16:51:10 +0000 | [diff] [blame] | 2919 | } |
| 2920 | |
Robert Khasanov | ed88297 | 2014-08-13 10:46:00 +0000 | [diff] [blame] | 2921 | multiclass avx512_movnt<bits<8> opc, string OpcodeStr, PatFrag st_frag, |
| 2922 | ValueType OpVT, RegisterClass RC, X86MemOperand memop, |
| 2923 | Domain d, InstrItinClass itin = IIC_SSE_MOVNT> { |
| 2924 | let SchedRW = [WriteStore], mayStore = 1, |
| 2925 | AddedComplexity = 400 in |
| 2926 | def mr : AVX512PI<opc, MRMDestMem, (outs), (ins memop:$dst, RC:$src), |
| 2927 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
| 2928 | [(st_frag (OpVT RC:$src), addr:$dst)], d, itin>, EVEX; |
| 2929 | } |
| 2930 | |
| 2931 | multiclass avx512_movnt_vl<bits<8> opc, string OpcodeStr, PatFrag st_frag, |
| 2932 | string elty, string elsz, string vsz512, |
| 2933 | string vsz256, string vsz128, Domain d, |
| 2934 | Predicate prd, InstrItinClass itin = IIC_SSE_MOVNT> { |
| 2935 | let Predicates = [prd] in |
| 2936 | defm Z : avx512_movnt<opc, OpcodeStr, st_frag, |
| 2937 | !cast<ValueType>("v"##vsz512##elty##elsz), VR512, |
| 2938 | !cast<X86MemOperand>(elty##"512mem"), d, itin>, |
| 2939 | EVEX_V512; |
| 2940 | |
| 2941 | let Predicates = [prd, HasVLX] in { |
| 2942 | defm Z256 : avx512_movnt<opc, OpcodeStr, st_frag, |
| 2943 | !cast<ValueType>("v"##vsz256##elty##elsz), VR256X, |
| 2944 | !cast<X86MemOperand>(elty##"256mem"), d, itin>, |
| 2945 | EVEX_V256; |
| 2946 | |
| 2947 | defm Z128 : avx512_movnt<opc, OpcodeStr, st_frag, |
| 2948 | !cast<ValueType>("v"##vsz128##elty##elsz), VR128X, |
| 2949 | !cast<X86MemOperand>(elty##"128mem"), d, itin>, |
| 2950 | EVEX_V128; |
| 2951 | } |
| 2952 | } |
| 2953 | |
| 2954 | defm VMOVNTDQ : avx512_movnt_vl<0xE7, "vmovntdq", alignednontemporalstore, |
| 2955 | "i", "64", "8", "4", "2", SSEPackedInt, |
| 2956 | HasAVX512>, PD, EVEX_CD8<64, CD8VF>; |
| 2957 | |
| 2958 | defm VMOVNTPD : avx512_movnt_vl<0x2B, "vmovntpd", alignednontemporalstore, |
| 2959 | "f", "64", "8", "4", "2", SSEPackedDouble, |
| 2960 | HasAVX512>, PD, VEX_W, EVEX_CD8<64, CD8VF>; |
| 2961 | |
| 2962 | defm VMOVNTPS : avx512_movnt_vl<0x2B, "vmovntps", alignednontemporalstore, |
| 2963 | "f", "32", "16", "8", "4", SSEPackedSingle, |
| 2964 | HasAVX512>, PS, EVEX_CD8<32, CD8VF>; |
| 2965 | |
Adam Nemet | 7f62b23 | 2014-06-10 16:39:53 +0000 | [diff] [blame] | 2966 | //===----------------------------------------------------------------------===// |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 2967 | // AVX-512 - Integer arithmetic |
| 2968 | // |
| 2969 | multiclass avx512_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, |
Robert Khasanov | 4424144 | 2014-10-08 14:37:45 +0000 | [diff] [blame] | 2970 | X86VectorVTInfo _, OpndItins itins, |
| 2971 | bit IsCommutable = 0> { |
Adam Nemet | 3480142 | 2014-10-08 23:25:39 +0000 | [diff] [blame] | 2972 | defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 2973 | (ins _.RC:$src1, _.RC:$src2), OpcodeStr, |
Robert Khasanov | 4424144 | 2014-10-08 14:37:45 +0000 | [diff] [blame] | 2974 | "$src2, $src1", "$src1, $src2", |
| 2975 | (_.VT (OpNode _.RC:$src1, _.RC:$src2)), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 2976 | itins.rr, IsCommutable>, |
Robert Khasanov | 4424144 | 2014-10-08 14:37:45 +0000 | [diff] [blame] | 2977 | AVX512BIBase, EVEX_4V; |
Elena Demikhovsky | bb2f6b7 | 2014-03-27 09:45:08 +0000 | [diff] [blame] | 2978 | |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 2979 | let mayLoad = 1 in |
Adam Nemet | 3480142 | 2014-10-08 23:25:39 +0000 | [diff] [blame] | 2980 | defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 2981 | (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr, |
Robert Khasanov | 4424144 | 2014-10-08 14:37:45 +0000 | [diff] [blame] | 2982 | "$src2, $src1", "$src1, $src2", |
| 2983 | (_.VT (OpNode _.RC:$src1, |
| 2984 | (bitconvert (_.LdFrag addr:$src2)))), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 2985 | itins.rm>, |
Robert Khasanov | 4424144 | 2014-10-08 14:37:45 +0000 | [diff] [blame] | 2986 | AVX512BIBase, EVEX_4V; |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 2987 | } |
| 2988 | |
| 2989 | multiclass avx512_binop_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 2990 | X86VectorVTInfo _, OpndItins itins, |
| 2991 | bit IsCommutable = 0> : |
| 2992 | avx512_binop_rm<opc, OpcodeStr, OpNode, _, itins, IsCommutable> { |
| 2993 | let mayLoad = 1 in |
Adam Nemet | 3480142 | 2014-10-08 23:25:39 +0000 | [diff] [blame] | 2994 | defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 2995 | (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr, |
Robert Khasanov | 4424144 | 2014-10-08 14:37:45 +0000 | [diff] [blame] | 2996 | "${src2}"##_.BroadcastStr##", $src1", |
| 2997 | "$src1, ${src2}"##_.BroadcastStr, |
| 2998 | (_.VT (OpNode _.RC:$src1, |
| 2999 | (X86VBroadcast |
| 3000 | (_.ScalarLdFrag addr:$src2)))), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3001 | itins.rm>, |
Robert Khasanov | 4424144 | 2014-10-08 14:37:45 +0000 | [diff] [blame] | 3002 | AVX512BIBase, EVEX_4V, EVEX_B; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3003 | } |
Elena Demikhovsky | bb2f6b7 | 2014-03-27 09:45:08 +0000 | [diff] [blame] | 3004 | |
Robert Khasanov | d5b14f7 | 2014-10-09 08:38:48 +0000 | [diff] [blame] | 3005 | multiclass avx512_binop_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3006 | AVX512VLVectorVTInfo VTInfo, OpndItins itins, |
| 3007 | Predicate prd, bit IsCommutable = 0> { |
| 3008 | let Predicates = [prd] in |
| 3009 | defm Z : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info512, itins, |
| 3010 | IsCommutable>, EVEX_V512; |
| 3011 | |
| 3012 | let Predicates = [prd, HasVLX] in { |
| 3013 | defm Z256 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info256, itins, |
| 3014 | IsCommutable>, EVEX_V256; |
| 3015 | defm Z128 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info128, itins, |
| 3016 | IsCommutable>, EVEX_V128; |
| 3017 | } |
| 3018 | } |
| 3019 | |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3020 | multiclass avx512_binop_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3021 | AVX512VLVectorVTInfo VTInfo, OpndItins itins, |
| 3022 | Predicate prd, bit IsCommutable = 0> { |
| 3023 | let Predicates = [prd] in |
| 3024 | defm Z : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info512, itins, |
| 3025 | IsCommutable>, EVEX_V512; |
| 3026 | |
| 3027 | let Predicates = [prd, HasVLX] in { |
| 3028 | defm Z256 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info256, itins, |
| 3029 | IsCommutable>, EVEX_V256; |
| 3030 | defm Z128 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info128, itins, |
| 3031 | IsCommutable>, EVEX_V128; |
| 3032 | } |
| 3033 | } |
| 3034 | |
| 3035 | multiclass avx512_binop_rm_vl_q<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3036 | OpndItins itins, Predicate prd, |
| 3037 | bit IsCommutable = 0> { |
| 3038 | defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i64_info, |
| 3039 | itins, prd, IsCommutable>, |
| 3040 | VEX_W, EVEX_CD8<64, CD8VF>; |
| 3041 | } |
| 3042 | |
| 3043 | multiclass avx512_binop_rm_vl_d<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3044 | OpndItins itins, Predicate prd, |
| 3045 | bit IsCommutable = 0> { |
| 3046 | defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i32_info, |
| 3047 | itins, prd, IsCommutable>, EVEX_CD8<32, CD8VF>; |
| 3048 | } |
| 3049 | |
| 3050 | multiclass avx512_binop_rm_vl_w<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3051 | OpndItins itins, Predicate prd, |
| 3052 | bit IsCommutable = 0> { |
| 3053 | defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i16_info, |
| 3054 | itins, prd, IsCommutable>, EVEX_CD8<16, CD8VF>; |
| 3055 | } |
| 3056 | |
| 3057 | multiclass avx512_binop_rm_vl_b<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3058 | OpndItins itins, Predicate prd, |
| 3059 | bit IsCommutable = 0> { |
| 3060 | defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i8_info, |
| 3061 | itins, prd, IsCommutable>, EVEX_CD8<8, CD8VF>; |
| 3062 | } |
| 3063 | |
| 3064 | multiclass avx512_binop_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr, |
| 3065 | SDNode OpNode, OpndItins itins, Predicate prd, |
| 3066 | bit IsCommutable = 0> { |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3067 | defm Q : avx512_binop_rm_vl_q<opc_q, OpcodeStr#"q", OpNode, itins, prd, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3068 | IsCommutable>; |
| 3069 | |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3070 | defm D : avx512_binop_rm_vl_d<opc_d, OpcodeStr#"d", OpNode, itins, prd, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3071 | IsCommutable>; |
| 3072 | } |
| 3073 | |
| 3074 | multiclass avx512_binop_rm_vl_bw<bits<8> opc_b, bits<8> opc_w, string OpcodeStr, |
| 3075 | SDNode OpNode, OpndItins itins, Predicate prd, |
| 3076 | bit IsCommutable = 0> { |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3077 | defm W : avx512_binop_rm_vl_w<opc_w, OpcodeStr#"w", OpNode, itins, prd, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3078 | IsCommutable>; |
| 3079 | |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3080 | defm B : avx512_binop_rm_vl_b<opc_b, OpcodeStr#"b", OpNode, itins, prd, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3081 | IsCommutable>; |
| 3082 | } |
| 3083 | |
| 3084 | multiclass avx512_binop_rm_vl_all<bits<8> opc_b, bits<8> opc_w, |
| 3085 | bits<8> opc_d, bits<8> opc_q, |
| 3086 | string OpcodeStr, SDNode OpNode, |
| 3087 | OpndItins itins, bit IsCommutable = 0> { |
| 3088 | defm NAME : avx512_binop_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode, |
| 3089 | itins, HasAVX512, IsCommutable>, |
| 3090 | avx512_binop_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode, |
| 3091 | itins, HasBWI, IsCommutable>; |
| 3092 | } |
| 3093 | |
Elena Demikhovsky | 1eeece1 | 2015-04-02 10:51:40 +0000 | [diff] [blame] | 3094 | multiclass avx512_binop_rm2<bits<8> opc, string OpcodeStr, OpndItins itins, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3095 | SDNode OpNode,X86VectorVTInfo _Src, |
Elena Demikhovsky | 1eeece1 | 2015-04-02 10:51:40 +0000 | [diff] [blame] | 3096 | X86VectorVTInfo _Dst, bit IsCommutable = 0> { |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3097 | defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst), |
Elena Demikhovsky | 1eeece1 | 2015-04-02 10:51:40 +0000 | [diff] [blame] | 3098 | (ins _Src.RC:$src1, _Src.RC:$src2), OpcodeStr, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3099 | "$src2, $src1","$src1, $src2", |
| 3100 | (_Dst.VT (OpNode |
| 3101 | (_Src.VT _Src.RC:$src1), |
Elena Demikhovsky | 1eeece1 | 2015-04-02 10:51:40 +0000 | [diff] [blame] | 3102 | (_Src.VT _Src.RC:$src2))), |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3103 | itins.rr, IsCommutable>, |
Elena Demikhovsky | 1eeece1 | 2015-04-02 10:51:40 +0000 | [diff] [blame] | 3104 | AVX512BIBase, EVEX_4V; |
Elena Demikhovsky | bb2f6b7 | 2014-03-27 09:45:08 +0000 | [diff] [blame] | 3105 | let mayLoad = 1 in { |
Elena Demikhovsky | 1eeece1 | 2015-04-02 10:51:40 +0000 | [diff] [blame] | 3106 | defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst), |
| 3107 | (ins _Src.RC:$src1, _Src.MemOp:$src2), OpcodeStr, |
| 3108 | "$src2, $src1", "$src1, $src2", |
| 3109 | (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), |
| 3110 | (bitconvert (_Src.LdFrag addr:$src2)))), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3111 | itins.rm>, |
Elena Demikhovsky | 1eeece1 | 2015-04-02 10:51:40 +0000 | [diff] [blame] | 3112 | AVX512BIBase, EVEX_4V; |
| 3113 | |
| 3114 | defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst), |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3115 | (ins _Src.RC:$src1, _Dst.ScalarMemOp:$src2), |
Elena Demikhovsky | 1eeece1 | 2015-04-02 10:51:40 +0000 | [diff] [blame] | 3116 | OpcodeStr, |
| 3117 | "${src2}"##_Dst.BroadcastStr##", $src1", |
| 3118 | "$src1, ${src2}"##_Dst.BroadcastStr, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3119 | (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert |
| 3120 | (_Dst.VT (X86VBroadcast |
Elena Demikhovsky | 1eeece1 | 2015-04-02 10:51:40 +0000 | [diff] [blame] | 3121 | (_Dst.ScalarLdFrag addr:$src2)))))), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3122 | itins.rm>, |
Elena Demikhovsky | 1eeece1 | 2015-04-02 10:51:40 +0000 | [diff] [blame] | 3123 | AVX512BIBase, EVEX_4V, EVEX_B; |
Elena Demikhovsky | bb2f6b7 | 2014-03-27 09:45:08 +0000 | [diff] [blame] | 3124 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3125 | } |
| 3126 | |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3127 | defm VPADD : avx512_binop_rm_vl_all<0xFC, 0xFD, 0xFE, 0xD4, "vpadd", add, |
| 3128 | SSE_INTALU_ITINS_P, 1>; |
| 3129 | defm VPSUB : avx512_binop_rm_vl_all<0xF8, 0xF9, 0xFA, 0xFB, "vpsub", sub, |
| 3130 | SSE_INTALU_ITINS_P, 0>; |
Elena Demikhovsky | 5226638 | 2015-05-04 12:35:55 +0000 | [diff] [blame] | 3131 | defm VPADDS : avx512_binop_rm_vl_bw<0xEC, 0xED, "vpadds", X86adds, |
| 3132 | SSE_INTALU_ITINS_P, HasBWI, 1>; |
| 3133 | defm VPSUBS : avx512_binop_rm_vl_bw<0xE8, 0xE9, "vpsubs", X86subs, |
| 3134 | SSE_INTALU_ITINS_P, HasBWI, 0>; |
| 3135 | defm VPADDUS : avx512_binop_rm_vl_bw<0xDC, 0xDD, "vpaddus", X86addus, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3136 | SSE_INTALU_ITINS_P, HasBWI, 1>; |
Elena Demikhovsky | 5226638 | 2015-05-04 12:35:55 +0000 | [diff] [blame] | 3137 | defm VPSUBUS : avx512_binop_rm_vl_bw<0xD8, 0xD9, "vpsubus", X86subus, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3138 | SSE_INTALU_ITINS_P, HasBWI, 0>; |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3139 | defm VPMULLD : avx512_binop_rm_vl_d<0x40, "vpmulld", mul, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3140 | SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3141 | defm VPMULLW : avx512_binop_rm_vl_w<0xD5, "vpmullw", mul, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3142 | SSE_INTALU_ITINS_P, HasBWI, 1>; |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3143 | defm VPMULLQ : avx512_binop_rm_vl_q<0x40, "vpmullq", mul, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3144 | SSE_INTALU_ITINS_P, HasDQI, 1>, T8PD; |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3145 | defm VPMULHW : avx512_binop_rm_vl_w<0xE5, "vpmulhw", mulhs, SSE_INTALU_ITINS_P, |
Asaf Badouh | 73f26f8 | 2015-07-05 12:23:20 +0000 | [diff] [blame] | 3146 | HasBWI, 1>; |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3147 | defm VPMULHUW : avx512_binop_rm_vl_w<0xE4, "vpmulhuw", mulhu, SSE_INTMUL_ITINS_P, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3148 | HasBWI, 1>; |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3149 | defm VPMULHRSW : avx512_binop_rm_vl_w<0x0B, "vpmulhrsw", X86mulhrs, SSE_INTMUL_ITINS_P, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3150 | HasBWI, 1>, T8PD; |
Asaf Badouh | 81f03c3 | 2015-06-18 12:30:53 +0000 | [diff] [blame] | 3151 | defm VPAVG : avx512_binop_rm_vl_bw<0xE0, 0xE3, "vpavg", X86avg, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3152 | SSE_INTALU_ITINS_P, HasBWI, 1>; |
| 3153 | |
Elena Demikhovsky | 50b88dd | 2015-04-21 10:27:40 +0000 | [diff] [blame] | 3154 | multiclass avx512_binop_all<bits<8> opc, string OpcodeStr, OpndItins itins, |
| 3155 | SDNode OpNode, bit IsCommutable = 0> { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3156 | |
Elena Demikhovsky | 50b88dd | 2015-04-21 10:27:40 +0000 | [diff] [blame] | 3157 | defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode, |
| 3158 | v16i32_info, v8i64_info, IsCommutable>, |
| 3159 | EVEX_V512, EVEX_CD8<64, CD8VF>, VEX_W; |
| 3160 | let Predicates = [HasVLX] in { |
| 3161 | defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode, |
| 3162 | v8i32x_info, v4i64x_info, IsCommutable>, |
| 3163 | EVEX_V256, EVEX_CD8<64, CD8VF>, VEX_W; |
| 3164 | defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode, |
| 3165 | v4i32x_info, v2i64x_info, IsCommutable>, |
| 3166 | EVEX_V128, EVEX_CD8<64, CD8VF>, VEX_W; |
| 3167 | } |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3168 | } |
Elena Demikhovsky | 50b88dd | 2015-04-21 10:27:40 +0000 | [diff] [blame] | 3169 | |
| 3170 | defm VPMULDQ : avx512_binop_all<0x28, "vpmuldq", SSE_INTALU_ITINS_P, |
| 3171 | X86pmuldq, 1>,T8PD; |
| 3172 | defm VPMULUDQ : avx512_binop_all<0xF4, "vpmuludq", SSE_INTMUL_ITINS_P, |
| 3173 | X86pmuludq, 1>; |
Elena Demikhovsky | 172a27c | 2014-01-08 10:54:22 +0000 | [diff] [blame] | 3174 | |
Elena Demikhovsky | 2557a22 | 2015-05-04 09:14:02 +0000 | [diff] [blame] | 3175 | multiclass avx512_packs_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3176 | X86VectorVTInfo _Src, X86VectorVTInfo _Dst> { |
| 3177 | let mayLoad = 1 in { |
| 3178 | defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst), |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3179 | (ins _Src.RC:$src1, _Src.ScalarMemOp:$src2), |
Elena Demikhovsky | 2557a22 | 2015-05-04 09:14:02 +0000 | [diff] [blame] | 3180 | OpcodeStr, |
| 3181 | "${src2}"##_Src.BroadcastStr##", $src1", |
| 3182 | "$src1, ${src2}"##_Src.BroadcastStr, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3183 | (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert |
| 3184 | (_Src.VT (X86VBroadcast |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3185 | (_Src.ScalarLdFrag addr:$src2))))))>, |
Elena Demikhovsky | 2557a22 | 2015-05-04 09:14:02 +0000 | [diff] [blame] | 3186 | EVEX_4V, EVEX_B, EVEX_CD8<_Src.EltSize, CD8VF>; |
| 3187 | } |
| 3188 | } |
| 3189 | |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3190 | multiclass avx512_packs_rm<bits<8> opc, string OpcodeStr, |
| 3191 | SDNode OpNode,X86VectorVTInfo _Src, |
Elena Demikhovsky | 2557a22 | 2015-05-04 09:14:02 +0000 | [diff] [blame] | 3192 | X86VectorVTInfo _Dst> { |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3193 | defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst), |
Elena Demikhovsky | 2557a22 | 2015-05-04 09:14:02 +0000 | [diff] [blame] | 3194 | (ins _Src.RC:$src1, _Src.RC:$src2), OpcodeStr, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3195 | "$src2, $src1","$src1, $src2", |
| 3196 | (_Dst.VT (OpNode |
| 3197 | (_Src.VT _Src.RC:$src1), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3198 | (_Src.VT _Src.RC:$src2)))>, |
| 3199 | EVEX_CD8<_Src.EltSize, CD8VF>, EVEX_4V; |
Elena Demikhovsky | 2557a22 | 2015-05-04 09:14:02 +0000 | [diff] [blame] | 3200 | let mayLoad = 1 in { |
| 3201 | defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst), |
| 3202 | (ins _Src.RC:$src1, _Src.MemOp:$src2), OpcodeStr, |
| 3203 | "$src2, $src1", "$src1, $src2", |
| 3204 | (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3205 | (bitconvert (_Src.LdFrag addr:$src2))))>, |
| 3206 | EVEX_4V, EVEX_CD8<_Src.EltSize, CD8VF>; |
Elena Demikhovsky | 2557a22 | 2015-05-04 09:14:02 +0000 | [diff] [blame] | 3207 | } |
| 3208 | } |
| 3209 | |
| 3210 | multiclass avx512_packs_all_i32_i16<bits<8> opc, string OpcodeStr, |
| 3211 | SDNode OpNode> { |
| 3212 | defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i32_info, |
| 3213 | v32i16_info>, |
| 3214 | avx512_packs_rmb<opc, OpcodeStr, OpNode, v16i32_info, |
| 3215 | v32i16_info>, EVEX_V512; |
| 3216 | let Predicates = [HasVLX] in { |
| 3217 | defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i32x_info, |
| 3218 | v16i16x_info>, |
| 3219 | avx512_packs_rmb<opc, OpcodeStr, OpNode, v8i32x_info, |
| 3220 | v16i16x_info>, EVEX_V256; |
| 3221 | defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v4i32x_info, |
| 3222 | v8i16x_info>, |
| 3223 | avx512_packs_rmb<opc, OpcodeStr, OpNode, v4i32x_info, |
| 3224 | v8i16x_info>, EVEX_V128; |
| 3225 | } |
| 3226 | } |
| 3227 | multiclass avx512_packs_all_i16_i8<bits<8> opc, string OpcodeStr, |
| 3228 | SDNode OpNode> { |
| 3229 | defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v32i16_info, |
| 3230 | v64i8_info>, EVEX_V512; |
| 3231 | let Predicates = [HasVLX] in { |
| 3232 | defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i16x_info, |
| 3233 | v32i8x_info>, EVEX_V256; |
| 3234 | defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i16x_info, |
| 3235 | v16i8x_info>, EVEX_V128; |
| 3236 | } |
| 3237 | } |
Igor Breger | f7fd547 | 2015-07-21 07:11:28 +0000 | [diff] [blame] | 3238 | |
| 3239 | multiclass avx512_vpmadd<bits<8> opc, string OpcodeStr, |
| 3240 | SDNode OpNode, AVX512VLVectorVTInfo _Src, |
| 3241 | AVX512VLVectorVTInfo _Dst> { |
| 3242 | defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info512, |
| 3243 | _Dst.info512>, EVEX_V512; |
| 3244 | let Predicates = [HasVLX] in { |
| 3245 | defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info256, |
| 3246 | _Dst.info256>, EVEX_V256; |
| 3247 | defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info128, |
| 3248 | _Dst.info128>, EVEX_V128; |
| 3249 | } |
| 3250 | } |
| 3251 | |
Elena Demikhovsky | 2557a22 | 2015-05-04 09:14:02 +0000 | [diff] [blame] | 3252 | let Predicates = [HasBWI] in { |
| 3253 | defm VPACKSSDW : avx512_packs_all_i32_i16<0x6B, "vpackssdw", X86Packss>, PD; |
| 3254 | defm VPACKUSDW : avx512_packs_all_i32_i16<0x2b, "vpackusdw", X86Packus>, T8PD; |
| 3255 | defm VPACKSSWB : avx512_packs_all_i16_i8 <0x63, "vpacksswb", X86Packss>, AVX512BIBase, VEX_W; |
| 3256 | defm VPACKUSWB : avx512_packs_all_i16_i8 <0x67, "vpackuswb", X86Packus>, AVX512BIBase, VEX_W; |
Igor Breger | f7fd547 | 2015-07-21 07:11:28 +0000 | [diff] [blame] | 3257 | |
| 3258 | defm VPMADDUBSW : avx512_vpmadd<0x04, "vpmaddubsw", X86vpmaddubsw, |
| 3259 | avx512vl_i8_info, avx512vl_i16_info>, AVX512BIBase, T8PD; |
| 3260 | defm VPMADDWD : avx512_vpmadd<0xF5, "vpmaddwd", X86vpmaddwd, |
| 3261 | avx512vl_i16_info, avx512vl_i32_info>, AVX512BIBase; |
Elena Demikhovsky | 2557a22 | 2015-05-04 09:14:02 +0000 | [diff] [blame] | 3262 | } |
| 3263 | |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3264 | defm VPMAXSB : avx512_binop_rm_vl_b<0x3C, "vpmaxsb", smax, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3265 | SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD; |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3266 | defm VPMAXSW : avx512_binop_rm_vl_w<0xEE, "vpmaxsw", smax, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3267 | SSE_INTALU_ITINS_P, HasBWI, 1>; |
Simon Pilgrim | 8b75659 | 2015-07-06 20:30:47 +0000 | [diff] [blame] | 3268 | defm VPMAXS : avx512_binop_rm_vl_dq<0x3D, 0x3D, "vpmaxs", smax, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3269 | SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; |
Elena Demikhovsky | 199c823 | 2013-10-27 08:18:37 +0000 | [diff] [blame] | 3270 | |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3271 | defm VPMAXUB : avx512_binop_rm_vl_b<0xDE, "vpmaxub", umax, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3272 | SSE_INTALU_ITINS_P, HasBWI, 1>; |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3273 | defm VPMAXUW : avx512_binop_rm_vl_w<0x3E, "vpmaxuw", umax, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3274 | SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD; |
Simon Pilgrim | 8b75659 | 2015-07-06 20:30:47 +0000 | [diff] [blame] | 3275 | defm VPMAXU : avx512_binop_rm_vl_dq<0x3F, 0x3F, "vpmaxu", umax, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3276 | SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; |
Elena Demikhovsky | 199c823 | 2013-10-27 08:18:37 +0000 | [diff] [blame] | 3277 | |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3278 | defm VPMINSB : avx512_binop_rm_vl_b<0x38, "vpminsb", smin, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3279 | SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD; |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3280 | defm VPMINSW : avx512_binop_rm_vl_w<0xEA, "vpminsw", smin, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3281 | SSE_INTALU_ITINS_P, HasBWI, 1>; |
Simon Pilgrim | 8b75659 | 2015-07-06 20:30:47 +0000 | [diff] [blame] | 3282 | defm VPMINS : avx512_binop_rm_vl_dq<0x39, 0x39, "vpmins", smin, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3283 | SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; |
Elena Demikhovsky | 199c823 | 2013-10-27 08:18:37 +0000 | [diff] [blame] | 3284 | |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3285 | defm VPMINUB : avx512_binop_rm_vl_b<0xDA, "vpminub", umin, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3286 | SSE_INTALU_ITINS_P, HasBWI, 1>; |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 3287 | defm VPMINUW : avx512_binop_rm_vl_w<0x3A, "vpminuw", umin, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3288 | SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD; |
Simon Pilgrim | 8b75659 | 2015-07-06 20:30:47 +0000 | [diff] [blame] | 3289 | defm VPMINU : avx512_binop_rm_vl_dq<0x3B, 0x3B, "vpminu", umin, |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3290 | SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3291 | //===----------------------------------------------------------------------===// |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3292 | // AVX-512 Logical Instructions |
| 3293 | //===----------------------------------------------------------------------===// |
| 3294 | |
Robert Khasanov | 545d1b7 | 2014-10-14 14:36:19 +0000 | [diff] [blame] | 3295 | defm VPAND : avx512_binop_rm_vl_dq<0xDB, 0xDB, "vpand", and, |
| 3296 | SSE_INTALU_ITINS_P, HasAVX512, 1>; |
| 3297 | defm VPOR : avx512_binop_rm_vl_dq<0xEB, 0xEB, "vpor", or, |
| 3298 | SSE_INTALU_ITINS_P, HasAVX512, 1>; |
| 3299 | defm VPXOR : avx512_binop_rm_vl_dq<0xEF, 0xEF, "vpxor", xor, |
| 3300 | SSE_INTALU_ITINS_P, HasAVX512, 1>; |
| 3301 | defm VPANDN : avx512_binop_rm_vl_dq<0xDF, 0xDF, "vpandn", X86andnp, |
Elena Demikhovsky | 72e3ccc | 2015-03-29 09:14:29 +0000 | [diff] [blame] | 3302 | SSE_INTALU_ITINS_P, HasAVX512, 0>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3303 | |
| 3304 | //===----------------------------------------------------------------------===// |
| 3305 | // AVX-512 FP arithmetic |
| 3306 | //===----------------------------------------------------------------------===// |
Elena Demikhovsky | 02ffd26 | 2015-03-01 07:44:04 +0000 | [diff] [blame] | 3307 | multiclass avx512_fp_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _, |
| 3308 | SDNode OpNode, SDNode VecNode, OpndItins itins, |
| 3309 | bit IsCommutable> { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3310 | |
Elena Demikhovsky | 02ffd26 | 2015-03-01 07:44:04 +0000 | [diff] [blame] | 3311 | defm rr_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 3312 | (ins _.RC:$src1, _.RC:$src2), OpcodeStr, |
| 3313 | "$src2, $src1", "$src1, $src2", |
| 3314 | (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), |
| 3315 | (i32 FROUND_CURRENT)), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3316 | itins.rr, IsCommutable>; |
Elena Demikhovsky | 02ffd26 | 2015-03-01 07:44:04 +0000 | [diff] [blame] | 3317 | |
| 3318 | defm rm_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 3319 | (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr, |
| 3320 | "$src2, $src1", "$src1, $src2", |
| 3321 | (VecNode (_.VT _.RC:$src1), |
| 3322 | (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))), |
| 3323 | (i32 FROUND_CURRENT)), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3324 | itins.rm, IsCommutable>; |
Elena Demikhovsky | 02ffd26 | 2015-03-01 07:44:04 +0000 | [diff] [blame] | 3325 | let isCodeGenOnly = 1, isCommutable = IsCommutable, |
| 3326 | Predicates = [HasAVX512] in { |
| 3327 | def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst), |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3328 | (ins _.FRC:$src1, _.FRC:$src2), |
Elena Demikhovsky | 02ffd26 | 2015-03-01 07:44:04 +0000 | [diff] [blame] | 3329 | OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", |
| 3330 | [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))], |
| 3331 | itins.rr>; |
| 3332 | def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst), |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3333 | (ins _.FRC:$src1, _.ScalarMemOp:$src2), |
Elena Demikhovsky | 02ffd26 | 2015-03-01 07:44:04 +0000 | [diff] [blame] | 3334 | OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", |
| 3335 | [(set _.FRC:$dst, (OpNode _.FRC:$src1, |
| 3336 | (_.ScalarLdFrag addr:$src2)))], itins.rr>; |
| 3337 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3338 | } |
| 3339 | |
Elena Demikhovsky | 02ffd26 | 2015-03-01 07:44:04 +0000 | [diff] [blame] | 3340 | multiclass avx512_fp_scalar_round<bits<8> opc, string OpcodeStr,X86VectorVTInfo _, |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 3341 | SDNode VecNode, OpndItins itins, bit IsCommutable = 0> { |
Elena Demikhovsky | 02ffd26 | 2015-03-01 07:44:04 +0000 | [diff] [blame] | 3342 | |
| 3343 | defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 3344 | (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr, |
| 3345 | "$rc, $src2, $src1", "$src1, $src2, $rc", |
| 3346 | (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3347 | (i32 imm:$rc)), itins.rr, IsCommutable>, |
Elena Demikhovsky | 02ffd26 | 2015-03-01 07:44:04 +0000 | [diff] [blame] | 3348 | EVEX_B, EVEX_RC; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3349 | } |
Elena Demikhovsky | 02ffd26 | 2015-03-01 07:44:04 +0000 | [diff] [blame] | 3350 | multiclass avx512_fp_scalar_sae<bits<8> opc, string OpcodeStr,X86VectorVTInfo _, |
| 3351 | SDNode VecNode, OpndItins itins, bit IsCommutable> { |
| 3352 | |
| 3353 | defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 3354 | (ins _.RC:$src1, _.RC:$src2), OpcodeStr, |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3355 | "{sae}, $src2, $src1", "$src1, $src2, {sae}", |
Elena Demikhovsky | 02ffd26 | 2015-03-01 07:44:04 +0000 | [diff] [blame] | 3356 | (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3357 | (i32 FROUND_NO_EXC))>, EVEX_B; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3358 | } |
| 3359 | |
Elena Demikhovsky | 02ffd26 | 2015-03-01 07:44:04 +0000 | [diff] [blame] | 3360 | multiclass avx512_binop_s_round<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3361 | SDNode VecNode, |
| 3362 | SizeItins itins, bit IsCommutable> { |
| 3363 | defm SSZ : avx512_fp_scalar<opc, OpcodeStr#"ss", f32x_info, OpNode, VecNode, |
| 3364 | itins.s, IsCommutable>, |
| 3365 | avx512_fp_scalar_round<opc, OpcodeStr#"ss", f32x_info, VecNode, |
| 3366 | itins.s, IsCommutable>, |
| 3367 | XS, EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>; |
| 3368 | defm SDZ : avx512_fp_scalar<opc, OpcodeStr#"sd", f64x_info, OpNode, VecNode, |
| 3369 | itins.d, IsCommutable>, |
| 3370 | avx512_fp_scalar_round<opc, OpcodeStr#"sd", f64x_info, VecNode, |
| 3371 | itins.d, IsCommutable>, |
| 3372 | XD, VEX_W, EVEX_4V, VEX_LIG, EVEX_CD8<64, CD8VT1>; |
| 3373 | } |
| 3374 | |
| 3375 | multiclass avx512_binop_s_sae<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3376 | SDNode VecNode, |
| 3377 | SizeItins itins, bit IsCommutable> { |
| 3378 | defm SSZ : avx512_fp_scalar<opc, OpcodeStr#"ss", f32x_info, OpNode, VecNode, |
| 3379 | itins.s, IsCommutable>, |
| 3380 | avx512_fp_scalar_sae<opc, OpcodeStr#"ss", f32x_info, VecNode, |
| 3381 | itins.s, IsCommutable>, |
| 3382 | XS, EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>; |
| 3383 | defm SDZ : avx512_fp_scalar<opc, OpcodeStr#"sd", f64x_info, OpNode, VecNode, |
| 3384 | itins.d, IsCommutable>, |
| 3385 | avx512_fp_scalar_sae<opc, OpcodeStr#"sd", f64x_info, VecNode, |
| 3386 | itins.d, IsCommutable>, |
| 3387 | XD, VEX_W, EVEX_4V, VEX_LIG, EVEX_CD8<64, CD8VT1>; |
| 3388 | } |
| 3389 | defm VADD : avx512_binop_s_round<0x58, "vadd", fadd, X86faddRnd, SSE_ALU_ITINS_S, 1>; |
| 3390 | defm VMUL : avx512_binop_s_round<0x59, "vmul", fmul, X86fmulRnd, SSE_ALU_ITINS_S, 1>; |
| 3391 | defm VSUB : avx512_binop_s_round<0x5C, "vsub", fsub, X86fsubRnd, SSE_ALU_ITINS_S, 0>; |
| 3392 | defm VDIV : avx512_binop_s_round<0x5E, "vdiv", fdiv, X86fdivRnd, SSE_ALU_ITINS_S, 0>; |
| 3393 | defm VMIN : avx512_binop_s_sae <0x5D, "vmin", X86fmin, X86fminRnd, SSE_ALU_ITINS_S, 1>; |
| 3394 | defm VMAX : avx512_binop_s_sae <0x5F, "vmax", X86fmax, X86fmaxRnd, SSE_ALU_ITINS_S, 1>; |
| 3395 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3396 | multiclass avx512_fp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode, |
Robert Khasanov | 595e598 | 2014-10-29 15:43:02 +0000 | [diff] [blame] | 3397 | X86VectorVTInfo _, bit IsCommutable> { |
| 3398 | defm rr: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 3399 | (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix, |
| 3400 | "$src2, $src1", "$src1, $src2", |
| 3401 | (_.VT (OpNode _.RC:$src1, _.RC:$src2))>, EVEX_4V; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3402 | let mayLoad = 1 in { |
Robert Khasanov | 595e598 | 2014-10-29 15:43:02 +0000 | [diff] [blame] | 3403 | defm rm: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 3404 | (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix, |
| 3405 | "$src2, $src1", "$src1, $src2", |
| 3406 | (OpNode _.RC:$src1, (_.LdFrag addr:$src2))>, EVEX_4V; |
| 3407 | defm rmb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 3408 | (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix, |
| 3409 | "${src2}"##_.BroadcastStr##", $src1", |
| 3410 | "$src1, ${src2}"##_.BroadcastStr, |
| 3411 | (OpNode _.RC:$src1, (_.VT (X86VBroadcast |
| 3412 | (_.ScalarLdFrag addr:$src2))))>, |
| 3413 | EVEX_4V, EVEX_B; |
| 3414 | }//let mayLoad = 1 |
| 3415 | } |
Elena Demikhovsky | f7c1b16 | 2014-03-06 08:45:30 +0000 | [diff] [blame] | 3416 | |
Elena Demikhovsky | 714f23b | 2015-02-18 07:59:20 +0000 | [diff] [blame] | 3417 | multiclass avx512_fp_round_packed<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd, |
Asaf Badouh | 7ec4b7a | 2015-06-28 14:30:39 +0000 | [diff] [blame] | 3418 | X86VectorVTInfo _> { |
Elena Demikhovsky | 714f23b | 2015-02-18 07:59:20 +0000 | [diff] [blame] | 3419 | defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 3420 | (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr##_.Suffix, |
| 3421 | "$rc, $src2, $src1", "$src1, $src2, $rc", |
| 3422 | (_.VT (OpNodeRnd _.RC:$src1, _.RC:$src2, (i32 imm:$rc)))>, |
| 3423 | EVEX_4V, EVEX_B, EVEX_RC; |
| 3424 | } |
| 3425 | |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3426 | |
| 3427 | multiclass avx512_fp_sae_packed<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd, |
Asaf Badouh | 7ec4b7a | 2015-06-28 14:30:39 +0000 | [diff] [blame] | 3428 | X86VectorVTInfo _> { |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3429 | defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 3430 | (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix, |
| 3431 | "{sae}, $src2, $src1", "$src1, $src2, {sae}", |
| 3432 | (_.VT (OpNodeRnd _.RC:$src1, _.RC:$src2, (i32 FROUND_NO_EXC)))>, |
| 3433 | EVEX_4V, EVEX_B; |
| 3434 | } |
| 3435 | |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3436 | multiclass avx512_fp_binop_p<bits<8> opc, string OpcodeStr, SDNode OpNode, |
Robert Khasanov | 595e598 | 2014-10-29 15:43:02 +0000 | [diff] [blame] | 3437 | bit IsCommutable = 0> { |
| 3438 | defm PSZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v16f32_info, |
| 3439 | IsCommutable>, EVEX_V512, PS, |
| 3440 | EVEX_CD8<32, CD8VF>; |
| 3441 | defm PDZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v8f64_info, |
| 3442 | IsCommutable>, EVEX_V512, PD, VEX_W, |
| 3443 | EVEX_CD8<64, CD8VF>; |
Elena Demikhovsky | f7c1b16 | 2014-03-06 08:45:30 +0000 | [diff] [blame] | 3444 | |
Robert Khasanov | 595e598 | 2014-10-29 15:43:02 +0000 | [diff] [blame] | 3445 | // Define only if AVX512VL feature is present. |
| 3446 | let Predicates = [HasVLX] in { |
| 3447 | defm PSZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, v4f32x_info, |
| 3448 | IsCommutable>, EVEX_V128, PS, |
| 3449 | EVEX_CD8<32, CD8VF>; |
| 3450 | defm PSZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, v8f32x_info, |
| 3451 | IsCommutable>, EVEX_V256, PS, |
| 3452 | EVEX_CD8<32, CD8VF>; |
| 3453 | defm PDZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, v2f64x_info, |
| 3454 | IsCommutable>, EVEX_V128, PD, VEX_W, |
| 3455 | EVEX_CD8<64, CD8VF>; |
| 3456 | defm PDZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, v4f64x_info, |
| 3457 | IsCommutable>, EVEX_V256, PD, VEX_W, |
| 3458 | EVEX_CD8<64, CD8VF>; |
Elena Demikhovsky | f7c1b16 | 2014-03-06 08:45:30 +0000 | [diff] [blame] | 3459 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3460 | } |
| 3461 | |
Elena Demikhovsky | 714f23b | 2015-02-18 07:59:20 +0000 | [diff] [blame] | 3462 | multiclass avx512_fp_binop_p_round<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd> { |
Asaf Badouh | 7ec4b7a | 2015-06-28 14:30:39 +0000 | [diff] [blame] | 3463 | defm PSZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, v16f32_info>, |
Elena Demikhovsky | 714f23b | 2015-02-18 07:59:20 +0000 | [diff] [blame] | 3464 | EVEX_V512, PS, EVEX_CD8<32, CD8VF>; |
Asaf Badouh | 7ec4b7a | 2015-06-28 14:30:39 +0000 | [diff] [blame] | 3465 | defm PDZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, v8f64_info>, |
Elena Demikhovsky | 714f23b | 2015-02-18 07:59:20 +0000 | [diff] [blame] | 3466 | EVEX_V512, PD, VEX_W,EVEX_CD8<64, CD8VF>; |
| 3467 | } |
| 3468 | |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3469 | multiclass avx512_fp_binop_p_sae<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd> { |
Asaf Badouh | 7ec4b7a | 2015-06-28 14:30:39 +0000 | [diff] [blame] | 3470 | defm PSZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, v16f32_info>, |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3471 | EVEX_V512, PS, EVEX_CD8<32, CD8VF>; |
Asaf Badouh | 7ec4b7a | 2015-06-28 14:30:39 +0000 | [diff] [blame] | 3472 | defm PDZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, v8f64_info>, |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3473 | EVEX_V512, PD, VEX_W,EVEX_CD8<64, CD8VF>; |
| 3474 | } |
| 3475 | |
Elena Demikhovsky | 714f23b | 2015-02-18 07:59:20 +0000 | [diff] [blame] | 3476 | defm VADD : avx512_fp_binop_p<0x58, "vadd", fadd, 1>, |
| 3477 | avx512_fp_binop_p_round<0x58, "vadd", X86faddRnd>; |
| 3478 | defm VMUL : avx512_fp_binop_p<0x59, "vmul", fmul, 1>, |
| 3479 | avx512_fp_binop_p_round<0x59, "vmul", X86fmulRnd>; |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3480 | defm VSUB : avx512_fp_binop_p<0x5C, "vsub", fsub>, |
Elena Demikhovsky | 714f23b | 2015-02-18 07:59:20 +0000 | [diff] [blame] | 3481 | avx512_fp_binop_p_round<0x5C, "vsub", X86fsubRnd>; |
| 3482 | defm VDIV : avx512_fp_binop_p<0x5E, "vdiv", fdiv>, |
| 3483 | avx512_fp_binop_p_round<0x5E, "vdiv", X86fdivRnd>; |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3484 | defm VMIN : avx512_fp_binop_p<0x5D, "vmin", X86fmin, 1>, |
| 3485 | avx512_fp_binop_p_sae<0x5D, "vmin", X86fminRnd>; |
| 3486 | defm VMAX : avx512_fp_binop_p<0x5F, "vmax", X86fmax, 1>, |
| 3487 | avx512_fp_binop_p_sae<0x5F, "vmax", X86fmaxRnd>; |
Elena Demikhovsky | 50b88dd | 2015-04-21 10:27:40 +0000 | [diff] [blame] | 3488 | let Predicates = [HasDQI] in { |
| 3489 | defm VAND : avx512_fp_binop_p<0x54, "vand", X86fand, 1>; |
| 3490 | defm VANDN : avx512_fp_binop_p<0x55, "vandn", X86fandn, 0>; |
| 3491 | defm VOR : avx512_fp_binop_p<0x56, "vor", X86for, 1>; |
| 3492 | defm VXOR : avx512_fp_binop_p<0x57, "vxor", X86fxor, 1>; |
| 3493 | } |
Elena Demikhovsky | 52e4a0e | 2014-01-05 10:46:09 +0000 | [diff] [blame] | 3494 | |
Asaf Badouh | 7ec4b7a | 2015-06-28 14:30:39 +0000 | [diff] [blame] | 3495 | multiclass avx512_fp_scalef_p<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3496 | X86VectorVTInfo _> { |
| 3497 | defm rr: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 3498 | (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix, |
| 3499 | "$src2, $src1", "$src1, $src2", |
| 3500 | (_.VT (OpNode _.RC:$src1, _.RC:$src2, (i32 FROUND_CURRENT)))>, EVEX_4V; |
| 3501 | let mayLoad = 1 in { |
| 3502 | defm rm: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 3503 | (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix, |
| 3504 | "$src2, $src1", "$src1, $src2", |
| 3505 | (OpNode _.RC:$src1, (_.LdFrag addr:$src2), (i32 FROUND_CURRENT))>, EVEX_4V; |
| 3506 | defm rmb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 3507 | (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix, |
| 3508 | "${src2}"##_.BroadcastStr##", $src1", |
| 3509 | "$src1, ${src2}"##_.BroadcastStr, |
| 3510 | (OpNode _.RC:$src1, (_.VT (X86VBroadcast |
| 3511 | (_.ScalarLdFrag addr:$src2))), (i32 FROUND_CURRENT))>, |
| 3512 | EVEX_4V, EVEX_B; |
| 3513 | }//let mayLoad = 1 |
| 3514 | } |
| 3515 | |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 3516 | multiclass avx512_fp_scalef_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3517 | X86VectorVTInfo _> { |
| 3518 | defm rr: AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 3519 | (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix, |
| 3520 | "$src2, $src1", "$src1, $src2", |
| 3521 | (_.VT (OpNode _.RC:$src1, _.RC:$src2, (i32 FROUND_CURRENT)))>; |
| 3522 | let mayLoad = 1 in { |
| 3523 | defm rm: AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 3524 | (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix, |
| 3525 | "$src2, $src1", "$src1, $src2", |
| 3526 | (OpNode _.RC:$src1, (_.LdFrag addr:$src2), (i32 FROUND_CURRENT))>; |
| 3527 | }//let mayLoad = 1 |
| 3528 | } |
| 3529 | |
| 3530 | multiclass avx512_fp_scalef_all<bits<8> opc, bits<8> opcScaler, string OpcodeStr, SDNode OpNode> { |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3531 | defm PSZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v16f32_info>, |
Asaf Badouh | 7ec4b7a | 2015-06-28 14:30:39 +0000 | [diff] [blame] | 3532 | avx512_fp_round_packed<opc, OpcodeStr, OpNode, v16f32_info>, |
| 3533 | EVEX_V512, EVEX_CD8<32, CD8VF>; |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3534 | defm PDZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v8f64_info>, |
Asaf Badouh | 7ec4b7a | 2015-06-28 14:30:39 +0000 | [diff] [blame] | 3535 | avx512_fp_round_packed<opc, OpcodeStr, OpNode, v8f64_info>, |
| 3536 | EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 3537 | defm SSZ128 : avx512_fp_scalef_scalar<opcScaler, OpcodeStr, OpNode, f32x_info>, |
| 3538 | avx512_fp_scalar_round<opcScaler, OpcodeStr##"ss", f32x_info, OpNode, SSE_ALU_ITINS_S.s>, |
| 3539 | EVEX_4V,EVEX_CD8<32, CD8VT1>; |
| 3540 | defm SDZ128 : avx512_fp_scalef_scalar<opcScaler, OpcodeStr, OpNode, f64x_info>, |
| 3541 | avx512_fp_scalar_round<opcScaler, OpcodeStr##"sd", f64x_info, OpNode, SSE_ALU_ITINS_S.d>, |
| 3542 | EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W; |
| 3543 | |
Asaf Badouh | 7ec4b7a | 2015-06-28 14:30:39 +0000 | [diff] [blame] | 3544 | // Define only if AVX512VL feature is present. |
| 3545 | let Predicates = [HasVLX] in { |
| 3546 | defm PSZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v4f32x_info>, |
| 3547 | EVEX_V128, EVEX_CD8<32, CD8VF>; |
| 3548 | defm PSZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v8f32x_info>, |
| 3549 | EVEX_V256, EVEX_CD8<32, CD8VF>; |
| 3550 | defm PDZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v2f64x_info>, |
| 3551 | EVEX_V128, VEX_W, EVEX_CD8<64, CD8VF>; |
| 3552 | defm PDZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v4f64x_info>, |
| 3553 | EVEX_V256, VEX_W, EVEX_CD8<64, CD8VF>; |
| 3554 | } |
| 3555 | } |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 3556 | defm VSCALEF : avx512_fp_scalef_all<0x2C, 0x2D, "vscalef", X86scalef>, T8PD; |
Asaf Badouh | 7ec4b7a | 2015-06-28 14:30:39 +0000 | [diff] [blame] | 3557 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3558 | //===----------------------------------------------------------------------===// |
| 3559 | // AVX-512 VPTESTM instructions |
| 3560 | //===----------------------------------------------------------------------===// |
| 3561 | |
Elena Demikhovsky | 431b81e | 2015-04-21 13:13:46 +0000 | [diff] [blame] | 3562 | multiclass avx512_vptest<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3563 | X86VectorVTInfo _> { |
| 3564 | defm rr : AVX512_maskable_cmp<opc, MRMSrcReg, _, (outs _.KRC:$dst), |
| 3565 | (ins _.RC:$src1, _.RC:$src2), OpcodeStr, |
| 3566 | "$src2, $src1", "$src1, $src2", |
| 3567 | (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>, |
| 3568 | EVEX_4V; |
| 3569 | let mayLoad = 1 in |
| 3570 | defm rm : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst), |
| 3571 | (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr, |
| 3572 | "$src2, $src1", "$src1, $src2", |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3573 | (OpNode (_.VT _.RC:$src1), |
Elena Demikhovsky | 431b81e | 2015-04-21 13:13:46 +0000 | [diff] [blame] | 3574 | (_.VT (bitconvert (_.LdFrag addr:$src2))))>, |
| 3575 | EVEX_4V, |
| 3576 | EVEX_CD8<_.EltSize, CD8VF>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3577 | } |
| 3578 | |
Elena Demikhovsky | 431b81e | 2015-04-21 13:13:46 +0000 | [diff] [blame] | 3579 | multiclass avx512_vptest_mb<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3580 | X86VectorVTInfo _> { |
| 3581 | let mayLoad = 1 in |
| 3582 | defm rmb : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst), |
| 3583 | (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr, |
| 3584 | "${src2}"##_.BroadcastStr##", $src1", |
| 3585 | "$src1, ${src2}"##_.BroadcastStr, |
| 3586 | (OpNode (_.VT _.RC:$src1), (_.VT (X86VBroadcast |
| 3587 | (_.ScalarLdFrag addr:$src2))))>, |
| 3588 | EVEX_B, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>; |
Elena Demikhovsky | a30e437 | 2014-02-05 07:05:03 +0000 | [diff] [blame] | 3589 | } |
Elena Demikhovsky | 431b81e | 2015-04-21 13:13:46 +0000 | [diff] [blame] | 3590 | multiclass avx512_vptest_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3591 | AVX512VLVectorVTInfo _> { |
| 3592 | let Predicates = [HasAVX512] in |
| 3593 | defm Z : avx512_vptest<opc, OpcodeStr, OpNode, _.info512>, |
| 3594 | avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512; |
| 3595 | |
| 3596 | let Predicates = [HasAVX512, HasVLX] in { |
| 3597 | defm Z256 : avx512_vptest<opc, OpcodeStr, OpNode, _.info256>, |
| 3598 | avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256; |
| 3599 | defm Z128 : avx512_vptest<opc, OpcodeStr, OpNode, _.info128>, |
| 3600 | avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128; |
| 3601 | } |
| 3602 | } |
| 3603 | |
| 3604 | multiclass avx512_vptest_dq<bits<8> opc, string OpcodeStr, SDNode OpNode> { |
| 3605 | defm D : avx512_vptest_dq_sizes<opc, OpcodeStr#"d", OpNode, |
| 3606 | avx512vl_i32_info>; |
| 3607 | defm Q : avx512_vptest_dq_sizes<opc, OpcodeStr#"q", OpNode, |
| 3608 | avx512vl_i64_info>, VEX_W; |
| 3609 | } |
| 3610 | |
| 3611 | multiclass avx512_vptest_wb<bits<8> opc, string OpcodeStr, |
| 3612 | SDNode OpNode> { |
| 3613 | let Predicates = [HasBWI] in { |
| 3614 | defm WZ: avx512_vptest<opc, OpcodeStr#"w", OpNode, v32i16_info>, |
| 3615 | EVEX_V512, VEX_W; |
| 3616 | defm BZ: avx512_vptest<opc, OpcodeStr#"b", OpNode, v64i8_info>, |
| 3617 | EVEX_V512; |
| 3618 | } |
| 3619 | let Predicates = [HasVLX, HasBWI] in { |
| 3620 | |
| 3621 | defm WZ256: avx512_vptest<opc, OpcodeStr#"w", OpNode, v16i16x_info>, |
| 3622 | EVEX_V256, VEX_W; |
| 3623 | defm WZ128: avx512_vptest<opc, OpcodeStr#"w", OpNode, v8i16x_info>, |
| 3624 | EVEX_V128, VEX_W; |
| 3625 | defm BZ256: avx512_vptest<opc, OpcodeStr#"b", OpNode, v32i8x_info>, |
| 3626 | EVEX_V256; |
| 3627 | defm BZ128: avx512_vptest<opc, OpcodeStr#"b", OpNode, v16i8x_info>, |
| 3628 | EVEX_V128; |
| 3629 | } |
| 3630 | } |
| 3631 | |
| 3632 | multiclass avx512_vptest_all_forms<bits<8> opc_wb, bits<8> opc_dq, string OpcodeStr, |
| 3633 | SDNode OpNode> : |
| 3634 | avx512_vptest_wb <opc_wb, OpcodeStr, OpNode>, |
| 3635 | avx512_vptest_dq<opc_dq, OpcodeStr, OpNode>; |
| 3636 | |
| 3637 | defm VPTESTM : avx512_vptest_all_forms<0x26, 0x27, "vptestm", X86testm>, T8PD; |
| 3638 | defm VPTESTNM : avx512_vptest_all_forms<0x26, 0x27, "vptestnm", X86testnm>, T8XS; |
Elena Demikhovsky | a30e437 | 2014-02-05 07:05:03 +0000 | [diff] [blame] | 3639 | |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 3640 | def : Pat <(i16 (int_x86_avx512_mask_ptestm_d_512 (v16i32 VR512:$src1), |
| 3641 | (v16i32 VR512:$src2), (i16 -1))), |
| 3642 | (COPY_TO_REGCLASS (VPTESTMDZrr VR512:$src1, VR512:$src2), GR16)>; |
| 3643 | |
| 3644 | def : Pat <(i8 (int_x86_avx512_mask_ptestm_q_512 (v8i64 VR512:$src1), |
| 3645 | (v8i64 VR512:$src2), (i8 -1))), |
Elena Demikhovsky | 3ebfe11 | 2014-02-23 14:28:35 +0000 | [diff] [blame] | 3646 | (COPY_TO_REGCLASS (VPTESTMQZrr VR512:$src1, VR512:$src2), GR8)>; |
Cameron McInally | 9b7c15a | 2014-11-25 20:41:51 +0000 | [diff] [blame] | 3647 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3648 | //===----------------------------------------------------------------------===// |
| 3649 | // AVX-512 Shift instructions |
| 3650 | //===----------------------------------------------------------------------===// |
| 3651 | multiclass avx512_shift_rmi<bits<8> opc, Format ImmFormR, Format ImmFormM, |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 3652 | string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { |
Cameron McInally | 0440044 | 2014-11-14 15:43:00 +0000 | [diff] [blame] | 3653 | defm ri : AVX512_maskable<opc, ImmFormR, _, (outs _.RC:$dst), |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 3654 | (ins _.RC:$src1, u8imm:$src2), OpcodeStr, |
Cameron McInally | 0440044 | 2014-11-14 15:43:00 +0000 | [diff] [blame] | 3655 | "$src2, $src1", "$src1, $src2", |
| 3656 | (_.VT (OpNode _.RC:$src1, (i8 imm:$src2))), |
Elena Demikhovsky | 75ede68 | 2015-06-01 07:17:23 +0000 | [diff] [blame] | 3657 | SSE_INTSHIFT_ITINS_P.rr>; |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3658 | let mayLoad = 1 in |
Cameron McInally | 0440044 | 2014-11-14 15:43:00 +0000 | [diff] [blame] | 3659 | defm mi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst), |
Craig Topper | 7ff6ab3 | 2015-01-21 08:43:49 +0000 | [diff] [blame] | 3660 | (ins _.MemOp:$src1, u8imm:$src2), OpcodeStr, |
Cameron McInally | 0440044 | 2014-11-14 15:43:00 +0000 | [diff] [blame] | 3661 | "$src2, $src1", "$src1, $src2", |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3662 | (_.VT (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))), |
| 3663 | (i8 imm:$src2))), |
Elena Demikhovsky | 75ede68 | 2015-06-01 07:17:23 +0000 | [diff] [blame] | 3664 | SSE_INTSHIFT_ITINS_P.rm>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3665 | } |
| 3666 | |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3667 | multiclass avx512_shift_rmbi<bits<8> opc, Format ImmFormM, |
| 3668 | string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { |
| 3669 | let mayLoad = 1 in |
| 3670 | defm mbi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst), |
| 3671 | (ins _.ScalarMemOp:$src1, u8imm:$src2), OpcodeStr, |
| 3672 | "$src2, ${src1}"##_.BroadcastStr, "${src1}"##_.BroadcastStr##", $src2", |
| 3673 | (_.VT (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src1)), (i8 imm:$src2))), |
Elena Demikhovsky | 75ede68 | 2015-06-01 07:17:23 +0000 | [diff] [blame] | 3674 | SSE_INTSHIFT_ITINS_P.rm>, EVEX_B; |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3675 | } |
| 3676 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3677 | multiclass avx512_shift_rrm<bits<8> opc, string OpcodeStr, SDNode OpNode, |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3678 | ValueType SrcVT, PatFrag bc_frag, X86VectorVTInfo _> { |
Cameron McInally | 9b7c15a | 2014-11-25 20:41:51 +0000 | [diff] [blame] | 3679 | // src2 is always 128-bit |
| 3680 | defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 3681 | (ins _.RC:$src1, VR128X:$src2), OpcodeStr, |
| 3682 | "$src2, $src1", "$src1, $src2", |
| 3683 | (_.VT (OpNode _.RC:$src1, (SrcVT VR128X:$src2))), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3684 | SSE_INTSHIFT_ITINS_P.rr>, AVX512BIBase, EVEX_4V; |
Cameron McInally | 9b7c15a | 2014-11-25 20:41:51 +0000 | [diff] [blame] | 3685 | defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 3686 | (ins _.RC:$src1, i128mem:$src2), OpcodeStr, |
| 3687 | "$src2, $src1", "$src1, $src2", |
Craig Topper | 820d492 | 2015-02-09 04:04:50 +0000 | [diff] [blame] | 3688 | (_.VT (OpNode _.RC:$src1, (bc_frag (loadv2i64 addr:$src2)))), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3689 | SSE_INTSHIFT_ITINS_P.rm>, AVX512BIBase, |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3690 | EVEX_4V; |
Cameron McInally | 9b7c15a | 2014-11-25 20:41:51 +0000 | [diff] [blame] | 3691 | } |
| 3692 | |
Cameron McInally | 5fb084e | 2014-12-11 17:13:05 +0000 | [diff] [blame] | 3693 | multiclass avx512_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode, |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3694 | ValueType SrcVT, PatFrag bc_frag, |
| 3695 | AVX512VLVectorVTInfo VTInfo, Predicate prd> { |
| 3696 | let Predicates = [prd] in |
| 3697 | defm Z : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag, |
| 3698 | VTInfo.info512>, EVEX_V512, |
| 3699 | EVEX_CD8<VTInfo.info512.EltSize, CD8VQ> ; |
| 3700 | let Predicates = [prd, HasVLX] in { |
| 3701 | defm Z256 : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag, |
| 3702 | VTInfo.info256>, EVEX_V256, |
| 3703 | EVEX_CD8<VTInfo.info256.EltSize, CD8VH>; |
| 3704 | defm Z128 : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag, |
| 3705 | VTInfo.info128>, EVEX_V128, |
| 3706 | EVEX_CD8<VTInfo.info128.EltSize, CD8VF>; |
| 3707 | } |
Cameron McInally | 9b7c15a | 2014-11-25 20:41:51 +0000 | [diff] [blame] | 3708 | } |
| 3709 | |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3710 | multiclass avx512_shift_types<bits<8> opcd, bits<8> opcq, bits<8> opcw, |
| 3711 | string OpcodeStr, SDNode OpNode> { |
Cameron McInally | 5fb084e | 2014-12-11 17:13:05 +0000 | [diff] [blame] | 3712 | defm D : avx512_shift_sizes<opcd, OpcodeStr#"d", OpNode, v4i32, bc_v4i32, |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3713 | avx512vl_i32_info, HasAVX512>; |
Cameron McInally | 5fb084e | 2014-12-11 17:13:05 +0000 | [diff] [blame] | 3714 | defm Q : avx512_shift_sizes<opcq, OpcodeStr#"q", OpNode, v2i64, bc_v2i64, |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3715 | avx512vl_i64_info, HasAVX512>, VEX_W; |
| 3716 | defm W : avx512_shift_sizes<opcw, OpcodeStr#"w", OpNode, v8i16, bc_v8i16, |
| 3717 | avx512vl_i16_info, HasBWI>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3718 | } |
| 3719 | |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3720 | multiclass avx512_shift_rmi_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM, |
| 3721 | string OpcodeStr, SDNode OpNode, |
| 3722 | AVX512VLVectorVTInfo VTInfo> { |
| 3723 | let Predicates = [HasAVX512] in |
| 3724 | defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode, |
| 3725 | VTInfo.info512>, |
| 3726 | avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, |
| 3727 | VTInfo.info512>, EVEX_V512; |
| 3728 | let Predicates = [HasAVX512, HasVLX] in { |
| 3729 | defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode, |
| 3730 | VTInfo.info256>, |
| 3731 | avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, |
| 3732 | VTInfo.info256>, EVEX_V256; |
| 3733 | defm Z128: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode, |
| 3734 | VTInfo.info128>, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3735 | avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3736 | VTInfo.info128>, EVEX_V128; |
| 3737 | } |
| 3738 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3739 | |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3740 | multiclass avx512_shift_rmi_w<bits<8> opcw, |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3741 | Format ImmFormR, Format ImmFormM, |
| 3742 | string OpcodeStr, SDNode OpNode> { |
| 3743 | let Predicates = [HasBWI] in |
| 3744 | defm WZ: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode, |
| 3745 | v32i16_info>, EVEX_V512; |
| 3746 | let Predicates = [HasVLX, HasBWI] in { |
| 3747 | defm WZ256: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode, |
| 3748 | v16i16x_info>, EVEX_V256; |
| 3749 | defm WZ128: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode, |
| 3750 | v8i16x_info>, EVEX_V128; |
| 3751 | } |
| 3752 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3753 | |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3754 | multiclass avx512_shift_rmi_dq<bits<8> opcd, bits<8> opcq, |
| 3755 | Format ImmFormR, Format ImmFormM, |
| 3756 | string OpcodeStr, SDNode OpNode> { |
| 3757 | defm D: avx512_shift_rmi_sizes<opcd, ImmFormR, ImmFormM, OpcodeStr#"d", OpNode, |
| 3758 | avx512vl_i32_info>, EVEX_CD8<32, CD8VF>; |
| 3759 | defm Q: avx512_shift_rmi_sizes<opcq, ImmFormR, ImmFormM, OpcodeStr#"q", OpNode, |
| 3760 | avx512vl_i64_info>, EVEX_CD8<64, CD8VF>, VEX_W; |
| 3761 | } |
Cameron McInally | 9b7c15a | 2014-11-25 20:41:51 +0000 | [diff] [blame] | 3762 | |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3763 | defm VPSRL : avx512_shift_rmi_dq<0x72, 0x73, MRM2r, MRM2m, "vpsrl", X86vsrli>, |
Elena Demikhovsky | 75ede68 | 2015-06-01 07:17:23 +0000 | [diff] [blame] | 3764 | avx512_shift_rmi_w<0x71, MRM2r, MRM2m, "vpsrlw", X86vsrli>, AVX512BIi8Base, EVEX_4V; |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3765 | |
| 3766 | defm VPSLL : avx512_shift_rmi_dq<0x72, 0x73, MRM6r, MRM6m, "vpsll", X86vshli>, |
Elena Demikhovsky | 75ede68 | 2015-06-01 07:17:23 +0000 | [diff] [blame] | 3767 | avx512_shift_rmi_w<0x71, MRM6r, MRM6m, "vpsllw", X86vshli>, AVX512BIi8Base, EVEX_4V; |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3768 | |
Elena Demikhovsky | 1b2f2f1 | 2015-05-13 07:35:05 +0000 | [diff] [blame] | 3769 | defm VPSRA : avx512_shift_rmi_dq<0x72, 0x72, MRM4r, MRM4m, "vpsra", X86vsrai>, |
Elena Demikhovsky | 75ede68 | 2015-06-01 07:17:23 +0000 | [diff] [blame] | 3770 | avx512_shift_rmi_w<0x71, MRM4r, MRM4m, "vpsraw", X86vsrai>, AVX512BIi8Base, EVEX_4V; |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3771 | |
Elena Demikhovsky | 75ede68 | 2015-06-01 07:17:23 +0000 | [diff] [blame] | 3772 | defm VPROR : avx512_shift_rmi_dq<0x72, 0x72, MRM0r, MRM0m, "vpror", rotr>, AVX512BIi8Base, EVEX_4V; |
| 3773 | defm VPROL : avx512_shift_rmi_dq<0x72, 0x72, MRM1r, MRM1m, "vprol", rotl>, AVX512BIi8Base, EVEX_4V; |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3774 | |
| 3775 | defm VPSLL : avx512_shift_types<0xF2, 0xF3, 0xF1, "vpsll", X86vshl>; |
| 3776 | defm VPSRA : avx512_shift_types<0xE2, 0xE2, 0xE1, "vpsra", X86vsra>; |
| 3777 | defm VPSRL : avx512_shift_types<0xD2, 0xD3, 0xD1, "vpsrl", X86vsrl>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3778 | |
| 3779 | //===-------------------------------------------------------------------===// |
| 3780 | // Variable Bit Shifts |
| 3781 | //===-------------------------------------------------------------------===// |
| 3782 | multiclass avx512_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode, |
Cameron McInally | 5fb084e | 2014-12-11 17:13:05 +0000 | [diff] [blame] | 3783 | X86VectorVTInfo _> { |
| 3784 | defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 3785 | (ins _.RC:$src1, _.RC:$src2), OpcodeStr, |
| 3786 | "$src2, $src1", "$src1, $src2", |
| 3787 | (_.VT (OpNode _.RC:$src1, (_.VT _.RC:$src2))), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3788 | SSE_INTSHIFT_ITINS_P.rr>, AVX5128IBase, EVEX_4V; |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3789 | let mayLoad = 1 in |
Cameron McInally | 5fb084e | 2014-12-11 17:13:05 +0000 | [diff] [blame] | 3790 | defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 3791 | (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr, |
| 3792 | "$src2, $src1", "$src1, $src2", |
Elena Demikhovsky | 4078c75 | 2015-06-04 07:07:13 +0000 | [diff] [blame] | 3793 | (_.VT (OpNode _.RC:$src1, |
| 3794 | (_.VT (bitconvert (_.LdFrag addr:$src2))))), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3795 | SSE_INTSHIFT_ITINS_P.rm>, AVX5128IBase, EVEX_4V, |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3796 | EVEX_CD8<_.EltSize, CD8VF>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3797 | } |
| 3798 | |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3799 | multiclass avx512_var_shift_mb<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3800 | X86VectorVTInfo _> { |
| 3801 | let mayLoad = 1 in |
| 3802 | defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 3803 | (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr, |
| 3804 | "${src2}"##_.BroadcastStr##", $src1", |
| 3805 | "$src1, ${src2}"##_.BroadcastStr, |
| 3806 | (_.VT (OpNode _.RC:$src1, (_.VT (X86VBroadcast |
| 3807 | (_.ScalarLdFrag addr:$src2))))), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 3808 | SSE_INTSHIFT_ITINS_P.rm>, AVX5128IBase, EVEX_B, |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3809 | EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>; |
| 3810 | } |
Cameron McInally | 5fb084e | 2014-12-11 17:13:05 +0000 | [diff] [blame] | 3811 | multiclass avx512_var_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3812 | AVX512VLVectorVTInfo _> { |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3813 | let Predicates = [HasAVX512] in |
| 3814 | defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>, |
| 3815 | avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512; |
| 3816 | |
| 3817 | let Predicates = [HasAVX512, HasVLX] in { |
| 3818 | defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>, |
| 3819 | avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256; |
| 3820 | defm Z128 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info128>, |
| 3821 | avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128; |
| 3822 | } |
Cameron McInally | 5fb084e | 2014-12-11 17:13:05 +0000 | [diff] [blame] | 3823 | } |
| 3824 | |
| 3825 | multiclass avx512_var_shift_types<bits<8> opc, string OpcodeStr, |
| 3826 | SDNode OpNode> { |
| 3827 | defm D : avx512_var_shift_sizes<opc, OpcodeStr#"d", OpNode, |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3828 | avx512vl_i32_info>; |
Cameron McInally | 5fb084e | 2014-12-11 17:13:05 +0000 | [diff] [blame] | 3829 | defm Q : avx512_var_shift_sizes<opc, OpcodeStr#"q", OpNode, |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3830 | avx512vl_i64_info>, VEX_W; |
Cameron McInally | 5fb084e | 2014-12-11 17:13:05 +0000 | [diff] [blame] | 3831 | } |
| 3832 | |
Elena Demikhovsky | 0b9dbe3 | 2015-03-11 10:25:42 +0000 | [diff] [blame] | 3833 | multiclass avx512_var_shift_w<bits<8> opc, string OpcodeStr, |
| 3834 | SDNode OpNode> { |
| 3835 | let Predicates = [HasBWI] in |
| 3836 | defm WZ: avx512_var_shift<opc, OpcodeStr, OpNode, v32i16_info>, |
| 3837 | EVEX_V512, VEX_W; |
| 3838 | let Predicates = [HasVLX, HasBWI] in { |
| 3839 | |
| 3840 | defm WZ256: avx512_var_shift<opc, OpcodeStr, OpNode, v16i16x_info>, |
| 3841 | EVEX_V256, VEX_W; |
| 3842 | defm WZ128: avx512_var_shift<opc, OpcodeStr, OpNode, v8i16x_info>, |
| 3843 | EVEX_V128, VEX_W; |
| 3844 | } |
| 3845 | } |
| 3846 | |
| 3847 | defm VPSLLV : avx512_var_shift_types<0x47, "vpsllv", shl>, |
| 3848 | avx512_var_shift_w<0x12, "vpsllvw", shl>; |
| 3849 | defm VPSRAV : avx512_var_shift_types<0x46, "vpsrav", sra>, |
| 3850 | avx512_var_shift_w<0x11, "vpsravw", sra>; |
| 3851 | defm VPSRLV : avx512_var_shift_types<0x45, "vpsrlv", srl>, |
| 3852 | avx512_var_shift_w<0x10, "vpsrlvw", srl>; |
| 3853 | defm VPRORV : avx512_var_shift_types<0x14, "vprorv", rotr>; |
| 3854 | defm VPROLV : avx512_var_shift_types<0x15, "vprolv", rotl>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3855 | |
Elena Demikhovsky | 4078c75 | 2015-06-04 07:07:13 +0000 | [diff] [blame] | 3856 | //===-------------------------------------------------------------------===// |
| 3857 | // 1-src variable permutation VPERMW/D/Q |
| 3858 | //===-------------------------------------------------------------------===// |
| 3859 | multiclass avx512_vperm_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 3860 | AVX512VLVectorVTInfo _> { |
| 3861 | let Predicates = [HasAVX512] in |
| 3862 | defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>, |
| 3863 | avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512; |
| 3864 | |
| 3865 | let Predicates = [HasAVX512, HasVLX] in |
| 3866 | defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>, |
| 3867 | avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256; |
| 3868 | } |
| 3869 | |
| 3870 | multiclass avx512_vpermi_dq_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM, |
| 3871 | string OpcodeStr, SDNode OpNode, |
| 3872 | AVX512VLVectorVTInfo VTInfo> { |
| 3873 | let Predicates = [HasAVX512] in |
| 3874 | defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode, |
| 3875 | VTInfo.info512>, |
| 3876 | avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, |
| 3877 | VTInfo.info512>, EVEX_V512; |
| 3878 | let Predicates = [HasAVX512, HasVLX] in |
| 3879 | defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode, |
| 3880 | VTInfo.info256>, |
| 3881 | avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, |
| 3882 | VTInfo.info256>, EVEX_V256; |
| 3883 | } |
| 3884 | |
| 3885 | |
| 3886 | defm VPERM : avx512_var_shift_w<0x8D, "vpermw", X86VPermv>; |
| 3887 | |
| 3888 | defm VPERMD : avx512_vperm_dq_sizes<0x36, "vpermd", X86VPermv, |
| 3889 | avx512vl_i32_info>; |
| 3890 | defm VPERMQ : avx512_vperm_dq_sizes<0x36, "vpermq", X86VPermv, |
| 3891 | avx512vl_i64_info>, VEX_W; |
| 3892 | defm VPERMPS : avx512_vperm_dq_sizes<0x16, "vpermps", X86VPermv, |
| 3893 | avx512vl_f32_info>; |
| 3894 | defm VPERMPD : avx512_vperm_dq_sizes<0x16, "vpermpd", X86VPermv, |
| 3895 | avx512vl_f64_info>, VEX_W; |
| 3896 | |
| 3897 | defm VPERMQ : avx512_vpermi_dq_sizes<0x00, MRMSrcReg, MRMSrcMem, "vpermq", |
| 3898 | X86VPermi, avx512vl_i64_info>, |
| 3899 | EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, VEX_W; |
| 3900 | defm VPERMPD : avx512_vpermi_dq_sizes<0x01, MRMSrcReg, MRMSrcMem, "vpermpd", |
| 3901 | X86VPermi, avx512vl_f64_info>, |
| 3902 | EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, VEX_W; |
| 3903 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3904 | //===----------------------------------------------------------------------===// |
Elena Demikhovsky | 75ede68 | 2015-06-01 07:17:23 +0000 | [diff] [blame] | 3905 | // AVX-512 - VPSHUFD, VPSHUFLW, VPSHUFHW |
| 3906 | //===----------------------------------------------------------------------===// |
| 3907 | |
| 3908 | defm VPSHUFD : avx512_shift_rmi_sizes<0x70, MRMSrcReg, MRMSrcMem, "vpshufd", |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3909 | X86PShufd, avx512vl_i32_info>, |
Elena Demikhovsky | 75ede68 | 2015-06-01 07:17:23 +0000 | [diff] [blame] | 3910 | EVEX, AVX512BIi8Base, EVEX_CD8<32, CD8VF>; |
| 3911 | defm VPSHUFH : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshufhw", |
| 3912 | X86PShufhw>, EVEX, AVX512XSIi8Base, VEX_W; |
| 3913 | defm VPSHUFL : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshuflw", |
| 3914 | X86PShuflw>, EVEX, AVX512XDIi8Base, VEX_W; |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 3915 | |
Elena Demikhovsky | 55a9974 | 2015-06-22 13:00:42 +0000 | [diff] [blame] | 3916 | multiclass avx512_pshufb_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode> { |
| 3917 | let Predicates = [HasBWI] in |
| 3918 | defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, v64i8_info>, EVEX_V512; |
| 3919 | |
| 3920 | let Predicates = [HasVLX, HasBWI] in { |
| 3921 | defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, v32i8x_info>, EVEX_V256; |
| 3922 | defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, v16i8x_info>, EVEX_V128; |
| 3923 | } |
| 3924 | } |
| 3925 | |
| 3926 | defm VPSHUFB: avx512_pshufb_sizes<0x00, "vpshufb", X86pshufb>; |
| 3927 | |
Elena Demikhovsky | 75ede68 | 2015-06-01 07:17:23 +0000 | [diff] [blame] | 3928 | //===----------------------------------------------------------------------===// |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3929 | // AVX-512 - MOVDDUP |
| 3930 | //===----------------------------------------------------------------------===// |
| 3931 | |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 3932 | multiclass avx512_movddup<string OpcodeStr, RegisterClass RC, ValueType VT, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3933 | X86MemOperand x86memop, PatFrag memop_frag> { |
| 3934 | def rr : AVX512PDI<0x12, MRMSrcReg, (outs RC:$dst), (ins RC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 3935 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3936 | [(set RC:$dst, (VT (X86Movddup RC:$src)))]>, EVEX; |
| 3937 | def rm : AVX512PDI<0x12, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 3938 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3939 | [(set RC:$dst, |
| 3940 | (VT (X86Movddup (memop_frag addr:$src))))]>, EVEX; |
| 3941 | } |
| 3942 | |
Craig Topper | 820d492 | 2015-02-09 04:04:50 +0000 | [diff] [blame] | 3943 | defm VMOVDDUPZ : avx512_movddup<"vmovddup", VR512, v8f64, f512mem, loadv8f64>, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3944 | VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>; |
| 3945 | def : Pat<(X86Movddup (v8f64 (scalar_to_vector (loadf64 addr:$src)))), |
| 3946 | (VMOVDDUPZrm addr:$src)>; |
| 3947 | |
Elena Demikhovsky | 0a74b7d | 2013-11-14 11:29:27 +0000 | [diff] [blame] | 3948 | //===---------------------------------------------------------------------===// |
| 3949 | // Replicate Single FP - MOVSHDUP and MOVSLDUP |
| 3950 | //===---------------------------------------------------------------------===// |
| 3951 | multiclass avx512_replicate_sfp<bits<8> op, SDNode OpNode, string OpcodeStr, |
| 3952 | ValueType vt, RegisterClass RC, PatFrag mem_frag, |
| 3953 | X86MemOperand x86memop> { |
| 3954 | def rr : AVX512XSI<op, MRMSrcReg, (outs RC:$dst), (ins RC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 3955 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | 0a74b7d | 2013-11-14 11:29:27 +0000 | [diff] [blame] | 3956 | [(set RC:$dst, (vt (OpNode RC:$src)))]>, EVEX; |
| 3957 | let mayLoad = 1 in |
| 3958 | def rm : AVX512XSI<op, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 3959 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | 0a74b7d | 2013-11-14 11:29:27 +0000 | [diff] [blame] | 3960 | [(set RC:$dst, (OpNode (mem_frag addr:$src)))]>, EVEX; |
| 3961 | } |
| 3962 | |
| 3963 | defm VMOVSHDUPZ : avx512_replicate_sfp<0x16, X86Movshdup, "vmovshdup", |
Craig Topper | 820d492 | 2015-02-09 04:04:50 +0000 | [diff] [blame] | 3964 | v16f32, VR512, loadv16f32, f512mem>, EVEX_V512, |
Elena Demikhovsky | 0a74b7d | 2013-11-14 11:29:27 +0000 | [diff] [blame] | 3965 | EVEX_CD8<32, CD8VF>; |
| 3966 | defm VMOVSLDUPZ : avx512_replicate_sfp<0x12, X86Movsldup, "vmovsldup", |
Craig Topper | 820d492 | 2015-02-09 04:04:50 +0000 | [diff] [blame] | 3967 | v16f32, VR512, loadv16f32, f512mem>, EVEX_V512, |
Elena Demikhovsky | 0a74b7d | 2013-11-14 11:29:27 +0000 | [diff] [blame] | 3968 | EVEX_CD8<32, CD8VF>; |
| 3969 | |
| 3970 | def : Pat<(v16i32 (X86Movshdup VR512:$src)), (VMOVSHDUPZrr VR512:$src)>; |
Craig Topper | 820d492 | 2015-02-09 04:04:50 +0000 | [diff] [blame] | 3971 | def : Pat<(v16i32 (X86Movshdup (loadv16i32 addr:$src))), |
Elena Demikhovsky | 0a74b7d | 2013-11-14 11:29:27 +0000 | [diff] [blame] | 3972 | (VMOVSHDUPZrm addr:$src)>; |
| 3973 | def : Pat<(v16i32 (X86Movsldup VR512:$src)), (VMOVSLDUPZrr VR512:$src)>; |
Craig Topper | 820d492 | 2015-02-09 04:04:50 +0000 | [diff] [blame] | 3974 | def : Pat<(v16i32 (X86Movsldup (loadv16i32 addr:$src))), |
Elena Demikhovsky | 0a74b7d | 2013-11-14 11:29:27 +0000 | [diff] [blame] | 3975 | (VMOVSLDUPZrm addr:$src)>; |
| 3976 | |
| 3977 | //===----------------------------------------------------------------------===// |
| 3978 | // Move Low to High and High to Low packed FP Instructions |
| 3979 | //===----------------------------------------------------------------------===// |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3980 | def VMOVLHPSZrr : AVX512PSI<0x16, MRMSrcReg, (outs VR128X:$dst), |
| 3981 | (ins VR128X:$src1, VR128X:$src2), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 3982 | "vmovlhps\t{$src2, $src1, $dst|$dst, $src1, $src2}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3983 | [(set VR128X:$dst, (v4f32 (X86Movlhps VR128X:$src1, VR128X:$src2)))], |
| 3984 | IIC_SSE_MOV_LH>, EVEX_4V; |
| 3985 | def VMOVHLPSZrr : AVX512PSI<0x12, MRMSrcReg, (outs VR128X:$dst), |
| 3986 | (ins VR128X:$src1, VR128X:$src2), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 3987 | "vmovhlps\t{$src2, $src1, $dst|$dst, $src1, $src2}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3988 | [(set VR128X:$dst, (v4f32 (X86Movhlps VR128X:$src1, VR128X:$src2)))], |
| 3989 | IIC_SSE_MOV_LH>, EVEX_4V; |
| 3990 | |
Craig Topper | dbe8b7d | 2013-09-27 07:20:47 +0000 | [diff] [blame] | 3991 | let Predicates = [HasAVX512] in { |
| 3992 | // MOVLHPS patterns |
| 3993 | def : Pat<(v4i32 (X86Movlhps VR128X:$src1, VR128X:$src2)), |
| 3994 | (VMOVLHPSZrr VR128X:$src1, VR128X:$src2)>; |
| 3995 | def : Pat<(v2i64 (X86Movlhps VR128X:$src1, VR128X:$src2)), |
| 3996 | (VMOVLHPSZrr (v2i64 VR128X:$src1), VR128X:$src2)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 3997 | |
Craig Topper | dbe8b7d | 2013-09-27 07:20:47 +0000 | [diff] [blame] | 3998 | // MOVHLPS patterns |
| 3999 | def : Pat<(v4i32 (X86Movhlps VR128X:$src1, VR128X:$src2)), |
| 4000 | (VMOVHLPSZrr VR128X:$src1, VR128X:$src2)>; |
| 4001 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4002 | |
| 4003 | //===----------------------------------------------------------------------===// |
| 4004 | // FMA - Fused Multiply Operations |
| 4005 | // |
Adam Nemet | 26371ce | 2014-10-24 00:02:55 +0000 | [diff] [blame] | 4006 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4007 | let Constraints = "$src1 = $dst" in { |
Igor Breger | a7a8e9a | 2015-06-29 09:10:00 +0000 | [diff] [blame] | 4008 | multiclass avx512_fma3p_213_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 4009 | X86VectorVTInfo _> { |
Adam Nemet | 3480142 | 2014-10-08 23:25:39 +0000 | [diff] [blame] | 4010 | defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst), |
Adam Nemet | 6bddb8c | 2014-09-29 22:54:41 +0000 | [diff] [blame] | 4011 | (ins _.RC:$src2, _.RC:$src3), |
Adam Nemet | 2e91ee5 | 2014-08-14 17:13:19 +0000 | [diff] [blame] | 4012 | OpcodeStr, "$src3, $src2", "$src2, $src3", |
Adam Nemet | 6bddb8c | 2014-09-29 22:54:41 +0000 | [diff] [blame] | 4013 | (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>, |
Adam Nemet | 2e91ee5 | 2014-08-14 17:13:19 +0000 | [diff] [blame] | 4014 | AVX512FMA3Base; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4015 | |
Igor Breger | a7a8e9a | 2015-06-29 09:10:00 +0000 | [diff] [blame] | 4016 | let mayLoad = 1 in { |
| 4017 | defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), |
Elena Demikhovsky | fcea06a | 2014-12-23 10:30:39 +0000 | [diff] [blame] | 4018 | (ins _.RC:$src2, _.MemOp:$src3), |
| 4019 | OpcodeStr, "$src3, $src2", "$src2, $src3", |
| 4020 | (_.VT (OpNode _.RC:$src1, _.RC:$src2, (_.LdFrag addr:$src3)))>, |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 4021 | AVX512FMA3Base; |
Elena Demikhovsky | fcea06a | 2014-12-23 10:30:39 +0000 | [diff] [blame] | 4022 | |
Igor Breger | a7a8e9a | 2015-06-29 09:10:00 +0000 | [diff] [blame] | 4023 | defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), |
Elena Demikhovsky | fcea06a | 2014-12-23 10:30:39 +0000 | [diff] [blame] | 4024 | (ins _.RC:$src2, _.ScalarMemOp:$src3), |
Elena Demikhovsky | d8fda62 | 2015-03-30 09:29:28 +0000 | [diff] [blame] | 4025 | OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"), |
| 4026 | !strconcat("$src2, ${src3}", _.BroadcastStr ), |
| 4027 | (OpNode _.RC:$src1, |
Simon Pilgrim | 8b75659 | 2015-07-06 20:30:47 +0000 | [diff] [blame] | 4028 | _.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))))>, |
Elena Demikhovsky | fcea06a | 2014-12-23 10:30:39 +0000 | [diff] [blame] | 4029 | AVX512FMA3Base, EVEX_B; |
Igor Breger | a7a8e9a | 2015-06-29 09:10:00 +0000 | [diff] [blame] | 4030 | } |
| 4031 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4032 | |
Igor Breger | a7a8e9a | 2015-06-29 09:10:00 +0000 | [diff] [blame] | 4033 | multiclass avx512_fma3_213_round<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 4034 | X86VectorVTInfo _> { |
| 4035 | defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst), |
Elena Demikhovsky | 7b0dd39 | 2015-01-28 10:21:27 +0000 | [diff] [blame] | 4036 | (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc), |
| 4037 | OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc", |
| 4038 | (_.VT ( OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3, (i32 imm:$rc)))>, |
| 4039 | AVX512FMA3Base, EVEX_B, EVEX_RC; |
Igor Breger | a7a8e9a | 2015-06-29 09:10:00 +0000 | [diff] [blame] | 4040 | } |
Elena Demikhovsky | 7b0dd39 | 2015-01-28 10:21:27 +0000 | [diff] [blame] | 4041 | } // Constraints = "$src1 = $dst" |
| 4042 | |
Igor Breger | a7a8e9a | 2015-06-29 09:10:00 +0000 | [diff] [blame] | 4043 | multiclass avx512_fma3p_213_common<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 4044 | SDNode OpNodeRnd, AVX512VLVectorVTInfo _> { |
| 4045 | let Predicates = [HasAVX512] in { |
| 4046 | defm Z : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info512>, |
| 4047 | avx512_fma3_213_round<opc, OpcodeStr, OpNodeRnd, _.info512>, |
| 4048 | EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>; |
Elena Demikhovsky | fcea06a | 2014-12-23 10:30:39 +0000 | [diff] [blame] | 4049 | } |
Igor Breger | a7a8e9a | 2015-06-29 09:10:00 +0000 | [diff] [blame] | 4050 | let Predicates = [HasVLX, HasAVX512] in { |
| 4051 | defm Z256 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info256>, |
| 4052 | EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>; |
| 4053 | defm Z128 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info128>, |
| 4054 | EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>; |
Elena Demikhovsky | fcea06a | 2014-12-23 10:30:39 +0000 | [diff] [blame] | 4055 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4056 | } |
| 4057 | |
Igor Breger | a7a8e9a | 2015-06-29 09:10:00 +0000 | [diff] [blame] | 4058 | multiclass avx512_fma3p_213_f<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 4059 | SDNode OpNodeRnd > { |
| 4060 | defm PS : avx512_fma3p_213_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd, |
| 4061 | avx512vl_f32_info>; |
| 4062 | defm PD : avx512_fma3p_213_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd, |
| 4063 | avx512vl_f64_info>, VEX_W; |
| 4064 | } |
| 4065 | |
| 4066 | defm VFMADD213 : avx512_fma3p_213_f<0xA8, "vfmadd213", X86Fmadd, X86FmaddRnd>; |
| 4067 | defm VFMSUB213 : avx512_fma3p_213_f<0xAA, "vfmsub213", X86Fmsub, X86FmsubRnd>; |
| 4068 | defm VFMADDSUB213 : avx512_fma3p_213_f<0xA6, "vfmaddsub213", X86Fmaddsub, X86FmaddsubRnd>; |
| 4069 | defm VFMSUBADD213 : avx512_fma3p_213_f<0xA7, "vfmsubadd213", X86Fmsubadd, X86FmsubaddRnd>; |
| 4070 | defm VFNMADD213 : avx512_fma3p_213_f<0xAC, "vfnmadd213", X86Fnmadd, X86FnmaddRnd>; |
| 4071 | defm VFNMSUB213 : avx512_fma3p_213_f<0xAE, "vfnmsub213", X86Fnmsub, X86FnmsubRnd>; |
| 4072 | |
Elena Demikhovsky | fcea06a | 2014-12-23 10:30:39 +0000 | [diff] [blame] | 4073 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4074 | let Constraints = "$src1 = $dst" in { |
Igor Breger | a7a8e9a | 2015-06-29 09:10:00 +0000 | [diff] [blame] | 4075 | multiclass avx512_fma3p_231_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 4076 | X86VectorVTInfo _> { |
| 4077 | defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 4078 | (ins _.RC:$src2, _.RC:$src3), |
| 4079 | OpcodeStr, "$src3, $src2", "$src2, $src3", |
| 4080 | (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1))>, |
| 4081 | AVX512FMA3Base; |
| 4082 | |
| 4083 | let mayLoad = 1 in { |
| 4084 | defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 4085 | (ins _.RC:$src2, _.MemOp:$src3), |
| 4086 | OpcodeStr, "$src3, $src2", "$src2, $src3", |
| 4087 | (_.VT (OpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1))>, |
| 4088 | AVX512FMA3Base; |
| 4089 | |
| 4090 | defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 4091 | (ins _.RC:$src2, _.ScalarMemOp:$src3), |
| 4092 | OpcodeStr, "${src3}"##_.BroadcastStr##", $src2", |
| 4093 | "$src2, ${src3}"##_.BroadcastStr, |
| 4094 | (_.VT (OpNode _.RC:$src2, |
| 4095 | (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))), |
| 4096 | _.RC:$src1))>, AVX512FMA3Base, EVEX_B; |
| 4097 | } |
| 4098 | } |
| 4099 | |
| 4100 | multiclass avx512_fma3_231_round<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 4101 | X86VectorVTInfo _> { |
| 4102 | defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 4103 | (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc), |
| 4104 | OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc", |
| 4105 | (_.VT ( OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1, (i32 imm:$rc)))>, |
| 4106 | AVX512FMA3Base, EVEX_B, EVEX_RC; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4107 | } |
| 4108 | } // Constraints = "$src1 = $dst" |
| 4109 | |
Igor Breger | a7a8e9a | 2015-06-29 09:10:00 +0000 | [diff] [blame] | 4110 | multiclass avx512_fma3p_231_common<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 4111 | SDNode OpNodeRnd, AVX512VLVectorVTInfo _> { |
| 4112 | let Predicates = [HasAVX512] in { |
| 4113 | defm Z : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info512>, |
| 4114 | avx512_fma3_231_round<opc, OpcodeStr, OpNodeRnd, _.info512>, |
| 4115 | EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>; |
Elena Demikhovsky | fcea06a | 2014-12-23 10:30:39 +0000 | [diff] [blame] | 4116 | } |
Igor Breger | a7a8e9a | 2015-06-29 09:10:00 +0000 | [diff] [blame] | 4117 | let Predicates = [HasVLX, HasAVX512] in { |
| 4118 | defm Z256 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info256>, |
| 4119 | EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>; |
| 4120 | defm Z128 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info128>, |
| 4121 | EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>; |
Elena Demikhovsky | fcea06a | 2014-12-23 10:30:39 +0000 | [diff] [blame] | 4122 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4123 | } |
| 4124 | |
Igor Breger | a7a8e9a | 2015-06-29 09:10:00 +0000 | [diff] [blame] | 4125 | multiclass avx512_fma3p_231_f<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 4126 | SDNode OpNodeRnd > { |
| 4127 | defm PS : avx512_fma3p_231_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd, |
| 4128 | avx512vl_f32_info>; |
| 4129 | defm PD : avx512_fma3p_231_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd, |
| 4130 | avx512vl_f64_info>, VEX_W; |
| 4131 | } |
| 4132 | |
| 4133 | defm VFMADD231 : avx512_fma3p_231_f<0xB8, "vfmadd231", X86Fmadd, X86FmaddRnd>; |
| 4134 | defm VFMSUB231 : avx512_fma3p_231_f<0xBA, "vfmsub231", X86Fmsub, X86FmsubRnd>; |
| 4135 | defm VFMADDSUB231 : avx512_fma3p_231_f<0xB6, "vfmaddsub231", X86Fmaddsub, X86FmaddsubRnd>; |
| 4136 | defm VFMSUBADD231 : avx512_fma3p_231_f<0xB7, "vfmsubadd231", X86Fmsubadd, X86FmsubaddRnd>; |
| 4137 | defm VFNMADD231 : avx512_fma3p_231_f<0xBC, "vfnmadd231", X86Fnmadd, X86FnmaddRnd>; |
| 4138 | defm VFNMSUB231 : avx512_fma3p_231_f<0xBE, "vfnmsub231", X86Fnmsub, X86FnmsubRnd>; |
| 4139 | |
| 4140 | let Constraints = "$src1 = $dst" in { |
| 4141 | multiclass avx512_fma3p_132_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 4142 | X86VectorVTInfo _> { |
| 4143 | defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 4144 | (ins _.RC:$src3, _.RC:$src2), |
| 4145 | OpcodeStr, "$src2, $src3", "$src3, $src2", |
| 4146 | (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>, |
| 4147 | AVX512FMA3Base; |
| 4148 | |
| 4149 | let mayLoad = 1 in { |
| 4150 | defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 4151 | (ins _.RC:$src3, _.MemOp:$src2), |
| 4152 | OpcodeStr, "$src2, $src3", "$src3, $src2", |
| 4153 | (_.VT (OpNode _.RC:$src1, (_.LdFrag addr:$src2), _.RC:$src3))>, |
| 4154 | AVX512FMA3Base; |
| 4155 | |
| 4156 | defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 4157 | (ins _.RC:$src3, _.ScalarMemOp:$src2), |
| 4158 | OpcodeStr, "${src2}"##_.BroadcastStr##", $src3", |
| 4159 | "$src3, ${src2}"##_.BroadcastStr, |
| 4160 | (_.VT (OpNode _.RC:$src1, |
| 4161 | (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))), |
| 4162 | _.RC:$src3))>, AVX512FMA3Base, EVEX_B; |
| 4163 | } |
| 4164 | } |
| 4165 | |
| 4166 | multiclass avx512_fma3_132_round<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 4167 | X86VectorVTInfo _> { |
| 4168 | defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 4169 | (ins _.RC:$src3, _.RC:$src2, AVX512RC:$rc), |
| 4170 | OpcodeStr, "$rc, $src2, $src3", "$src3, $src2, $rc", |
| 4171 | (_.VT ( OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3, (i32 imm:$rc)))>, |
| 4172 | AVX512FMA3Base, EVEX_B, EVEX_RC; |
| 4173 | } |
| 4174 | } // Constraints = "$src1 = $dst" |
| 4175 | |
| 4176 | multiclass avx512_fma3p_132_common<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 4177 | SDNode OpNodeRnd, AVX512VLVectorVTInfo _> { |
| 4178 | let Predicates = [HasAVX512] in { |
| 4179 | defm Z : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info512>, |
| 4180 | avx512_fma3_132_round<opc, OpcodeStr, OpNodeRnd, _.info512>, |
| 4181 | EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>; |
| 4182 | } |
| 4183 | let Predicates = [HasVLX, HasAVX512] in { |
| 4184 | defm Z256 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info256>, |
| 4185 | EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>; |
| 4186 | defm Z128 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info128>, |
| 4187 | EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>; |
| 4188 | } |
| 4189 | } |
| 4190 | |
| 4191 | multiclass avx512_fma3p_132_f<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 4192 | SDNode OpNodeRnd > { |
| 4193 | defm PS : avx512_fma3p_132_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd, |
| 4194 | avx512vl_f32_info>; |
| 4195 | defm PD : avx512_fma3p_132_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd, |
| 4196 | avx512vl_f64_info>, VEX_W; |
| 4197 | } |
| 4198 | |
| 4199 | defm VFMADD132 : avx512_fma3p_132_f<0x98, "vfmadd132", X86Fmadd, X86FmaddRnd>; |
| 4200 | defm VFMSUB132 : avx512_fma3p_132_f<0x9A, "vfmsub132", X86Fmsub, X86FmsubRnd>; |
| 4201 | defm VFMADDSUB132 : avx512_fma3p_132_f<0x96, "vfmaddsub132", X86Fmaddsub, X86FmaddsubRnd>; |
| 4202 | defm VFMSUBADD132 : avx512_fma3p_132_f<0x97, "vfmsubadd132", X86Fmsubadd, X86FmsubaddRnd>; |
| 4203 | defm VFNMADD132 : avx512_fma3p_132_f<0x9C, "vfnmadd132", X86Fnmadd, X86FnmaddRnd>; |
| 4204 | defm VFNMSUB132 : avx512_fma3p_132_f<0x9E, "vfnmsub132", X86Fnmsub, X86FnmsubRnd>; |
Elena Demikhovsky | fcea06a | 2014-12-23 10:30:39 +0000 | [diff] [blame] | 4205 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4206 | // Scalar FMA |
| 4207 | let Constraints = "$src1 = $dst" in { |
Igor Breger | 15820b0 | 2015-07-01 13:24:28 +0000 | [diff] [blame] | 4208 | multiclass avx512_fma3s_common<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, |
| 4209 | dag RHS_VEC_r, dag RHS_VEC_m, dag RHS_VEC_rb, |
| 4210 | dag RHS_r, dag RHS_m > { |
| 4211 | defm r_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 4212 | (ins _.RC:$src2, _.RC:$src3), OpcodeStr, |
| 4213 | "$src3, $src2", "$src2, $src3", RHS_VEC_r>, AVX512FMA3Base; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4214 | |
Igor Breger | 15820b0 | 2015-07-01 13:24:28 +0000 | [diff] [blame] | 4215 | let mayLoad = 1 in |
| 4216 | defm m_Int: AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 4217 | (ins _.RC:$src2, _.MemOp:$src3), OpcodeStr, |
| 4218 | "$src3, $src2", "$src2, $src3", RHS_VEC_m>, AVX512FMA3Base; |
| 4219 | |
| 4220 | defm rb_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 4221 | (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc), |
| 4222 | OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc", RHS_VEC_rb>, |
| 4223 | AVX512FMA3Base, EVEX_B, EVEX_RC; |
| 4224 | |
| 4225 | let isCodeGenOnly = 1 in { |
| 4226 | def r : AVX512FMA3<opc, MRMSrcReg, (outs _.FRC:$dst), |
| 4227 | (ins _.FRC:$src1, _.FRC:$src2, _.FRC:$src3), |
| 4228 | !strconcat(OpcodeStr, |
| 4229 | "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), |
| 4230 | [RHS_r]>; |
| 4231 | let mayLoad = 1 in |
| 4232 | def m : AVX512FMA3<opc, MRMSrcMem, (outs _.FRC:$dst), |
| 4233 | (ins _.FRC:$src1, _.FRC:$src2, _.ScalarMemOp:$src3), |
| 4234 | !strconcat(OpcodeStr, |
| 4235 | "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), |
| 4236 | [RHS_m]>; |
| 4237 | }// isCodeGenOnly = 1 |
| 4238 | } |
| 4239 | }// Constraints = "$src1 = $dst" |
| 4240 | |
| 4241 | multiclass avx512_fma3s_all<bits<8> opc213, bits<8> opc231, bits<8> opc132, |
| 4242 | string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd, X86VectorVTInfo _ , |
| 4243 | string SUFF> { |
| 4244 | |
| 4245 | defm NAME#213#SUFF: avx512_fma3s_common<opc213, OpcodeStr#"213"#_.Suffix , _ , |
| 4246 | (_.VT (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3)), |
| 4247 | (_.VT (OpNode _.RC:$src2, _.RC:$src1, |
| 4248 | (_.VT (scalar_to_vector(_.ScalarLdFrag addr:$src3))))), |
| 4249 | (_.VT ( OpNodeRnd _.RC:$src2, _.RC:$src1, _.RC:$src3, |
| 4250 | (i32 imm:$rc))), |
| 4251 | (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1, |
| 4252 | _.FRC:$src3))), |
| 4253 | (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1, |
| 4254 | (_.ScalarLdFrag addr:$src3))))>; |
| 4255 | |
| 4256 | defm NAME#231#SUFF: avx512_fma3s_common<opc231, OpcodeStr#"231"#_.Suffix , _ , |
| 4257 | (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1)), |
| 4258 | (_.VT (OpNode _.RC:$src2, |
| 4259 | (_.VT (scalar_to_vector(_.ScalarLdFrag addr:$src3))), |
| 4260 | _.RC:$src1)), |
| 4261 | (_.VT ( OpNodeRnd _.RC:$src2, _.RC:$src3, _.RC:$src1, |
| 4262 | (i32 imm:$rc))), |
| 4263 | (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src3, |
| 4264 | _.FRC:$src1))), |
| 4265 | (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, |
| 4266 | (_.ScalarLdFrag addr:$src3), _.FRC:$src1)))>; |
| 4267 | |
| 4268 | defm NAME#132#SUFF: avx512_fma3s_common<opc132, OpcodeStr#"132"#_.Suffix , _ , |
| 4269 | (_.VT (OpNode _.RC:$src1, _.RC:$src3, _.RC:$src2)), |
| 4270 | (_.VT (OpNode _.RC:$src1, |
| 4271 | (_.VT (scalar_to_vector(_.ScalarLdFrag addr:$src3))), |
| 4272 | _.RC:$src2)), |
| 4273 | (_.VT ( OpNodeRnd _.RC:$src1, _.RC:$src3, _.RC:$src2, |
| 4274 | (i32 imm:$rc))), |
| 4275 | (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1, _.FRC:$src3, |
| 4276 | _.FRC:$src2))), |
| 4277 | (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1, |
| 4278 | (_.ScalarLdFrag addr:$src3), _.FRC:$src2)))>; |
| 4279 | } |
| 4280 | |
| 4281 | multiclass avx512_fma3s<bits<8> opc213, bits<8> opc231, bits<8> opc132, |
| 4282 | string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd>{ |
| 4283 | let Predicates = [HasAVX512] in { |
| 4284 | defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode, |
| 4285 | OpNodeRnd, f32x_info, "SS">, |
| 4286 | EVEX_CD8<32, CD8VT1>, VEX_LIG; |
| 4287 | defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode, |
| 4288 | OpNodeRnd, f64x_info, "SD">, |
| 4289 | EVEX_CD8<64, CD8VT1>, VEX_LIG, VEX_W; |
| 4290 | } |
| 4291 | } |
| 4292 | |
| 4293 | defm VFMADD : avx512_fma3s<0xA9, 0xB9, 0x99, "vfmadd", X86Fmadd, X86FmaddRnd>; |
| 4294 | defm VFMSUB : avx512_fma3s<0xAB, 0xBB, 0x9B, "vfmsub", X86Fmsub, X86FmsubRnd>; |
| 4295 | defm VFNMADD : avx512_fma3s<0xAD, 0xBD, 0x9D, "vfnmadd", X86Fnmadd, X86FnmaddRnd>; |
| 4296 | defm VFNMSUB : avx512_fma3s<0xAF, 0xBF, 0x9F, "vfnmsub", X86Fnmsub, X86FnmsubRnd>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4297 | |
| 4298 | //===----------------------------------------------------------------------===// |
| 4299 | // AVX-512 Scalar convert from sign integer to float/double |
| 4300 | //===----------------------------------------------------------------------===// |
| 4301 | |
Igor Breger | dfcc3d3 | 2015-06-17 07:23:57 +0000 | [diff] [blame] | 4302 | multiclass avx512_vcvtsi<bits<8> opc, SDNode OpNode, RegisterClass SrcRC, |
| 4303 | X86VectorVTInfo DstVT, X86MemOperand x86memop, |
| 4304 | PatFrag ld_frag, string asm> { |
| 4305 | let hasSideEffects = 0 in { |
| 4306 | def rr : SI<opc, MRMSrcReg, (outs DstVT.FRC:$dst), |
| 4307 | (ins DstVT.FRC:$src1, SrcRC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 4308 | !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4309 | EVEX_4V; |
Igor Breger | dfcc3d3 | 2015-06-17 07:23:57 +0000 | [diff] [blame] | 4310 | let mayLoad = 1 in |
| 4311 | def rm : SI<opc, MRMSrcMem, (outs DstVT.FRC:$dst), |
| 4312 | (ins DstVT.FRC:$src1, x86memop:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 4313 | !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4314 | EVEX_4V; |
Igor Breger | dfcc3d3 | 2015-06-17 07:23:57 +0000 | [diff] [blame] | 4315 | } // hasSideEffects = 0 |
| 4316 | let isCodeGenOnly = 1 in { |
| 4317 | def rr_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst), |
| 4318 | (ins DstVT.RC:$src1, SrcRC:$src2), |
| 4319 | !strconcat(asm,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
| 4320 | [(set DstVT.RC:$dst, |
| 4321 | (OpNode (DstVT.VT DstVT.RC:$src1), |
| 4322 | SrcRC:$src2, |
| 4323 | (i32 FROUND_CURRENT)))]>, EVEX_4V; |
| 4324 | |
| 4325 | def rm_Int : SI<opc, MRMSrcMem, (outs DstVT.RC:$dst), |
| 4326 | (ins DstVT.RC:$src1, x86memop:$src2), |
| 4327 | !strconcat(asm,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
| 4328 | [(set DstVT.RC:$dst, |
| 4329 | (OpNode (DstVT.VT DstVT.RC:$src1), |
| 4330 | (ld_frag addr:$src2), |
| 4331 | (i32 FROUND_CURRENT)))]>, EVEX_4V; |
| 4332 | }//isCodeGenOnly = 1 |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4333 | } |
Elena Demikhovsky | d8fda62 | 2015-03-30 09:29:28 +0000 | [diff] [blame] | 4334 | |
Igor Breger | abe4a79 | 2015-06-14 12:44:55 +0000 | [diff] [blame] | 4335 | multiclass avx512_vcvtsi_round<bits<8> opc, SDNode OpNode, RegisterClass SrcRC, |
Igor Breger | dfcc3d3 | 2015-06-17 07:23:57 +0000 | [diff] [blame] | 4336 | X86VectorVTInfo DstVT, string asm> { |
Igor Breger | abe4a79 | 2015-06-14 12:44:55 +0000 | [diff] [blame] | 4337 | def rrb_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst), |
| 4338 | (ins DstVT.RC:$src1, SrcRC:$src2, AVX512RC:$rc), |
Igor Breger | dfcc3d3 | 2015-06-17 07:23:57 +0000 | [diff] [blame] | 4339 | !strconcat(asm, |
| 4340 | "\t{$src2, $rc, $src1, $dst|$dst, $src1, $rc, $src2}"), |
Igor Breger | abe4a79 | 2015-06-14 12:44:55 +0000 | [diff] [blame] | 4341 | [(set DstVT.RC:$dst, |
| 4342 | (OpNode (DstVT.VT DstVT.RC:$src1), |
| 4343 | SrcRC:$src2, |
| 4344 | (i32 imm:$rc)))]>, EVEX_4V, EVEX_B, EVEX_RC; |
| 4345 | } |
| 4346 | |
| 4347 | multiclass avx512_vcvtsi_common<bits<8> opc, SDNode OpNode, RegisterClass SrcRC, |
Igor Breger | dfcc3d3 | 2015-06-17 07:23:57 +0000 | [diff] [blame] | 4348 | X86VectorVTInfo DstVT, X86MemOperand x86memop, |
| 4349 | PatFrag ld_frag, string asm> { |
| 4350 | defm NAME : avx512_vcvtsi_round<opc, OpNode, SrcRC, DstVT, asm>, |
| 4351 | avx512_vcvtsi<opc, OpNode, SrcRC, DstVT, x86memop, ld_frag, asm>, |
| 4352 | VEX_LIG; |
Igor Breger | abe4a79 | 2015-06-14 12:44:55 +0000 | [diff] [blame] | 4353 | } |
| 4354 | |
Andrew Trick | 15a4774 | 2013-10-09 05:11:10 +0000 | [diff] [blame] | 4355 | let Predicates = [HasAVX512] in { |
Igor Breger | abe4a79 | 2015-06-14 12:44:55 +0000 | [diff] [blame] | 4356 | defm VCVTSI2SSZ : avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR32, |
Igor Breger | dfcc3d3 | 2015-06-17 07:23:57 +0000 | [diff] [blame] | 4357 | v4f32x_info, i32mem, loadi32, "cvtsi2ss{l}">, |
| 4358 | XS, EVEX_CD8<32, CD8VT1>; |
Igor Breger | abe4a79 | 2015-06-14 12:44:55 +0000 | [diff] [blame] | 4359 | defm VCVTSI642SSZ: avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR64, |
Igor Breger | dfcc3d3 | 2015-06-17 07:23:57 +0000 | [diff] [blame] | 4360 | v4f32x_info, i64mem, loadi64, "cvtsi2ss{q}">, |
| 4361 | XS, VEX_W, EVEX_CD8<64, CD8VT1>; |
Igor Breger | abe4a79 | 2015-06-14 12:44:55 +0000 | [diff] [blame] | 4362 | defm VCVTSI2SDZ : avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR32, |
Igor Breger | dfcc3d3 | 2015-06-17 07:23:57 +0000 | [diff] [blame] | 4363 | v2f64x_info, i32mem, loadi32, "cvtsi2sd{l}">, |
| 4364 | XD, EVEX_CD8<32, CD8VT1>; |
Igor Breger | abe4a79 | 2015-06-14 12:44:55 +0000 | [diff] [blame] | 4365 | defm VCVTSI642SDZ: avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR64, |
Igor Breger | dfcc3d3 | 2015-06-17 07:23:57 +0000 | [diff] [blame] | 4366 | v2f64x_info, i64mem, loadi64, "cvtsi2sd{q}">, |
| 4367 | XD, VEX_W, EVEX_CD8<64, CD8VT1>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4368 | |
| 4369 | def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))), |
| 4370 | (VCVTSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>; |
| 4371 | def : Pat<(f32 (sint_to_fp (loadi64 addr:$src))), |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4372 | (VCVTSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4373 | def : Pat<(f64 (sint_to_fp (loadi32 addr:$src))), |
| 4374 | (VCVTSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>; |
| 4375 | def : Pat<(f64 (sint_to_fp (loadi64 addr:$src))), |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4376 | (VCVTSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4377 | |
| 4378 | def : Pat<(f32 (sint_to_fp GR32:$src)), |
| 4379 | (VCVTSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>; |
| 4380 | def : Pat<(f32 (sint_to_fp GR64:$src)), |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4381 | (VCVTSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4382 | def : Pat<(f64 (sint_to_fp GR32:$src)), |
| 4383 | (VCVTSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>; |
| 4384 | def : Pat<(f64 (sint_to_fp GR64:$src)), |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4385 | (VCVTSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>; |
| 4386 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4387 | defm VCVTUSI2SSZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR32, |
Igor Breger | dfcc3d3 | 2015-06-17 07:23:57 +0000 | [diff] [blame] | 4388 | v4f32x_info, i32mem, loadi32, |
| 4389 | "cvtusi2ss{l}">, XS, EVEX_CD8<32, CD8VT1>; |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4390 | defm VCVTUSI642SSZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR64, |
Igor Breger | dfcc3d3 | 2015-06-17 07:23:57 +0000 | [diff] [blame] | 4391 | v4f32x_info, i64mem, loadi64, "cvtusi2ss{q}">, |
| 4392 | XS, VEX_W, EVEX_CD8<64, CD8VT1>; |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4393 | defm VCVTUSI2SDZ : avx512_vcvtsi<0x7B, X86UintToFpRnd, GR32, v2f64x_info, |
Igor Breger | dfcc3d3 | 2015-06-17 07:23:57 +0000 | [diff] [blame] | 4394 | i32mem, loadi32, "cvtusi2sd{l}">, |
| 4395 | XD, VEX_LIG, EVEX_CD8<32, CD8VT1>; |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4396 | defm VCVTUSI642SDZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR64, |
Igor Breger | dfcc3d3 | 2015-06-17 07:23:57 +0000 | [diff] [blame] | 4397 | v2f64x_info, i64mem, loadi64, "cvtusi2sd{q}">, |
| 4398 | XD, VEX_W, EVEX_CD8<64, CD8VT1>; |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4399 | |
| 4400 | def : Pat<(f32 (uint_to_fp (loadi32 addr:$src))), |
| 4401 | (VCVTUSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>; |
| 4402 | def : Pat<(f32 (uint_to_fp (loadi64 addr:$src))), |
| 4403 | (VCVTUSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>; |
| 4404 | def : Pat<(f64 (uint_to_fp (loadi32 addr:$src))), |
| 4405 | (VCVTUSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>; |
| 4406 | def : Pat<(f64 (uint_to_fp (loadi64 addr:$src))), |
| 4407 | (VCVTUSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>; |
| 4408 | |
| 4409 | def : Pat<(f32 (uint_to_fp GR32:$src)), |
| 4410 | (VCVTUSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>; |
| 4411 | def : Pat<(f32 (uint_to_fp GR64:$src)), |
| 4412 | (VCVTUSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>; |
| 4413 | def : Pat<(f64 (uint_to_fp GR32:$src)), |
| 4414 | (VCVTUSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>; |
| 4415 | def : Pat<(f64 (uint_to_fp GR64:$src)), |
| 4416 | (VCVTUSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>; |
Andrew Trick | 15a4774 | 2013-10-09 05:11:10 +0000 | [diff] [blame] | 4417 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4418 | |
| 4419 | //===----------------------------------------------------------------------===// |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4420 | // AVX-512 Scalar convert from float/double to integer |
| 4421 | //===----------------------------------------------------------------------===// |
| 4422 | multiclass avx512_cvt_s_int<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, |
| 4423 | Intrinsic Int, Operand memop, ComplexPattern mem_cpat, |
| 4424 | string asm> { |
Elena Demikhovsky | f404e05 | 2014-01-05 14:21:07 +0000 | [diff] [blame] | 4425 | let hasSideEffects = 0 in { |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4426 | def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 4427 | !strconcat(asm,"\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4428 | [(set DstRC:$dst, (Int SrcRC:$src))]>, EVEX, VEX_LIG, |
| 4429 | Requires<[HasAVX512]>; |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4430 | let mayLoad = 1 in |
| 4431 | def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins memop:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 4432 | !strconcat(asm,"\t{$src, $dst|$dst, $src}"), []>, EVEX, VEX_LIG, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4433 | Requires<[HasAVX512]>; |
Elena Demikhovsky | f404e05 | 2014-01-05 14:21:07 +0000 | [diff] [blame] | 4434 | } // hasSideEffects = 0 |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4435 | } |
| 4436 | let Predicates = [HasAVX512] in { |
| 4437 | // Convert float/double to signed/unsigned int 32/64 |
| 4438 | defm VCVTSS2SIZ: avx512_cvt_s_int<0x2D, VR128X, GR32, int_x86_sse_cvtss2si, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4439 | ssmem, sse_load_f32, "cvtss2si">, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4440 | XS, EVEX_CD8<32, CD8VT1>; |
| 4441 | defm VCVTSS2SI64Z: avx512_cvt_s_int<0x2D, VR128X, GR64, int_x86_sse_cvtss2si64, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4442 | ssmem, sse_load_f32, "cvtss2si">, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4443 | XS, VEX_W, EVEX_CD8<32, CD8VT1>; |
| 4444 | defm VCVTSS2USIZ: avx512_cvt_s_int<0x79, VR128X, GR32, int_x86_avx512_cvtss2usi, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4445 | ssmem, sse_load_f32, "cvtss2usi">, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4446 | XS, EVEX_CD8<32, CD8VT1>; |
| 4447 | defm VCVTSS2USI64Z: avx512_cvt_s_int<0x79, VR128X, GR64, |
| 4448 | int_x86_avx512_cvtss2usi64, ssmem, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4449 | sse_load_f32, "cvtss2usi">, XS, VEX_W, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4450 | EVEX_CD8<32, CD8VT1>; |
| 4451 | defm VCVTSD2SIZ: avx512_cvt_s_int<0x2D, VR128X, GR32, int_x86_sse2_cvtsd2si, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4452 | sdmem, sse_load_f64, "cvtsd2si">, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4453 | XD, EVEX_CD8<64, CD8VT1>; |
| 4454 | defm VCVTSD2SI64Z: avx512_cvt_s_int<0x2D, VR128X, GR64, int_x86_sse2_cvtsd2si64, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4455 | sdmem, sse_load_f64, "cvtsd2si">, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4456 | XD, VEX_W, EVEX_CD8<64, CD8VT1>; |
| 4457 | defm VCVTSD2USIZ: avx512_cvt_s_int<0x79, VR128X, GR32, int_x86_avx512_cvtsd2usi, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4458 | sdmem, sse_load_f64, "cvtsd2usi">, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4459 | XD, EVEX_CD8<64, CD8VT1>; |
| 4460 | defm VCVTSD2USI64Z: avx512_cvt_s_int<0x79, VR128X, GR64, |
| 4461 | int_x86_avx512_cvtsd2usi64, sdmem, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4462 | sse_load_f64, "cvtsd2usi">, XD, VEX_W, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4463 | EVEX_CD8<64, CD8VT1>; |
| 4464 | |
Craig Topper | 9dd48c8 | 2014-01-02 17:28:14 +0000 | [diff] [blame] | 4465 | let isCodeGenOnly = 1 in { |
| 4466 | defm Int_VCVTSI2SSZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, |
| 4467 | int_x86_sse_cvtsi2ss, i32mem, loadi32, "cvtsi2ss{l}", |
| 4468 | SSE_CVT_Scalar, 0>, XS, EVEX_4V; |
| 4469 | defm Int_VCVTSI2SS64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X, |
| 4470 | int_x86_sse_cvtsi642ss, i64mem, loadi64, "cvtsi2ss{q}", |
| 4471 | SSE_CVT_Scalar, 0>, XS, EVEX_4V, VEX_W; |
| 4472 | defm Int_VCVTSI2SDZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, |
| 4473 | int_x86_sse2_cvtsi2sd, i32mem, loadi32, "cvtsi2sd{l}", |
| 4474 | SSE_CVT_Scalar, 0>, XD, EVEX_4V; |
| 4475 | defm Int_VCVTSI2SD64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X, |
| 4476 | int_x86_sse2_cvtsi642sd, i64mem, loadi64, "cvtsi2sd{q}", |
| 4477 | SSE_CVT_Scalar, 0>, XD, EVEX_4V, VEX_W; |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4478 | |
Craig Topper | 9dd48c8 | 2014-01-02 17:28:14 +0000 | [diff] [blame] | 4479 | defm Int_VCVTUSI2SDZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, |
| 4480 | int_x86_avx512_cvtusi2sd, i32mem, loadi32, "cvtusi2sd{l}", |
| 4481 | SSE_CVT_Scalar, 0>, XD, EVEX_4V; |
Craig Topper | 9dd48c8 | 2014-01-02 17:28:14 +0000 | [diff] [blame] | 4482 | } // isCodeGenOnly = 1 |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4483 | |
| 4484 | // Convert float/double to signed/unsigned int 32/64 with truncation |
Craig Topper | 9dd48c8 | 2014-01-02 17:28:14 +0000 | [diff] [blame] | 4485 | let isCodeGenOnly = 1 in { |
| 4486 | defm Int_VCVTTSS2SIZ : avx512_cvt_s_int<0x2C, VR128X, GR32, int_x86_sse_cvttss2si, |
| 4487 | ssmem, sse_load_f32, "cvttss2si">, |
| 4488 | XS, EVEX_CD8<32, CD8VT1>; |
| 4489 | defm Int_VCVTTSS2SI64Z : avx512_cvt_s_int<0x2C, VR128X, GR64, |
| 4490 | int_x86_sse_cvttss2si64, ssmem, sse_load_f32, |
| 4491 | "cvttss2si">, XS, VEX_W, |
| 4492 | EVEX_CD8<32, CD8VT1>; |
| 4493 | defm Int_VCVTTSD2SIZ : avx512_cvt_s_int<0x2C, VR128X, GR32, int_x86_sse2_cvttsd2si, |
| 4494 | sdmem, sse_load_f64, "cvttsd2si">, XD, |
| 4495 | EVEX_CD8<64, CD8VT1>; |
| 4496 | defm Int_VCVTTSD2SI64Z : avx512_cvt_s_int<0x2C, VR128X, GR64, |
| 4497 | int_x86_sse2_cvttsd2si64, sdmem, sse_load_f64, |
| 4498 | "cvttsd2si">, XD, VEX_W, |
| 4499 | EVEX_CD8<64, CD8VT1>; |
| 4500 | defm Int_VCVTTSS2USIZ : avx512_cvt_s_int<0x78, VR128X, GR32, |
| 4501 | int_x86_avx512_cvttss2usi, ssmem, sse_load_f32, |
| 4502 | "cvttss2usi">, XS, EVEX_CD8<32, CD8VT1>; |
| 4503 | defm Int_VCVTTSS2USI64Z : avx512_cvt_s_int<0x78, VR128X, GR64, |
| 4504 | int_x86_avx512_cvttss2usi64, ssmem, |
| 4505 | sse_load_f32, "cvttss2usi">, XS, VEX_W, |
| 4506 | EVEX_CD8<32, CD8VT1>; |
| 4507 | defm Int_VCVTTSD2USIZ : avx512_cvt_s_int<0x78, VR128X, GR32, |
| 4508 | int_x86_avx512_cvttsd2usi, |
| 4509 | sdmem, sse_load_f64, "cvttsd2usi">, XD, |
| 4510 | EVEX_CD8<64, CD8VT1>; |
| 4511 | defm Int_VCVTTSD2USI64Z : avx512_cvt_s_int<0x78, VR128X, GR64, |
| 4512 | int_x86_avx512_cvttsd2usi64, sdmem, |
| 4513 | sse_load_f64, "cvttsd2usi">, XD, VEX_W, |
| 4514 | EVEX_CD8<64, CD8VT1>; |
| 4515 | } // isCodeGenOnly = 1 |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4516 | |
| 4517 | multiclass avx512_cvt_s<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, |
| 4518 | SDNode OpNode, X86MemOperand x86memop, PatFrag ld_frag, |
| 4519 | string asm> { |
| 4520 | def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 4521 | !strconcat(asm,"\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4522 | [(set DstRC:$dst, (OpNode SrcRC:$src))]>, EVEX; |
| 4523 | def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 4524 | !strconcat(asm,"\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4525 | [(set DstRC:$dst, (OpNode (ld_frag addr:$src)))]>, EVEX; |
| 4526 | } |
| 4527 | |
| 4528 | defm VCVTTSS2SIZ : avx512_cvt_s<0x2C, FR32X, GR32, fp_to_sint, f32mem, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4529 | loadf32, "cvttss2si">, XS, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4530 | EVEX_CD8<32, CD8VT1>; |
| 4531 | defm VCVTTSS2USIZ : avx512_cvt_s<0x78, FR32X, GR32, fp_to_uint, f32mem, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4532 | loadf32, "cvttss2usi">, XS, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4533 | EVEX_CD8<32, CD8VT1>; |
| 4534 | defm VCVTTSS2SI64Z : avx512_cvt_s<0x2C, FR32X, GR64, fp_to_sint, f32mem, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4535 | loadf32, "cvttss2si">, XS, VEX_W, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4536 | EVEX_CD8<32, CD8VT1>; |
| 4537 | defm VCVTTSS2USI64Z : avx512_cvt_s<0x78, FR32X, GR64, fp_to_uint, f32mem, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4538 | loadf32, "cvttss2usi">, XS, VEX_W, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4539 | EVEX_CD8<32, CD8VT1>; |
| 4540 | defm VCVTTSD2SIZ : avx512_cvt_s<0x2C, FR64X, GR32, fp_to_sint, f64mem, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4541 | loadf64, "cvttsd2si">, XD, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4542 | EVEX_CD8<64, CD8VT1>; |
| 4543 | defm VCVTTSD2USIZ : avx512_cvt_s<0x78, FR64X, GR32, fp_to_uint, f64mem, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4544 | loadf64, "cvttsd2usi">, XD, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4545 | EVEX_CD8<64, CD8VT1>; |
| 4546 | defm VCVTTSD2SI64Z : avx512_cvt_s<0x2C, FR64X, GR64, fp_to_sint, f64mem, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4547 | loadf64, "cvttsd2si">, XD, VEX_W, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4548 | EVEX_CD8<64, CD8VT1>; |
| 4549 | defm VCVTTSD2USI64Z : avx512_cvt_s<0x78, FR64X, GR64, fp_to_uint, f64mem, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4550 | loadf64, "cvttsd2usi">, XD, VEX_W, |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4551 | EVEX_CD8<64, CD8VT1>; |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4552 | } // HasAVX512 |
Elena Demikhovsky | 2e408ae | 2013-10-06 13:11:09 +0000 | [diff] [blame] | 4553 | //===----------------------------------------------------------------------===// |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4554 | // AVX-512 Convert form float to double and back |
| 4555 | //===----------------------------------------------------------------------===// |
Elena Demikhovsky | f404e05 | 2014-01-05 14:21:07 +0000 | [diff] [blame] | 4556 | let hasSideEffects = 0 in { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4557 | def VCVTSS2SDZrr : AVX512XSI<0x5A, MRMSrcReg, (outs FR64X:$dst), |
| 4558 | (ins FR32X:$src1, FR32X:$src2), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4559 | "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4560 | []>, EVEX_4V, VEX_LIG, Sched<[WriteCvtF2F]>; |
| 4561 | let mayLoad = 1 in |
| 4562 | def VCVTSS2SDZrm : AVX512XSI<0x5A, MRMSrcMem, (outs FR64X:$dst), |
| 4563 | (ins FR32X:$src1, f32mem:$src2), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4564 | "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4565 | []>, EVEX_4V, VEX_LIG, Sched<[WriteCvtF2FLd, ReadAfterLd]>, |
| 4566 | EVEX_CD8<32, CD8VT1>; |
| 4567 | |
| 4568 | // Convert scalar double to scalar single |
| 4569 | def VCVTSD2SSZrr : AVX512XDI<0x5A, MRMSrcReg, (outs FR32X:$dst), |
| 4570 | (ins FR64X:$src1, FR64X:$src2), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4571 | "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4572 | []>, EVEX_4V, VEX_LIG, VEX_W, Sched<[WriteCvtF2F]>; |
| 4573 | let mayLoad = 1 in |
| 4574 | def VCVTSD2SSZrm : AVX512XDI<0x5A, MRMSrcMem, (outs FR32X:$dst), |
| 4575 | (ins FR64X:$src1, f64mem:$src2), |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 4576 | "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4577 | []>, EVEX_4V, VEX_LIG, VEX_W, |
| 4578 | Sched<[WriteCvtF2FLd, ReadAfterLd]>, EVEX_CD8<64, CD8VT1>; |
| 4579 | } |
| 4580 | |
| 4581 | def : Pat<(f64 (fextend FR32X:$src)), (VCVTSS2SDZrr FR32X:$src, FR32X:$src)>, |
| 4582 | Requires<[HasAVX512]>; |
| 4583 | def : Pat<(fextend (loadf32 addr:$src)), |
| 4584 | (VCVTSS2SDZrm (f32 (IMPLICIT_DEF)), addr:$src)>, Requires<[HasAVX512]>; |
| 4585 | |
| 4586 | def : Pat<(extloadf32 addr:$src), |
| 4587 | (VCVTSS2SDZrm (f32 (IMPLICIT_DEF)), addr:$src)>, |
| 4588 | Requires<[HasAVX512, OptForSize]>; |
| 4589 | |
| 4590 | def : Pat<(extloadf32 addr:$src), |
| 4591 | (VCVTSS2SDZrr (f32 (IMPLICIT_DEF)), (VMOVSSZrm addr:$src))>, |
| 4592 | Requires<[HasAVX512, OptForSpeed]>; |
| 4593 | |
| 4594 | def : Pat<(f32 (fround FR64X:$src)), (VCVTSD2SSZrr FR64X:$src, FR64X:$src)>, |
| 4595 | Requires<[HasAVX512]>; |
| 4596 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4597 | //===----------------------------------------------------------------------===// |
| 4598 | // AVX-512 Vector convert from signed/unsigned integer to float/double |
| 4599 | // and from float/double to signed/unsigned integer |
| 4600 | //===----------------------------------------------------------------------===// |
| 4601 | |
| 4602 | multiclass avx512_vcvt_fp<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, |
| 4603 | X86VectorVTInfo _Src, SDNode OpNode, |
| 4604 | string Broadcast = _.BroadcastStr, |
| 4605 | string Alias = ""> { |
| 4606 | |
| 4607 | defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 4608 | (ins _Src.RC:$src), OpcodeStr, "$src", "$src", |
| 4609 | (_.VT (OpNode (_Src.VT _Src.RC:$src)))>, EVEX; |
| 4610 | |
| 4611 | defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 4612 | (ins _Src.MemOp:$src), OpcodeStr#Alias, "$src", "$src", |
| 4613 | (_.VT (OpNode (_Src.VT |
| 4614 | (bitconvert (_Src.LdFrag addr:$src)))))>, EVEX; |
| 4615 | |
| 4616 | defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 4617 | (ins _Src.MemOp:$src), OpcodeStr, |
| 4618 | "${src}"##Broadcast, "${src}"##Broadcast, |
| 4619 | (_.VT (OpNode (_Src.VT |
| 4620 | (X86VBroadcast (_Src.ScalarLdFrag addr:$src))) |
| 4621 | ))>, EVEX, EVEX_B; |
| 4622 | } |
| 4623 | // Coversion with SAE - suppress all exceptions |
| 4624 | multiclass avx512_vcvt_fp_sae<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, |
| 4625 | X86VectorVTInfo _Src, SDNode OpNodeRnd> { |
| 4626 | defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 4627 | (ins _Src.RC:$src), OpcodeStr, |
| 4628 | "{sae}, $src", "$src, {sae}", |
| 4629 | (_.VT (OpNodeRnd (_Src.VT _Src.RC:$src), |
| 4630 | (i32 FROUND_NO_EXC)))>, |
| 4631 | EVEX, EVEX_B; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4632 | } |
| 4633 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4634 | // Conversion with rounding control (RC) |
| 4635 | multiclass avx512_vcvt_fp_rc<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, |
| 4636 | X86VectorVTInfo _Src, SDNode OpNodeRnd> { |
| 4637 | defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 4638 | (ins _Src.RC:$src, AVX512RC:$rc), OpcodeStr, |
| 4639 | "$rc, $src", "$src, $rc", |
| 4640 | (_.VT (OpNodeRnd (_Src.VT _Src.RC:$src), (i32 imm:$rc)))>, |
| 4641 | EVEX, EVEX_B, EVEX_RC; |
Elena Demikhovsky | de3f751 | 2014-01-01 15:12:34 +0000 | [diff] [blame] | 4642 | } |
| 4643 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4644 | // Extend Float to Double |
| 4645 | multiclass avx512_cvtps2pd<bits<8> opc, string OpcodeStr> { |
| 4646 | let Predicates = [HasAVX512] in { |
| 4647 | defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8f32x_info, fextend>, |
| 4648 | avx512_vcvt_fp_sae<opc, OpcodeStr, v8f64_info, v8f32x_info, |
| 4649 | X86vfpextRnd>, EVEX_V512; |
| 4650 | } |
| 4651 | let Predicates = [HasVLX] in { |
| 4652 | defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4f32x_info, |
| 4653 | X86vfpext, "{1to2}">, EVEX_V128; |
| 4654 | defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4f32x_info, fextend>, |
| 4655 | EVEX_V256; |
| 4656 | } |
| 4657 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4658 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4659 | // Truncate Double to Float |
| 4660 | multiclass avx512_cvtpd2ps<bits<8> opc, string OpcodeStr> { |
| 4661 | let Predicates = [HasAVX512] in { |
| 4662 | defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8f64_info, fround>, |
| 4663 | avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8f64_info, |
| 4664 | X86vfproundRnd>, EVEX_V512; |
| 4665 | } |
| 4666 | let Predicates = [HasVLX] in { |
| 4667 | defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2f64x_info, |
| 4668 | X86vfpround, "{1to2}", "{x}">, EVEX_V128; |
| 4669 | defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4f64x_info, fround, |
| 4670 | "{1to4}", "{y}">, EVEX_V256; |
| 4671 | } |
| 4672 | } |
| 4673 | |
| 4674 | defm VCVTPD2PS : avx512_cvtpd2ps<0x5A, "vcvtpd2ps">, |
| 4675 | VEX_W, PD, EVEX_CD8<64, CD8VF>; |
| 4676 | defm VCVTPS2PD : avx512_cvtps2pd<0x5A, "vcvtps2pd">, |
| 4677 | PS, EVEX_CD8<32, CD8VH>; |
| 4678 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4679 | def : Pat<(v8f64 (extloadv8f32 addr:$src)), |
| 4680 | (VCVTPS2PDZrm addr:$src)>; |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 4681 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4682 | let Predicates = [HasVLX] in { |
| 4683 | def : Pat<(v4f64 (extloadv4f32 addr:$src)), |
| 4684 | (VCVTPS2PDZ256rm addr:$src)>; |
| 4685 | } |
Elena Demikhovsky | 3629b4a | 2014-01-06 08:45:54 +0000 | [diff] [blame] | 4686 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4687 | // Convert Signed/Unsigned Doubleword to Double |
| 4688 | multiclass avx512_cvtdq2pd<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 4689 | SDNode OpNode128> { |
| 4690 | // No rounding in this op |
| 4691 | let Predicates = [HasAVX512] in |
| 4692 | defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i32x_info, OpNode>, |
| 4693 | EVEX_V512; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4694 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4695 | let Predicates = [HasVLX] in { |
| 4696 | defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4i32x_info, |
| 4697 | OpNode128, "{1to2}">, EVEX_V128; |
| 4698 | defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i32x_info, OpNode>, |
| 4699 | EVEX_V256; |
| 4700 | } |
| 4701 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4702 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4703 | // Convert Signed/Unsigned Doubleword to Float |
| 4704 | multiclass avx512_cvtdq2ps<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 4705 | SDNode OpNodeRnd> { |
| 4706 | let Predicates = [HasAVX512] in |
| 4707 | defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16f32_info, v16i32_info, OpNode>, |
| 4708 | avx512_vcvt_fp_rc<opc, OpcodeStr, v16f32_info, v16i32_info, |
| 4709 | OpNodeRnd>, EVEX_V512; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4710 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4711 | let Predicates = [HasVLX] in { |
| 4712 | defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i32x_info, OpNode>, |
| 4713 | EVEX_V128; |
| 4714 | defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i32x_info, OpNode>, |
| 4715 | EVEX_V256; |
| 4716 | } |
| 4717 | } |
| 4718 | |
| 4719 | // Convert Float to Signed/Unsigned Doubleword with truncation |
| 4720 | multiclass avx512_cvttps2dq<bits<8> opc, string OpcodeStr, |
| 4721 | SDNode OpNode, SDNode OpNodeRnd> { |
| 4722 | let Predicates = [HasAVX512] in { |
| 4723 | defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode>, |
| 4724 | avx512_vcvt_fp_sae<opc, OpcodeStr, v16i32_info, v16f32_info, |
| 4725 | OpNodeRnd>, EVEX_V512; |
| 4726 | } |
| 4727 | let Predicates = [HasVLX] in { |
| 4728 | defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode>, |
| 4729 | EVEX_V128; |
| 4730 | defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode>, |
| 4731 | EVEX_V256; |
| 4732 | } |
| 4733 | } |
| 4734 | |
| 4735 | // Convert Float to Signed/Unsigned Doubleword |
| 4736 | multiclass avx512_cvtps2dq<bits<8> opc, string OpcodeStr, |
| 4737 | SDNode OpNode, SDNode OpNodeRnd> { |
| 4738 | let Predicates = [HasAVX512] in { |
| 4739 | defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode>, |
| 4740 | avx512_vcvt_fp_rc<opc, OpcodeStr, v16i32_info, v16f32_info, |
| 4741 | OpNodeRnd>, EVEX_V512; |
| 4742 | } |
| 4743 | let Predicates = [HasVLX] in { |
| 4744 | defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode>, |
| 4745 | EVEX_V128; |
| 4746 | defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode>, |
| 4747 | EVEX_V256; |
| 4748 | } |
| 4749 | } |
| 4750 | |
| 4751 | // Convert Double to Signed/Unsigned Doubleword with truncation |
| 4752 | multiclass avx512_cvttpd2dq<bits<8> opc, string OpcodeStr, |
| 4753 | SDNode OpNode, SDNode OpNodeRnd> { |
| 4754 | let Predicates = [HasAVX512] in { |
| 4755 | defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode>, |
| 4756 | avx512_vcvt_fp_sae<opc, OpcodeStr, v8i32x_info, v8f64_info, |
| 4757 | OpNodeRnd>, EVEX_V512; |
| 4758 | } |
| 4759 | let Predicates = [HasVLX] in { |
| 4760 | // we need "x"/"y" suffixes in order to distinguish between 128 and 256 |
| 4761 | // memory forms of these instructions in Asm Parcer. They have the same |
| 4762 | // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly |
| 4763 | // due to the same reason. |
| 4764 | defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode, |
| 4765 | "{1to2}", "{x}">, EVEX_V128; |
| 4766 | defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode, |
| 4767 | "{1to4}", "{y}">, EVEX_V256; |
| 4768 | } |
| 4769 | } |
| 4770 | |
| 4771 | // Convert Double to Signed/Unsigned Doubleword |
| 4772 | multiclass avx512_cvtpd2dq<bits<8> opc, string OpcodeStr, |
| 4773 | SDNode OpNode, SDNode OpNodeRnd> { |
| 4774 | let Predicates = [HasAVX512] in { |
| 4775 | defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode>, |
| 4776 | avx512_vcvt_fp_rc<opc, OpcodeStr, v8i32x_info, v8f64_info, |
| 4777 | OpNodeRnd>, EVEX_V512; |
| 4778 | } |
| 4779 | let Predicates = [HasVLX] in { |
| 4780 | // we need "x"/"y" suffixes in order to distinguish between 128 and 256 |
| 4781 | // memory forms of these instructions in Asm Parcer. They have the same |
| 4782 | // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly |
| 4783 | // due to the same reason. |
| 4784 | defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode, |
| 4785 | "{1to2}", "{x}">, EVEX_V128; |
| 4786 | defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode, |
| 4787 | "{1to4}", "{y}">, EVEX_V256; |
| 4788 | } |
| 4789 | } |
| 4790 | |
| 4791 | // Convert Double to Signed/Unsigned Quardword |
| 4792 | multiclass avx512_cvtpd2qq<bits<8> opc, string OpcodeStr, |
| 4793 | SDNode OpNode, SDNode OpNodeRnd> { |
| 4794 | let Predicates = [HasDQI] in { |
| 4795 | defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode>, |
| 4796 | avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f64_info, |
| 4797 | OpNodeRnd>, EVEX_V512; |
| 4798 | } |
| 4799 | let Predicates = [HasDQI, HasVLX] in { |
| 4800 | defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode>, |
| 4801 | EVEX_V128; |
| 4802 | defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode>, |
| 4803 | EVEX_V256; |
| 4804 | } |
| 4805 | } |
| 4806 | |
| 4807 | // Convert Double to Signed/Unsigned Quardword with truncation |
| 4808 | multiclass avx512_cvttpd2qq<bits<8> opc, string OpcodeStr, |
| 4809 | SDNode OpNode, SDNode OpNodeRnd> { |
| 4810 | let Predicates = [HasDQI] in { |
| 4811 | defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode>, |
| 4812 | avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f64_info, |
| 4813 | OpNodeRnd>, EVEX_V512; |
| 4814 | } |
| 4815 | let Predicates = [HasDQI, HasVLX] in { |
| 4816 | defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode>, |
| 4817 | EVEX_V128; |
| 4818 | defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode>, |
| 4819 | EVEX_V256; |
| 4820 | } |
| 4821 | } |
| 4822 | |
| 4823 | // Convert Signed/Unsigned Quardword to Double |
| 4824 | multiclass avx512_cvtqq2pd<bits<8> opc, string OpcodeStr, |
| 4825 | SDNode OpNode, SDNode OpNodeRnd> { |
| 4826 | let Predicates = [HasDQI] in { |
| 4827 | defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i64_info, OpNode>, |
| 4828 | avx512_vcvt_fp_rc<opc, OpcodeStr, v8f64_info, v8i64_info, |
| 4829 | OpNodeRnd>, EVEX_V512; |
| 4830 | } |
| 4831 | let Predicates = [HasDQI, HasVLX] in { |
| 4832 | defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v2i64x_info, OpNode>, |
| 4833 | EVEX_V128; |
| 4834 | defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i64x_info, OpNode>, |
| 4835 | EVEX_V256; |
| 4836 | } |
| 4837 | } |
| 4838 | |
| 4839 | // Convert Float to Signed/Unsigned Quardword |
| 4840 | multiclass avx512_cvtps2qq<bits<8> opc, string OpcodeStr, |
| 4841 | SDNode OpNode, SDNode OpNodeRnd> { |
| 4842 | let Predicates = [HasDQI] in { |
| 4843 | defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode>, |
| 4844 | avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f32x_info, |
| 4845 | OpNodeRnd>, EVEX_V512; |
| 4846 | } |
| 4847 | let Predicates = [HasDQI, HasVLX] in { |
| 4848 | // Explicitly specified broadcast string, since we take only 2 elements |
| 4849 | // from v4f32x_info source |
| 4850 | defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode, |
| 4851 | "{1to2}">, EVEX_V128; |
| 4852 | defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode>, |
| 4853 | EVEX_V256; |
| 4854 | } |
| 4855 | } |
| 4856 | |
| 4857 | // Convert Float to Signed/Unsigned Quardword with truncation |
| 4858 | multiclass avx512_cvttps2qq<bits<8> opc, string OpcodeStr, |
| 4859 | SDNode OpNode, SDNode OpNodeRnd> { |
| 4860 | let Predicates = [HasDQI] in { |
| 4861 | defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode>, |
| 4862 | avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f32x_info, |
| 4863 | OpNodeRnd>, EVEX_V512; |
| 4864 | } |
| 4865 | let Predicates = [HasDQI, HasVLX] in { |
| 4866 | // Explicitly specified broadcast string, since we take only 2 elements |
| 4867 | // from v4f32x_info source |
| 4868 | defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode, |
| 4869 | "{1to2}">, EVEX_V128; |
| 4870 | defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode>, |
| 4871 | EVEX_V256; |
| 4872 | } |
| 4873 | } |
| 4874 | |
| 4875 | // Convert Signed/Unsigned Quardword to Float |
| 4876 | multiclass avx512_cvtqq2ps<bits<8> opc, string OpcodeStr, |
| 4877 | SDNode OpNode, SDNode OpNodeRnd> { |
| 4878 | let Predicates = [HasDQI] in { |
| 4879 | defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i64_info, OpNode>, |
| 4880 | avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8i64_info, |
| 4881 | OpNodeRnd>, EVEX_V512; |
| 4882 | } |
| 4883 | let Predicates = [HasDQI, HasVLX] in { |
| 4884 | // we need "x"/"y" suffixes in order to distinguish between 128 and 256 |
| 4885 | // memory forms of these instructions in Asm Parcer. They have the same |
| 4886 | // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly |
| 4887 | // due to the same reason. |
| 4888 | defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2i64x_info, OpNode, |
| 4889 | "{1to2}", "{x}">, EVEX_V128; |
| 4890 | defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i64x_info, OpNode, |
| 4891 | "{1to4}", "{y}">, EVEX_V256; |
| 4892 | } |
| 4893 | } |
| 4894 | |
| 4895 | defm VCVTDQ2PD : avx512_cvtdq2pd<0xE6, "vcvtdq2pd", sint_to_fp, X86cvtdq2pd>, XS, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4896 | EVEX_CD8<32, CD8VH>; |
| 4897 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4898 | defm VCVTDQ2PS : avx512_cvtdq2ps<0x5B, "vcvtdq2ps", sint_to_fp, |
| 4899 | X86VSintToFpRnd>, |
| 4900 | PS, EVEX_CD8<32, CD8VF>; |
| 4901 | |
| 4902 | defm VCVTTPS2DQ : avx512_cvttps2dq<0x5B, "vcvttps2dq", fp_to_sint, |
| 4903 | X86VFpToSintRnd>, |
| 4904 | XS, EVEX_CD8<32, CD8VF>; |
| 4905 | |
| 4906 | defm VCVTTPD2DQ : avx512_cvttpd2dq<0xE6, "vcvttpd2dq", fp_to_sint, |
| 4907 | X86VFpToSintRnd>, |
| 4908 | PD, VEX_W, EVEX_CD8<64, CD8VF>; |
| 4909 | |
| 4910 | defm VCVTTPS2UDQ : avx512_cvttps2dq<0x78, "vcvttps2udq", fp_to_uint, |
| 4911 | X86VFpToUintRnd>, PS, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4912 | EVEX_CD8<32, CD8VF>; |
| 4913 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4914 | defm VCVTTPD2UDQ : avx512_cvttpd2dq<0x78, "vcvttpd2udq", fp_to_uint, |
| 4915 | X86VFpToUintRnd>, PS, VEX_W, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4916 | EVEX_CD8<64, CD8VF>; |
| 4917 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4918 | defm VCVTUDQ2PD : avx512_cvtdq2pd<0x7A, "vcvtudq2pd", uint_to_fp, X86cvtudq2pd>, |
| 4919 | XS, EVEX_CD8<32, CD8VH>; |
| 4920 | |
| 4921 | defm VCVTUDQ2PS : avx512_cvtdq2ps<0x7A, "vcvtudq2ps", uint_to_fp, |
| 4922 | X86VUintToFpRnd>, XD, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4923 | EVEX_CD8<32, CD8VF>; |
| 4924 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4925 | defm VCVTPS2DQ : avx512_cvtps2dq<0x5B, "vcvtps2dq", X86cvtps2Int, |
| 4926 | X86cvtps2IntRnd>, PD, EVEX_CD8<32, CD8VF>; |
Elena Demikhovsky | de3f751 | 2014-01-01 15:12:34 +0000 | [diff] [blame] | 4927 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4928 | defm VCVTPD2DQ : avx512_cvtpd2dq<0xE6, "vcvtpd2dq", X86cvtpd2Int, |
| 4929 | X86cvtpd2IntRnd>, XD, VEX_W, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4930 | EVEX_CD8<64, CD8VF>; |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 4931 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4932 | defm VCVTPS2UDQ : avx512_cvtps2dq<0x79, "vcvtps2udq", X86cvtps2UInt, |
| 4933 | X86cvtps2UIntRnd>, |
| 4934 | PS, EVEX_CD8<32, CD8VF>; |
| 4935 | defm VCVTPD2UDQ : avx512_cvtpd2dq<0x79, "vcvtpd2udq", X86cvtpd2UInt, |
| 4936 | X86cvtpd2UIntRnd>, VEX_W, |
| 4937 | PS, EVEX_CD8<64, CD8VF>; |
Elena Demikhovsky | de3f751 | 2014-01-01 15:12:34 +0000 | [diff] [blame] | 4938 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4939 | defm VCVTPD2QQ : avx512_cvtpd2qq<0x7B, "vcvtpd2qq", X86cvtpd2Int, |
| 4940 | X86cvtpd2IntRnd>, VEX_W, |
| 4941 | PD, EVEX_CD8<64, CD8VF>; |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 4942 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4943 | defm VCVTPS2QQ : avx512_cvtps2qq<0x7B, "vcvtps2qq", X86cvtps2Int, |
| 4944 | X86cvtps2IntRnd>, PD, EVEX_CD8<32, CD8VH>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4945 | |
Elena Demikhovsky | 0f37093 | 2015-07-13 13:26:20 +0000 | [diff] [blame] | 4946 | defm VCVTPD2UQQ : avx512_cvtpd2qq<0x79, "vcvtpd2uqq", X86cvtpd2UInt, |
| 4947 | X86cvtpd2UIntRnd>, VEX_W, |
| 4948 | PD, EVEX_CD8<64, CD8VF>; |
| 4949 | |
| 4950 | defm VCVTPS2UQQ : avx512_cvtps2qq<0x79, "vcvtps2uqq", X86cvtps2UInt, |
| 4951 | X86cvtps2UIntRnd>, PD, EVEX_CD8<32, CD8VH>; |
| 4952 | |
| 4953 | defm VCVTTPD2QQ : avx512_cvttpd2qq<0x7A, "vcvttpd2qq", fp_to_sint, |
| 4954 | X86VFpToSlongRnd>, VEX_W, |
| 4955 | PD, EVEX_CD8<64, CD8VF>; |
| 4956 | |
| 4957 | defm VCVTTPS2QQ : avx512_cvttps2qq<0x7A, "vcvttps2qq", fp_to_sint, |
| 4958 | X86VFpToSlongRnd>, PD, EVEX_CD8<32, CD8VH>; |
| 4959 | |
| 4960 | defm VCVTTPD2UQQ : avx512_cvttpd2qq<0x78, "vcvttpd2uqq", fp_to_uint, |
| 4961 | X86VFpToUlongRnd>, VEX_W, |
| 4962 | PD, EVEX_CD8<64, CD8VF>; |
| 4963 | |
| 4964 | defm VCVTTPS2UQQ : avx512_cvttps2qq<0x78, "vcvttps2uqq", fp_to_uint, |
| 4965 | X86VFpToUlongRnd>, PD, EVEX_CD8<32, CD8VH>; |
| 4966 | |
| 4967 | defm VCVTQQ2PD : avx512_cvtqq2pd<0xE6, "vcvtqq2pd", sint_to_fp, |
| 4968 | X86VSlongToFpRnd>, VEX_W, XS, EVEX_CD8<64, CD8VF>; |
| 4969 | |
| 4970 | defm VCVTUQQ2PD : avx512_cvtqq2pd<0x7A, "vcvtuqq2pd", uint_to_fp, |
| 4971 | X86VUlongToFpRnd>, VEX_W, XS, EVEX_CD8<64, CD8VF>; |
| 4972 | |
| 4973 | defm VCVTQQ2PS : avx512_cvtqq2ps<0x5B, "vcvtqq2ps", sint_to_fp, |
| 4974 | X86VSlongToFpRnd>, VEX_W, PS, EVEX_CD8<64, CD8VF>; |
| 4975 | |
| 4976 | defm VCVTUQQ2PS : avx512_cvtqq2ps<0x7A, "vcvtuqq2ps", uint_to_fp, |
| 4977 | X86VUlongToFpRnd>, VEX_W, XD, EVEX_CD8<64, CD8VF>; |
| 4978 | |
| 4979 | let Predicates = [NoVLX] in { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4980 | def : Pat<(v8i32 (fp_to_uint (v8f32 VR256X:$src1))), |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 4981 | (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4982 | (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>; |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 4983 | |
Elena Demikhovsky | 3dcfbdf | 2014-04-08 07:24:02 +0000 | [diff] [blame] | 4984 | def : Pat<(v4i32 (fp_to_uint (v4f32 VR128X:$src1))), |
| 4985 | (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr |
| 4986 | (v16f32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_xmm)>; |
| 4987 | |
| 4988 | def : Pat<(v8f32 (uint_to_fp (v8i32 VR256X:$src1))), |
| 4989 | (EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr |
| 4990 | (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>; |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 4991 | |
Elena Demikhovsky | 3dcfbdf | 2014-04-08 07:24:02 +0000 | [diff] [blame] | 4992 | def : Pat<(v4f32 (uint_to_fp (v4i32 VR128X:$src1))), |
| 4993 | (EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr |
| 4994 | (v16i32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_xmm)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 4995 | |
Cameron McInally | f10a7c9 | 2014-06-18 14:04:37 +0000 | [diff] [blame] | 4996 | def : Pat<(v4f64 (uint_to_fp (v4i32 VR128X:$src1))), |
| 4997 | (EXTRACT_SUBREG (v8f64 (VCVTUDQ2PDZrr |
| 4998 | (v8i32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_ymm)>; |
Elena Demikhovsky | de3f751 | 2014-01-01 15:12:34 +0000 | [diff] [blame] | 4999 | } |
| 5000 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5001 | let Predicates = [HasAVX512] in { |
| 5002 | def : Pat<(v8f32 (fround (loadv8f64 addr:$src))), |
| 5003 | (VCVTPD2PSZrm addr:$src)>; |
| 5004 | def : Pat<(v8f64 (extloadv8f32 addr:$src)), |
| 5005 | (VCVTPS2PDZrm addr:$src)>; |
| 5006 | } |
| 5007 | |
Elena Demikhovsky | dd0794e | 2013-10-24 07:16:35 +0000 | [diff] [blame] | 5008 | //===----------------------------------------------------------------------===// |
| 5009 | // Half precision conversion instructions |
| 5010 | //===----------------------------------------------------------------------===// |
Elena Demikhovsky | a30e437 | 2014-02-05 07:05:03 +0000 | [diff] [blame] | 5011 | multiclass avx512_cvtph2ps<RegisterClass destRC, RegisterClass srcRC, |
| 5012 | X86MemOperand x86memop> { |
Elena Demikhovsky | dd0794e | 2013-10-24 07:16:35 +0000 | [diff] [blame] | 5013 | def rr : AVX5128I<0x13, MRMSrcReg, (outs destRC:$dst), (ins srcRC:$src), |
| 5014 | "vcvtph2ps\t{$src, $dst|$dst, $src}", |
Elena Demikhovsky | a30e437 | 2014-02-05 07:05:03 +0000 | [diff] [blame] | 5015 | []>, EVEX; |
Elena Demikhovsky | f404e05 | 2014-01-05 14:21:07 +0000 | [diff] [blame] | 5016 | let hasSideEffects = 0, mayLoad = 1 in |
Elena Demikhovsky | dd0794e | 2013-10-24 07:16:35 +0000 | [diff] [blame] | 5017 | def rm : AVX5128I<0x13, MRMSrcMem, (outs destRC:$dst), (ins x86memop:$src), |
| 5018 | "vcvtph2ps\t{$src, $dst|$dst, $src}", []>, EVEX; |
| 5019 | } |
| 5020 | |
Elena Demikhovsky | a30e437 | 2014-02-05 07:05:03 +0000 | [diff] [blame] | 5021 | multiclass avx512_cvtps2ph<RegisterClass destRC, RegisterClass srcRC, |
| 5022 | X86MemOperand x86memop> { |
Elena Demikhovsky | dd0794e | 2013-10-24 07:16:35 +0000 | [diff] [blame] | 5023 | def rr : AVX512AIi8<0x1D, MRMDestReg, (outs destRC:$dst), |
Craig Topper | 53a8467 | 2015-01-25 02:21:16 +0000 | [diff] [blame] | 5024 | (ins srcRC:$src1, i32u8imm:$src2), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 5025 | "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", |
Elena Demikhovsky | a30e437 | 2014-02-05 07:05:03 +0000 | [diff] [blame] | 5026 | []>, EVEX; |
Elena Demikhovsky | f404e05 | 2014-01-05 14:21:07 +0000 | [diff] [blame] | 5027 | let hasSideEffects = 0, mayStore = 1 in |
Elena Demikhovsky | dd0794e | 2013-10-24 07:16:35 +0000 | [diff] [blame] | 5028 | def mr : AVX512AIi8<0x1D, MRMDestMem, (outs), |
Craig Topper | 53a8467 | 2015-01-25 02:21:16 +0000 | [diff] [blame] | 5029 | (ins x86memop:$dst, srcRC:$src1, i32u8imm:$src2), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 5030 | "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, EVEX; |
Elena Demikhovsky | dd0794e | 2013-10-24 07:16:35 +0000 | [diff] [blame] | 5031 | } |
| 5032 | |
Elena Demikhovsky | a30e437 | 2014-02-05 07:05:03 +0000 | [diff] [blame] | 5033 | defm VCVTPH2PSZ : avx512_cvtph2ps<VR512, VR256X, f256mem>, EVEX_V512, |
Elena Demikhovsky | dd0794e | 2013-10-24 07:16:35 +0000 | [diff] [blame] | 5034 | EVEX_CD8<32, CD8VH>; |
Elena Demikhovsky | a30e437 | 2014-02-05 07:05:03 +0000 | [diff] [blame] | 5035 | defm VCVTPS2PHZ : avx512_cvtps2ph<VR256X, VR512, f256mem>, EVEX_V512, |
Elena Demikhovsky | dd0794e | 2013-10-24 07:16:35 +0000 | [diff] [blame] | 5036 | EVEX_CD8<32, CD8VH>; |
| 5037 | |
Elena Demikhovsky | a30e437 | 2014-02-05 07:05:03 +0000 | [diff] [blame] | 5038 | def : Pat<(v16i16 (int_x86_avx512_mask_vcvtps2ph_512 (v16f32 VR512:$src), |
| 5039 | imm:$rc, (bc_v16i16(v8i32 immAllZerosV)), (i16 -1))), |
| 5040 | (VCVTPS2PHZrr VR512:$src, imm:$rc)>; |
| 5041 | |
| 5042 | def : Pat<(v16f32 (int_x86_avx512_mask_vcvtph2ps_512 (v16i16 VR256X:$src), |
| 5043 | (bc_v16f32(v16i32 immAllZerosV)), (i16 -1), (i32 FROUND_CURRENT))), |
| 5044 | (VCVTPH2PSZrr VR256X:$src)>; |
| 5045 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5046 | let Defs = [EFLAGS], Predicates = [HasAVX512] in { |
| 5047 | defm VUCOMISSZ : sse12_ord_cmp<0x2E, FR32X, X86cmp, f32, f32mem, loadf32, |
Craig Topper | 5ccb617 | 2014-02-18 00:21:49 +0000 | [diff] [blame] | 5048 | "ucomiss">, PS, EVEX, VEX_LIG, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5049 | EVEX_CD8<32, CD8VT1>; |
| 5050 | defm VUCOMISDZ : sse12_ord_cmp<0x2E, FR64X, X86cmp, f64, f64mem, loadf64, |
Craig Topper | ae11aed | 2014-01-14 07:41:20 +0000 | [diff] [blame] | 5051 | "ucomisd">, PD, EVEX, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5052 | VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; |
| 5053 | let Pattern = []<dag> in { |
Marina Yatsina | 7a4e1ba | 2015-08-20 11:21:36 +0000 | [diff] [blame] | 5054 | defm VCOMISSZ : sse12_ord_cmp<0x2F, FR32X, undef, f32, f32mem, loadf32, |
Craig Topper | 5ccb617 | 2014-02-18 00:21:49 +0000 | [diff] [blame] | 5055 | "comiss">, PS, EVEX, VEX_LIG, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5056 | EVEX_CD8<32, CD8VT1>; |
Marina Yatsina | 7a4e1ba | 2015-08-20 11:21:36 +0000 | [diff] [blame] | 5057 | defm VCOMISDZ : sse12_ord_cmp<0x2F, FR64X, undef, f64, f64mem, loadf64, |
Craig Topper | ae11aed | 2014-01-14 07:41:20 +0000 | [diff] [blame] | 5058 | "comisd">, PD, EVEX, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5059 | VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; |
| 5060 | } |
Craig Topper | 9dd48c8 | 2014-01-02 17:28:14 +0000 | [diff] [blame] | 5061 | let isCodeGenOnly = 1 in { |
| 5062 | defm Int_VUCOMISSZ : sse12_ord_cmp<0x2E, VR128X, X86ucomi, v4f32, f128mem, |
Craig Topper | 5ccb617 | 2014-02-18 00:21:49 +0000 | [diff] [blame] | 5063 | load, "ucomiss">, PS, EVEX, VEX_LIG, |
Craig Topper | 9dd48c8 | 2014-01-02 17:28:14 +0000 | [diff] [blame] | 5064 | EVEX_CD8<32, CD8VT1>; |
| 5065 | defm Int_VUCOMISDZ : sse12_ord_cmp<0x2E, VR128X, X86ucomi, v2f64, f128mem, |
Craig Topper | ae11aed | 2014-01-14 07:41:20 +0000 | [diff] [blame] | 5066 | load, "ucomisd">, PD, EVEX, |
Craig Topper | 9dd48c8 | 2014-01-02 17:28:14 +0000 | [diff] [blame] | 5067 | VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5068 | |
Craig Topper | 9dd48c8 | 2014-01-02 17:28:14 +0000 | [diff] [blame] | 5069 | defm Int_VCOMISSZ : sse12_ord_cmp<0x2F, VR128X, X86comi, v4f32, f128mem, |
Craig Topper | 5ccb617 | 2014-02-18 00:21:49 +0000 | [diff] [blame] | 5070 | load, "comiss">, PS, EVEX, VEX_LIG, |
Craig Topper | 9dd48c8 | 2014-01-02 17:28:14 +0000 | [diff] [blame] | 5071 | EVEX_CD8<32, CD8VT1>; |
| 5072 | defm Int_VCOMISDZ : sse12_ord_cmp<0x2F, VR128X, X86comi, v2f64, f128mem, |
Craig Topper | ae11aed | 2014-01-14 07:41:20 +0000 | [diff] [blame] | 5073 | load, "comisd">, PD, EVEX, |
Craig Topper | 9dd48c8 | 2014-01-02 17:28:14 +0000 | [diff] [blame] | 5074 | VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; |
| 5075 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5076 | } |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 5077 | |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5078 | /// avx512_fp14_s rcp14ss, rcp14sd, rsqrt14ss, rsqrt14sd |
| 5079 | multiclass avx512_fp14_s<bits<8> opc, string OpcodeStr, RegisterClass RC, |
| 5080 | X86MemOperand x86memop> { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5081 | let hasSideEffects = 0 in { |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5082 | def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), |
| 5083 | (ins RC:$src1, RC:$src2), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5084 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 5085 | "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, EVEX_4V; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5086 | let mayLoad = 1 in { |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5087 | def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), |
| 5088 | (ins RC:$src1, x86memop:$src2), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5089 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 5090 | "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, EVEX_4V; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5091 | } |
| 5092 | } |
| 5093 | } |
| 5094 | |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5095 | defm VRCP14SS : avx512_fp14_s<0x4D, "vrcp14ss", FR32X, f32mem>, |
| 5096 | EVEX_CD8<32, CD8VT1>; |
| 5097 | defm VRCP14SD : avx512_fp14_s<0x4D, "vrcp14sd", FR64X, f64mem>, |
| 5098 | VEX_W, EVEX_CD8<64, CD8VT1>; |
| 5099 | defm VRSQRT14SS : avx512_fp14_s<0x4F, "vrsqrt14ss", FR32X, f32mem>, |
| 5100 | EVEX_CD8<32, CD8VT1>; |
| 5101 | defm VRSQRT14SD : avx512_fp14_s<0x4F, "vrsqrt14sd", FR64X, f64mem>, |
| 5102 | VEX_W, EVEX_CD8<64, CD8VT1>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5103 | |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5104 | def : Pat <(v4f32 (int_x86_avx512_rcp14_ss (v4f32 VR128X:$src1), |
| 5105 | (v4f32 VR128X:$src2), (bc_v4f32 (v4i32 immAllZerosV)), (i8 -1))), |
| 5106 | (COPY_TO_REGCLASS (VRCP14SSrr (COPY_TO_REGCLASS VR128X:$src1, FR32X), |
| 5107 | (COPY_TO_REGCLASS VR128X:$src2, FR32X)), VR128X)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5108 | |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5109 | def : Pat <(v2f64 (int_x86_avx512_rcp14_sd (v2f64 VR128X:$src1), |
| 5110 | (v2f64 VR128X:$src2), (bc_v2f64 (v4i32 immAllZerosV)), (i8 -1))), |
| 5111 | (COPY_TO_REGCLASS (VRCP14SDrr (COPY_TO_REGCLASS VR128X:$src1, FR64X), |
| 5112 | (COPY_TO_REGCLASS VR128X:$src2, FR64X)), VR128X)>; |
Elena Demikhovsky | a3a7140 | 2013-10-09 08:16:14 +0000 | [diff] [blame] | 5113 | |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5114 | def : Pat <(v4f32 (int_x86_avx512_rsqrt14_ss (v4f32 VR128X:$src1), |
| 5115 | (v4f32 VR128X:$src2), (bc_v4f32 (v4i32 immAllZerosV)), (i8 -1))), |
| 5116 | (COPY_TO_REGCLASS (VRSQRT14SSrr (COPY_TO_REGCLASS VR128X:$src1, FR32X), |
| 5117 | (COPY_TO_REGCLASS VR128X:$src2, FR32X)), VR128X)>; |
Elena Demikhovsky | a3a7140 | 2013-10-09 08:16:14 +0000 | [diff] [blame] | 5118 | |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5119 | def : Pat <(v2f64 (int_x86_avx512_rsqrt14_sd (v2f64 VR128X:$src1), |
| 5120 | (v2f64 VR128X:$src2), (bc_v2f64 (v4i32 immAllZerosV)), (i8 -1))), |
| 5121 | (COPY_TO_REGCLASS (VRSQRT14SDrr (COPY_TO_REGCLASS VR128X:$src1, FR64X), |
| 5122 | (COPY_TO_REGCLASS VR128X:$src2, FR64X)), VR128X)>; |
Elena Demikhovsky | a3a7140 | 2013-10-09 08:16:14 +0000 | [diff] [blame] | 5123 | |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5124 | /// avx512_fp14_p rcp14ps, rcp14pd, rsqrt14ps, rsqrt14pd |
| 5125 | multiclass avx512_fp14_p<bits<8> opc, string OpcodeStr, SDNode OpNode, |
Robert Khasanov | 3e534c9 | 2014-10-28 16:37:13 +0000 | [diff] [blame] | 5126 | X86VectorVTInfo _> { |
| 5127 | defm r: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 5128 | (ins _.RC:$src), OpcodeStr, "$src", "$src", |
| 5129 | (_.FloatVT (OpNode _.RC:$src))>, EVEX, T8PD; |
| 5130 | let mayLoad = 1 in { |
| 5131 | defm m: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 5132 | (ins _.MemOp:$src), OpcodeStr, "$src", "$src", |
| 5133 | (OpNode (_.FloatVT |
| 5134 | (bitconvert (_.LdFrag addr:$src))))>, EVEX, T8PD; |
| 5135 | defm mb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 5136 | (ins _.ScalarMemOp:$src), OpcodeStr, |
| 5137 | "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr, |
| 5138 | (OpNode (_.FloatVT |
| 5139 | (X86VBroadcast (_.ScalarLdFrag addr:$src))))>, |
| 5140 | EVEX, T8PD, EVEX_B; |
| 5141 | } |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5142 | } |
Robert Khasanov | 3e534c9 | 2014-10-28 16:37:13 +0000 | [diff] [blame] | 5143 | |
| 5144 | multiclass avx512_fp14_p_vl_all<bits<8> opc, string OpcodeStr, SDNode OpNode> { |
| 5145 | defm PSZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"), OpNode, v16f32_info>, |
| 5146 | EVEX_V512, EVEX_CD8<32, CD8VF>; |
| 5147 | defm PDZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"), OpNode, v8f64_info>, |
| 5148 | EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; |
| 5149 | |
| 5150 | // Define only if AVX512VL feature is present. |
| 5151 | let Predicates = [HasVLX] in { |
| 5152 | defm PSZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"), |
| 5153 | OpNode, v4f32x_info>, |
| 5154 | EVEX_V128, EVEX_CD8<32, CD8VF>; |
| 5155 | defm PSZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"), |
| 5156 | OpNode, v8f32x_info>, |
| 5157 | EVEX_V256, EVEX_CD8<32, CD8VF>; |
| 5158 | defm PDZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"), |
| 5159 | OpNode, v2f64x_info>, |
| 5160 | EVEX_V128, VEX_W, EVEX_CD8<64, CD8VF>; |
| 5161 | defm PDZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"), |
| 5162 | OpNode, v4f64x_info>, |
| 5163 | EVEX_V256, VEX_W, EVEX_CD8<64, CD8VF>; |
| 5164 | } |
| 5165 | } |
| 5166 | |
| 5167 | defm VRSQRT14 : avx512_fp14_p_vl_all<0x4E, "vrsqrt14", X86frsqrt>; |
| 5168 | defm VRCP14 : avx512_fp14_p_vl_all<0x4C, "vrcp14", X86frcp>; |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5169 | |
| 5170 | def : Pat <(v16f32 (int_x86_avx512_rsqrt14_ps_512 (v16f32 VR512:$src), |
| 5171 | (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1))), |
| 5172 | (VRSQRT14PSZr VR512:$src)>; |
| 5173 | def : Pat <(v8f64 (int_x86_avx512_rsqrt14_pd_512 (v8f64 VR512:$src), |
| 5174 | (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))), |
| 5175 | (VRSQRT14PDZr VR512:$src)>; |
| 5176 | |
| 5177 | def : Pat <(v16f32 (int_x86_avx512_rcp14_ps_512 (v16f32 VR512:$src), |
| 5178 | (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1))), |
| 5179 | (VRCP14PSZr VR512:$src)>; |
| 5180 | def : Pat <(v8f64 (int_x86_avx512_rcp14_pd_512 (v8f64 VR512:$src), |
| 5181 | (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))), |
| 5182 | (VRCP14PDZr VR512:$src)>; |
| 5183 | |
| 5184 | /// avx512_fp28_s rcp28ss, rcp28sd, rsqrt28ss, rsqrt28sd |
Elena Demikhovsky | 905a5a6 | 2014-11-26 10:46:49 +0000 | [diff] [blame] | 5185 | multiclass avx512_fp28_s<bits<8> opc, string OpcodeStr,X86VectorVTInfo _, |
| 5186 | SDNode OpNode> { |
| 5187 | |
| 5188 | defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 5189 | (ins _.RC:$src1, _.RC:$src2), OpcodeStr, |
| 5190 | "$src2, $src1", "$src1, $src2", |
| 5191 | (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), |
| 5192 | (i32 FROUND_CURRENT))>; |
| 5193 | |
| 5194 | defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 5195 | (ins _.RC:$src1, _.RC:$src2), OpcodeStr, |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 5196 | "{sae}, $src2, $src1", "$src1, $src2, {sae}", |
Elena Demikhovsky | 905a5a6 | 2014-11-26 10:46:49 +0000 | [diff] [blame] | 5197 | (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 5198 | (i32 FROUND_NO_EXC))>, EVEX_B; |
Elena Demikhovsky | 905a5a6 | 2014-11-26 10:46:49 +0000 | [diff] [blame] | 5199 | |
| 5200 | defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 5201 | (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr, |
| 5202 | "$src2, $src1", "$src1, $src2", |
| 5203 | (OpNode (_.VT _.RC:$src1), |
| 5204 | (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))), |
| 5205 | (i32 FROUND_CURRENT))>; |
Elena Demikhovsky | a3a7140 | 2013-10-09 08:16:14 +0000 | [diff] [blame] | 5206 | } |
| 5207 | |
Elena Demikhovsky | 905a5a6 | 2014-11-26 10:46:49 +0000 | [diff] [blame] | 5208 | multiclass avx512_eri_s<bits<8> opc, string OpcodeStr, SDNode OpNode> { |
| 5209 | defm SS : avx512_fp28_s<opc, OpcodeStr#"ss", f32x_info, OpNode>, |
| 5210 | EVEX_CD8<32, CD8VT1>; |
| 5211 | defm SD : avx512_fp28_s<opc, OpcodeStr#"sd", f64x_info, OpNode>, |
| 5212 | EVEX_CD8<64, CD8VT1>, VEX_W; |
| 5213 | } |
Elena Demikhovsky | a3a7140 | 2013-10-09 08:16:14 +0000 | [diff] [blame] | 5214 | |
Elena Demikhovsky | 905a5a6 | 2014-11-26 10:46:49 +0000 | [diff] [blame] | 5215 | let hasSideEffects = 0, Predicates = [HasERI] in { |
| 5216 | defm VRCP28 : avx512_eri_s<0xCB, "vrcp28", X86rcp28s>, T8PD, EVEX_4V; |
| 5217 | defm VRSQRT28 : avx512_eri_s<0xCD, "vrsqrt28", X86rsqrt28s>, T8PD, EVEX_4V; |
| 5218 | } |
Igor Breger | 8352a0d | 2015-07-28 06:53:28 +0000 | [diff] [blame] | 5219 | |
| 5220 | defm VGETEXP : avx512_eri_s<0x43, "vgetexp", X86fgetexpRnds>, T8PD, EVEX_4V; |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5221 | /// avx512_fp28_p rcp28ps, rcp28pd, rsqrt28ps, rsqrt28pd |
Elena Demikhovsky | be8808d | 2014-11-12 07:31:03 +0000 | [diff] [blame] | 5222 | |
| 5223 | multiclass avx512_fp28_p<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, |
| 5224 | SDNode OpNode> { |
| 5225 | |
| 5226 | defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 5227 | (ins _.RC:$src), OpcodeStr, "$src", "$src", |
| 5228 | (OpNode (_.VT _.RC:$src), (i32 FROUND_CURRENT))>; |
| 5229 | |
Elena Demikhovsky | be8808d | 2014-11-12 07:31:03 +0000 | [diff] [blame] | 5230 | defm m : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 5231 | (ins _.MemOp:$src), OpcodeStr, "$src", "$src", |
| 5232 | (OpNode (_.FloatVT |
Elena Demikhovsky | 905a5a6 | 2014-11-26 10:46:49 +0000 | [diff] [blame] | 5233 | (bitconvert (_.LdFrag addr:$src))), |
| 5234 | (i32 FROUND_CURRENT))>; |
Elena Demikhovsky | be8808d | 2014-11-12 07:31:03 +0000 | [diff] [blame] | 5235 | |
| 5236 | defm mb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
Asaf Badouh | 402ebb3 | 2015-06-03 13:41:48 +0000 | [diff] [blame] | 5237 | (ins _.MemOp:$src), OpcodeStr, |
| 5238 | "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr, |
Elena Demikhovsky | be8808d | 2014-11-12 07:31:03 +0000 | [diff] [blame] | 5239 | (OpNode (_.FloatVT |
| 5240 | (X86VBroadcast (_.ScalarLdFrag addr:$src))), |
| 5241 | (i32 FROUND_CURRENT))>, EVEX_B; |
Elena Demikhovsky | a3a7140 | 2013-10-09 08:16:14 +0000 | [diff] [blame] | 5242 | } |
Asaf Badouh | 402ebb3 | 2015-06-03 13:41:48 +0000 | [diff] [blame] | 5243 | multiclass avx512_fp28_p_round<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, |
| 5244 | SDNode OpNode> { |
| 5245 | defm rb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 5246 | (ins _.RC:$src), OpcodeStr, |
| 5247 | "{sae}, $src", "$src, {sae}", |
| 5248 | (OpNode (_.VT _.RC:$src), (i32 FROUND_NO_EXC))>, EVEX_B; |
| 5249 | } |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5250 | |
Elena Demikhovsky | be8808d | 2014-11-12 07:31:03 +0000 | [diff] [blame] | 5251 | multiclass avx512_eri<bits<8> opc, string OpcodeStr, SDNode OpNode> { |
| 5252 | defm PS : avx512_fp28_p<opc, OpcodeStr#"ps", v16f32_info, OpNode>, |
Asaf Badouh | 402ebb3 | 2015-06-03 13:41:48 +0000 | [diff] [blame] | 5253 | avx512_fp28_p_round<opc, OpcodeStr#"ps", v16f32_info, OpNode>, |
| 5254 | T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>; |
Elena Demikhovsky | be8808d | 2014-11-12 07:31:03 +0000 | [diff] [blame] | 5255 | defm PD : avx512_fp28_p<opc, OpcodeStr#"pd", v8f64_info, OpNode>, |
Asaf Badouh | 402ebb3 | 2015-06-03 13:41:48 +0000 | [diff] [blame] | 5256 | avx512_fp28_p_round<opc, OpcodeStr#"pd", v8f64_info, OpNode>, |
| 5257 | T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; |
Elena Demikhovsky | be8808d | 2014-11-12 07:31:03 +0000 | [diff] [blame] | 5258 | } |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5259 | |
Asaf Badouh | 402ebb3 | 2015-06-03 13:41:48 +0000 | [diff] [blame] | 5260 | multiclass avx512_fp_unaryop_packed<bits<8> opc, string OpcodeStr, |
| 5261 | SDNode OpNode> { |
| 5262 | // Define only if AVX512VL feature is present. |
| 5263 | let Predicates = [HasVLX] in { |
| 5264 | defm PSZ128 : avx512_fp28_p<opc, OpcodeStr#"ps", v4f32x_info, OpNode>, |
| 5265 | EVEX_V128, T8PD, EVEX_CD8<32, CD8VF>; |
| 5266 | defm PSZ256 : avx512_fp28_p<opc, OpcodeStr#"ps", v8f32x_info, OpNode>, |
| 5267 | EVEX_V256, T8PD, EVEX_CD8<32, CD8VF>; |
| 5268 | defm PDZ128 : avx512_fp28_p<opc, OpcodeStr#"pd", v2f64x_info, OpNode>, |
| 5269 | EVEX_V128, VEX_W, T8PD, EVEX_CD8<64, CD8VF>; |
| 5270 | defm PDZ256 : avx512_fp28_p<opc, OpcodeStr#"pd", v4f64x_info, OpNode>, |
| 5271 | EVEX_V256, VEX_W, T8PD, EVEX_CD8<64, CD8VF>; |
| 5272 | } |
| 5273 | } |
Elena Demikhovsky | be8808d | 2014-11-12 07:31:03 +0000 | [diff] [blame] | 5274 | let Predicates = [HasERI], hasSideEffects = 0 in { |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 5275 | |
Asaf Badouh | 402ebb3 | 2015-06-03 13:41:48 +0000 | [diff] [blame] | 5276 | defm VRSQRT28 : avx512_eri<0xCC, "vrsqrt28", X86rsqrt28>, EVEX; |
| 5277 | defm VRCP28 : avx512_eri<0xCA, "vrcp28", X86rcp28>, EVEX; |
| 5278 | defm VEXP2 : avx512_eri<0xC8, "vexp2", X86exp2>, EVEX; |
| 5279 | } |
| 5280 | defm VGETEXP : avx512_eri<0x42, "vgetexp", X86fgetexpRnd>, |
| 5281 | avx512_fp_unaryop_packed<0x42, "vgetexp", X86fgetexpRnd> , EVEX; |
| 5282 | |
| 5283 | multiclass avx512_sqrt_packed_round<bits<8> opc, string OpcodeStr, |
| 5284 | SDNode OpNodeRnd, X86VectorVTInfo _>{ |
| 5285 | defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 5286 | (ins _.RC:$src, AVX512RC:$rc), OpcodeStr, "$rc, $src", "$src, $rc", |
| 5287 | (_.VT (OpNodeRnd _.RC:$src, (i32 imm:$rc)))>, |
| 5288 | EVEX, EVEX_B, EVEX_RC; |
Elena Demikhovsky | be8808d | 2014-11-12 07:31:03 +0000 | [diff] [blame] | 5289 | } |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5290 | |
Robert Khasanov | eb12639 | 2014-10-28 18:15:20 +0000 | [diff] [blame] | 5291 | multiclass avx512_sqrt_packed<bits<8> opc, string OpcodeStr, |
| 5292 | SDNode OpNode, X86VectorVTInfo _>{ |
Robert Khasanov | 1cf354c | 2014-10-28 18:22:41 +0000 | [diff] [blame] | 5293 | defm r: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
Robert Khasanov | eb12639 | 2014-10-28 18:15:20 +0000 | [diff] [blame] | 5294 | (ins _.RC:$src), OpcodeStr, "$src", "$src", |
| 5295 | (_.FloatVT (OpNode _.RC:$src))>, EVEX; |
| 5296 | let mayLoad = 1 in { |
Robert Khasanov | 1cf354c | 2014-10-28 18:22:41 +0000 | [diff] [blame] | 5297 | defm m: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
Robert Khasanov | eb12639 | 2014-10-28 18:15:20 +0000 | [diff] [blame] | 5298 | (ins _.MemOp:$src), OpcodeStr, "$src", "$src", |
| 5299 | (OpNode (_.FloatVT |
| 5300 | (bitconvert (_.LdFrag addr:$src))))>, EVEX; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5301 | |
Robert Khasanov | 1cf354c | 2014-10-28 18:22:41 +0000 | [diff] [blame] | 5302 | defm mb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
Robert Khasanov | eb12639 | 2014-10-28 18:15:20 +0000 | [diff] [blame] | 5303 | (ins _.ScalarMemOp:$src), OpcodeStr, |
| 5304 | "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr, |
| 5305 | (OpNode (_.FloatVT |
| 5306 | (X86VBroadcast (_.ScalarLdFrag addr:$src))))>, |
| 5307 | EVEX, EVEX_B; |
| 5308 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5309 | } |
| 5310 | |
| 5311 | multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr, |
| 5312 | Intrinsic F32Int, Intrinsic F64Int, |
| 5313 | OpndItins itins_s, OpndItins itins_d> { |
| 5314 | def SSZr : SI<opc, MRMSrcReg, (outs FR32X:$dst), |
| 5315 | (ins FR32X:$src1, FR32X:$src2), |
| 5316 | !strconcat(OpcodeStr, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 5317 | "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5318 | [], itins_s.rr>, XS, EVEX_4V; |
Craig Topper | 9dd48c8 | 2014-01-02 17:28:14 +0000 | [diff] [blame] | 5319 | let isCodeGenOnly = 1 in |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5320 | def SSZr_Int : SIi8<opc, MRMSrcReg, (outs VR128X:$dst), |
| 5321 | (ins VR128X:$src1, VR128X:$src2), |
| 5322 | !strconcat(OpcodeStr, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 5323 | "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 5324 | [(set VR128X:$dst, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5325 | (F32Int VR128X:$src1, VR128X:$src2))], |
| 5326 | itins_s.rr>, XS, EVEX_4V; |
| 5327 | let mayLoad = 1 in { |
| 5328 | def SSZm : SI<opc, MRMSrcMem, (outs FR32X:$dst), |
| 5329 | (ins FR32X:$src1, f32mem:$src2), |
| 5330 | !strconcat(OpcodeStr, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 5331 | "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5332 | [], itins_s.rm>, XS, EVEX_4V, EVEX_CD8<32, CD8VT1>; |
Craig Topper | 9dd48c8 | 2014-01-02 17:28:14 +0000 | [diff] [blame] | 5333 | let isCodeGenOnly = 1 in |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5334 | def SSZm_Int : SIi8<opc, MRMSrcMem, (outs VR128X:$dst), |
| 5335 | (ins VR128X:$src1, ssmem:$src2), |
| 5336 | !strconcat(OpcodeStr, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 5337 | "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 5338 | [(set VR128X:$dst, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5339 | (F32Int VR128X:$src1, sse_load_f32:$src2))], |
| 5340 | itins_s.rm>, XS, EVEX_4V, EVEX_CD8<32, CD8VT1>; |
| 5341 | } |
| 5342 | def SDZr : SI<opc, MRMSrcReg, (outs FR64X:$dst), |
| 5343 | (ins FR64X:$src1, FR64X:$src2), |
| 5344 | !strconcat(OpcodeStr, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 5345 | "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5346 | XD, EVEX_4V, VEX_W; |
Craig Topper | 9dd48c8 | 2014-01-02 17:28:14 +0000 | [diff] [blame] | 5347 | let isCodeGenOnly = 1 in |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5348 | def SDZr_Int : SIi8<opc, MRMSrcReg, (outs VR128X:$dst), |
| 5349 | (ins VR128X:$src1, VR128X:$src2), |
| 5350 | !strconcat(OpcodeStr, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 5351 | "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 5352 | [(set VR128X:$dst, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5353 | (F64Int VR128X:$src1, VR128X:$src2))], |
| 5354 | itins_s.rr>, XD, EVEX_4V, VEX_W; |
| 5355 | let mayLoad = 1 in { |
| 5356 | def SDZm : SI<opc, MRMSrcMem, (outs FR64X:$dst), |
| 5357 | (ins FR64X:$src1, f64mem:$src2), |
| 5358 | !strconcat(OpcodeStr, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 5359 | "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5360 | XD, EVEX_4V, VEX_W, EVEX_CD8<64, CD8VT1>; |
Craig Topper | 9dd48c8 | 2014-01-02 17:28:14 +0000 | [diff] [blame] | 5361 | let isCodeGenOnly = 1 in |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5362 | def SDZm_Int : SIi8<opc, MRMSrcMem, (outs VR128X:$dst), |
| 5363 | (ins VR128X:$src1, sdmem:$src2), |
| 5364 | !strconcat(OpcodeStr, |
Elena Demikhovsky | cf08809 | 2013-12-11 14:31:04 +0000 | [diff] [blame] | 5365 | "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 5366 | [(set VR128X:$dst, |
| 5367 | (F64Int VR128X:$src1, sse_load_f64:$src2))]>, |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5368 | XD, EVEX_4V, VEX_W, EVEX_CD8<64, CD8VT1>; |
| 5369 | } |
| 5370 | } |
| 5371 | |
Robert Khasanov | eb12639 | 2014-10-28 18:15:20 +0000 | [diff] [blame] | 5372 | multiclass avx512_sqrt_packed_all<bits<8> opc, string OpcodeStr, |
| 5373 | SDNode OpNode> { |
| 5374 | defm PSZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, |
| 5375 | v16f32_info>, |
| 5376 | EVEX_V512, PS, EVEX_CD8<32, CD8VF>; |
| 5377 | defm PDZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, |
| 5378 | v8f64_info>, |
| 5379 | EVEX_V512, VEX_W, PD, EVEX_CD8<64, CD8VF>; |
| 5380 | // Define only if AVX512VL feature is present. |
| 5381 | let Predicates = [HasVLX] in { |
| 5382 | defm PSZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"), |
| 5383 | OpNode, v4f32x_info>, |
| 5384 | EVEX_V128, PS, EVEX_CD8<32, CD8VF>; |
| 5385 | defm PSZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"), |
| 5386 | OpNode, v8f32x_info>, |
| 5387 | EVEX_V256, PS, EVEX_CD8<32, CD8VF>; |
| 5388 | defm PDZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"), |
| 5389 | OpNode, v2f64x_info>, |
| 5390 | EVEX_V128, VEX_W, PD, EVEX_CD8<64, CD8VF>; |
| 5391 | defm PDZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"), |
| 5392 | OpNode, v4f64x_info>, |
| 5393 | EVEX_V256, VEX_W, PD, EVEX_CD8<64, CD8VF>; |
| 5394 | } |
| 5395 | } |
| 5396 | |
Asaf Badouh | 402ebb3 | 2015-06-03 13:41:48 +0000 | [diff] [blame] | 5397 | multiclass avx512_sqrt_packed_all_round<bits<8> opc, string OpcodeStr, |
| 5398 | SDNode OpNodeRnd> { |
| 5399 | defm PSZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "ps"), OpNodeRnd, |
| 5400 | v16f32_info>, EVEX_V512, PS, EVEX_CD8<32, CD8VF>; |
| 5401 | defm PDZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "pd"), OpNodeRnd, |
| 5402 | v8f64_info>, EVEX_V512, VEX_W, PD, EVEX_CD8<64, CD8VF>; |
| 5403 | } |
| 5404 | |
| 5405 | defm VSQRT : avx512_sqrt_packed_all<0x51, "vsqrt", fsqrt>, |
| 5406 | avx512_sqrt_packed_all_round<0x51, "vsqrt", X86fsqrtRnd>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5407 | |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 5408 | defm VSQRT : avx512_sqrt_scalar<0x51, "sqrt", |
| 5409 | int_x86_avx512_sqrt_ss, int_x86_avx512_sqrt_sd, |
Robert Khasanov | eb12639 | 2014-10-28 18:15:20 +0000 | [diff] [blame] | 5410 | SSE_SQRTSS, SSE_SQRTSD>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5411 | |
Elena Demikhovsky | a3a7140 | 2013-10-09 08:16:14 +0000 | [diff] [blame] | 5412 | let Predicates = [HasAVX512] in { |
| 5413 | def : Pat<(f32 (fsqrt FR32X:$src)), |
| 5414 | (VSQRTSSZr (f32 (IMPLICIT_DEF)), FR32X:$src)>; |
| 5415 | def : Pat<(f32 (fsqrt (load addr:$src))), |
| 5416 | (VSQRTSSZm (f32 (IMPLICIT_DEF)), addr:$src)>, |
| 5417 | Requires<[OptForSize]>; |
| 5418 | def : Pat<(f64 (fsqrt FR64X:$src)), |
| 5419 | (VSQRTSDZr (f64 (IMPLICIT_DEF)), FR64X:$src)>; |
| 5420 | def : Pat<(f64 (fsqrt (load addr:$src))), |
| 5421 | (VSQRTSDZm (f64 (IMPLICIT_DEF)), addr:$src)>, |
| 5422 | Requires<[OptForSize]>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5423 | |
Elena Demikhovsky | a3a7140 | 2013-10-09 08:16:14 +0000 | [diff] [blame] | 5424 | def : Pat<(f32 (X86frsqrt FR32X:$src)), |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5425 | (VRSQRT14SSrr (f32 (IMPLICIT_DEF)), FR32X:$src)>; |
Elena Demikhovsky | a3a7140 | 2013-10-09 08:16:14 +0000 | [diff] [blame] | 5426 | def : Pat<(f32 (X86frsqrt (load addr:$src))), |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5427 | (VRSQRT14SSrm (f32 (IMPLICIT_DEF)), addr:$src)>, |
Elena Demikhovsky | a3a7140 | 2013-10-09 08:16:14 +0000 | [diff] [blame] | 5428 | Requires<[OptForSize]>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5429 | |
Elena Demikhovsky | a3a7140 | 2013-10-09 08:16:14 +0000 | [diff] [blame] | 5430 | def : Pat<(f32 (X86frcp FR32X:$src)), |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5431 | (VRCP14SSrr (f32 (IMPLICIT_DEF)), FR32X:$src)>; |
Elena Demikhovsky | a3a7140 | 2013-10-09 08:16:14 +0000 | [diff] [blame] | 5432 | def : Pat<(f32 (X86frcp (load addr:$src))), |
Elena Demikhovsky | b19c9dc | 2014-01-13 12:55:03 +0000 | [diff] [blame] | 5433 | (VRCP14SSrm (f32 (IMPLICIT_DEF)), addr:$src)>, |
Elena Demikhovsky | a3a7140 | 2013-10-09 08:16:14 +0000 | [diff] [blame] | 5434 | Requires<[OptForSize]>; |
| 5435 | |
| 5436 | def : Pat<(int_x86_sse_sqrt_ss VR128X:$src), |
| 5437 | (COPY_TO_REGCLASS (VSQRTSSZr (f32 (IMPLICIT_DEF)), |
| 5438 | (COPY_TO_REGCLASS VR128X:$src, FR32)), |
| 5439 | VR128X)>; |
| 5440 | def : Pat<(int_x86_sse_sqrt_ss sse_load_f32:$src), |
| 5441 | (VSQRTSSZm_Int (v4f32 (IMPLICIT_DEF)), sse_load_f32:$src)>; |
| 5442 | |
| 5443 | def : Pat<(int_x86_sse2_sqrt_sd VR128X:$src), |
| 5444 | (COPY_TO_REGCLASS (VSQRTSDZr (f64 (IMPLICIT_DEF)), |
| 5445 | (COPY_TO_REGCLASS VR128X:$src, FR64)), |
| 5446 | VR128X)>; |
| 5447 | def : Pat<(int_x86_sse2_sqrt_sd sse_load_f64:$src), |
| 5448 | (VSQRTSDZm_Int (v2f64 (IMPLICIT_DEF)), sse_load_f64:$src)>; |
| 5449 | } |
| 5450 | |
Elena Demikhovsky | 52e81bc | 2015-02-23 15:12:31 +0000 | [diff] [blame] | 5451 | multiclass |
| 5452 | avx512_rndscale_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> { |
Elena Demikhovsky | de3f751 | 2014-01-01 15:12:34 +0000 | [diff] [blame] | 5453 | |
Elena Demikhovsky | 52e81bc | 2015-02-23 15:12:31 +0000 | [diff] [blame] | 5454 | let ExeDomain = _.ExeDomain in { |
| 5455 | defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 5456 | (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr, |
| 5457 | "$src3, $src2, $src1", "$src1, $src2, $src3", |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 5458 | (_.VT (X86RndScales (_.VT _.RC:$src1), (_.VT _.RC:$src2), |
Elena Demikhovsky | 52e81bc | 2015-02-23 15:12:31 +0000 | [diff] [blame] | 5459 | (i32 imm:$src3), (i32 FROUND_CURRENT)))>; |
| 5460 | |
| 5461 | defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 5462 | (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr, |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 5463 | "$src3, {sae}, $src2, $src1", "$src1, $src2, {sae}, $src3", |
| 5464 | (_.VT (X86RndScales (_.VT _.RC:$src1), (_.VT _.RC:$src2), |
Elena Demikhovsky | 0d7e936 | 2015-05-11 06:05:05 +0000 | [diff] [blame] | 5465 | (i32 imm:$src3), (i32 FROUND_NO_EXC)))>, EVEX_B; |
Elena Demikhovsky | 52e81bc | 2015-02-23 15:12:31 +0000 | [diff] [blame] | 5466 | |
| 5467 | let mayLoad = 1 in |
| 5468 | defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 5469 | (ins _.RC:$src1, _.MemOp:$src2, i32u8imm:$src3), OpcodeStr, |
| 5470 | "$src3, $src2, $src1", "$src1, $src2, $src3", |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 5471 | (_.VT (X86RndScales (_.VT _.RC:$src1), |
Elena Demikhovsky | 52e81bc | 2015-02-23 15:12:31 +0000 | [diff] [blame] | 5472 | (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))), |
| 5473 | (i32 imm:$src3), (i32 FROUND_CURRENT)))>; |
| 5474 | } |
| 5475 | let Predicates = [HasAVX512] in { |
| 5476 | def : Pat<(ffloor _.FRC:$src), (COPY_TO_REGCLASS |
| 5477 | (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)), |
| 5478 | (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x1))), _.FRC)>; |
| 5479 | def : Pat<(fceil _.FRC:$src), (COPY_TO_REGCLASS |
| 5480 | (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)), |
| 5481 | (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x2))), _.FRC)>; |
| 5482 | def : Pat<(ftrunc _.FRC:$src), (COPY_TO_REGCLASS |
| 5483 | (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)), |
| 5484 | (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x3))), _.FRC)>; |
| 5485 | def : Pat<(frint _.FRC:$src), (COPY_TO_REGCLASS |
| 5486 | (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)), |
| 5487 | (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x4))), _.FRC)>; |
| 5488 | def : Pat<(fnearbyint _.FRC:$src), (COPY_TO_REGCLASS |
| 5489 | (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)), |
| 5490 | (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0xc))), _.FRC)>; |
| 5491 | |
| 5492 | def : Pat<(ffloor (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS |
| 5493 | (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)), |
| 5494 | addr:$src, (i32 0x1))), _.FRC)>; |
| 5495 | def : Pat<(fceil (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS |
| 5496 | (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)), |
| 5497 | addr:$src, (i32 0x2))), _.FRC)>; |
| 5498 | def : Pat<(ftrunc (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS |
| 5499 | (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)), |
| 5500 | addr:$src, (i32 0x3))), _.FRC)>; |
| 5501 | def : Pat<(frint (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS |
| 5502 | (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)), |
| 5503 | addr:$src, (i32 0x4))), _.FRC)>; |
| 5504 | def : Pat<(fnearbyint (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS |
| 5505 | (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)), |
| 5506 | addr:$src, (i32 0xc))), _.FRC)>; |
| 5507 | } |
Elena Demikhovsky | de3f751 | 2014-01-01 15:12:34 +0000 | [diff] [blame] | 5508 | } |
| 5509 | |
Elena Demikhovsky | 52e81bc | 2015-02-23 15:12:31 +0000 | [diff] [blame] | 5510 | defm VRNDSCALESS : avx512_rndscale_scalar<0x0A, "vrndscaless", f32x_info>, |
| 5511 | AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VT1>; |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 5512 | |
Elena Demikhovsky | 52e81bc | 2015-02-23 15:12:31 +0000 | [diff] [blame] | 5513 | defm VRNDSCALESD : avx512_rndscale_scalar<0x0B, "vrndscalesd", f64x_info>, VEX_W, |
| 5514 | AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VT1>; |
Eric Christopher | 0d94fa9 | 2015-02-20 00:45:28 +0000 | [diff] [blame] | 5515 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5516 | //------------------------------------------------- |
| 5517 | // Integer truncate and extend operations |
| 5518 | //------------------------------------------------- |
| 5519 | |
Igor Breger | 074a64e | 2015-07-24 17:24:15 +0000 | [diff] [blame] | 5520 | multiclass avx512_trunc_common<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 5521 | X86VectorVTInfo SrcInfo, X86VectorVTInfo DestInfo, |
| 5522 | X86MemOperand x86memop> { |
| 5523 | |
| 5524 | defm rr : AVX512_maskable<opc, MRMDestReg, DestInfo, (outs DestInfo.RC:$dst), |
| 5525 | (ins SrcInfo.RC:$src1), OpcodeStr ,"$src1", "$src1", |
| 5526 | (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1)))>, |
| 5527 | EVEX, T8XS; |
| 5528 | |
| 5529 | // for intrinsic patter match |
| 5530 | def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask, |
| 5531 | (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))), |
| 5532 | undef)), |
| 5533 | (!cast<Instruction>(NAME#SrcInfo.ZSuffix##rrkz) DestInfo.KRCWM:$mask , |
| 5534 | SrcInfo.RC:$src1)>; |
| 5535 | |
| 5536 | def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask, |
| 5537 | (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))), |
| 5538 | DestInfo.ImmAllZerosV)), |
| 5539 | (!cast<Instruction>(NAME#SrcInfo.ZSuffix##rrkz) DestInfo.KRCWM:$mask , |
| 5540 | SrcInfo.RC:$src1)>; |
| 5541 | |
| 5542 | def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask, |
| 5543 | (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))), |
| 5544 | DestInfo.RC:$src0)), |
| 5545 | (!cast<Instruction>(NAME#SrcInfo.ZSuffix##rrk) DestInfo.RC:$src0, |
| 5546 | DestInfo.KRCWM:$mask , |
| 5547 | SrcInfo.RC:$src1)>; |
| 5548 | |
| 5549 | let mayStore = 1 in { |
| 5550 | def mr : AVX512XS8I<opc, MRMDestMem, (outs), |
| 5551 | (ins x86memop:$dst, SrcInfo.RC:$src), |
| 5552 | OpcodeStr # "\t{$src, $dst |$dst, $src}", |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5553 | []>, EVEX; |
| 5554 | |
Igor Breger | 074a64e | 2015-07-24 17:24:15 +0000 | [diff] [blame] | 5555 | def mrk : AVX512XS8I<opc, MRMDestMem, (outs), |
| 5556 | (ins x86memop:$dst, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src), |
| 5557 | OpcodeStr # "\t{$src, $dst {${mask}} |$dst {${mask}}, $src}", |
Robert Khasanov | 189e7fd | 2014-04-22 11:36:19 +0000 | [diff] [blame] | 5558 | []>, EVEX, EVEX_K; |
Igor Breger | 074a64e | 2015-07-24 17:24:15 +0000 | [diff] [blame] | 5559 | }//mayStore = 1 |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5560 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5561 | |
Igor Breger | 074a64e | 2015-07-24 17:24:15 +0000 | [diff] [blame] | 5562 | multiclass avx512_trunc_mr_lowering<X86VectorVTInfo SrcInfo, |
| 5563 | X86VectorVTInfo DestInfo, |
| 5564 | PatFrag truncFrag, PatFrag mtruncFrag > { |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5565 | |
Igor Breger | 074a64e | 2015-07-24 17:24:15 +0000 | [diff] [blame] | 5566 | def : Pat<(truncFrag (SrcInfo.VT SrcInfo.RC:$src), addr:$dst), |
| 5567 | (!cast<Instruction>(NAME#SrcInfo.ZSuffix##mr) |
| 5568 | addr:$dst, SrcInfo.RC:$src)>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5569 | |
Igor Breger | 074a64e | 2015-07-24 17:24:15 +0000 | [diff] [blame] | 5570 | def : Pat<(mtruncFrag addr:$dst, SrcInfo.KRCWM:$mask, |
| 5571 | (SrcInfo.VT SrcInfo.RC:$src)), |
| 5572 | (!cast<Instruction>(NAME#SrcInfo.ZSuffix##mrk) |
| 5573 | addr:$dst, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src)>; |
| 5574 | } |
| 5575 | |
| 5576 | multiclass avx512_trunc_sat_mr_lowering<X86VectorVTInfo SrcInfo, |
| 5577 | X86VectorVTInfo DestInfo, string sat > { |
| 5578 | |
| 5579 | def: Pat<(!cast<Intrinsic>("int_x86_avx512_mask_pmov"#sat#"_"#SrcInfo.Suffix# |
| 5580 | DestInfo.Suffix#"_mem_"#SrcInfo.Size) |
| 5581 | addr:$ptr, (SrcInfo.VT SrcInfo.RC:$src), SrcInfo.MRC:$mask), |
| 5582 | (!cast<Instruction>(NAME#SrcInfo.ZSuffix##mrk) addr:$ptr, |
| 5583 | (COPY_TO_REGCLASS SrcInfo.MRC:$mask, SrcInfo.KRCWM), |
| 5584 | (SrcInfo.VT SrcInfo.RC:$src))>; |
| 5585 | |
| 5586 | def: Pat<(!cast<Intrinsic>("int_x86_avx512_mask_pmov"#sat#"_"#SrcInfo.Suffix# |
| 5587 | DestInfo.Suffix#"_mem_"#SrcInfo.Size) |
| 5588 | addr:$ptr, (SrcInfo.VT SrcInfo.RC:$src), -1), |
| 5589 | (!cast<Instruction>(NAME#SrcInfo.ZSuffix##mr) addr:$ptr, |
| 5590 | (SrcInfo.VT SrcInfo.RC:$src))>; |
| 5591 | } |
| 5592 | |
| 5593 | multiclass avx512_trunc<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 5594 | AVX512VLVectorVTInfo VTSrcInfo, X86VectorVTInfo DestInfoZ128, |
| 5595 | X86VectorVTInfo DestInfoZ256, X86VectorVTInfo DestInfoZ, |
| 5596 | X86MemOperand x86memopZ128, X86MemOperand x86memopZ256, |
| 5597 | X86MemOperand x86memopZ, PatFrag truncFrag, PatFrag mtruncFrag, |
| 5598 | Predicate prd = HasAVX512>{ |
| 5599 | |
| 5600 | let Predicates = [HasVLX, prd] in { |
| 5601 | defm Z128: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info128, |
| 5602 | DestInfoZ128, x86memopZ128>, |
| 5603 | avx512_trunc_mr_lowering<VTSrcInfo.info128, DestInfoZ128, |
| 5604 | truncFrag, mtruncFrag>, EVEX_V128; |
| 5605 | |
| 5606 | defm Z256: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info256, |
| 5607 | DestInfoZ256, x86memopZ256>, |
| 5608 | avx512_trunc_mr_lowering<VTSrcInfo.info256, DestInfoZ256, |
| 5609 | truncFrag, mtruncFrag>, EVEX_V256; |
| 5610 | } |
| 5611 | let Predicates = [prd] in |
| 5612 | defm Z: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info512, |
| 5613 | DestInfoZ, x86memopZ>, |
| 5614 | avx512_trunc_mr_lowering<VTSrcInfo.info512, DestInfoZ, |
| 5615 | truncFrag, mtruncFrag>, EVEX_V512; |
| 5616 | } |
| 5617 | |
| 5618 | multiclass avx512_trunc_sat<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 5619 | AVX512VLVectorVTInfo VTSrcInfo, X86VectorVTInfo DestInfoZ128, |
| 5620 | X86VectorVTInfo DestInfoZ256, X86VectorVTInfo DestInfoZ, |
| 5621 | X86MemOperand x86memopZ128, X86MemOperand x86memopZ256, |
| 5622 | X86MemOperand x86memopZ, string sat, Predicate prd = HasAVX512>{ |
| 5623 | |
| 5624 | let Predicates = [HasVLX, prd] in { |
| 5625 | defm Z128: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info128, |
| 5626 | DestInfoZ128, x86memopZ128>, |
| 5627 | avx512_trunc_sat_mr_lowering<VTSrcInfo.info128, DestInfoZ128, |
| 5628 | sat>, EVEX_V128; |
| 5629 | |
| 5630 | defm Z256: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info256, |
| 5631 | DestInfoZ256, x86memopZ256>, |
| 5632 | avx512_trunc_sat_mr_lowering<VTSrcInfo.info256, DestInfoZ256, |
| 5633 | sat>, EVEX_V256; |
| 5634 | } |
| 5635 | let Predicates = [prd] in |
| 5636 | defm Z: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info512, |
| 5637 | DestInfoZ, x86memopZ>, |
| 5638 | avx512_trunc_sat_mr_lowering<VTSrcInfo.info512, DestInfoZ, |
| 5639 | sat>, EVEX_V512; |
| 5640 | } |
| 5641 | |
| 5642 | multiclass avx512_trunc_qb<bits<8> opc, string OpcodeStr, SDNode OpNode> { |
| 5643 | defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info, |
| 5644 | v16i8x_info, v16i8x_info, v16i8x_info, i16mem, i32mem, i64mem, |
| 5645 | truncstorevi8, masked_truncstorevi8>, EVEX_CD8<8, CD8VO>; |
| 5646 | } |
| 5647 | multiclass avx512_trunc_sat_qb<bits<8> opc, string sat, SDNode OpNode> { |
| 5648 | defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qb", OpNode, avx512vl_i64_info, |
| 5649 | v16i8x_info, v16i8x_info, v16i8x_info, i16mem, i32mem, i64mem, |
| 5650 | sat>, EVEX_CD8<8, CD8VO>; |
| 5651 | } |
| 5652 | |
| 5653 | multiclass avx512_trunc_qw<bits<8> opc, string OpcodeStr, SDNode OpNode> { |
| 5654 | defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info, |
| 5655 | v8i16x_info, v8i16x_info, v8i16x_info, i32mem, i64mem, i128mem, |
| 5656 | truncstorevi16, masked_truncstorevi16>, EVEX_CD8<16, CD8VQ>; |
| 5657 | } |
| 5658 | multiclass avx512_trunc_sat_qw<bits<8> opc, string sat, SDNode OpNode> { |
| 5659 | defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qw", OpNode, avx512vl_i64_info, |
| 5660 | v8i16x_info, v8i16x_info, v8i16x_info, i32mem, i64mem, i128mem, |
| 5661 | sat>, EVEX_CD8<16, CD8VQ>; |
| 5662 | } |
| 5663 | |
| 5664 | multiclass avx512_trunc_qd<bits<8> opc, string OpcodeStr, SDNode OpNode> { |
| 5665 | defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info, |
| 5666 | v4i32x_info, v4i32x_info, v8i32x_info, i64mem, i128mem, i256mem, |
| 5667 | truncstorevi32, masked_truncstorevi32>, EVEX_CD8<32, CD8VH>; |
| 5668 | } |
| 5669 | multiclass avx512_trunc_sat_qd<bits<8> opc, string sat, SDNode OpNode> { |
| 5670 | defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qd", OpNode, avx512vl_i64_info, |
| 5671 | v4i32x_info, v4i32x_info, v8i32x_info, i64mem, i128mem, i256mem, |
| 5672 | sat>, EVEX_CD8<32, CD8VH>; |
| 5673 | } |
| 5674 | |
| 5675 | multiclass avx512_trunc_db<bits<8> opc, string OpcodeStr, SDNode OpNode> { |
| 5676 | defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i32_info, |
| 5677 | v16i8x_info, v16i8x_info, v16i8x_info, i32mem, i64mem, i128mem, |
| 5678 | truncstorevi8, masked_truncstorevi8>, EVEX_CD8<8, CD8VQ>; |
| 5679 | } |
| 5680 | multiclass avx512_trunc_sat_db<bits<8> opc, string sat, SDNode OpNode> { |
| 5681 | defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"db", OpNode, avx512vl_i32_info, |
| 5682 | v16i8x_info, v16i8x_info, v16i8x_info, i32mem, i64mem, i128mem, |
| 5683 | sat>, EVEX_CD8<8, CD8VQ>; |
| 5684 | } |
| 5685 | |
| 5686 | multiclass avx512_trunc_dw<bits<8> opc, string OpcodeStr, SDNode OpNode> { |
| 5687 | defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i32_info, |
| 5688 | v8i16x_info, v8i16x_info, v16i16x_info, i64mem, i128mem, i256mem, |
| 5689 | truncstorevi16, masked_truncstorevi16>, EVEX_CD8<16, CD8VH>; |
| 5690 | } |
| 5691 | multiclass avx512_trunc_sat_dw<bits<8> opc, string sat, SDNode OpNode> { |
| 5692 | defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"dw", OpNode, avx512vl_i32_info, |
| 5693 | v8i16x_info, v8i16x_info, v16i16x_info, i64mem, i128mem, i256mem, |
| 5694 | sat>, EVEX_CD8<16, CD8VH>; |
| 5695 | } |
| 5696 | |
| 5697 | multiclass avx512_trunc_wb<bits<8> opc, string OpcodeStr, SDNode OpNode> { |
| 5698 | defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i16_info, |
| 5699 | v16i8x_info, v16i8x_info, v32i8x_info, i64mem, i128mem, i256mem, |
| 5700 | truncstorevi8, masked_truncstorevi8,HasBWI>, EVEX_CD8<16, CD8VH>; |
| 5701 | } |
| 5702 | multiclass avx512_trunc_sat_wb<bits<8> opc, string sat, SDNode OpNode> { |
| 5703 | defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"wb", OpNode, avx512vl_i16_info, |
| 5704 | v16i8x_info, v16i8x_info, v32i8x_info, i64mem, i128mem, i256mem, |
| 5705 | sat, HasBWI>, EVEX_CD8<16, CD8VH>; |
| 5706 | } |
| 5707 | |
| 5708 | defm VPMOVQB : avx512_trunc_qb<0x32, "vpmovqb", X86vtrunc>; |
| 5709 | defm VPMOVSQB : avx512_trunc_sat_qb<0x22, "s", X86vtruncs>; |
| 5710 | defm VPMOVUSQB : avx512_trunc_sat_qb<0x12, "us", X86vtruncus>; |
| 5711 | |
| 5712 | defm VPMOVQW : avx512_trunc_qw<0x34, "vpmovqw", X86vtrunc>; |
| 5713 | defm VPMOVSQW : avx512_trunc_sat_qw<0x24, "s", X86vtruncs>; |
| 5714 | defm VPMOVUSQW : avx512_trunc_sat_qw<0x14, "us", X86vtruncus>; |
| 5715 | |
| 5716 | defm VPMOVQD : avx512_trunc_qd<0x35, "vpmovqd", X86vtrunc>; |
| 5717 | defm VPMOVSQD : avx512_trunc_sat_qd<0x25, "s", X86vtruncs>; |
| 5718 | defm VPMOVUSQD : avx512_trunc_sat_qd<0x15, "us", X86vtruncus>; |
| 5719 | |
| 5720 | defm VPMOVDB : avx512_trunc_db<0x31, "vpmovdb", X86vtrunc>; |
| 5721 | defm VPMOVSDB : avx512_trunc_sat_db<0x21, "s", X86vtruncs>; |
| 5722 | defm VPMOVUSDB : avx512_trunc_sat_db<0x11, "us", X86vtruncus>; |
| 5723 | |
| 5724 | defm VPMOVDW : avx512_trunc_dw<0x33, "vpmovdw", X86vtrunc>; |
| 5725 | defm VPMOVSDW : avx512_trunc_sat_dw<0x23, "s", X86vtruncs>; |
| 5726 | defm VPMOVUSDW : avx512_trunc_sat_dw<0x13, "us", X86vtruncus>; |
| 5727 | |
| 5728 | defm VPMOVWB : avx512_trunc_wb<0x30, "vpmovwb", X86vtrunc>; |
| 5729 | defm VPMOVSWB : avx512_trunc_sat_wb<0x20, "s", X86vtruncs>; |
| 5730 | defm VPMOVUSWB : avx512_trunc_sat_wb<0x10, "us", X86vtruncus>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5731 | |
Elena Demikhovsky | 3948c59 | 2015-05-27 08:15:19 +0000 | [diff] [blame] | 5732 | multiclass avx512_extend_common<bits<8> opc, string OpcodeStr, |
| 5733 | X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo, |
| 5734 | X86MemOperand x86memop, PatFrag LdFrag, SDNode OpNode>{ |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5735 | |
Elena Demikhovsky | 3948c59 | 2015-05-27 08:15:19 +0000 | [diff] [blame] | 5736 | defm rr : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst), |
| 5737 | (ins SrcInfo.RC:$src), OpcodeStr ,"$src", "$src", |
| 5738 | (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src)))>, |
| 5739 | EVEX; |
Robert Khasanov | 189e7fd | 2014-04-22 11:36:19 +0000 | [diff] [blame] | 5740 | |
| 5741 | let mayLoad = 1 in { |
Elena Demikhovsky | 3948c59 | 2015-05-27 08:15:19 +0000 | [diff] [blame] | 5742 | defm rm : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst), |
| 5743 | (ins x86memop:$src), OpcodeStr ,"$src", "$src", |
| 5744 | (DestInfo.VT (LdFrag addr:$src))>, |
| 5745 | EVEX; |
Robert Khasanov | 189e7fd | 2014-04-22 11:36:19 +0000 | [diff] [blame] | 5746 | } |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5747 | } |
| 5748 | |
Elena Demikhovsky | 3948c59 | 2015-05-27 08:15:19 +0000 | [diff] [blame] | 5749 | multiclass avx512_extend_BW<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 5750 | string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> { |
| 5751 | let Predicates = [HasVLX, HasBWI] in { |
| 5752 | defm Z128: avx512_extend_common<opc, OpcodeStr, v8i16x_info, |
| 5753 | v16i8x_info, i64mem, LdFrag, OpNode>, |
| 5754 | EVEX_CD8<8, CD8VH>, T8PD, EVEX_V128; |
Robert Khasanov | 189e7fd | 2014-04-22 11:36:19 +0000 | [diff] [blame] | 5755 | |
Elena Demikhovsky | 3948c59 | 2015-05-27 08:15:19 +0000 | [diff] [blame] | 5756 | defm Z256: avx512_extend_common<opc, OpcodeStr, v16i16x_info, |
| 5757 | v16i8x_info, i128mem, LdFrag, OpNode>, |
| 5758 | EVEX_CD8<8, CD8VH>, T8PD, EVEX_V256; |
| 5759 | } |
| 5760 | let Predicates = [HasBWI] in { |
| 5761 | defm Z : avx512_extend_common<opc, OpcodeStr, v32i16_info, |
| 5762 | v32i8x_info, i256mem, LdFrag, OpNode>, |
| 5763 | EVEX_CD8<8, CD8VH>, T8PD, EVEX_V512; |
| 5764 | } |
| 5765 | } |
| 5766 | |
| 5767 | multiclass avx512_extend_BD<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 5768 | string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> { |
| 5769 | let Predicates = [HasVLX, HasAVX512] in { |
| 5770 | defm Z128: avx512_extend_common<opc, OpcodeStr, v4i32x_info, |
| 5771 | v16i8x_info, i32mem, LdFrag, OpNode>, |
| 5772 | EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V128; |
| 5773 | |
| 5774 | defm Z256: avx512_extend_common<opc, OpcodeStr, v8i32x_info, |
| 5775 | v16i8x_info, i64mem, LdFrag, OpNode>, |
| 5776 | EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V256; |
| 5777 | } |
| 5778 | let Predicates = [HasAVX512] in { |
| 5779 | defm Z : avx512_extend_common<opc, OpcodeStr, v16i32_info, |
| 5780 | v16i8x_info, i128mem, LdFrag, OpNode>, |
| 5781 | EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V512; |
| 5782 | } |
| 5783 | } |
| 5784 | |
| 5785 | multiclass avx512_extend_BQ<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 5786 | string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> { |
| 5787 | let Predicates = [HasVLX, HasAVX512] in { |
| 5788 | defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info, |
| 5789 | v16i8x_info, i16mem, LdFrag, OpNode>, |
| 5790 | EVEX_CD8<8, CD8VO>, T8PD, EVEX_V128; |
| 5791 | |
| 5792 | defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info, |
| 5793 | v16i8x_info, i32mem, LdFrag, OpNode>, |
| 5794 | EVEX_CD8<8, CD8VO>, T8PD, EVEX_V256; |
| 5795 | } |
| 5796 | let Predicates = [HasAVX512] in { |
| 5797 | defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info, |
| 5798 | v16i8x_info, i64mem, LdFrag, OpNode>, |
| 5799 | EVEX_CD8<8, CD8VO>, T8PD, EVEX_V512; |
| 5800 | } |
| 5801 | } |
| 5802 | |
| 5803 | multiclass avx512_extend_WD<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 5804 | string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> { |
| 5805 | let Predicates = [HasVLX, HasAVX512] in { |
| 5806 | defm Z128: avx512_extend_common<opc, OpcodeStr, v4i32x_info, |
| 5807 | v8i16x_info, i64mem, LdFrag, OpNode>, |
| 5808 | EVEX_CD8<16, CD8VH>, T8PD, EVEX_V128; |
| 5809 | |
| 5810 | defm Z256: avx512_extend_common<opc, OpcodeStr, v8i32x_info, |
| 5811 | v8i16x_info, i128mem, LdFrag, OpNode>, |
| 5812 | EVEX_CD8<16, CD8VH>, T8PD, EVEX_V256; |
| 5813 | } |
| 5814 | let Predicates = [HasAVX512] in { |
| 5815 | defm Z : avx512_extend_common<opc, OpcodeStr, v16i32_info, |
| 5816 | v16i16x_info, i256mem, LdFrag, OpNode>, |
| 5817 | EVEX_CD8<16, CD8VH>, T8PD, EVEX_V512; |
| 5818 | } |
| 5819 | } |
| 5820 | |
| 5821 | multiclass avx512_extend_WQ<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 5822 | string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> { |
| 5823 | let Predicates = [HasVLX, HasAVX512] in { |
| 5824 | defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info, |
| 5825 | v8i16x_info, i32mem, LdFrag, OpNode>, |
| 5826 | EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V128; |
| 5827 | |
| 5828 | defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info, |
| 5829 | v8i16x_info, i64mem, LdFrag, OpNode>, |
| 5830 | EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V256; |
| 5831 | } |
| 5832 | let Predicates = [HasAVX512] in { |
| 5833 | defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info, |
| 5834 | v8i16x_info, i128mem, LdFrag, OpNode>, |
| 5835 | EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V512; |
| 5836 | } |
| 5837 | } |
| 5838 | |
| 5839 | multiclass avx512_extend_DQ<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 5840 | string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi32")> { |
| 5841 | |
| 5842 | let Predicates = [HasVLX, HasAVX512] in { |
| 5843 | defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info, |
| 5844 | v4i32x_info, i64mem, LdFrag, OpNode>, |
| 5845 | EVEX_CD8<32, CD8VH>, T8PD, EVEX_V128; |
| 5846 | |
| 5847 | defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info, |
| 5848 | v4i32x_info, i128mem, LdFrag, OpNode>, |
| 5849 | EVEX_CD8<32, CD8VH>, T8PD, EVEX_V256; |
| 5850 | } |
| 5851 | let Predicates = [HasAVX512] in { |
| 5852 | defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info, |
| 5853 | v8i32x_info, i256mem, LdFrag, OpNode>, |
| 5854 | EVEX_CD8<32, CD8VH>, T8PD, EVEX_V512; |
| 5855 | } |
| 5856 | } |
| 5857 | |
| 5858 | defm VPMOVZXBW : avx512_extend_BW<0x30, "vpmovzxbw", X86vzext, "z">; |
| 5859 | defm VPMOVZXBD : avx512_extend_BD<0x31, "vpmovzxbd", X86vzext, "z">; |
| 5860 | defm VPMOVZXBQ : avx512_extend_BQ<0x32, "vpmovzxbq", X86vzext, "z">; |
| 5861 | defm VPMOVZXWD : avx512_extend_WD<0x33, "vpmovzxwd", X86vzext, "z">; |
| 5862 | defm VPMOVZXWQ : avx512_extend_WQ<0x34, "vpmovzxwq", X86vzext, "z">; |
| 5863 | defm VPMOVZXDQ : avx512_extend_DQ<0x35, "vpmovzxdq", X86vzext, "z">; |
| 5864 | |
| 5865 | |
| 5866 | defm VPMOVSXBW: avx512_extend_BW<0x20, "vpmovsxbw", X86vsext, "s">; |
| 5867 | defm VPMOVSXBD: avx512_extend_BD<0x21, "vpmovsxbd", X86vsext, "s">; |
| 5868 | defm VPMOVSXBQ: avx512_extend_BQ<0x22, "vpmovsxbq", X86vsext, "s">; |
| 5869 | defm VPMOVSXWD: avx512_extend_WD<0x23, "vpmovsxwd", X86vsext, "s">; |
| 5870 | defm VPMOVSXWQ: avx512_extend_WQ<0x24, "vpmovsxwq", X86vsext, "s">; |
| 5871 | defm VPMOVSXDQ: avx512_extend_DQ<0x25, "vpmovsxdq", X86vsext, "s">; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5872 | |
| 5873 | //===----------------------------------------------------------------------===// |
| 5874 | // GATHER - SCATTER Operations |
| 5875 | |
Elena Demikhovsky | e1eda8a | 2015-04-30 08:38:48 +0000 | [diff] [blame] | 5876 | multiclass avx512_gather<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, |
| 5877 | X86MemOperand memop, PatFrag GatherNode> { |
Elena Demikhovsky | 6a1a357 | 2015-06-28 10:53:29 +0000 | [diff] [blame] | 5878 | let Constraints = "@earlyclobber $dst, $src1 = $dst, $mask = $mask_wb", |
| 5879 | ExeDomain = _.ExeDomain in |
Elena Demikhovsky | e1eda8a | 2015-04-30 08:38:48 +0000 | [diff] [blame] | 5880 | def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst, _.KRCWM:$mask_wb), |
| 5881 | (ins _.RC:$src1, _.KRCWM:$mask, memop:$src2), |
Elena Demikhovsky | 6a1a357 | 2015-06-28 10:53:29 +0000 | [diff] [blame] | 5882 | !strconcat(OpcodeStr#_.Suffix, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 5883 | "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), |
Elena Demikhovsky | e1eda8a | 2015-04-30 08:38:48 +0000 | [diff] [blame] | 5884 | [(set _.RC:$dst, _.KRCWM:$mask_wb, |
| 5885 | (GatherNode (_.VT _.RC:$src1), _.KRCWM:$mask, |
| 5886 | vectoraddr:$src2))]>, EVEX, EVEX_K, |
| 5887 | EVEX_CD8<_.EltSize, CD8VT1>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5888 | } |
Cameron McInally | 4532596 | 2014-03-26 13:50:50 +0000 | [diff] [blame] | 5889 | |
Elena Demikhovsky | 6a1a357 | 2015-06-28 10:53:29 +0000 | [diff] [blame] | 5890 | multiclass avx512_gather_q_pd<bits<8> dopc, bits<8> qopc, |
| 5891 | AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> { |
| 5892 | defm NAME##D##SUFF##Z: avx512_gather<dopc, OpcodeStr##"d", _.info512, |
| 5893 | vy32xmem, mgatherv8i32>, EVEX_V512, VEX_W; |
| 5894 | defm NAME##Q##SUFF##Z: avx512_gather<qopc, OpcodeStr##"q", _.info512, |
| 5895 | vz64mem, mgatherv8i64>, EVEX_V512, VEX_W; |
| 5896 | let Predicates = [HasVLX] in { |
| 5897 | defm NAME##D##SUFF##Z256: avx512_gather<dopc, OpcodeStr##"d", _.info256, |
| 5898 | vx32xmem, mgatherv4i32>, EVEX_V256, VEX_W; |
| 5899 | defm NAME##Q##SUFF##Z256: avx512_gather<qopc, OpcodeStr##"q", _.info256, |
| 5900 | vy64xmem, mgatherv4i64>, EVEX_V256, VEX_W; |
| 5901 | defm NAME##D##SUFF##Z128: avx512_gather<dopc, OpcodeStr##"d", _.info128, |
| 5902 | vx32xmem, mgatherv4i32>, EVEX_V128, VEX_W; |
| 5903 | defm NAME##Q##SUFF##Z128: avx512_gather<qopc, OpcodeStr##"q", _.info128, |
| 5904 | vx64xmem, mgatherv2i64>, EVEX_V128, VEX_W; |
| 5905 | } |
Cameron McInally | 4532596 | 2014-03-26 13:50:50 +0000 | [diff] [blame] | 5906 | } |
| 5907 | |
Elena Demikhovsky | 6a1a357 | 2015-06-28 10:53:29 +0000 | [diff] [blame] | 5908 | multiclass avx512_gather_d_ps<bits<8> dopc, bits<8> qopc, |
| 5909 | AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> { |
| 5910 | defm NAME##D##SUFF##Z: avx512_gather<dopc, OpcodeStr##"d", _.info512, vz32mem, |
| 5911 | mgatherv16i32>, EVEX_V512; |
| 5912 | defm NAME##Q##SUFF##Z: avx512_gather<qopc, OpcodeStr##"q", _.info256, vz64mem, |
| 5913 | mgatherv8i64>, EVEX_V512; |
| 5914 | let Predicates = [HasVLX] in { |
| 5915 | defm NAME##D##SUFF##Z256: avx512_gather<dopc, OpcodeStr##"d", _.info256, |
| 5916 | vy32xmem, mgatherv8i32>, EVEX_V256; |
| 5917 | defm NAME##Q##SUFF##Z256: avx512_gather<qopc, OpcodeStr##"q", _.info128, |
| 5918 | vy64xmem, mgatherv4i64>, EVEX_V256; |
| 5919 | defm NAME##D##SUFF##Z128: avx512_gather<dopc, OpcodeStr##"d", _.info128, |
| 5920 | vx32xmem, mgatherv4i32>, EVEX_V128; |
| 5921 | defm NAME##Q##SUFF##Z128: avx512_gather<qopc, OpcodeStr##"q", _.info128, |
| 5922 | vx64xmem, mgatherv2i64>, EVEX_V128; |
| 5923 | } |
Cameron McInally | 4532596 | 2014-03-26 13:50:50 +0000 | [diff] [blame] | 5924 | } |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 5925 | |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5926 | |
Elena Demikhovsky | 6a1a357 | 2015-06-28 10:53:29 +0000 | [diff] [blame] | 5927 | defm VGATHER : avx512_gather_q_pd<0x92, 0x93, avx512vl_f64_info, "vgather", "PD">, |
| 5928 | avx512_gather_d_ps<0x92, 0x93, avx512vl_f32_info, "vgather", "PS">; |
| 5929 | |
| 5930 | defm VPGATHER : avx512_gather_q_pd<0x90, 0x91, avx512vl_i64_info, "vpgather", "Q">, |
| 5931 | avx512_gather_d_ps<0x90, 0x91, avx512vl_i32_info, "vpgather", "D">; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5932 | |
Elena Demikhovsky | e1eda8a | 2015-04-30 08:38:48 +0000 | [diff] [blame] | 5933 | multiclass avx512_scatter<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, |
| 5934 | X86MemOperand memop, PatFrag ScatterNode> { |
| 5935 | |
Elena Demikhovsky | 30bc4ca | 2015-06-29 12:14:24 +0000 | [diff] [blame] | 5936 | let mayStore = 1, Constraints = "$mask = $mask_wb", ExeDomain = _.ExeDomain in |
Elena Demikhovsky | e1eda8a | 2015-04-30 08:38:48 +0000 | [diff] [blame] | 5937 | |
| 5938 | def mr : AVX5128I<opc, MRMDestMem, (outs _.KRCWM:$mask_wb), |
| 5939 | (ins memop:$dst, _.KRCWM:$mask, _.RC:$src), |
Elena Demikhovsky | 30bc4ca | 2015-06-29 12:14:24 +0000 | [diff] [blame] | 5940 | !strconcat(OpcodeStr#_.Suffix, |
Elena Demikhovsky | e1eda8a | 2015-04-30 08:38:48 +0000 | [diff] [blame] | 5941 | "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"), |
| 5942 | [(set _.KRCWM:$mask_wb, (ScatterNode (_.VT _.RC:$src), |
| 5943 | _.KRCWM:$mask, vectoraddr:$dst))]>, |
| 5944 | EVEX, EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5945 | } |
| 5946 | |
Elena Demikhovsky | 30bc4ca | 2015-06-29 12:14:24 +0000 | [diff] [blame] | 5947 | multiclass avx512_scatter_q_pd<bits<8> dopc, bits<8> qopc, |
| 5948 | AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> { |
| 5949 | defm NAME##D##SUFF##Z: avx512_scatter<dopc, OpcodeStr##"d", _.info512, |
| 5950 | vy32xmem, mscatterv8i32>, EVEX_V512, VEX_W; |
| 5951 | defm NAME##Q##SUFF##Z: avx512_scatter<qopc, OpcodeStr##"q", _.info512, |
| 5952 | vz64mem, mscatterv8i64>, EVEX_V512, VEX_W; |
| 5953 | let Predicates = [HasVLX] in { |
| 5954 | defm NAME##D##SUFF##Z256: avx512_scatter<dopc, OpcodeStr##"d", _.info256, |
| 5955 | vx32xmem, mscatterv4i32>, EVEX_V256, VEX_W; |
| 5956 | defm NAME##Q##SUFF##Z256: avx512_scatter<qopc, OpcodeStr##"q", _.info256, |
| 5957 | vy64xmem, mscatterv4i64>, EVEX_V256, VEX_W; |
| 5958 | defm NAME##D##SUFF##Z128: avx512_scatter<dopc, OpcodeStr##"d", _.info128, |
| 5959 | vx32xmem, mscatterv4i32>, EVEX_V128, VEX_W; |
| 5960 | defm NAME##Q##SUFF##Z128: avx512_scatter<qopc, OpcodeStr##"q", _.info128, |
| 5961 | vx64xmem, mscatterv2i64>, EVEX_V128, VEX_W; |
| 5962 | } |
Cameron McInally | 4532596 | 2014-03-26 13:50:50 +0000 | [diff] [blame] | 5963 | } |
| 5964 | |
Elena Demikhovsky | 30bc4ca | 2015-06-29 12:14:24 +0000 | [diff] [blame] | 5965 | multiclass avx512_scatter_d_ps<bits<8> dopc, bits<8> qopc, |
| 5966 | AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> { |
| 5967 | defm NAME##D##SUFF##Z: avx512_scatter<dopc, OpcodeStr##"d", _.info512, vz32mem, |
| 5968 | mscatterv16i32>, EVEX_V512; |
| 5969 | defm NAME##Q##SUFF##Z: avx512_scatter<qopc, OpcodeStr##"q", _.info256, vz64mem, |
| 5970 | mscatterv8i64>, EVEX_V512; |
| 5971 | let Predicates = [HasVLX] in { |
| 5972 | defm NAME##D##SUFF##Z256: avx512_scatter<dopc, OpcodeStr##"d", _.info256, |
| 5973 | vy32xmem, mscatterv8i32>, EVEX_V256; |
| 5974 | defm NAME##Q##SUFF##Z256: avx512_scatter<qopc, OpcodeStr##"q", _.info128, |
| 5975 | vy64xmem, mscatterv4i64>, EVEX_V256; |
| 5976 | defm NAME##D##SUFF##Z128: avx512_scatter<dopc, OpcodeStr##"d", _.info128, |
| 5977 | vx32xmem, mscatterv4i32>, EVEX_V128; |
| 5978 | defm NAME##Q##SUFF##Z128: avx512_scatter<qopc, OpcodeStr##"q", _.info128, |
| 5979 | vx64xmem, mscatterv2i64>, EVEX_V128; |
| 5980 | } |
Cameron McInally | 4532596 | 2014-03-26 13:50:50 +0000 | [diff] [blame] | 5981 | } |
| 5982 | |
Elena Demikhovsky | 30bc4ca | 2015-06-29 12:14:24 +0000 | [diff] [blame] | 5983 | defm VSCATTER : avx512_scatter_q_pd<0xA2, 0xA3, avx512vl_f64_info, "vscatter", "PD">, |
| 5984 | avx512_scatter_d_ps<0xA2, 0xA3, avx512vl_f32_info, "vscatter", "PS">; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5985 | |
Elena Demikhovsky | 30bc4ca | 2015-06-29 12:14:24 +0000 | [diff] [blame] | 5986 | defm VPSCATTER : avx512_scatter_q_pd<0xA0, 0xA1, avx512vl_i64_info, "vpscatter", "Q">, |
| 5987 | avx512_scatter_d_ps<0xA0, 0xA1, avx512vl_i32_info, "vpscatter", "D">; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 5988 | |
Elena Demikhovsky | 8e8fde8 | 2014-05-12 07:18:51 +0000 | [diff] [blame] | 5989 | // prefetch |
| 5990 | multiclass avx512_gather_scatter_prefetch<bits<8> opc, Format F, string OpcodeStr, |
| 5991 | RegisterClass KRC, X86MemOperand memop> { |
| 5992 | let Predicates = [HasPFI], hasSideEffects = 1 in |
| 5993 | def m : AVX5128I<opc, F, (outs), (ins KRC:$mask, memop:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 5994 | !strconcat(OpcodeStr, "\t{$src {${mask}}|{${mask}}, $src}"), |
Elena Demikhovsky | 8e8fde8 | 2014-05-12 07:18:51 +0000 | [diff] [blame] | 5995 | []>, EVEX, EVEX_K; |
| 5996 | } |
| 5997 | |
| 5998 | defm VGATHERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dps", |
| 5999 | VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; |
| 6000 | |
| 6001 | defm VGATHERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qps", |
| 6002 | VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>; |
| 6003 | |
| 6004 | defm VGATHERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dpd", |
| 6005 | VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>; |
| 6006 | |
| 6007 | defm VGATHERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qpd", |
| 6008 | VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 6009 | |
Elena Demikhovsky | 8e8fde8 | 2014-05-12 07:18:51 +0000 | [diff] [blame] | 6010 | defm VGATHERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dps", |
| 6011 | VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; |
| 6012 | |
| 6013 | defm VGATHERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qps", |
| 6014 | VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>; |
| 6015 | |
| 6016 | defm VGATHERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dpd", |
| 6017 | VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>; |
| 6018 | |
| 6019 | defm VGATHERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qpd", |
| 6020 | VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; |
| 6021 | |
| 6022 | defm VSCATTERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dps", |
| 6023 | VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; |
| 6024 | |
| 6025 | defm VSCATTERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qps", |
| 6026 | VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>; |
| 6027 | |
| 6028 | defm VSCATTERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dpd", |
| 6029 | VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>; |
| 6030 | |
| 6031 | defm VSCATTERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qpd", |
| 6032 | VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; |
| 6033 | |
| 6034 | defm VSCATTERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dps", |
| 6035 | VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; |
| 6036 | |
| 6037 | defm VSCATTERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qps", |
| 6038 | VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>; |
| 6039 | |
| 6040 | defm VSCATTERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dpd", |
| 6041 | VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>; |
| 6042 | |
| 6043 | defm VSCATTERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qpd", |
| 6044 | VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; |
Elena Demikhovsky | ac3e8eb | 2013-09-17 07:34:34 +0000 | [diff] [blame] | 6045 | |
Elena Demikhovsky | bb2f6b7 | 2014-03-27 09:45:08 +0000 | [diff] [blame] | 6046 | // Helper fragments to match sext vXi1 to vXiY. |
| 6047 | def v16i1sextv16i32 : PatLeaf<(v16i32 (X86vsrai VR512:$src, (i8 31)))>; |
| 6048 | def v8i1sextv8i64 : PatLeaf<(v8i64 (X86vsrai VR512:$src, (i8 63)))>; |
| 6049 | |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 6050 | multiclass avx512_conflict<bits<8> opc, string OpcodeStr, |
Elena Demikhovsky | 6270b38 | 2013-12-10 11:58:35 +0000 | [diff] [blame] | 6051 | RegisterClass RC, RegisterClass KRC, |
| 6052 | X86MemOperand x86memop, |
| 6053 | X86MemOperand x86scalar_mop, string BrdcstStr> { |
Craig Topper | 46469aa | 2015-01-23 06:11:45 +0000 | [diff] [blame] | 6054 | let hasSideEffects = 0 in { |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6055 | def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), |
| 6056 | (ins RC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 6057 | !strconcat(OpcodeStr, "\t{$src, ${dst} |${dst}, $src}"), |
Elena Demikhovsky | 6270b38 | 2013-12-10 11:58:35 +0000 | [diff] [blame] | 6058 | []>, EVEX; |
Craig Topper | 46469aa | 2015-01-23 06:11:45 +0000 | [diff] [blame] | 6059 | let mayLoad = 1 in |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6060 | def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), |
| 6061 | (ins x86memop:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 6062 | !strconcat(OpcodeStr, "\t{$src, ${dst}|${dst}, $src}"), |
Elena Demikhovsky | 6270b38 | 2013-12-10 11:58:35 +0000 | [diff] [blame] | 6063 | []>, EVEX; |
Craig Topper | 46469aa | 2015-01-23 06:11:45 +0000 | [diff] [blame] | 6064 | let mayLoad = 1 in |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6065 | def rmb : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), |
| 6066 | (ins x86scalar_mop:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 6067 | !strconcat(OpcodeStr, "\t{${src}", BrdcstStr, |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6068 | ", ${dst}|${dst}, ${src}", BrdcstStr, "}"), |
| 6069 | []>, EVEX, EVEX_B; |
| 6070 | def rrkz : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), |
| 6071 | (ins KRC:$mask, RC:$src), |
| 6072 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 6073 | "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), |
Elena Demikhovsky | 6270b38 | 2013-12-10 11:58:35 +0000 | [diff] [blame] | 6074 | []>, EVEX, EVEX_KZ; |
Craig Topper | 46469aa | 2015-01-23 06:11:45 +0000 | [diff] [blame] | 6075 | let mayLoad = 1 in |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6076 | def rmkz : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), |
| 6077 | (ins KRC:$mask, x86memop:$src), |
| 6078 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 6079 | "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), |
Elena Demikhovsky | 6270b38 | 2013-12-10 11:58:35 +0000 | [diff] [blame] | 6080 | []>, EVEX, EVEX_KZ; |
Craig Topper | 46469aa | 2015-01-23 06:11:45 +0000 | [diff] [blame] | 6081 | let mayLoad = 1 in |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6082 | def rmbkz : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), |
| 6083 | (ins KRC:$mask, x86scalar_mop:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 6084 | !strconcat(OpcodeStr, "\t{${src}", BrdcstStr, |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6085 | ", ${dst} {${mask}} {z}|${dst} {${mask}} {z}, ${src}", |
| 6086 | BrdcstStr, "}"), |
| 6087 | []>, EVEX, EVEX_KZ, EVEX_B; |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 6088 | |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6089 | let Constraints = "$src1 = $dst" in { |
| 6090 | def rrk : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), |
| 6091 | (ins RC:$src1, KRC:$mask, RC:$src2), |
| 6092 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 6093 | "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), |
Elena Demikhovsky | 6270b38 | 2013-12-10 11:58:35 +0000 | [diff] [blame] | 6094 | []>, EVEX, EVEX_K; |
Craig Topper | 46469aa | 2015-01-23 06:11:45 +0000 | [diff] [blame] | 6095 | let mayLoad = 1 in |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6096 | def rmk : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), |
| 6097 | (ins RC:$src1, KRC:$mask, x86memop:$src2), |
| 6098 | !strconcat(OpcodeStr, |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 6099 | "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), |
Elena Demikhovsky | 6270b38 | 2013-12-10 11:58:35 +0000 | [diff] [blame] | 6100 | []>, EVEX, EVEX_K; |
Craig Topper | 46469aa | 2015-01-23 06:11:45 +0000 | [diff] [blame] | 6101 | let mayLoad = 1 in |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6102 | def rmbk : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), |
| 6103 | (ins RC:$src1, KRC:$mask, x86scalar_mop:$src2), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 6104 | !strconcat(OpcodeStr, "\t{${src2}", BrdcstStr, |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6105 | ", ${dst} {${mask}}|${dst} {${mask}}, ${src2}", BrdcstStr, "}"), |
| 6106 | []>, EVEX, EVEX_K, EVEX_B; |
Craig Topper | 46469aa | 2015-01-23 06:11:45 +0000 | [diff] [blame] | 6107 | } |
| 6108 | } |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6109 | } |
| 6110 | |
| 6111 | let Predicates = [HasCDI] in { |
| 6112 | defm VPCONFLICTD : avx512_conflict<0xC4, "vpconflictd", VR512, VK16WM, |
Elena Demikhovsky | 6270b38 | 2013-12-10 11:58:35 +0000 | [diff] [blame] | 6113 | i512mem, i32mem, "{1to16}">, |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6114 | EVEX_V512, EVEX_CD8<32, CD8VF>; |
| 6115 | |
Elena Demikhovsky | 6270b38 | 2013-12-10 11:58:35 +0000 | [diff] [blame] | 6116 | |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6117 | defm VPCONFLICTQ : avx512_conflict<0xC4, "vpconflictq", VR512, VK8WM, |
Elena Demikhovsky | 6270b38 | 2013-12-10 11:58:35 +0000 | [diff] [blame] | 6118 | i512mem, i64mem, "{1to8}">, |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6119 | EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; |
Elena Demikhovsky | 6270b38 | 2013-12-10 11:58:35 +0000 | [diff] [blame] | 6120 | |
Elena Demikhovsky | dacddb0 | 2013-11-03 13:46:31 +0000 | [diff] [blame] | 6121 | } |
Elena Demikhovsky | 6270b38 | 2013-12-10 11:58:35 +0000 | [diff] [blame] | 6122 | |
| 6123 | def : Pat<(int_x86_avx512_mask_conflict_d_512 VR512:$src2, VR512:$src1, |
| 6124 | GR16:$mask), |
| 6125 | (VPCONFLICTDrrk VR512:$src1, |
| 6126 | (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), VR512:$src2)>; |
| 6127 | |
| 6128 | def : Pat<(int_x86_avx512_mask_conflict_q_512 VR512:$src2, VR512:$src1, |
| 6129 | GR8:$mask), |
| 6130 | (VPCONFLICTQrrk VR512:$src1, |
| 6131 | (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), VR512:$src2)>; |
Elena Demikhovsky | cf0b9ba | 2014-04-09 12:37:50 +0000 | [diff] [blame] | 6132 | |
Cameron McInally | 5d1b7b9 | 2014-06-11 12:54:45 +0000 | [diff] [blame] | 6133 | let Predicates = [HasCDI] in { |
| 6134 | defm VPLZCNTD : avx512_conflict<0x44, "vplzcntd", VR512, VK16WM, |
| 6135 | i512mem, i32mem, "{1to16}">, |
| 6136 | EVEX_V512, EVEX_CD8<32, CD8VF>; |
| 6137 | |
| 6138 | |
| 6139 | defm VPLZCNTQ : avx512_conflict<0x44, "vplzcntq", VR512, VK8WM, |
| 6140 | i512mem, i64mem, "{1to8}">, |
| 6141 | EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; |
| 6142 | |
| 6143 | } |
| 6144 | |
| 6145 | def : Pat<(int_x86_avx512_mask_lzcnt_d_512 VR512:$src2, VR512:$src1, |
| 6146 | GR16:$mask), |
| 6147 | (VPLZCNTDrrk VR512:$src1, |
| 6148 | (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), VR512:$src2)>; |
| 6149 | |
| 6150 | def : Pat<(int_x86_avx512_mask_lzcnt_q_512 VR512:$src2, VR512:$src1, |
| 6151 | GR8:$mask), |
| 6152 | (VPLZCNTQrrk VR512:$src1, |
| 6153 | (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), VR512:$src2)>; |
| 6154 | |
Craig Topper | 820d492 | 2015-02-09 04:04:50 +0000 | [diff] [blame] | 6155 | def : Pat<(v16i32 (ctlz (loadv16i32 addr:$src))), |
Cameron McInally | 0d0489c | 2014-06-16 14:12:28 +0000 | [diff] [blame] | 6156 | (VPLZCNTDrm addr:$src)>; |
| 6157 | def : Pat<(v16i32 (ctlz (v16i32 VR512:$src))), |
| 6158 | (VPLZCNTDrr VR512:$src)>; |
Craig Topper | 820d492 | 2015-02-09 04:04:50 +0000 | [diff] [blame] | 6159 | def : Pat<(v8i64 (ctlz (loadv8i64 addr:$src))), |
Cameron McInally | 0d0489c | 2014-06-16 14:12:28 +0000 | [diff] [blame] | 6160 | (VPLZCNTQrm addr:$src)>; |
| 6161 | def : Pat<(v8i64 (ctlz (v8i64 VR512:$src))), |
| 6162 | (VPLZCNTQrr VR512:$src)>; |
| 6163 | |
Elena Demikhovsky | cf0b9ba | 2014-04-09 12:37:50 +0000 | [diff] [blame] | 6164 | def : Pat<(store (i1 -1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>; |
| 6165 | def : Pat<(store (i1 1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>; |
| 6166 | def : Pat<(store (i1 0), addr:$dst), (MOV8mi addr:$dst, (i8 0))>; |
Elena Demikhovsky | acc5c9e | 2014-04-22 14:13:10 +0000 | [diff] [blame] | 6167 | |
| 6168 | def : Pat<(store VK1:$src, addr:$dst), |
Elena Demikhovsky | 1a603b3 | 2015-01-25 12:47:15 +0000 | [diff] [blame] | 6169 | (MOV8mr addr:$dst, |
| 6170 | (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), |
| 6171 | sub_8bit))>, Requires<[HasAVX512, NoDQI]>; |
| 6172 | |
| 6173 | def : Pat<(store VK8:$src, addr:$dst), |
| 6174 | (MOV8mr addr:$dst, |
| 6175 | (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16)), |
| 6176 | sub_8bit))>, Requires<[HasAVX512, NoDQI]>; |
Elena Demikhovsky | acc5c9e | 2014-04-22 14:13:10 +0000 | [diff] [blame] | 6177 | |
| 6178 | def truncstorei1 : PatFrag<(ops node:$val, node:$ptr), |
| 6179 | (truncstore node:$val, node:$ptr), [{ |
| 6180 | return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i1; |
| 6181 | }]>; |
| 6182 | |
| 6183 | def : Pat<(truncstorei1 GR8:$src, addr:$dst), |
| 6184 | (MOV8mr addr:$dst, GR8:$src)>; |
| 6185 | |
Elena Demikhovsky | 44bf063 | 2014-10-05 14:11:08 +0000 | [diff] [blame] | 6186 | multiclass cvt_by_vec_width<bits<8> opc, X86VectorVTInfo Vec, string OpcodeStr > { |
Elena Demikhovsky | 0e6d6d5 | 2015-04-21 14:38:31 +0000 | [diff] [blame] | 6187 | def rr : AVX512XS8I<opc, MRMSrcReg, (outs Vec.RC:$dst), (ins Vec.KRC:$src), |
Craig Topper | edb0911 | 2014-11-25 20:11:23 +0000 | [diff] [blame] | 6188 | !strconcat(OpcodeStr##Vec.Suffix, "\t{$src, $dst|$dst, $src}"), |
Elena Demikhovsky | 44bf063 | 2014-10-05 14:11:08 +0000 | [diff] [blame] | 6189 | [(set Vec.RC:$dst, (Vec.VT (X86vsext Vec.KRC:$src)))]>, EVEX; |
| 6190 | } |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 6191 | |
Elena Demikhovsky | 44bf063 | 2014-10-05 14:11:08 +0000 | [diff] [blame] | 6192 | multiclass cvt_mask_by_elt_width<bits<8> opc, AVX512VLVectorVTInfo VTInfo, |
| 6193 | string OpcodeStr, Predicate prd> { |
| 6194 | let Predicates = [prd] in |
| 6195 | defm Z : cvt_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512; |
| 6196 | |
| 6197 | let Predicates = [prd, HasVLX] in { |
| 6198 | defm Z256 : cvt_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256; |
| 6199 | defm Z128 : cvt_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128; |
| 6200 | } |
| 6201 | } |
| 6202 | |
| 6203 | multiclass avx512_convert_mask_to_vector<string OpcodeStr> { |
| 6204 | defm NAME##B : cvt_mask_by_elt_width<0x28, avx512vl_i8_info, OpcodeStr, |
| 6205 | HasBWI>; |
| 6206 | defm NAME##W : cvt_mask_by_elt_width<0x28, avx512vl_i16_info, OpcodeStr, |
| 6207 | HasBWI>, VEX_W; |
| 6208 | defm NAME##D : cvt_mask_by_elt_width<0x38, avx512vl_i32_info, OpcodeStr, |
| 6209 | HasDQI>; |
| 6210 | defm NAME##Q : cvt_mask_by_elt_width<0x38, avx512vl_i64_info, OpcodeStr, |
| 6211 | HasDQI>, VEX_W; |
| 6212 | } |
Michael Liao | 5bf9578 | 2014-12-04 05:20:33 +0000 | [diff] [blame] | 6213 | |
Elena Demikhovsky | 44bf063 | 2014-10-05 14:11:08 +0000 | [diff] [blame] | 6214 | defm VPMOVM2 : avx512_convert_mask_to_vector<"vpmovm2">; |
Elena Demikhovsky | 908dbf4 | 2014-12-11 15:02:24 +0000 | [diff] [blame] | 6215 | |
Elena Demikhovsky | 0e6d6d5 | 2015-04-21 14:38:31 +0000 | [diff] [blame] | 6216 | multiclass convert_vector_to_mask_common<bits<8> opc, X86VectorVTInfo _, string OpcodeStr > { |
| 6217 | def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.KRC:$dst), (ins _.RC:$src), |
| 6218 | !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), |
| 6219 | [(set _.KRC:$dst, (trunc (_.VT _.RC:$src)))]>, EVEX; |
| 6220 | } |
| 6221 | |
| 6222 | multiclass avx512_convert_vector_to_mask<bits<8> opc, string OpcodeStr, |
| 6223 | AVX512VLVectorVTInfo VTInfo, Predicate prd> { |
| 6224 | let Predicates = [prd] in |
| 6225 | defm Z : convert_vector_to_mask_common <opc, VTInfo.info512, OpcodeStr>, |
| 6226 | EVEX_V512; |
| 6227 | |
| 6228 | let Predicates = [prd, HasVLX] in { |
| 6229 | defm Z256 : convert_vector_to_mask_common<opc, VTInfo.info256, OpcodeStr>, |
| 6230 | EVEX_V256; |
| 6231 | defm Z128 : convert_vector_to_mask_common<opc, VTInfo.info128, OpcodeStr>, |
| 6232 | EVEX_V128; |
| 6233 | } |
| 6234 | } |
| 6235 | |
| 6236 | defm VPMOVB2M : avx512_convert_vector_to_mask<0x29, "vpmovb2m", |
| 6237 | avx512vl_i8_info, HasBWI>; |
| 6238 | defm VPMOVW2M : avx512_convert_vector_to_mask<0x29, "vpmovw2m", |
| 6239 | avx512vl_i16_info, HasBWI>, VEX_W; |
| 6240 | defm VPMOVD2M : avx512_convert_vector_to_mask<0x39, "vpmovd2m", |
| 6241 | avx512vl_i32_info, HasDQI>; |
| 6242 | defm VPMOVQ2M : avx512_convert_vector_to_mask<0x39, "vpmovq2m", |
| 6243 | avx512vl_i64_info, HasDQI>, VEX_W; |
| 6244 | |
Elena Demikhovsky | 908dbf4 | 2014-12-11 15:02:24 +0000 | [diff] [blame] | 6245 | //===----------------------------------------------------------------------===// |
| 6246 | // AVX-512 - COMPRESS and EXPAND |
| 6247 | // |
Elena Demikhovsky | ba5ab32 | 2015-06-22 11:16:30 +0000 | [diff] [blame] | 6248 | |
Elena Demikhovsky | 908dbf4 | 2014-12-11 15:02:24 +0000 | [diff] [blame] | 6249 | multiclass compress_by_vec_width<bits<8> opc, X86VectorVTInfo _, |
| 6250 | string OpcodeStr> { |
Elena Demikhovsky | ba5ab32 | 2015-06-22 11:16:30 +0000 | [diff] [blame] | 6251 | defm rr : AVX512_maskable<opc, MRMDestReg, _, (outs _.RC:$dst), |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 6252 | (ins _.RC:$src1), OpcodeStr, "$src1", "$src1", |
Elena Demikhovsky | ba5ab32 | 2015-06-22 11:16:30 +0000 | [diff] [blame] | 6253 | (_.VT (X86compress _.RC:$src1))>, AVX5128IBase; |
Elena Demikhovsky | 908dbf4 | 2014-12-11 15:02:24 +0000 | [diff] [blame] | 6254 | |
| 6255 | let mayStore = 1 in { |
Elena Demikhovsky | ba5ab32 | 2015-06-22 11:16:30 +0000 | [diff] [blame] | 6256 | def mr : AVX5128I<opc, MRMDestMem, (outs), |
| 6257 | (ins _.MemOp:$dst, _.RC:$src), |
| 6258 | OpcodeStr # "\t{$src, $dst |$dst, $src}", |
| 6259 | []>, EVEX_CD8<_.EltSize, CD8VT1>; |
| 6260 | |
Elena Demikhovsky | 908dbf4 | 2014-12-11 15:02:24 +0000 | [diff] [blame] | 6261 | def mrk : AVX5128I<opc, MRMDestMem, (outs), |
| 6262 | (ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src), |
| 6263 | OpcodeStr # "\t{$src, $dst {${mask}} |$dst {${mask}}, $src}", |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 6264 | [(store (_.VT (vselect _.KRCWM:$mask, |
Elena Demikhovsky | ba5ab32 | 2015-06-22 11:16:30 +0000 | [diff] [blame] | 6265 | (_.VT (X86compress _.RC:$src)), _.ImmAllZerosV)), |
Elena Demikhovsky | 908dbf4 | 2014-12-11 15:02:24 +0000 | [diff] [blame] | 6266 | addr:$dst)]>, |
| 6267 | EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>; |
| 6268 | } |
| 6269 | } |
| 6270 | |
| 6271 | multiclass compress_by_elt_width<bits<8> opc, string OpcodeStr, |
| 6272 | AVX512VLVectorVTInfo VTInfo> { |
| 6273 | defm Z : compress_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512; |
| 6274 | |
| 6275 | let Predicates = [HasVLX] in { |
| 6276 | defm Z256 : compress_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256; |
| 6277 | defm Z128 : compress_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128; |
| 6278 | } |
| 6279 | } |
| 6280 | |
| 6281 | defm VPCOMPRESSD : compress_by_elt_width <0x8B, "vpcompressd", avx512vl_i32_info>, |
| 6282 | EVEX; |
| 6283 | defm VPCOMPRESSQ : compress_by_elt_width <0x8B, "vpcompressq", avx512vl_i64_info>, |
| 6284 | EVEX, VEX_W; |
| 6285 | defm VCOMPRESSPS : compress_by_elt_width <0x8A, "vcompressps", avx512vl_f32_info>, |
| 6286 | EVEX; |
| 6287 | defm VCOMPRESSPD : compress_by_elt_width <0x8A, "vcompresspd", avx512vl_f64_info>, |
| 6288 | EVEX, VEX_W; |
| 6289 | |
Elena Demikhovsky | 72860c3 | 2014-12-15 10:03:52 +0000 | [diff] [blame] | 6290 | // expand |
| 6291 | multiclass expand_by_vec_width<bits<8> opc, X86VectorVTInfo _, |
| 6292 | string OpcodeStr> { |
Elena Demikhovsky | ba5ab32 | 2015-06-22 11:16:30 +0000 | [diff] [blame] | 6293 | defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
Michael Liao | 66233b7 | 2015-08-06 09:06:20 +0000 | [diff] [blame] | 6294 | (ins _.RC:$src1), OpcodeStr, "$src1", "$src1", |
Elena Demikhovsky | ba5ab32 | 2015-06-22 11:16:30 +0000 | [diff] [blame] | 6295 | (_.VT (X86expand _.RC:$src1))>, AVX5128IBase; |
Elena Demikhovsky | 75ede68 | 2015-06-01 07:17:23 +0000 | [diff] [blame] | 6296 | |
Elena Demikhovsky | 72860c3 | 2014-12-15 10:03:52 +0000 | [diff] [blame] | 6297 | let mayLoad = 1 in |
Elena Demikhovsky | ba5ab32 | 2015-06-22 11:16:30 +0000 | [diff] [blame] | 6298 | defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 6299 | (ins _.MemOp:$src1), OpcodeStr, "$src1", "$src1", |
| 6300 | (_.VT (X86expand (_.VT (bitconvert |
| 6301 | (_.LdFrag addr:$src1)))))>, |
| 6302 | AVX5128IBase, EVEX_CD8<_.EltSize, CD8VT1>; |
Elena Demikhovsky | 72860c3 | 2014-12-15 10:03:52 +0000 | [diff] [blame] | 6303 | } |
| 6304 | |
| 6305 | multiclass expand_by_elt_width<bits<8> opc, string OpcodeStr, |
| 6306 | AVX512VLVectorVTInfo VTInfo> { |
| 6307 | defm Z : expand_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512; |
| 6308 | |
| 6309 | let Predicates = [HasVLX] in { |
| 6310 | defm Z256 : expand_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256; |
| 6311 | defm Z128 : expand_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128; |
| 6312 | } |
| 6313 | } |
| 6314 | |
| 6315 | defm VPEXPANDD : expand_by_elt_width <0x89, "vpexpandd", avx512vl_i32_info>, |
| 6316 | EVEX; |
| 6317 | defm VPEXPANDQ : expand_by_elt_width <0x89, "vpexpandq", avx512vl_i64_info>, |
| 6318 | EVEX, VEX_W; |
| 6319 | defm VEXPANDPS : expand_by_elt_width <0x88, "vexpandps", avx512vl_f32_info>, |
| 6320 | EVEX; |
| 6321 | defm VEXPANDPD : expand_by_elt_width <0x88, "vexpandpd", avx512vl_f64_info>, |
| 6322 | EVEX, VEX_W; |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6323 | |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6324 | //handle instruction reg_vec1 = op(reg_vec,imm) |
| 6325 | // op(mem_vec,imm) |
| 6326 | // op(broadcast(eltVt),imm) |
| 6327 | //all instruction created with FROUND_CURRENT |
| 6328 | multiclass avx512_unary_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 6329 | X86VectorVTInfo _>{ |
| 6330 | defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 6331 | (ins _.RC:$src1, i32u8imm:$src2), |
| 6332 | OpcodeStr##_.Suffix, "$src2, $src1", "$src2, $src2", |
| 6333 | (OpNode (_.VT _.RC:$src1), |
| 6334 | (i32 imm:$src2), |
| 6335 | (i32 FROUND_CURRENT))>; |
| 6336 | let mayLoad = 1 in { |
| 6337 | defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 6338 | (ins _.MemOp:$src1, i32u8imm:$src2), |
| 6339 | OpcodeStr##_.Suffix, "$src2, $src1", "$src1, $src2", |
| 6340 | (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))), |
| 6341 | (i32 imm:$src2), |
| 6342 | (i32 FROUND_CURRENT))>; |
| 6343 | defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 6344 | (ins _.ScalarMemOp:$src1, i32u8imm:$src2), |
| 6345 | OpcodeStr##_.Suffix, "$src2, ${src1}"##_.BroadcastStr, |
| 6346 | "${src1}"##_.BroadcastStr##", $src2", |
| 6347 | (OpNode (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src1))), |
| 6348 | (i32 imm:$src2), |
| 6349 | (i32 FROUND_CURRENT))>, EVEX_B; |
| 6350 | } |
| 6351 | } |
| 6352 | |
| 6353 | //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae} |
| 6354 | multiclass avx512_unary_fp_sae_packed_imm<bits<8> opc, string OpcodeStr, |
| 6355 | SDNode OpNode, X86VectorVTInfo _>{ |
| 6356 | defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 6357 | (ins _.RC:$src1, i32u8imm:$src2), |
| 6358 | OpcodeStr##_.Suffix, "$src2,{sae}, $src1", |
| 6359 | "$src1, {sae}, $src2", |
| 6360 | (OpNode (_.VT _.RC:$src1), |
| 6361 | (i32 imm:$src2), |
| 6362 | (i32 FROUND_NO_EXC))>, EVEX_B; |
| 6363 | } |
| 6364 | |
| 6365 | multiclass avx512_common_unary_fp_sae_packed_imm<string OpcodeStr, |
| 6366 | AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{ |
| 6367 | let Predicates = [prd] in { |
| 6368 | defm Z : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info512>, |
| 6369 | avx512_unary_fp_sae_packed_imm<opc, OpcodeStr, OpNode, _.info512>, |
| 6370 | EVEX_V512; |
| 6371 | } |
| 6372 | let Predicates = [prd, HasVLX] in { |
| 6373 | defm Z128 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info128>, |
| 6374 | EVEX_V128; |
| 6375 | defm Z256 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info256>, |
| 6376 | EVEX_V256; |
| 6377 | } |
| 6378 | } |
| 6379 | |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6380 | //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm) |
| 6381 | // op(reg_vec2,mem_vec,imm) |
| 6382 | // op(reg_vec2,broadcast(eltVt),imm) |
| 6383 | //all instruction created with FROUND_CURRENT |
| 6384 | multiclass avx512_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 6385 | X86VectorVTInfo _>{ |
| 6386 | defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6387 | (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6388 | OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", |
| 6389 | (OpNode (_.VT _.RC:$src1), |
| 6390 | (_.VT _.RC:$src2), |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6391 | (i32 imm:$src3), |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6392 | (i32 FROUND_CURRENT))>; |
| 6393 | let mayLoad = 1 in { |
| 6394 | defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6395 | (ins _.RC:$src1, _.MemOp:$src2, i32u8imm:$src3), |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6396 | OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", |
| 6397 | (OpNode (_.VT _.RC:$src1), |
| 6398 | (_.VT (bitconvert (_.LdFrag addr:$src2))), |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6399 | (i32 imm:$src3), |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6400 | (i32 FROUND_CURRENT))>; |
| 6401 | defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6402 | (ins _.RC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3), |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6403 | OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1", |
| 6404 | "$src1, ${src2}"##_.BroadcastStr##", $src3", |
| 6405 | (OpNode (_.VT _.RC:$src1), |
| 6406 | (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))), |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6407 | (i32 imm:$src3), |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6408 | (i32 FROUND_CURRENT))>, EVEX_B; |
| 6409 | } |
| 6410 | } |
| 6411 | |
Elena Demikhovsky | 9e38086 | 2015-06-03 10:56:40 +0000 | [diff] [blame] | 6412 | //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm) |
| 6413 | // op(reg_vec2,mem_vec,imm) |
Igor Breger | 2ae0fe3 | 2015-08-31 11:14:02 +0000 | [diff] [blame] | 6414 | multiclass avx512_3Op_rm_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 6415 | X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo>{ |
| 6416 | |
| 6417 | defm rri : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst), |
| 6418 | (ins SrcInfo.RC:$src1, SrcInfo.RC:$src2, u8imm:$src3), |
| 6419 | OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", |
| 6420 | (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1), |
| 6421 | (SrcInfo.VT SrcInfo.RC:$src2), |
| 6422 | (i8 imm:$src3)))>; |
| 6423 | let mayLoad = 1 in |
| 6424 | defm rmi : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst), |
| 6425 | (ins SrcInfo.RC:$src1, SrcInfo.MemOp:$src2, u8imm:$src3), |
| 6426 | OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", |
| 6427 | (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1), |
| 6428 | (SrcInfo.VT (bitconvert |
| 6429 | (SrcInfo.LdFrag addr:$src2))), |
| 6430 | (i8 imm:$src3)))>; |
| 6431 | } |
| 6432 | |
| 6433 | //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm) |
| 6434 | // op(reg_vec2,mem_vec,imm) |
Elena Demikhovsky | 9e38086 | 2015-06-03 10:56:40 +0000 | [diff] [blame] | 6435 | // op(reg_vec2,broadcast(eltVt),imm) |
| 6436 | multiclass avx512_3Op_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode, |
Igor Breger | 2ae0fe3 | 2015-08-31 11:14:02 +0000 | [diff] [blame] | 6437 | X86VectorVTInfo _>: |
| 6438 | avx512_3Op_rm_imm8<opc, OpcodeStr, OpNode, _, _>{ |
| 6439 | |
| 6440 | let mayLoad = 1 in |
Elena Demikhovsky | 9e38086 | 2015-06-03 10:56:40 +0000 | [diff] [blame] | 6441 | defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 6442 | (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3), |
| 6443 | OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1", |
| 6444 | "$src1, ${src2}"##_.BroadcastStr##", $src3", |
| 6445 | (OpNode (_.VT _.RC:$src1), |
| 6446 | (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))), |
| 6447 | (i8 imm:$src3))>, EVEX_B; |
Elena Demikhovsky | 9e38086 | 2015-06-03 10:56:40 +0000 | [diff] [blame] | 6448 | } |
| 6449 | |
Elena Demikhovsky | 3425c93 | 2015-06-02 08:28:57 +0000 | [diff] [blame] | 6450 | //handle scalar instruction reg_vec1 = op(reg_vec2,reg_vec3,imm) |
| 6451 | // op(reg_vec2,mem_scalar,imm) |
| 6452 | //all instruction created with FROUND_CURRENT |
| 6453 | multiclass avx512_fp_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 6454 | X86VectorVTInfo _> { |
| 6455 | |
| 6456 | defm rri : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6457 | (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), |
Elena Demikhovsky | 3425c93 | 2015-06-02 08:28:57 +0000 | [diff] [blame] | 6458 | OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", |
| 6459 | (OpNode (_.VT _.RC:$src1), |
| 6460 | (_.VT _.RC:$src2), |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6461 | (i32 imm:$src3), |
Elena Demikhovsky | 3425c93 | 2015-06-02 08:28:57 +0000 | [diff] [blame] | 6462 | (i32 FROUND_CURRENT))>; |
| 6463 | let mayLoad = 1 in { |
| 6464 | defm rmi : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6465 | (ins _.RC:$src1, _.MemOp:$src2, i32u8imm:$src3), |
Elena Demikhovsky | 3425c93 | 2015-06-02 08:28:57 +0000 | [diff] [blame] | 6466 | OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", |
| 6467 | (OpNode (_.VT _.RC:$src1), |
| 6468 | (_.VT (scalar_to_vector |
| 6469 | (_.ScalarLdFrag addr:$src2))), |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6470 | (i32 imm:$src3), |
Elena Demikhovsky | 3425c93 | 2015-06-02 08:28:57 +0000 | [diff] [blame] | 6471 | (i32 FROUND_CURRENT))>; |
| 6472 | |
| 6473 | let isAsmParserOnly = 1 in { |
| 6474 | defm rmi_alt :AVX512_maskable_in_asm<opc, MRMSrcMem, _, (outs _.FRC:$dst), |
| 6475 | (ins _.FRC:$src1, _.ScalarMemOp:$src2, u8imm:$src3), |
| 6476 | OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", |
| 6477 | []>; |
| 6478 | } |
| 6479 | } |
| 6480 | } |
| 6481 | |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6482 | //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae} |
| 6483 | multiclass avx512_fp_sae_packed_imm<bits<8> opc, string OpcodeStr, |
| 6484 | SDNode OpNode, X86VectorVTInfo _>{ |
| 6485 | defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6486 | (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6487 | OpcodeStr, "$src3,{sae}, $src2, $src1", |
| 6488 | "$src1, $src2,{sae}, $src3", |
| 6489 | (OpNode (_.VT _.RC:$src1), |
| 6490 | (_.VT _.RC:$src2), |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6491 | (i32 imm:$src3), |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6492 | (i32 FROUND_NO_EXC))>, EVEX_B; |
| 6493 | } |
Elena Demikhovsky | 3425c93 | 2015-06-02 08:28:57 +0000 | [diff] [blame] | 6494 | //handle scalar instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae} |
| 6495 | multiclass avx512_fp_sae_scalar_imm<bits<8> opc, string OpcodeStr, |
| 6496 | SDNode OpNode, X86VectorVTInfo _> { |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6497 | defm NAME#rrib : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 6498 | (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), |
| 6499 | OpcodeStr, "$src3,{sae}, $src2, $src1", |
| 6500 | "$src1, $src2,{sae}, $src3", |
| 6501 | (OpNode (_.VT _.RC:$src1), |
| 6502 | (_.VT _.RC:$src2), |
| 6503 | (i32 imm:$src3), |
| 6504 | (i32 FROUND_NO_EXC))>, EVEX_B; |
Elena Demikhovsky | 3425c93 | 2015-06-02 08:28:57 +0000 | [diff] [blame] | 6505 | } |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6506 | |
Elena Demikhovsky | 3582eb3 | 2015-06-01 11:05:34 +0000 | [diff] [blame] | 6507 | multiclass avx512_common_fp_sae_packed_imm<string OpcodeStr, |
| 6508 | AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{ |
Elena Demikhovsky | 9e38086 | 2015-06-03 10:56:40 +0000 | [diff] [blame] | 6509 | let Predicates = [prd] in { |
| 6510 | defm Z : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info512>, |
Igor Breger | 00d9f84 | 2015-06-08 14:03:17 +0000 | [diff] [blame] | 6511 | avx512_fp_sae_packed_imm<opc, OpcodeStr, OpNode, _.info512>, |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6512 | EVEX_V512; |
| 6513 | |
Elena Demikhovsky | 9e38086 | 2015-06-03 10:56:40 +0000 | [diff] [blame] | 6514 | } |
| 6515 | let Predicates = [prd, HasVLX] in { |
| 6516 | defm Z128 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info128>, |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6517 | EVEX_V128; |
Elena Demikhovsky | 9e38086 | 2015-06-03 10:56:40 +0000 | [diff] [blame] | 6518 | defm Z256 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info256>, |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6519 | EVEX_V256; |
Elena Demikhovsky | 9e38086 | 2015-06-03 10:56:40 +0000 | [diff] [blame] | 6520 | } |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6521 | } |
| 6522 | |
Igor Breger | 2ae0fe3 | 2015-08-31 11:14:02 +0000 | [diff] [blame] | 6523 | multiclass avx512_common_3Op_rm_imm8<bits<8> opc, SDNode OpNode, string OpStr, |
| 6524 | AVX512VLVectorVTInfo DestInfo, AVX512VLVectorVTInfo SrcInfo>{ |
| 6525 | let Predicates = [HasBWI] in { |
| 6526 | defm Z : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info512, |
| 6527 | SrcInfo.info512>, EVEX_V512, AVX512AIi8Base, EVEX_4V; |
| 6528 | } |
| 6529 | let Predicates = [HasBWI, HasVLX] in { |
| 6530 | defm Z128 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info128, |
| 6531 | SrcInfo.info128>, EVEX_V128, AVX512AIi8Base, EVEX_4V; |
| 6532 | defm Z256 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info256, |
| 6533 | SrcInfo.info256>, EVEX_V256, AVX512AIi8Base, EVEX_4V; |
| 6534 | } |
| 6535 | } |
| 6536 | |
Igor Breger | 00d9f84 | 2015-06-08 14:03:17 +0000 | [diff] [blame] | 6537 | multiclass avx512_common_3Op_imm8<string OpcodeStr, AVX512VLVectorVTInfo _, |
| 6538 | bits<8> opc, SDNode OpNode>{ |
| 6539 | let Predicates = [HasAVX512] in { |
| 6540 | defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512; |
| 6541 | } |
| 6542 | let Predicates = [HasAVX512, HasVLX] in { |
| 6543 | defm Z128 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128; |
| 6544 | defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256; |
| 6545 | } |
| 6546 | } |
| 6547 | |
Elena Demikhovsky | 3425c93 | 2015-06-02 08:28:57 +0000 | [diff] [blame] | 6548 | multiclass avx512_common_fp_sae_scalar_imm<string OpcodeStr, |
| 6549 | X86VectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{ |
| 6550 | let Predicates = [prd] in { |
| 6551 | defm Z128 : avx512_fp_scalar_imm<opc, OpcodeStr, OpNode, _>, |
| 6552 | avx512_fp_sae_scalar_imm<opc, OpcodeStr, OpNode, _>; |
Elena Demikhovsky | 9e38086 | 2015-06-03 10:56:40 +0000 | [diff] [blame] | 6553 | } |
Elena Demikhovsky | 3425c93 | 2015-06-02 08:28:57 +0000 | [diff] [blame] | 6554 | } |
| 6555 | |
Igor Breger | 1e58e8a | 2015-09-02 11:18:55 +0000 | [diff] [blame] | 6556 | multiclass avx512_common_unary_fp_sae_packed_imm_all<string OpcodeStr, |
| 6557 | bits<8> opcPs, bits<8> opcPd, SDNode OpNode, Predicate prd>{ |
| 6558 | defm PS : avx512_common_unary_fp_sae_packed_imm<OpcodeStr, avx512vl_f32_info, |
| 6559 | opcPs, OpNode, prd>, EVEX_CD8<32, CD8VF>; |
| 6560 | defm PD : avx512_common_unary_fp_sae_packed_imm<OpcodeStr, avx512vl_f64_info, |
| 6561 | opcPd, OpNode, prd>, EVEX_CD8<64, CD8VF>, VEX_W; |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6562 | } |
| 6563 | |
Elena Demikhovsky | 3582eb3 | 2015-06-01 11:05:34 +0000 | [diff] [blame] | 6564 | defm VFIXUPIMMPD : avx512_common_fp_sae_packed_imm<"vfixupimmpd", |
| 6565 | avx512vl_f64_info, 0x54, X86VFixupimm, HasAVX512>, |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6566 | AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; |
Elena Demikhovsky | 3582eb3 | 2015-06-01 11:05:34 +0000 | [diff] [blame] | 6567 | defm VFIXUPIMMPS : avx512_common_fp_sae_packed_imm<"vfixupimmps", |
| 6568 | avx512vl_f32_info, 0x54, X86VFixupimm, HasAVX512>, |
Elena Demikhovsky | 42c96d9 | 2015-06-01 06:50:49 +0000 | [diff] [blame] | 6569 | AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; |
| 6570 | |
Elena Demikhovsky | 3425c93 | 2015-06-02 08:28:57 +0000 | [diff] [blame] | 6571 | defm VFIXUPIMMSD: avx512_common_fp_sae_scalar_imm<"vfixupimmsd", f64x_info, |
| 6572 | 0x55, X86VFixupimm, HasAVX512>, |
| 6573 | AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W; |
| 6574 | defm VFIXUPIMMSS: avx512_common_fp_sae_scalar_imm<"vfixupimmss", f32x_info, |
| 6575 | 0x55, X86VFixupimm, HasAVX512>, |
| 6576 | AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>; |
Elena Demikhovsky | 3582eb3 | 2015-06-01 11:05:34 +0000 | [diff] [blame] | 6577 | |
Igor Breger | 1e58e8a | 2015-09-02 11:18:55 +0000 | [diff] [blame] | 6578 | defm VREDUCE : avx512_common_unary_fp_sae_packed_imm_all<"vreduce", 0x56, 0x56, |
| 6579 | X86VReduce, HasDQI>, AVX512AIi8Base, EVEX; |
| 6580 | defm VRNDSCALE : avx512_common_unary_fp_sae_packed_imm_all<"vrndscale", 0x08, 0x09, |
| 6581 | X86VRndScale, HasAVX512>, AVX512AIi8Base, EVEX; |
| 6582 | defm VGETMANT : avx512_common_unary_fp_sae_packed_imm_all<"vgetmant", 0x26, 0x26, |
| 6583 | X86VGetMant, HasAVX512>, AVX512AIi8Base, EVEX; |
| 6584 | |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6585 | |
Elena Demikhovsky | 3582eb3 | 2015-06-01 11:05:34 +0000 | [diff] [blame] | 6586 | defm VRANGEPD : avx512_common_fp_sae_packed_imm<"vrangepd", avx512vl_f64_info, |
| 6587 | 0x50, X86VRange, HasDQI>, |
| 6588 | AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; |
| 6589 | defm VRANGEPS : avx512_common_fp_sae_packed_imm<"vrangeps", avx512vl_f32_info, |
| 6590 | 0x50, X86VRange, HasDQI>, |
| 6591 | AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; |
| 6592 | |
Elena Demikhovsky | 8938f5a | 2015-06-02 14:12:54 +0000 | [diff] [blame] | 6593 | defm VRANGESD: avx512_common_fp_sae_scalar_imm<"vrangesd", f64x_info, |
| 6594 | 0x51, X86VRange, HasDQI>, |
| 6595 | AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W; |
| 6596 | defm VRANGESS: avx512_common_fp_sae_scalar_imm<"vrangess", f32x_info, |
| 6597 | 0x51, X86VRange, HasDQI>, |
| 6598 | AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>; |
| 6599 | |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6600 | defm VREDUCESD: avx512_common_fp_sae_scalar_imm<"vreducesd", f64x_info, |
| 6601 | 0x57, X86Reduces, HasDQI>, |
| 6602 | AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W; |
| 6603 | defm VREDUCESS: avx512_common_fp_sae_scalar_imm<"vreducess", f32x_info, |
| 6604 | 0x57, X86Reduces, HasDQI>, |
| 6605 | AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>; |
Elena Demikhovsky | 9e38086 | 2015-06-03 10:56:40 +0000 | [diff] [blame] | 6606 | |
Igor Breger | 1e58e8a | 2015-09-02 11:18:55 +0000 | [diff] [blame] | 6607 | defm VGETMANTSD: avx512_common_fp_sae_scalar_imm<"vgetmantsd", f64x_info, |
| 6608 | 0x27, X86GetMants, HasAVX512>, |
| 6609 | AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W; |
| 6610 | defm VGETMANTSS: avx512_common_fp_sae_scalar_imm<"vgetmantss", f32x_info, |
| 6611 | 0x27, X86GetMants, HasAVX512>, |
| 6612 | AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>; |
| 6613 | |
Elena Demikhovsky | 9e38086 | 2015-06-03 10:56:40 +0000 | [diff] [blame] | 6614 | multiclass avx512_shuff_packed_128<string OpcodeStr, AVX512VLVectorVTInfo _, |
| 6615 | bits<8> opc, SDNode OpNode = X86Shuf128>{ |
| 6616 | let Predicates = [HasAVX512] in { |
| 6617 | defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512; |
| 6618 | |
| 6619 | } |
| 6620 | let Predicates = [HasAVX512, HasVLX] in { |
| 6621 | defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256; |
| 6622 | } |
| 6623 | } |
Asaf Badouh | a5b2e5e | 2015-07-22 12:00:43 +0000 | [diff] [blame] | 6624 | let Predicates = [HasAVX512] in { |
| 6625 | def : Pat<(v16f32 (ffloor VR512:$src)), |
| 6626 | (VRNDSCALEPSZrri VR512:$src, (i32 0x1))>; |
| 6627 | def : Pat<(v16f32 (fnearbyint VR512:$src)), |
| 6628 | (VRNDSCALEPSZrri VR512:$src, (i32 0xC))>; |
| 6629 | def : Pat<(v16f32 (fceil VR512:$src)), |
| 6630 | (VRNDSCALEPSZrri VR512:$src, (i32 0x2))>; |
| 6631 | def : Pat<(v16f32 (frint VR512:$src)), |
| 6632 | (VRNDSCALEPSZrri VR512:$src, (i32 0x4))>; |
| 6633 | def : Pat<(v16f32 (ftrunc VR512:$src)), |
| 6634 | (VRNDSCALEPSZrri VR512:$src, (i32 0x3))>; |
| 6635 | |
| 6636 | def : Pat<(v8f64 (ffloor VR512:$src)), |
| 6637 | (VRNDSCALEPDZrri VR512:$src, (i32 0x1))>; |
| 6638 | def : Pat<(v8f64 (fnearbyint VR512:$src)), |
| 6639 | (VRNDSCALEPDZrri VR512:$src, (i32 0xC))>; |
| 6640 | def : Pat<(v8f64 (fceil VR512:$src)), |
| 6641 | (VRNDSCALEPDZrri VR512:$src, (i32 0x2))>; |
| 6642 | def : Pat<(v8f64 (frint VR512:$src)), |
| 6643 | (VRNDSCALEPDZrri VR512:$src, (i32 0x4))>; |
| 6644 | def : Pat<(v8f64 (ftrunc VR512:$src)), |
| 6645 | (VRNDSCALEPDZrri VR512:$src, (i32 0x3))>; |
| 6646 | } |
Elena Demikhovsky | 9e38086 | 2015-06-03 10:56:40 +0000 | [diff] [blame] | 6647 | |
| 6648 | defm VSHUFF32X4 : avx512_shuff_packed_128<"vshuff32x4",avx512vl_f32_info, 0x23>, |
| 6649 | AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; |
| 6650 | defm VSHUFF64X2 : avx512_shuff_packed_128<"vshuff64x2",avx512vl_f64_info, 0x23>, |
| 6651 | AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; |
| 6652 | defm VSHUFI32X4 : avx512_shuff_packed_128<"vshufi32x4",avx512vl_i32_info, 0x43>, |
| 6653 | AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; |
| 6654 | defm VSHUFI64X2 : avx512_shuff_packed_128<"vshufi64x2",avx512vl_i64_info, 0x43>, |
| 6655 | AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; |
Igor Breger | 00d9f84 | 2015-06-08 14:03:17 +0000 | [diff] [blame] | 6656 | |
| 6657 | multiclass avx512_valign<string OpcodeStr, AVX512VLVectorVTInfo VTInfo_I, |
| 6658 | AVX512VLVectorVTInfo VTInfo_FP>{ |
| 6659 | defm NAME: avx512_common_3Op_imm8<OpcodeStr, VTInfo_I, 0x03, X86VAlign>, |
| 6660 | AVX512AIi8Base, EVEX_4V; |
| 6661 | let isCodeGenOnly = 1 in { |
| 6662 | defm NAME#_FP: avx512_common_3Op_imm8<OpcodeStr, VTInfo_FP, 0x03, X86VAlign>, |
| 6663 | AVX512AIi8Base, EVEX_4V; |
| 6664 | } |
| 6665 | } |
| 6666 | |
| 6667 | defm VALIGND: avx512_valign<"valignd", avx512vl_i32_info, avx512vl_f32_info>, |
| 6668 | EVEX_CD8<32, CD8VF>; |
| 6669 | defm VALIGNQ: avx512_valign<"valignq", avx512vl_i64_info, avx512vl_f64_info>, |
| 6670 | EVEX_CD8<64, CD8VF>, VEX_W; |
Elena Demikhovsky | 5e2f8c4 | 2015-06-23 08:19:46 +0000 | [diff] [blame] | 6671 | |
Igor Breger | 2ae0fe3 | 2015-08-31 11:14:02 +0000 | [diff] [blame] | 6672 | multiclass avx512_vpalign_lowering<X86VectorVTInfo _ , list<Predicate> p>{ |
| 6673 | let Predicates = p in |
| 6674 | def NAME#_.VTName#rri: |
| 6675 | Pat<(_.VT (X86PAlignr _.RC:$src1, _.RC:$src2, (i8 imm:$imm))), |
| 6676 | (!cast<Instruction>(NAME#_.ZSuffix#rri) |
| 6677 | _.RC:$src1, _.RC:$src2, imm:$imm)>; |
| 6678 | } |
| 6679 | |
| 6680 | multiclass avx512_vpalign_lowering_common<AVX512VLVectorVTInfo _>: |
| 6681 | avx512_vpalign_lowering<_.info512, [HasBWI]>, |
| 6682 | avx512_vpalign_lowering<_.info128, [HasBWI, HasVLX]>, |
| 6683 | avx512_vpalign_lowering<_.info256, [HasBWI, HasVLX]>; |
| 6684 | |
| 6685 | defm VPALIGN: avx512_common_3Op_rm_imm8<0x0F, X86PAlignr, "vpalignr" , |
| 6686 | avx512vl_i8_info, avx512vl_i8_info>, |
| 6687 | avx512_vpalign_lowering_common<avx512vl_i16_info>, |
| 6688 | avx512_vpalign_lowering_common<avx512vl_i32_info>, |
| 6689 | avx512_vpalign_lowering_common<avx512vl_f32_info>, |
| 6690 | avx512_vpalign_lowering_common<avx512vl_i64_info>, |
| 6691 | avx512_vpalign_lowering_common<avx512vl_f64_info>, |
| 6692 | EVEX_CD8<8, CD8VF>; |
| 6693 | |
Igor Breger | f3ded81 | 2015-08-31 13:09:30 +0000 | [diff] [blame] | 6694 | defm VDBPSADBW: avx512_common_3Op_rm_imm8<0x42, X86dbpsadbw, "vdbpsadbw" , |
| 6695 | avx512vl_i16_info, avx512vl_i8_info>, EVEX_CD8<8, CD8VF>; |
| 6696 | |
Elena Demikhovsky | 5e2f8c4 | 2015-06-23 08:19:46 +0000 | [diff] [blame] | 6697 | multiclass avx512_unary_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 6698 | X86VectorVTInfo _> { |
| 6699 | defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), |
| 6700 | (ins _.RC:$src1), OpcodeStr##_.Suffix, |
| 6701 | "$src1", "$src1", |
| 6702 | (_.VT (OpNode _.RC:$src1))>, EVEX, AVX5128IBase; |
| 6703 | |
| 6704 | let mayLoad = 1 in |
| 6705 | defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 6706 | (ins _.MemOp:$src1), OpcodeStr##_.Suffix, |
| 6707 | "$src1", "$src1", |
| 6708 | (_.VT (OpNode (bitconvert (_.LdFrag addr:$src1))))>, |
| 6709 | EVEX, AVX5128IBase, EVEX_CD8<_.EltSize, CD8VF>; |
| 6710 | } |
| 6711 | |
| 6712 | multiclass avx512_unary_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 6713 | X86VectorVTInfo _> : |
| 6714 | avx512_unary_rm<opc, OpcodeStr, OpNode, _> { |
| 6715 | let mayLoad = 1 in |
| 6716 | defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), |
| 6717 | (ins _.ScalarMemOp:$src1), OpcodeStr##_.Suffix, |
| 6718 | "${src1}"##_.BroadcastStr, |
| 6719 | "${src1}"##_.BroadcastStr, |
| 6720 | (_.VT (OpNode (X86VBroadcast |
| 6721 | (_.ScalarLdFrag addr:$src1))))>, |
| 6722 | EVEX, AVX5128IBase, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>; |
| 6723 | } |
| 6724 | |
| 6725 | multiclass avx512_unary_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 6726 | AVX512VLVectorVTInfo VTInfo, Predicate prd> { |
| 6727 | let Predicates = [prd] in |
| 6728 | defm Z : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info512>, EVEX_V512; |
| 6729 | |
| 6730 | let Predicates = [prd, HasVLX] in { |
| 6731 | defm Z256 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info256>, |
| 6732 | EVEX_V256; |
| 6733 | defm Z128 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info128>, |
| 6734 | EVEX_V128; |
| 6735 | } |
| 6736 | } |
| 6737 | |
| 6738 | multiclass avx512_unary_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode, |
| 6739 | AVX512VLVectorVTInfo VTInfo, Predicate prd> { |
| 6740 | let Predicates = [prd] in |
| 6741 | defm Z : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info512>, |
| 6742 | EVEX_V512; |
| 6743 | |
| 6744 | let Predicates = [prd, HasVLX] in { |
| 6745 | defm Z256 : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info256>, |
| 6746 | EVEX_V256; |
| 6747 | defm Z128 : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info128>, |
| 6748 | EVEX_V128; |
| 6749 | } |
| 6750 | } |
| 6751 | |
| 6752 | multiclass avx512_unary_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr, |
| 6753 | SDNode OpNode, Predicate prd> { |
| 6754 | defm Q : avx512_unary_rmb_vl<opc_q, OpcodeStr, OpNode, avx512vl_i64_info, |
| 6755 | prd>, VEX_W; |
| 6756 | defm D : avx512_unary_rmb_vl<opc_d, OpcodeStr, OpNode, avx512vl_i32_info, prd>; |
| 6757 | } |
| 6758 | |
| 6759 | multiclass avx512_unary_rm_vl_bw<bits<8> opc_b, bits<8> opc_w, string OpcodeStr, |
| 6760 | SDNode OpNode, Predicate prd> { |
| 6761 | defm W : avx512_unary_rm_vl<opc_w, OpcodeStr, OpNode, avx512vl_i16_info, prd>; |
| 6762 | defm B : avx512_unary_rm_vl<opc_b, OpcodeStr, OpNode, avx512vl_i8_info, prd>; |
| 6763 | } |
| 6764 | |
| 6765 | multiclass avx512_unary_rm_vl_all<bits<8> opc_b, bits<8> opc_w, |
| 6766 | bits<8> opc_d, bits<8> opc_q, |
| 6767 | string OpcodeStr, SDNode OpNode> { |
| 6768 | defm NAME : avx512_unary_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode, |
| 6769 | HasAVX512>, |
| 6770 | avx512_unary_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode, |
| 6771 | HasBWI>; |
| 6772 | } |
| 6773 | |
| 6774 | defm VPABS : avx512_unary_rm_vl_all<0x1C, 0x1D, 0x1E, 0x1F, "vpabs", X86Abs>; |
| 6775 | |
| 6776 | def : Pat<(xor |
| 6777 | (bc_v16i32 (v16i1sextv16i32)), |
| 6778 | (bc_v16i32 (add (v16i32 VR512:$src), (v16i1sextv16i32)))), |
| 6779 | (VPABSDZrr VR512:$src)>; |
| 6780 | def : Pat<(xor |
| 6781 | (bc_v8i64 (v8i1sextv8i64)), |
| 6782 | (bc_v8i64 (add (v8i64 VR512:$src), (v8i1sextv8i64)))), |
| 6783 | (VPABSQZrr VR512:$src)>; |
Igor Breger | f246011 | 2015-07-26 14:41:44 +0000 | [diff] [blame] | 6784 | |
| 6785 | //===----------------------------------------------------------------------===// |
| 6786 | // AVX-512 - Unpack Instructions |
| 6787 | //===----------------------------------------------------------------------===// |
| 6788 | defm VUNPCKH : avx512_fp_binop_p<0x15, "vunpckh", X86Unpckh>; |
| 6789 | defm VUNPCKL : avx512_fp_binop_p<0x14, "vunpckl", X86Unpckl>; |
| 6790 | |
| 6791 | defm VPUNPCKLBW : avx512_binop_rm_vl_b<0x60, "vpunpcklbw", X86Unpckl, |
| 6792 | SSE_INTALU_ITINS_P, HasBWI>; |
| 6793 | defm VPUNPCKHBW : avx512_binop_rm_vl_b<0x68, "vpunpckhbw", X86Unpckh, |
| 6794 | SSE_INTALU_ITINS_P, HasBWI>; |
| 6795 | defm VPUNPCKLWD : avx512_binop_rm_vl_w<0x61, "vpunpcklwd", X86Unpckl, |
| 6796 | SSE_INTALU_ITINS_P, HasBWI>; |
| 6797 | defm VPUNPCKHWD : avx512_binop_rm_vl_w<0x69, "vpunpckhwd", X86Unpckh, |
| 6798 | SSE_INTALU_ITINS_P, HasBWI>; |
| 6799 | |
| 6800 | defm VPUNPCKLDQ : avx512_binop_rm_vl_d<0x62, "vpunpckldq", X86Unpckl, |
| 6801 | SSE_INTALU_ITINS_P, HasAVX512>; |
| 6802 | defm VPUNPCKHDQ : avx512_binop_rm_vl_d<0x6A, "vpunpckhdq", X86Unpckh, |
| 6803 | SSE_INTALU_ITINS_P, HasAVX512>; |
| 6804 | defm VPUNPCKLQDQ : avx512_binop_rm_vl_q<0x6C, "vpunpcklqdq", X86Unpckl, |
| 6805 | SSE_INTALU_ITINS_P, HasAVX512>; |
| 6806 | defm VPUNPCKHQDQ : avx512_binop_rm_vl_q<0x6D, "vpunpckhqdq", X86Unpckh, |
| 6807 | SSE_INTALU_ITINS_P, HasAVX512>; |
Igor Breger | a6297c7 | 2015-09-02 10:50:58 +0000 | [diff] [blame] | 6808 | //===----------------------------------------------------------------------===// |
| 6809 | // VSHUFPS - VSHUFPD Operations |
| 6810 | //===----------------------------------------------------------------------===// |
| 6811 | multiclass avx512_shufp<string OpcodeStr, AVX512VLVectorVTInfo VTInfo_I, |
| 6812 | AVX512VLVectorVTInfo VTInfo_FP>{ |
| 6813 | defm NAME: avx512_common_3Op_imm8<OpcodeStr, VTInfo_FP, 0xC6, X86Shufp>, |
| 6814 | EVEX_CD8<VTInfo_FP.info512.EltSize, CD8VF>, |
| 6815 | AVX512AIi8Base, EVEX_4V; |
| 6816 | let isCodeGenOnly = 1 in { |
| 6817 | defm NAME#_I: avx512_common_3Op_imm8<OpcodeStr, VTInfo_I, 0xC6, X86Shufp>, |
| 6818 | EVEX_CD8<VTInfo_I.info512.EltSize, CD8VF>, |
| 6819 | AVX512AIi8Base, EVEX_4V; |
| 6820 | } |
| 6821 | } |
| 6822 | |
| 6823 | defm VSHUFPS: avx512_shufp<"vshufps", avx512vl_i32_info, avx512vl_f32_info>, PS; |
| 6824 | defm VSHUFPD: avx512_shufp<"vshufpd", avx512vl_i64_info, avx512vl_f64_info>, PD, VEX_W; |
Asaf Badouh | d2c3599 | 2015-09-02 14:21:54 +0000 | [diff] [blame^] | 6825 | //===----------------------------------------------------------------------===// |
| 6826 | // AVX-512 - Byte shift Left/Right |
| 6827 | //===----------------------------------------------------------------------===// |
| 6828 | |
| 6829 | multiclass avx512_shift_packed<bits<8> opc, SDNode OpNode, Format MRMr, |
| 6830 | Format MRMm, string OpcodeStr, X86VectorVTInfo _>{ |
| 6831 | def rr : AVX512<opc, MRMr, |
| 6832 | (outs _.RC:$dst), (ins _.RC:$src1, u8imm:$src2), |
| 6833 | !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
| 6834 | [(set _.RC:$dst,(_.VT (OpNode _.RC:$src1, (i8 imm:$src2))))]>; |
| 6835 | let mayLoad = 1 in |
| 6836 | def rm : AVX512<opc, MRMm, |
| 6837 | (outs _.RC:$dst), (ins _.MemOp:$src1, u8imm:$src2), |
| 6838 | !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
| 6839 | [(set _.RC:$dst,(_.VT (OpNode |
| 6840 | (_.LdFrag addr:$src1), (i8 imm:$src2))))]>; |
| 6841 | } |
| 6842 | |
| 6843 | multiclass avx512_shift_packed_all<bits<8> opc, SDNode OpNode, Format MRMr, |
| 6844 | Format MRMm, string OpcodeStr, Predicate prd>{ |
| 6845 | let Predicates = [prd] in |
| 6846 | defm Z512 : avx512_shift_packed<opc, OpNode, MRMr, MRMm, |
| 6847 | OpcodeStr, v8i64_info>, EVEX_V512; |
| 6848 | let Predicates = [prd, HasVLX] in { |
| 6849 | defm Z256 : avx512_shift_packed<opc, OpNode, MRMr, MRMm, |
| 6850 | OpcodeStr, v4i64x_info>, EVEX_V256; |
| 6851 | defm Z128 : avx512_shift_packed<opc, OpNode, MRMr, MRMm, |
| 6852 | OpcodeStr, v2i64x_info>, EVEX_V128; |
| 6853 | } |
| 6854 | } |
| 6855 | defm VPSLLDQ : avx512_shift_packed_all<0x73, X86vshldq, MRM7r, MRM7m, "vpslldq", |
| 6856 | HasBWI>, AVX512PDIi8Base, EVEX_4V; |
| 6857 | defm VPSRLDQ : avx512_shift_packed_all<0x73, X86vshrdq, MRM3r, MRM3m, "vpsrldq", |
| 6858 | HasBWI>, AVX512PDIi8Base, EVEX_4V; |
| 6859 | |
| 6860 | |
| 6861 | multiclass avx512_psadbw_packed<bits<8> opc, SDNode OpNode, |
| 6862 | string OpcodeStr, X86VectorVTInfo _src>{ |
| 6863 | def rr : AVX512BI<opc, MRMSrcReg, |
| 6864 | (outs _src.RC:$dst), (ins _src.RC:$src1, _src.RC:$src2), |
| 6865 | !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
| 6866 | [(set _src.RC:$dst,(_src.VT |
| 6867 | (OpNode _src.RC:$src1, _src.RC:$src2)))]>; |
| 6868 | let mayLoad = 1 in |
| 6869 | def rm : AVX512BI<opc, MRMSrcMem, |
| 6870 | (outs _src.RC:$dst), (ins _src.RC:$src1, _src.MemOp:$src2), |
| 6871 | !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), |
| 6872 | [(set _src.RC:$dst,(_src.VT |
| 6873 | (OpNode _src.RC:$src1, |
| 6874 | (_src.VT (bitconvert |
| 6875 | (_src.LdFrag addr:$src2))))))]>; |
| 6876 | } |
| 6877 | |
| 6878 | multiclass avx512_psadbw_packed_all<bits<8> opc, SDNode OpNode, |
| 6879 | string OpcodeStr, Predicate prd> { |
| 6880 | let Predicates = [prd] in |
| 6881 | defm Z512 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v64i8_info>, |
| 6882 | EVEX_V512; |
| 6883 | let Predicates = [prd, HasVLX] in { |
| 6884 | defm Z256 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v32i8x_info>, |
| 6885 | EVEX_V256; |
| 6886 | defm Z128 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v16i8x_info>, |
| 6887 | EVEX_V128; |
| 6888 | } |
| 6889 | } |
| 6890 | |
| 6891 | defm VPSADBW : avx512_psadbw_packed_all<0xf6, X86psadbw, "vpsadbw", |
| 6892 | HasBWI>, EVEX_4V; |