blob: 9024168483600a4b19643597c850dcc74a3dc295 [file] [log] [blame]
Craig Topper991d4992015-11-03 06:16:31 +00001/*===---- avx512fintrin.h - AVX512F intrinsics -----------------------------===
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00002 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy
4 * of this software and associated documentation files (the "Software"), to deal
5 * in the Software without restriction, including without limitation the rights
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *
21 *===-----------------------------------------------------------------------===
22 */
23#ifndef __IMMINTRIN_H
24#error "Never use <avx512fintrin.h> directly; include <immintrin.h> instead."
25#endif
26
27#ifndef __AVX512FINTRIN_H
28#define __AVX512FINTRIN_H
29
Reid Kleckner89fbd552018-06-04 21:39:20 +000030typedef char __v64qi __attribute__((__vector_size__(64)));
31typedef short __v32hi __attribute__((__vector_size__(64)));
32typedef double __v8df __attribute__((__vector_size__(64)));
33typedef float __v16sf __attribute__((__vector_size__(64)));
34typedef long long __v8di __attribute__((__vector_size__(64)));
35typedef int __v16si __attribute__((__vector_size__(64)));
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +000036
Craig Topper6a77b622016-06-04 05:43:41 +000037/* Unsigned types */
Reid Kleckner89fbd552018-06-04 21:39:20 +000038typedef unsigned char __v64qu __attribute__((__vector_size__(64)));
39typedef unsigned short __v32hu __attribute__((__vector_size__(64)));
40typedef unsigned long long __v8du __attribute__((__vector_size__(64)));
41typedef unsigned int __v16su __attribute__((__vector_size__(64)));
Craig Topper6a77b622016-06-04 05:43:41 +000042
Reid Kleckner89fbd552018-06-04 21:39:20 +000043typedef float __m512 __attribute__((__vector_size__(64)));
44typedef double __m512d __attribute__((__vector_size__(64)));
45typedef long long __m512i __attribute__((__vector_size__(64)));
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +000046
47typedef unsigned char __mmask8;
48typedef unsigned short __mmask16;
49
50/* Rounding mode macros. */
51#define _MM_FROUND_TO_NEAREST_INT 0x00
52#define _MM_FROUND_TO_NEG_INF 0x01
53#define _MM_FROUND_TO_POS_INF 0x02
54#define _MM_FROUND_TO_ZERO 0x03
55#define _MM_FROUND_CUR_DIRECTION 0x04
56
Asaf Badouh2f344b72016-08-07 10:43:04 +000057/* Constants for integer comparison predicates */
58typedef enum {
59 _MM_CMPINT_EQ, /* Equal */
60 _MM_CMPINT_LT, /* Less than */
61 _MM_CMPINT_LE, /* Less than or Equal */
62 _MM_CMPINT_UNUSED,
63 _MM_CMPINT_NE, /* Not Equal */
64 _MM_CMPINT_NLT, /* Not Less than */
65#define _MM_CMPINT_GE _MM_CMPINT_NLT /* Greater than or Equal */
66 _MM_CMPINT_NLE /* Not Less than or Equal */
67#define _MM_CMPINT_GT _MM_CMPINT_NLE /* Greater than */
68} _MM_CMPINT_ENUM;
69
Michael Zuckermandef78752016-03-28 12:23:09 +000070typedef enum
71{
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +000072 _MM_PERM_AAAA = 0x00, _MM_PERM_AAAB = 0x01, _MM_PERM_AAAC = 0x02,
73 _MM_PERM_AAAD = 0x03, _MM_PERM_AABA = 0x04, _MM_PERM_AABB = 0x05,
74 _MM_PERM_AABC = 0x06, _MM_PERM_AABD = 0x07, _MM_PERM_AACA = 0x08,
75 _MM_PERM_AACB = 0x09, _MM_PERM_AACC = 0x0A, _MM_PERM_AACD = 0x0B,
76 _MM_PERM_AADA = 0x0C, _MM_PERM_AADB = 0x0D, _MM_PERM_AADC = 0x0E,
77 _MM_PERM_AADD = 0x0F, _MM_PERM_ABAA = 0x10, _MM_PERM_ABAB = 0x11,
78 _MM_PERM_ABAC = 0x12, _MM_PERM_ABAD = 0x13, _MM_PERM_ABBA = 0x14,
79 _MM_PERM_ABBB = 0x15, _MM_PERM_ABBC = 0x16, _MM_PERM_ABBD = 0x17,
80 _MM_PERM_ABCA = 0x18, _MM_PERM_ABCB = 0x19, _MM_PERM_ABCC = 0x1A,
81 _MM_PERM_ABCD = 0x1B, _MM_PERM_ABDA = 0x1C, _MM_PERM_ABDB = 0x1D,
82 _MM_PERM_ABDC = 0x1E, _MM_PERM_ABDD = 0x1F, _MM_PERM_ACAA = 0x20,
83 _MM_PERM_ACAB = 0x21, _MM_PERM_ACAC = 0x22, _MM_PERM_ACAD = 0x23,
84 _MM_PERM_ACBA = 0x24, _MM_PERM_ACBB = 0x25, _MM_PERM_ACBC = 0x26,
85 _MM_PERM_ACBD = 0x27, _MM_PERM_ACCA = 0x28, _MM_PERM_ACCB = 0x29,
86 _MM_PERM_ACCC = 0x2A, _MM_PERM_ACCD = 0x2B, _MM_PERM_ACDA = 0x2C,
87 _MM_PERM_ACDB = 0x2D, _MM_PERM_ACDC = 0x2E, _MM_PERM_ACDD = 0x2F,
88 _MM_PERM_ADAA = 0x30, _MM_PERM_ADAB = 0x31, _MM_PERM_ADAC = 0x32,
89 _MM_PERM_ADAD = 0x33, _MM_PERM_ADBA = 0x34, _MM_PERM_ADBB = 0x35,
90 _MM_PERM_ADBC = 0x36, _MM_PERM_ADBD = 0x37, _MM_PERM_ADCA = 0x38,
91 _MM_PERM_ADCB = 0x39, _MM_PERM_ADCC = 0x3A, _MM_PERM_ADCD = 0x3B,
92 _MM_PERM_ADDA = 0x3C, _MM_PERM_ADDB = 0x3D, _MM_PERM_ADDC = 0x3E,
93 _MM_PERM_ADDD = 0x3F, _MM_PERM_BAAA = 0x40, _MM_PERM_BAAB = 0x41,
94 _MM_PERM_BAAC = 0x42, _MM_PERM_BAAD = 0x43, _MM_PERM_BABA = 0x44,
95 _MM_PERM_BABB = 0x45, _MM_PERM_BABC = 0x46, _MM_PERM_BABD = 0x47,
96 _MM_PERM_BACA = 0x48, _MM_PERM_BACB = 0x49, _MM_PERM_BACC = 0x4A,
97 _MM_PERM_BACD = 0x4B, _MM_PERM_BADA = 0x4C, _MM_PERM_BADB = 0x4D,
98 _MM_PERM_BADC = 0x4E, _MM_PERM_BADD = 0x4F, _MM_PERM_BBAA = 0x50,
99 _MM_PERM_BBAB = 0x51, _MM_PERM_BBAC = 0x52, _MM_PERM_BBAD = 0x53,
100 _MM_PERM_BBBA = 0x54, _MM_PERM_BBBB = 0x55, _MM_PERM_BBBC = 0x56,
101 _MM_PERM_BBBD = 0x57, _MM_PERM_BBCA = 0x58, _MM_PERM_BBCB = 0x59,
102 _MM_PERM_BBCC = 0x5A, _MM_PERM_BBCD = 0x5B, _MM_PERM_BBDA = 0x5C,
103 _MM_PERM_BBDB = 0x5D, _MM_PERM_BBDC = 0x5E, _MM_PERM_BBDD = 0x5F,
104 _MM_PERM_BCAA = 0x60, _MM_PERM_BCAB = 0x61, _MM_PERM_BCAC = 0x62,
105 _MM_PERM_BCAD = 0x63, _MM_PERM_BCBA = 0x64, _MM_PERM_BCBB = 0x65,
106 _MM_PERM_BCBC = 0x66, _MM_PERM_BCBD = 0x67, _MM_PERM_BCCA = 0x68,
107 _MM_PERM_BCCB = 0x69, _MM_PERM_BCCC = 0x6A, _MM_PERM_BCCD = 0x6B,
108 _MM_PERM_BCDA = 0x6C, _MM_PERM_BCDB = 0x6D, _MM_PERM_BCDC = 0x6E,
109 _MM_PERM_BCDD = 0x6F, _MM_PERM_BDAA = 0x70, _MM_PERM_BDAB = 0x71,
110 _MM_PERM_BDAC = 0x72, _MM_PERM_BDAD = 0x73, _MM_PERM_BDBA = 0x74,
111 _MM_PERM_BDBB = 0x75, _MM_PERM_BDBC = 0x76, _MM_PERM_BDBD = 0x77,
112 _MM_PERM_BDCA = 0x78, _MM_PERM_BDCB = 0x79, _MM_PERM_BDCC = 0x7A,
113 _MM_PERM_BDCD = 0x7B, _MM_PERM_BDDA = 0x7C, _MM_PERM_BDDB = 0x7D,
114 _MM_PERM_BDDC = 0x7E, _MM_PERM_BDDD = 0x7F, _MM_PERM_CAAA = 0x80,
115 _MM_PERM_CAAB = 0x81, _MM_PERM_CAAC = 0x82, _MM_PERM_CAAD = 0x83,
116 _MM_PERM_CABA = 0x84, _MM_PERM_CABB = 0x85, _MM_PERM_CABC = 0x86,
117 _MM_PERM_CABD = 0x87, _MM_PERM_CACA = 0x88, _MM_PERM_CACB = 0x89,
118 _MM_PERM_CACC = 0x8A, _MM_PERM_CACD = 0x8B, _MM_PERM_CADA = 0x8C,
119 _MM_PERM_CADB = 0x8D, _MM_PERM_CADC = 0x8E, _MM_PERM_CADD = 0x8F,
120 _MM_PERM_CBAA = 0x90, _MM_PERM_CBAB = 0x91, _MM_PERM_CBAC = 0x92,
121 _MM_PERM_CBAD = 0x93, _MM_PERM_CBBA = 0x94, _MM_PERM_CBBB = 0x95,
122 _MM_PERM_CBBC = 0x96, _MM_PERM_CBBD = 0x97, _MM_PERM_CBCA = 0x98,
123 _MM_PERM_CBCB = 0x99, _MM_PERM_CBCC = 0x9A, _MM_PERM_CBCD = 0x9B,
124 _MM_PERM_CBDA = 0x9C, _MM_PERM_CBDB = 0x9D, _MM_PERM_CBDC = 0x9E,
125 _MM_PERM_CBDD = 0x9F, _MM_PERM_CCAA = 0xA0, _MM_PERM_CCAB = 0xA1,
126 _MM_PERM_CCAC = 0xA2, _MM_PERM_CCAD = 0xA3, _MM_PERM_CCBA = 0xA4,
127 _MM_PERM_CCBB = 0xA5, _MM_PERM_CCBC = 0xA6, _MM_PERM_CCBD = 0xA7,
128 _MM_PERM_CCCA = 0xA8, _MM_PERM_CCCB = 0xA9, _MM_PERM_CCCC = 0xAA,
129 _MM_PERM_CCCD = 0xAB, _MM_PERM_CCDA = 0xAC, _MM_PERM_CCDB = 0xAD,
130 _MM_PERM_CCDC = 0xAE, _MM_PERM_CCDD = 0xAF, _MM_PERM_CDAA = 0xB0,
131 _MM_PERM_CDAB = 0xB1, _MM_PERM_CDAC = 0xB2, _MM_PERM_CDAD = 0xB3,
132 _MM_PERM_CDBA = 0xB4, _MM_PERM_CDBB = 0xB5, _MM_PERM_CDBC = 0xB6,
133 _MM_PERM_CDBD = 0xB7, _MM_PERM_CDCA = 0xB8, _MM_PERM_CDCB = 0xB9,
134 _MM_PERM_CDCC = 0xBA, _MM_PERM_CDCD = 0xBB, _MM_PERM_CDDA = 0xBC,
135 _MM_PERM_CDDB = 0xBD, _MM_PERM_CDDC = 0xBE, _MM_PERM_CDDD = 0xBF,
136 _MM_PERM_DAAA = 0xC0, _MM_PERM_DAAB = 0xC1, _MM_PERM_DAAC = 0xC2,
137 _MM_PERM_DAAD = 0xC3, _MM_PERM_DABA = 0xC4, _MM_PERM_DABB = 0xC5,
138 _MM_PERM_DABC = 0xC6, _MM_PERM_DABD = 0xC7, _MM_PERM_DACA = 0xC8,
139 _MM_PERM_DACB = 0xC9, _MM_PERM_DACC = 0xCA, _MM_PERM_DACD = 0xCB,
140 _MM_PERM_DADA = 0xCC, _MM_PERM_DADB = 0xCD, _MM_PERM_DADC = 0xCE,
141 _MM_PERM_DADD = 0xCF, _MM_PERM_DBAA = 0xD0, _MM_PERM_DBAB = 0xD1,
142 _MM_PERM_DBAC = 0xD2, _MM_PERM_DBAD = 0xD3, _MM_PERM_DBBA = 0xD4,
143 _MM_PERM_DBBB = 0xD5, _MM_PERM_DBBC = 0xD6, _MM_PERM_DBBD = 0xD7,
144 _MM_PERM_DBCA = 0xD8, _MM_PERM_DBCB = 0xD9, _MM_PERM_DBCC = 0xDA,
145 _MM_PERM_DBCD = 0xDB, _MM_PERM_DBDA = 0xDC, _MM_PERM_DBDB = 0xDD,
146 _MM_PERM_DBDC = 0xDE, _MM_PERM_DBDD = 0xDF, _MM_PERM_DCAA = 0xE0,
147 _MM_PERM_DCAB = 0xE1, _MM_PERM_DCAC = 0xE2, _MM_PERM_DCAD = 0xE3,
148 _MM_PERM_DCBA = 0xE4, _MM_PERM_DCBB = 0xE5, _MM_PERM_DCBC = 0xE6,
149 _MM_PERM_DCBD = 0xE7, _MM_PERM_DCCA = 0xE8, _MM_PERM_DCCB = 0xE9,
150 _MM_PERM_DCCC = 0xEA, _MM_PERM_DCCD = 0xEB, _MM_PERM_DCDA = 0xEC,
151 _MM_PERM_DCDB = 0xED, _MM_PERM_DCDC = 0xEE, _MM_PERM_DCDD = 0xEF,
152 _MM_PERM_DDAA = 0xF0, _MM_PERM_DDAB = 0xF1, _MM_PERM_DDAC = 0xF2,
153 _MM_PERM_DDAD = 0xF3, _MM_PERM_DDBA = 0xF4, _MM_PERM_DDBB = 0xF5,
154 _MM_PERM_DDBC = 0xF6, _MM_PERM_DDBD = 0xF7, _MM_PERM_DDCA = 0xF8,
155 _MM_PERM_DDCB = 0xF9, _MM_PERM_DDCC = 0xFA, _MM_PERM_DDCD = 0xFB,
156 _MM_PERM_DDDA = 0xFC, _MM_PERM_DDDB = 0xFD, _MM_PERM_DDDC = 0xFE,
157 _MM_PERM_DDDD = 0xFF
158} _MM_PERM_ENUM;
159
160typedef enum
161{
Michael Zuckermandef78752016-03-28 12:23:09 +0000162 _MM_MANT_NORM_1_2, /* interval [1, 2) */
163 _MM_MANT_NORM_p5_2, /* interval [0.5, 2) */
164 _MM_MANT_NORM_p5_1, /* interval [0.5, 1) */
165 _MM_MANT_NORM_p75_1p5 /* interval [0.75, 1.5) */
166} _MM_MANTISSA_NORM_ENUM;
167
168typedef enum
169{
170 _MM_MANT_SIGN_src, /* sign = sign(SRC) */
171 _MM_MANT_SIGN_zero, /* sign = 0 */
172 _MM_MANT_SIGN_nan /* DEST = NaN if sign(SRC) = 1 */
173} _MM_MANTISSA_SIGN_ENUM;
174
Eric Christopher4d1851682015-06-17 07:09:20 +0000175/* Define the default attributes for the functions in this file. */
Craig Topper74c10e32018-07-09 19:00:16 +0000176#define __DEFAULT_FN_ATTRS512 __attribute__((__always_inline__, __nodebug__, __target__("avx512f"), __min_vector_width__(512)))
177#define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avx512f"), __min_vector_width__(128)))
Eric Christopher4d1851682015-06-17 07:09:20 +0000178
Adam Nemet0d5bb552014-07-28 17:14:40 +0000179/* Create vectors with repeated elements */
180
Craig Topper74c10e32018-07-09 19:00:16 +0000181static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +0000182_mm512_setzero_si512(void)
183{
Craig Topper63ec0ea2018-05-30 21:08:27 +0000184 return __extension__ (__m512i)(__v8di){ 0, 0, 0, 0, 0, 0, 0, 0 };
Adam Nemet0d5bb552014-07-28 17:14:40 +0000185}
186
Michael Zuckermanf36f6eb2016-06-05 15:12:52 +0000187#define _mm512_setzero_epi32 _mm512_setzero_si512
188
Craig Topper74c10e32018-07-09 19:00:16 +0000189static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper3a0c7262016-06-09 05:14:28 +0000190_mm512_undefined_pd(void)
Simon Pilgrim5aba9922015-08-26 21:17:12 +0000191{
192 return (__m512d)__builtin_ia32_undef512();
193}
194
Craig Topper74c10e32018-07-09 19:00:16 +0000195static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper3a0c7262016-06-09 05:14:28 +0000196_mm512_undefined(void)
Simon Pilgrim5aba9922015-08-26 21:17:12 +0000197{
198 return (__m512)__builtin_ia32_undef512();
199}
200
Craig Topper74c10e32018-07-09 19:00:16 +0000201static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper3a0c7262016-06-09 05:14:28 +0000202_mm512_undefined_ps(void)
Simon Pilgrim5aba9922015-08-26 21:17:12 +0000203{
204 return (__m512)__builtin_ia32_undef512();
205}
206
Craig Topper74c10e32018-07-09 19:00:16 +0000207static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper3a0c7262016-06-09 05:14:28 +0000208_mm512_undefined_epi32(void)
Simon Pilgrim5aba9922015-08-26 21:17:12 +0000209{
210 return (__m512i)__builtin_ia32_undef512();
211}
Simon Pilgrimf5a88372016-07-05 12:59:33 +0000212
Craig Topper74c10e32018-07-09 19:00:16 +0000213static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman8c2900f2016-04-27 11:43:14 +0000214_mm512_broadcastd_epi32 (__m128i __A)
215{
Craig Topper95ed88a2018-06-04 19:28:09 +0000216 return (__m512i)__builtin_shufflevector((__v4si) __A, (__v4si) __A,
Simon Pilgrimf5a88372016-07-05 12:59:33 +0000217 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
Michael Zuckerman8c2900f2016-04-27 11:43:14 +0000218}
219
Craig Topper74c10e32018-07-09 19:00:16 +0000220static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman8c2900f2016-04-27 11:43:14 +0000221_mm512_mask_broadcastd_epi32 (__m512i __O, __mmask16 __M, __m128i __A)
222{
Simon Pilgrimf5a88372016-07-05 12:59:33 +0000223 return (__m512i)__builtin_ia32_selectd_512(__M,
224 (__v16si) _mm512_broadcastd_epi32(__A),
225 (__v16si) __O);
Michael Zuckerman8c2900f2016-04-27 11:43:14 +0000226}
227
Craig Topper74c10e32018-07-09 19:00:16 +0000228static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman8c2900f2016-04-27 11:43:14 +0000229_mm512_maskz_broadcastd_epi32 (__mmask16 __M, __m128i __A)
230{
Simon Pilgrimf5a88372016-07-05 12:59:33 +0000231 return (__m512i)__builtin_ia32_selectd_512(__M,
232 (__v16si) _mm512_broadcastd_epi32(__A),
233 (__v16si) _mm512_setzero_si512());
Michael Zuckerman8c2900f2016-04-27 11:43:14 +0000234}
235
Craig Topper74c10e32018-07-09 19:00:16 +0000236static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman8c2900f2016-04-27 11:43:14 +0000237_mm512_broadcastq_epi64 (__m128i __A)
238{
Craig Topper95ed88a2018-06-04 19:28:09 +0000239 return (__m512i)__builtin_shufflevector((__v2di) __A, (__v2di) __A,
Simon Pilgrimf5a88372016-07-05 12:59:33 +0000240 0, 0, 0, 0, 0, 0, 0, 0);
Michael Zuckerman8c2900f2016-04-27 11:43:14 +0000241}
242
Craig Topper74c10e32018-07-09 19:00:16 +0000243static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman8c2900f2016-04-27 11:43:14 +0000244_mm512_mask_broadcastq_epi64 (__m512i __O, __mmask8 __M, __m128i __A)
245{
Simon Pilgrimf5a88372016-07-05 12:59:33 +0000246 return (__m512i)__builtin_ia32_selectq_512(__M,
247 (__v8di) _mm512_broadcastq_epi64(__A),
248 (__v8di) __O);
249
Michael Zuckerman8c2900f2016-04-27 11:43:14 +0000250}
251
Craig Topper74c10e32018-07-09 19:00:16 +0000252static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman8c2900f2016-04-27 11:43:14 +0000253_mm512_maskz_broadcastq_epi64 (__mmask8 __M, __m128i __A)
254{
Simon Pilgrimf5a88372016-07-05 12:59:33 +0000255 return (__m512i)__builtin_ia32_selectq_512(__M,
256 (__v8di) _mm512_broadcastq_epi64(__A),
257 (__v8di) _mm512_setzero_si512());
Michael Zuckerman8c2900f2016-04-27 11:43:14 +0000258}
Simon Pilgrim5aba9922015-08-26 21:17:12 +0000259
Adam Nemet0d5bb552014-07-28 17:14:40 +0000260
Craig Topper74c10e32018-07-09 19:00:16 +0000261static __inline __m512 __DEFAULT_FN_ATTRS512
Adam Nemet9a3ea602014-07-28 17:14:38 +0000262_mm512_setzero_ps(void)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +0000263{
Craig Topper63ec0ea2018-05-30 21:08:27 +0000264 return __extension__ (__m512){ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
265 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 };
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +0000266}
Michael Zuckermanf36f6eb2016-06-05 15:12:52 +0000267
268#define _mm512_setzero _mm512_setzero_ps
269
Craig Topper74c10e32018-07-09 19:00:16 +0000270static __inline __m512d __DEFAULT_FN_ATTRS512
Adam Nemet9a3ea602014-07-28 17:14:38 +0000271_mm512_setzero_pd(void)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +0000272{
Craig Topper63ec0ea2018-05-30 21:08:27 +0000273 return __extension__ (__m512d){ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 };
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +0000274}
Adam Nemet0d5bb552014-07-28 17:14:40 +0000275
Craig Topper74c10e32018-07-09 19:00:16 +0000276static __inline __m512 __DEFAULT_FN_ATTRS512
Adam Nemetf42e7a22014-07-30 16:51:22 +0000277_mm512_set1_ps(float __w)
278{
Craig Topper63ec0ea2018-05-30 21:08:27 +0000279 return __extension__ (__m512){ __w, __w, __w, __w, __w, __w, __w, __w,
280 __w, __w, __w, __w, __w, __w, __w, __w };
Adam Nemetf42e7a22014-07-30 16:51:22 +0000281}
282
Craig Topper74c10e32018-07-09 19:00:16 +0000283static __inline __m512d __DEFAULT_FN_ATTRS512
Adam Nemetf42e7a22014-07-30 16:51:22 +0000284_mm512_set1_pd(double __w)
285{
Craig Topper63ec0ea2018-05-30 21:08:27 +0000286 return __extension__ (__m512d){ __w, __w, __w, __w, __w, __w, __w, __w };
Adam Nemetf42e7a22014-07-30 16:51:22 +0000287}
288
Craig Topper74c10e32018-07-09 19:00:16 +0000289static __inline __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman6f08ceb2016-05-26 06:54:52 +0000290_mm512_set1_epi8(char __w)
291{
Craig Topper63ec0ea2018-05-30 21:08:27 +0000292 return __extension__ (__m512i)(__v64qi){
293 __w, __w, __w, __w, __w, __w, __w, __w,
294 __w, __w, __w, __w, __w, __w, __w, __w,
295 __w, __w, __w, __w, __w, __w, __w, __w,
296 __w, __w, __w, __w, __w, __w, __w, __w,
297 __w, __w, __w, __w, __w, __w, __w, __w,
298 __w, __w, __w, __w, __w, __w, __w, __w,
299 __w, __w, __w, __w, __w, __w, __w, __w,
300 __w, __w, __w, __w, __w, __w, __w, __w };
Michael Zuckerman6f08ceb2016-05-26 06:54:52 +0000301}
302
Craig Topper74c10e32018-07-09 19:00:16 +0000303static __inline __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman6f08ceb2016-05-26 06:54:52 +0000304_mm512_set1_epi16(short __w)
305{
Craig Topper63ec0ea2018-05-30 21:08:27 +0000306 return __extension__ (__m512i)(__v32hi){
307 __w, __w, __w, __w, __w, __w, __w, __w,
308 __w, __w, __w, __w, __w, __w, __w, __w,
309 __w, __w, __w, __w, __w, __w, __w, __w,
310 __w, __w, __w, __w, __w, __w, __w, __w };
Michael Zuckerman6f08ceb2016-05-26 06:54:52 +0000311}
312
Craig Topper74c10e32018-07-09 19:00:16 +0000313static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemetf42e7a22014-07-30 16:51:22 +0000314_mm512_set1_epi32(int __s)
315{
Craig Topper63ec0ea2018-05-30 21:08:27 +0000316 return __extension__ (__m512i)(__v16si){
317 __s, __s, __s, __s, __s, __s, __s, __s,
318 __s, __s, __s, __s, __s, __s, __s, __s };
Adam Nemetf42e7a22014-07-30 16:51:22 +0000319}
320
Craig Topper74c10e32018-07-09 19:00:16 +0000321static __inline __m512i __DEFAULT_FN_ATTRS512
Jina Nahias3ad702a2017-09-19 11:00:27 +0000322_mm512_maskz_set1_epi32(__mmask16 __M, int __A)
323{
324 return (__m512i)__builtin_ia32_selectd_512(__M,
325 (__v16si)_mm512_set1_epi32(__A),
326 (__v16si)_mm512_setzero_si512());
327}
328
Craig Topper74c10e32018-07-09 19:00:16 +0000329static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemetf42e7a22014-07-30 16:51:22 +0000330_mm512_set1_epi64(long long __d)
331{
Craig Topper63ec0ea2018-05-30 21:08:27 +0000332 return __extension__(__m512i)(__v8di){ __d, __d, __d, __d, __d, __d, __d, __d };
Adam Nemetf42e7a22014-07-30 16:51:22 +0000333}
334
Craig Topper74c10e32018-07-09 19:00:16 +0000335static __inline __m512i __DEFAULT_FN_ATTRS512
Jina Nahias3ad702a2017-09-19 11:00:27 +0000336_mm512_maskz_set1_epi64(__mmask8 __M, long long __A)
337{
338 return (__m512i)__builtin_ia32_selectq_512(__M,
339 (__v8di)_mm512_set1_epi64(__A),
340 (__v8di)_mm512_setzero_si512());
341}
Jina Nahias3ad702a2017-09-19 11:00:27 +0000342
Craig Topper74c10e32018-07-09 19:00:16 +0000343static __inline__ __m512 __DEFAULT_FN_ATTRS512
Simon Pilgrimf5a88372016-07-05 12:59:33 +0000344_mm512_broadcastss_ps(__m128 __A)
Adam Nemet4abc07c2014-08-13 00:29:01 +0000345{
Craig Topper95ed88a2018-06-04 19:28:09 +0000346 return (__m512)__builtin_shufflevector((__v4sf) __A, (__v4sf) __A,
Simon Pilgrimf5a88372016-07-05 12:59:33 +0000347 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
Adam Nemet4abc07c2014-08-13 00:29:01 +0000348}
349
Craig Topper74c10e32018-07-09 19:00:16 +0000350static __inline __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman95721ac2016-06-05 15:43:30 +0000351_mm512_set4_epi32 (int __A, int __B, int __C, int __D)
352{
Craig Topper63ec0ea2018-05-30 21:08:27 +0000353 return __extension__ (__m512i)(__v16si)
Michael Zuckerman95721ac2016-06-05 15:43:30 +0000354 { __D, __C, __B, __A, __D, __C, __B, __A,
355 __D, __C, __B, __A, __D, __C, __B, __A };
356}
357
Craig Topper74c10e32018-07-09 19:00:16 +0000358static __inline __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman95721ac2016-06-05 15:43:30 +0000359_mm512_set4_epi64 (long long __A, long long __B, long long __C,
360 long long __D)
361{
Craig Topper63ec0ea2018-05-30 21:08:27 +0000362 return __extension__ (__m512i) (__v8di)
Michael Zuckerman95721ac2016-06-05 15:43:30 +0000363 { __D, __C, __B, __A, __D, __C, __B, __A };
364}
365
Craig Topper74c10e32018-07-09 19:00:16 +0000366static __inline __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman95721ac2016-06-05 15:43:30 +0000367_mm512_set4_pd (double __A, double __B, double __C, double __D)
368{
Craig Topper63ec0ea2018-05-30 21:08:27 +0000369 return __extension__ (__m512d)
Michael Zuckerman95721ac2016-06-05 15:43:30 +0000370 { __D, __C, __B, __A, __D, __C, __B, __A };
371}
372
Craig Topper74c10e32018-07-09 19:00:16 +0000373static __inline __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman95721ac2016-06-05 15:43:30 +0000374_mm512_set4_ps (float __A, float __B, float __C, float __D)
375{
Craig Topper63ec0ea2018-05-30 21:08:27 +0000376 return __extension__ (__m512)
Michael Zuckerman95721ac2016-06-05 15:43:30 +0000377 { __D, __C, __B, __A, __D, __C, __B, __A,
378 __D, __C, __B, __A, __D, __C, __B, __A };
379}
380
381#define _mm512_setr4_epi32(e0,e1,e2,e3) \
Craig Topper60589492016-06-08 06:08:04 +0000382 _mm512_set4_epi32((e3),(e2),(e1),(e0))
Michael Zuckerman95721ac2016-06-05 15:43:30 +0000383
384#define _mm512_setr4_epi64(e0,e1,e2,e3) \
Craig Topper60589492016-06-08 06:08:04 +0000385 _mm512_set4_epi64((e3),(e2),(e1),(e0))
Michael Zuckerman95721ac2016-06-05 15:43:30 +0000386
387#define _mm512_setr4_pd(e0,e1,e2,e3) \
Craig Topper60589492016-06-08 06:08:04 +0000388 _mm512_set4_pd((e3),(e2),(e1),(e0))
Michael Zuckerman95721ac2016-06-05 15:43:30 +0000389
390#define _mm512_setr4_ps(e0,e1,e2,e3) \
Craig Topper60589492016-06-08 06:08:04 +0000391 _mm512_set4_ps((e3),(e2),(e1),(e0))
Michael Zuckerman95721ac2016-06-05 15:43:30 +0000392
Craig Topper74c10e32018-07-09 19:00:16 +0000393static __inline__ __m512d __DEFAULT_FN_ATTRS512
Simon Pilgrimf5a88372016-07-05 12:59:33 +0000394_mm512_broadcastsd_pd(__m128d __A)
Adam Nemet4abc07c2014-08-13 00:29:01 +0000395{
Craig Topper95ed88a2018-06-04 19:28:09 +0000396 return (__m512d)__builtin_shufflevector((__v2df) __A, (__v2df) __A,
Simon Pilgrimf5a88372016-07-05 12:59:33 +0000397 0, 0, 0, 0, 0, 0, 0, 0);
Adam Nemet4abc07c2014-08-13 00:29:01 +0000398}
399
Adam Nemetc871ff92014-07-30 16:51:24 +0000400/* Cast between vector types */
401
Craig Topper74c10e32018-07-09 19:00:16 +0000402static __inline __m512d __DEFAULT_FN_ATTRS512
Adam Nemetc871ff92014-07-30 16:51:24 +0000403_mm512_castpd256_pd512(__m256d __a)
404{
405 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, -1, -1, -1, -1);
406}
407
Craig Topper74c10e32018-07-09 19:00:16 +0000408static __inline __m512 __DEFAULT_FN_ATTRS512
Adam Nemetc871ff92014-07-30 16:51:24 +0000409_mm512_castps256_ps512(__m256 __a)
410{
411 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7,
412 -1, -1, -1, -1, -1, -1, -1, -1);
413}
414
Craig Topper74c10e32018-07-09 19:00:16 +0000415static __inline __m128d __DEFAULT_FN_ATTRS512
Adam Nemetc871ff92014-07-30 16:51:24 +0000416_mm512_castpd512_pd128(__m512d __a)
417{
418 return __builtin_shufflevector(__a, __a, 0, 1);
419}
420
Craig Topper74c10e32018-07-09 19:00:16 +0000421static __inline __m256d __DEFAULT_FN_ATTRS512
Michael Zuckermand5cc6cd2016-05-25 14:04:21 +0000422_mm512_castpd512_pd256 (__m512d __A)
423{
424 return __builtin_shufflevector(__A, __A, 0, 1, 2, 3);
425}
426
Craig Topper74c10e32018-07-09 19:00:16 +0000427static __inline __m128 __DEFAULT_FN_ATTRS512
Adam Nemetc871ff92014-07-30 16:51:24 +0000428_mm512_castps512_ps128(__m512 __a)
429{
430 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3);
431}
432
Craig Topper74c10e32018-07-09 19:00:16 +0000433static __inline __m256 __DEFAULT_FN_ATTRS512
Michael Zuckermand5cc6cd2016-05-25 14:04:21 +0000434_mm512_castps512_ps256 (__m512 __A)
435{
436 return __builtin_shufflevector(__A, __A, 0, 1, 2, 3, 4, 5, 6, 7);
437}
438
Craig Topper74c10e32018-07-09 19:00:16 +0000439static __inline __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermand5cc6cd2016-05-25 14:04:21 +0000440_mm512_castpd_ps (__m512d __A)
441{
442 return (__m512) (__A);
443}
444
Craig Topper74c10e32018-07-09 19:00:16 +0000445static __inline __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermand5cc6cd2016-05-25 14:04:21 +0000446_mm512_castpd_si512 (__m512d __A)
447{
448 return (__m512i) (__A);
449}
Michael Zuckermanc6677032016-05-03 14:26:52 +0000450
Craig Topper74c10e32018-07-09 19:00:16 +0000451static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermanc6677032016-05-03 14:26:52 +0000452_mm512_castpd128_pd512 (__m128d __A)
453{
454 return __builtin_shufflevector( __A, __A, 0, 1, -1, -1, -1, -1, -1, -1);
455}
456
Craig Topper74c10e32018-07-09 19:00:16 +0000457static __inline __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermand5cc6cd2016-05-25 14:04:21 +0000458_mm512_castps_pd (__m512 __A)
459{
460 return (__m512d) (__A);
461}
462
Craig Topper74c10e32018-07-09 19:00:16 +0000463static __inline __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermand5cc6cd2016-05-25 14:04:21 +0000464_mm512_castps_si512 (__m512 __A)
465{
466 return (__m512i) (__A);
467}
468
Craig Topper74c10e32018-07-09 19:00:16 +0000469static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanc6677032016-05-03 14:26:52 +0000470_mm512_castps128_ps512 (__m128 __A)
471{
472 return __builtin_shufflevector( __A, __A, 0, 1, 2, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
473}
474
Craig Topper74c10e32018-07-09 19:00:16 +0000475static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanc6677032016-05-03 14:26:52 +0000476_mm512_castsi128_si512 (__m128i __A)
477{
478 return __builtin_shufflevector( __A, __A, 0, 1, -1, -1, -1, -1, -1, -1);
479}
480
Craig Topper74c10e32018-07-09 19:00:16 +0000481static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanc6677032016-05-03 14:26:52 +0000482_mm512_castsi256_si512 (__m256i __A)
483{
484 return __builtin_shufflevector( __A, __A, 0, 1, 2, 3, -1, -1, -1, -1);
485}
486
Craig Topper74c10e32018-07-09 19:00:16 +0000487static __inline __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermand5cc6cd2016-05-25 14:04:21 +0000488_mm512_castsi512_ps (__m512i __A)
489{
490 return (__m512) (__A);
491}
492
Craig Topper74c10e32018-07-09 19:00:16 +0000493static __inline __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermand5cc6cd2016-05-25 14:04:21 +0000494_mm512_castsi512_pd (__m512i __A)
495{
496 return (__m512d) (__A);
497}
498
Craig Topper74c10e32018-07-09 19:00:16 +0000499static __inline __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermand5cc6cd2016-05-25 14:04:21 +0000500_mm512_castsi512_si128 (__m512i __A)
501{
502 return (__m128i)__builtin_shufflevector(__A, __A , 0, 1);
503}
504
Craig Topper74c10e32018-07-09 19:00:16 +0000505static __inline __m256i __DEFAULT_FN_ATTRS512
Michael Zuckerman22c47e62016-05-26 14:32:11 +0000506_mm512_castsi512_si256 (__m512i __A)
507{
508 return (__m256i)__builtin_shufflevector(__A, __A , 0, 1, 2, 3);
509}
510
Craig Topper74c10e32018-07-09 19:00:16 +0000511static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
Ayman Musae60a41c2016-11-08 12:00:30 +0000512_mm512_int2mask(int __a)
513{
514 return (__mmask16)__a;
515}
516
Craig Topper74c10e32018-07-09 19:00:16 +0000517static __inline__ int __DEFAULT_FN_ATTRS512
Ayman Musae60a41c2016-11-08 12:00:30 +0000518_mm512_mask2int(__mmask16 __a)
519{
520 return (int)__a;
521}
522
Adrian Prantl9fc8faf2018-05-09 01:00:01 +0000523/// Constructs a 512-bit floating-point vector of [8 x double] from a
Simon Pilgrim96d02f52017-04-29 17:17:06 +0000524/// 128-bit floating-point vector of [2 x double]. The lower 128 bits
525/// contain the value of the source vector. The upper 384 bits are set
526/// to zero.
527///
528/// \headerfile <x86intrin.h>
529///
530/// This intrinsic has no corresponding instruction.
531///
532/// \param __a
533/// A 128-bit vector of [2 x double].
534/// \returns A 512-bit floating-point vector of [8 x double]. The lower 128 bits
535/// contain the value of the parameter. The upper 384 bits are set to zero.
Craig Topper74c10e32018-07-09 19:00:16 +0000536static __inline __m512d __DEFAULT_FN_ATTRS512
Simon Pilgrim96d02f52017-04-29 17:17:06 +0000537_mm512_zextpd128_pd512(__m128d __a)
538{
539 return __builtin_shufflevector((__v2df)__a, (__v2df)_mm_setzero_pd(), 0, 1, 2, 3, 2, 3, 2, 3);
540}
541
Adrian Prantl9fc8faf2018-05-09 01:00:01 +0000542/// Constructs a 512-bit floating-point vector of [8 x double] from a
Simon Pilgrim96d02f52017-04-29 17:17:06 +0000543/// 256-bit floating-point vector of [4 x double]. The lower 256 bits
544/// contain the value of the source vector. The upper 256 bits are set
545/// to zero.
546///
547/// \headerfile <x86intrin.h>
548///
549/// This intrinsic has no corresponding instruction.
550///
551/// \param __a
552/// A 256-bit vector of [4 x double].
553/// \returns A 512-bit floating-point vector of [8 x double]. The lower 256 bits
554/// contain the value of the parameter. The upper 256 bits are set to zero.
Craig Topper74c10e32018-07-09 19:00:16 +0000555static __inline __m512d __DEFAULT_FN_ATTRS512
Simon Pilgrim96d02f52017-04-29 17:17:06 +0000556_mm512_zextpd256_pd512(__m256d __a)
557{
558 return __builtin_shufflevector((__v4df)__a, (__v4df)_mm256_setzero_pd(), 0, 1, 2, 3, 4, 5, 6, 7);
559}
560
Adrian Prantl9fc8faf2018-05-09 01:00:01 +0000561/// Constructs a 512-bit floating-point vector of [16 x float] from a
Simon Pilgrim96d02f52017-04-29 17:17:06 +0000562/// 128-bit floating-point vector of [4 x float]. The lower 128 bits contain
563/// the value of the source vector. The upper 384 bits are set to zero.
564///
565/// \headerfile <x86intrin.h>
566///
567/// This intrinsic has no corresponding instruction.
568///
569/// \param __a
570/// A 128-bit vector of [4 x float].
571/// \returns A 512-bit floating-point vector of [16 x float]. The lower 128 bits
572/// contain the value of the parameter. The upper 384 bits are set to zero.
Craig Topper74c10e32018-07-09 19:00:16 +0000573static __inline __m512 __DEFAULT_FN_ATTRS512
Simon Pilgrim96d02f52017-04-29 17:17:06 +0000574_mm512_zextps128_ps512(__m128 __a)
575{
576 return __builtin_shufflevector((__v4sf)__a, (__v4sf)_mm_setzero_ps(), 0, 1, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7, 4, 5, 6, 7);
577}
578
Adrian Prantl9fc8faf2018-05-09 01:00:01 +0000579/// Constructs a 512-bit floating-point vector of [16 x float] from a
Simon Pilgrim96d02f52017-04-29 17:17:06 +0000580/// 256-bit floating-point vector of [8 x float]. The lower 256 bits contain
581/// the value of the source vector. The upper 256 bits are set to zero.
582///
583/// \headerfile <x86intrin.h>
584///
585/// This intrinsic has no corresponding instruction.
586///
587/// \param __a
588/// A 256-bit vector of [8 x float].
589/// \returns A 512-bit floating-point vector of [16 x float]. The lower 256 bits
590/// contain the value of the parameter. The upper 256 bits are set to zero.
Craig Topper74c10e32018-07-09 19:00:16 +0000591static __inline __m512 __DEFAULT_FN_ATTRS512
Simon Pilgrim96d02f52017-04-29 17:17:06 +0000592_mm512_zextps256_ps512(__m256 __a)
593{
594 return __builtin_shufflevector((__v8sf)__a, (__v8sf)_mm256_setzero_ps(), 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
595}
596
Adrian Prantl9fc8faf2018-05-09 01:00:01 +0000597/// Constructs a 512-bit integer vector from a 128-bit integer vector.
Simon Pilgrim96d02f52017-04-29 17:17:06 +0000598/// The lower 128 bits contain the value of the source vector. The upper
599/// 384 bits are set to zero.
600///
601/// \headerfile <x86intrin.h>
602///
603/// This intrinsic has no corresponding instruction.
604///
605/// \param __a
606/// A 128-bit integer vector.
607/// \returns A 512-bit integer vector. The lower 128 bits contain the value of
608/// the parameter. The upper 384 bits are set to zero.
Craig Topper74c10e32018-07-09 19:00:16 +0000609static __inline __m512i __DEFAULT_FN_ATTRS512
Simon Pilgrim96d02f52017-04-29 17:17:06 +0000610_mm512_zextsi128_si512(__m128i __a)
611{
612 return __builtin_shufflevector((__v2di)__a, (__v2di)_mm_setzero_si128(), 0, 1, 2, 3, 2, 3, 2, 3);
613}
614
Adrian Prantl9fc8faf2018-05-09 01:00:01 +0000615/// Constructs a 512-bit integer vector from a 256-bit integer vector.
Simon Pilgrim96d02f52017-04-29 17:17:06 +0000616/// The lower 256 bits contain the value of the source vector. The upper
617/// 256 bits are set to zero.
618///
619/// \headerfile <x86intrin.h>
620///
621/// This intrinsic has no corresponding instruction.
622///
623/// \param __a
624/// A 256-bit integer vector.
625/// \returns A 512-bit integer vector. The lower 256 bits contain the value of
626/// the parameter. The upper 256 bits are set to zero.
Craig Topper74c10e32018-07-09 19:00:16 +0000627static __inline __m512i __DEFAULT_FN_ATTRS512
Simon Pilgrim96d02f52017-04-29 17:17:06 +0000628_mm512_zextsi256_si512(__m256i __a)
629{
630 return __builtin_shufflevector((__v4di)__a, (__v4di)_mm256_setzero_si256(), 0, 1, 2, 3, 4, 5, 6, 7);
631}
632
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000633/* Bitwise operators */
Craig Topper74c10e32018-07-09 19:00:16 +0000634static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000635_mm512_and_epi32(__m512i __a, __m512i __b)
636{
Craig Topper6a77b622016-06-04 05:43:41 +0000637 return (__m512i)((__v16su)__a & (__v16su)__b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000638}
639
Craig Topper74c10e32018-07-09 19:00:16 +0000640static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000641_mm512_mask_and_epi32(__m512i __src, __mmask16 __k, __m512i __a, __m512i __b)
642{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000643 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__k,
Asaf Badouh13633282016-07-05 12:24:14 +0000644 (__v16si) _mm512_and_epi32(__a, __b),
645 (__v16si) __src);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000646}
Asaf Badouh13633282016-07-05 12:24:14 +0000647
Craig Topper74c10e32018-07-09 19:00:16 +0000648static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000649_mm512_maskz_and_epi32(__mmask16 __k, __m512i __a, __m512i __b)
650{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000651 return (__m512i) _mm512_mask_and_epi32(_mm512_setzero_si512 (),
Asaf Badouh13633282016-07-05 12:24:14 +0000652 __k, __a, __b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000653}
654
Craig Topper74c10e32018-07-09 19:00:16 +0000655static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000656_mm512_and_epi64(__m512i __a, __m512i __b)
657{
Craig Topper6a77b622016-06-04 05:43:41 +0000658 return (__m512i)((__v8du)__a & (__v8du)__b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000659}
660
Craig Topper74c10e32018-07-09 19:00:16 +0000661static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000662_mm512_mask_and_epi64(__m512i __src, __mmask8 __k, __m512i __a, __m512i __b)
663{
Asaf Badouh13633282016-07-05 12:24:14 +0000664 return (__m512i) __builtin_ia32_selectq_512 ((__mmask8) __k,
665 (__v8di) _mm512_and_epi64(__a, __b),
666 (__v8di) __src);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000667}
Asaf Badouh13633282016-07-05 12:24:14 +0000668
Craig Topper74c10e32018-07-09 19:00:16 +0000669static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000670_mm512_maskz_and_epi64(__mmask8 __k, __m512i __a, __m512i __b)
671{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000672 return (__m512i) _mm512_mask_and_epi64(_mm512_setzero_si512 (),
Asaf Badouh13633282016-07-05 12:24:14 +0000673 __k, __a, __b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000674}
675
Craig Topper74c10e32018-07-09 19:00:16 +0000676static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman2cacc352016-05-18 15:25:53 +0000677_mm512_andnot_si512 (__m512i __A, __m512i __B)
678{
Craig Toppercbf39292018-05-31 01:24:40 +0000679 return (__m512i)(~(__v8du)__A & (__v8du)__B);
Michael Zuckerman2cacc352016-05-18 15:25:53 +0000680}
681
Craig Topper74c10e32018-07-09 19:00:16 +0000682static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000683_mm512_andnot_epi32 (__m512i __A, __m512i __B)
684{
Craig Toppercbf39292018-05-31 01:24:40 +0000685 return (__m512i)(~(__v16su)__A & (__v16su)__B);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000686}
687
Craig Topper74c10e32018-07-09 19:00:16 +0000688static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper4d61a3c2016-07-11 06:14:18 +0000689_mm512_mask_andnot_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000690{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000691 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
692 (__v16si)_mm512_andnot_epi32(__A, __B),
693 (__v16si)__W);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000694}
695
Craig Topper74c10e32018-07-09 19:00:16 +0000696static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper4d61a3c2016-07-11 06:14:18 +0000697_mm512_maskz_andnot_epi32(__mmask16 __U, __m512i __A, __m512i __B)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000698{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000699 return (__m512i)_mm512_mask_andnot_epi32(_mm512_setzero_si512(),
700 __U, __A, __B);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000701}
702
Craig Topper74c10e32018-07-09 19:00:16 +0000703static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper4d61a3c2016-07-11 06:14:18 +0000704_mm512_andnot_epi64(__m512i __A, __m512i __B)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000705{
Craig Toppercbf39292018-05-31 01:24:40 +0000706 return (__m512i)(~(__v8du)__A & (__v8du)__B);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000707}
708
Craig Topper74c10e32018-07-09 19:00:16 +0000709static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper4d61a3c2016-07-11 06:14:18 +0000710_mm512_mask_andnot_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000711{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000712 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
713 (__v8di)_mm512_andnot_epi64(__A, __B),
714 (__v8di)__W);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000715}
716
Craig Topper74c10e32018-07-09 19:00:16 +0000717static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper4d61a3c2016-07-11 06:14:18 +0000718_mm512_maskz_andnot_epi64(__mmask8 __U, __m512i __A, __m512i __B)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000719{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000720 return (__m512i)_mm512_mask_andnot_epi64(_mm512_setzero_si512(),
721 __U, __A, __B);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000722}
Craig Topper4d61a3c2016-07-11 06:14:18 +0000723
Craig Topper74c10e32018-07-09 19:00:16 +0000724static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000725_mm512_or_epi32(__m512i __a, __m512i __b)
726{
Craig Topper6a77b622016-06-04 05:43:41 +0000727 return (__m512i)((__v16su)__a | (__v16su)__b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000728}
729
Craig Topper74c10e32018-07-09 19:00:16 +0000730static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000731_mm512_mask_or_epi32(__m512i __src, __mmask16 __k, __m512i __a, __m512i __b)
732{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000733 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__k,
734 (__v16si)_mm512_or_epi32(__a, __b),
735 (__v16si)__src);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000736}
Craig Topper4d61a3c2016-07-11 06:14:18 +0000737
Craig Topper74c10e32018-07-09 19:00:16 +0000738static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000739_mm512_maskz_or_epi32(__mmask16 __k, __m512i __a, __m512i __b)
740{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000741 return (__m512i)_mm512_mask_or_epi32(_mm512_setzero_si512(), __k, __a, __b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000742}
743
Craig Topper74c10e32018-07-09 19:00:16 +0000744static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000745_mm512_or_epi64(__m512i __a, __m512i __b)
746{
Craig Topper6a77b622016-06-04 05:43:41 +0000747 return (__m512i)((__v8du)__a | (__v8du)__b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000748}
749
Craig Topper74c10e32018-07-09 19:00:16 +0000750static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000751_mm512_mask_or_epi64(__m512i __src, __mmask8 __k, __m512i __a, __m512i __b)
752{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000753 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__k,
754 (__v8di)_mm512_or_epi64(__a, __b),
755 (__v8di)__src);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000756}
Craig Topper4d61a3c2016-07-11 06:14:18 +0000757
Craig Topper74c10e32018-07-09 19:00:16 +0000758static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000759_mm512_maskz_or_epi64(__mmask8 __k, __m512i __a, __m512i __b)
760{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000761 return (__m512i)_mm512_mask_or_epi64(_mm512_setzero_si512(), __k, __a, __b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000762}
763
Craig Topper74c10e32018-07-09 19:00:16 +0000764static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000765_mm512_xor_epi32(__m512i __a, __m512i __b)
766{
Craig Topper6a77b622016-06-04 05:43:41 +0000767 return (__m512i)((__v16su)__a ^ (__v16su)__b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000768}
769
Craig Topper74c10e32018-07-09 19:00:16 +0000770static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000771_mm512_mask_xor_epi32(__m512i __src, __mmask16 __k, __m512i __a, __m512i __b)
772{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000773 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__k,
774 (__v16si)_mm512_xor_epi32(__a, __b),
775 (__v16si)__src);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000776}
Craig Topper4d61a3c2016-07-11 06:14:18 +0000777
Craig Topper74c10e32018-07-09 19:00:16 +0000778static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000779_mm512_maskz_xor_epi32(__mmask16 __k, __m512i __a, __m512i __b)
780{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000781 return (__m512i)_mm512_mask_xor_epi32(_mm512_setzero_si512(), __k, __a, __b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000782}
783
Craig Topper74c10e32018-07-09 19:00:16 +0000784static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000785_mm512_xor_epi64(__m512i __a, __m512i __b)
786{
Craig Topper6a77b622016-06-04 05:43:41 +0000787 return (__m512i)((__v8du)__a ^ (__v8du)__b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000788}
789
Craig Topper74c10e32018-07-09 19:00:16 +0000790static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000791_mm512_mask_xor_epi64(__m512i __src, __mmask8 __k, __m512i __a, __m512i __b)
792{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000793 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__k,
794 (__v8di)_mm512_xor_epi64(__a, __b),
795 (__v8di)__src);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000796}
Craig Topper4d61a3c2016-07-11 06:14:18 +0000797
Craig Topper74c10e32018-07-09 19:00:16 +0000798static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000799_mm512_maskz_xor_epi64(__mmask8 __k, __m512i __a, __m512i __b)
800{
Craig Topper4d61a3c2016-07-11 06:14:18 +0000801 return (__m512i)_mm512_mask_xor_epi64(_mm512_setzero_si512(), __k, __a, __b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000802}
803
Craig Topper74c10e32018-07-09 19:00:16 +0000804static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000805_mm512_and_si512(__m512i __a, __m512i __b)
806{
Craig Topper6a77b622016-06-04 05:43:41 +0000807 return (__m512i)((__v8du)__a & (__v8du)__b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000808}
809
Craig Topper74c10e32018-07-09 19:00:16 +0000810static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000811_mm512_or_si512(__m512i __a, __m512i __b)
812{
Craig Topper6a77b622016-06-04 05:43:41 +0000813 return (__m512i)((__v8du)__a | (__v8du)__b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000814}
815
Craig Topper74c10e32018-07-09 19:00:16 +0000816static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000817_mm512_xor_si512(__m512i __a, __m512i __b)
818{
Craig Topper6a77b622016-06-04 05:43:41 +0000819 return (__m512i)((__v8du)__a ^ (__v8du)__b);
Elena Demikhovsky29da2fb2015-04-01 06:54:16 +0000820}
Craig Topper4d61a3c2016-07-11 06:14:18 +0000821
Adam Nemet0d5bb552014-07-28 17:14:40 +0000822/* Arithmetic */
823
Craig Topper74c10e32018-07-09 19:00:16 +0000824static __inline __m512d __DEFAULT_FN_ATTRS512
Adam Nemeta3ebe622014-07-28 17:14:42 +0000825_mm512_add_pd(__m512d __a, __m512d __b)
826{
Craig Topper9c6c85f2016-05-16 06:38:36 +0000827 return (__m512d)((__v8df)__a + (__v8df)__b);
Adam Nemeta3ebe622014-07-28 17:14:42 +0000828}
829
Craig Topper74c10e32018-07-09 19:00:16 +0000830static __inline __m512 __DEFAULT_FN_ATTRS512
Adam Nemeta3ebe622014-07-28 17:14:42 +0000831_mm512_add_ps(__m512 __a, __m512 __b)
832{
Craig Topper9c6c85f2016-05-16 06:38:36 +0000833 return (__m512)((__v16sf)__a + (__v16sf)__b);
Adam Nemeta3ebe622014-07-28 17:14:42 +0000834}
835
Craig Topper74c10e32018-07-09 19:00:16 +0000836static __inline __m512d __DEFAULT_FN_ATTRS512
Adam Nemeta3ebe622014-07-28 17:14:42 +0000837_mm512_mul_pd(__m512d __a, __m512d __b)
838{
Craig Topper9c6c85f2016-05-16 06:38:36 +0000839 return (__m512d)((__v8df)__a * (__v8df)__b);
Adam Nemeta3ebe622014-07-28 17:14:42 +0000840}
841
Craig Topper74c10e32018-07-09 19:00:16 +0000842static __inline __m512 __DEFAULT_FN_ATTRS512
Adam Nemeta3ebe622014-07-28 17:14:42 +0000843_mm512_mul_ps(__m512 __a, __m512 __b)
844{
Craig Topper9c6c85f2016-05-16 06:38:36 +0000845 return (__m512)((__v16sf)__a * (__v16sf)__b);
Adam Nemeta3ebe622014-07-28 17:14:42 +0000846}
847
Craig Topper74c10e32018-07-09 19:00:16 +0000848static __inline __m512d __DEFAULT_FN_ATTRS512
Adam Nemeta3ebe622014-07-28 17:14:42 +0000849_mm512_sub_pd(__m512d __a, __m512d __b)
850{
Craig Topper9c6c85f2016-05-16 06:38:36 +0000851 return (__m512d)((__v8df)__a - (__v8df)__b);
Adam Nemeta3ebe622014-07-28 17:14:42 +0000852}
853
Craig Topper74c10e32018-07-09 19:00:16 +0000854static __inline __m512 __DEFAULT_FN_ATTRS512
Adam Nemeta3ebe622014-07-28 17:14:42 +0000855_mm512_sub_ps(__m512 __a, __m512 __b)
856{
Craig Topper9c6c85f2016-05-16 06:38:36 +0000857 return (__m512)((__v16sf)__a - (__v16sf)__b);
Adam Nemeta3ebe622014-07-28 17:14:42 +0000858}
859
Craig Topper74c10e32018-07-09 19:00:16 +0000860static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000861_mm512_add_epi64 (__m512i __A, __m512i __B)
862{
Craig Topper6a77b622016-06-04 05:43:41 +0000863 return (__m512i) ((__v8du) __A + (__v8du) __B);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000864}
865
Craig Topper74c10e32018-07-09 19:00:16 +0000866static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0e189762016-09-03 18:29:35 +0000867_mm512_mask_add_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000868{
Craig Topper0e189762016-09-03 18:29:35 +0000869 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
870 (__v8di)_mm512_add_epi64(__A, __B),
871 (__v8di)__W);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000872}
873
Craig Topper74c10e32018-07-09 19:00:16 +0000874static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0e189762016-09-03 18:29:35 +0000875_mm512_maskz_add_epi64(__mmask8 __U, __m512i __A, __m512i __B)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000876{
Craig Topper0e189762016-09-03 18:29:35 +0000877 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
878 (__v8di)_mm512_add_epi64(__A, __B),
879 (__v8di)_mm512_setzero_si512());
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000880}
881
Craig Topper74c10e32018-07-09 19:00:16 +0000882static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000883_mm512_sub_epi64 (__m512i __A, __m512i __B)
884{
Craig Topper6a77b622016-06-04 05:43:41 +0000885 return (__m512i) ((__v8du) __A - (__v8du) __B);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000886}
887
Craig Topper74c10e32018-07-09 19:00:16 +0000888static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0e189762016-09-03 18:29:35 +0000889_mm512_mask_sub_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000890{
Craig Topper0e189762016-09-03 18:29:35 +0000891 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
892 (__v8di)_mm512_sub_epi64(__A, __B),
893 (__v8di)__W);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000894}
895
Craig Topper74c10e32018-07-09 19:00:16 +0000896static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0e189762016-09-03 18:29:35 +0000897_mm512_maskz_sub_epi64(__mmask8 __U, __m512i __A, __m512i __B)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000898{
Craig Topper0e189762016-09-03 18:29:35 +0000899 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
900 (__v8di)_mm512_sub_epi64(__A, __B),
901 (__v8di)_mm512_setzero_si512());
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000902}
903
Craig Topper74c10e32018-07-09 19:00:16 +0000904static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000905_mm512_add_epi32 (__m512i __A, __m512i __B)
906{
Craig Topper6a77b622016-06-04 05:43:41 +0000907 return (__m512i) ((__v16su) __A + (__v16su) __B);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000908}
909
Craig Topper74c10e32018-07-09 19:00:16 +0000910static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0e189762016-09-03 18:29:35 +0000911_mm512_mask_add_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000912{
Craig Topper0e189762016-09-03 18:29:35 +0000913 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
914 (__v16si)_mm512_add_epi32(__A, __B),
915 (__v16si)__W);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000916}
917
Craig Topper74c10e32018-07-09 19:00:16 +0000918static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000919_mm512_maskz_add_epi32 (__mmask16 __U, __m512i __A, __m512i __B)
920{
Craig Topper0e189762016-09-03 18:29:35 +0000921 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
922 (__v16si)_mm512_add_epi32(__A, __B),
923 (__v16si)_mm512_setzero_si512());
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000924}
925
Craig Topper74c10e32018-07-09 19:00:16 +0000926static __inline__ __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000927_mm512_sub_epi32 (__m512i __A, __m512i __B)
928{
Craig Topper6a77b622016-06-04 05:43:41 +0000929 return (__m512i) ((__v16su) __A - (__v16su) __B);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000930}
931
Craig Topper74c10e32018-07-09 19:00:16 +0000932static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0e189762016-09-03 18:29:35 +0000933_mm512_mask_sub_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000934{
Craig Topper0e189762016-09-03 18:29:35 +0000935 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
936 (__v16si)_mm512_sub_epi32(__A, __B),
937 (__v16si)__W);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000938}
939
Craig Topper74c10e32018-07-09 19:00:16 +0000940static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0e189762016-09-03 18:29:35 +0000941_mm512_maskz_sub_epi32(__mmask16 __U, __m512i __A, __m512i __B)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000942{
Craig Topper0e189762016-09-03 18:29:35 +0000943 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
944 (__v16si)_mm512_sub_epi32(__A, __B),
945 (__v16si)_mm512_setzero_si512());
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +0000946}
947
Craig Topper2da60bc2018-06-21 05:01:01 +0000948#define _mm512_max_round_pd(A, B, R) \
949 (__m512d)__builtin_ia32_maxpd512((__v8df)(__m512d)(A), \
950 (__v8df)(__m512d)(B), (int)(R))
951
Craig Topperc6338672018-05-31 00:51:20 +0000952#define _mm512_mask_max_round_pd(W, U, A, B, R) \
Craig Topper2da60bc2018-06-21 05:01:01 +0000953 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
954 (__v8df)_mm512_max_round_pd((A), (B), (R)), \
955 (__v8df)(W))
Michael Zuckermane6aa66a2016-06-01 08:34:03 +0000956
Craig Topperc6338672018-05-31 00:51:20 +0000957#define _mm512_maskz_max_round_pd(U, A, B, R) \
Craig Topper2da60bc2018-06-21 05:01:01 +0000958 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
959 (__v8df)_mm512_max_round_pd((A), (B), (R)), \
960 (__v8df)_mm512_setzero_pd())
Michael Zuckermane6aa66a2016-06-01 08:34:03 +0000961
Craig Topper74c10e32018-07-09 19:00:16 +0000962static __inline__ __m512d __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +0000963_mm512_max_pd(__m512d __A, __m512d __B)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +0000964{
Craig Topper2da60bc2018-06-21 05:01:01 +0000965 return (__m512d) __builtin_ia32_maxpd512((__v8df) __A, (__v8df) __B,
966 _MM_FROUND_CUR_DIRECTION);
Adam Nemet0d5bb552014-07-28 17:14:40 +0000967}
968
Craig Topper74c10e32018-07-09 19:00:16 +0000969static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +0000970_mm512_mask_max_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B)
971{
Craig Topper2da60bc2018-06-21 05:01:01 +0000972 return (__m512d)__builtin_ia32_selectpd_512(__U,
973 (__v8df)_mm512_max_pd(__A, __B),
974 (__v8df)__W);
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +0000975}
976
Craig Topper74c10e32018-07-09 19:00:16 +0000977static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +0000978_mm512_maskz_max_pd (__mmask8 __U, __m512d __A, __m512d __B)
979{
Craig Topper2da60bc2018-06-21 05:01:01 +0000980 return (__m512d)__builtin_ia32_selectpd_512(__U,
981 (__v8df)_mm512_max_pd(__A, __B),
982 (__v8df)_mm512_setzero_pd());
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +0000983}
984
Craig Topper2da60bc2018-06-21 05:01:01 +0000985#define _mm512_max_round_ps(A, B, R) \
986 (__m512)__builtin_ia32_maxps512((__v16sf)(__m512)(A), \
987 (__v16sf)(__m512)(B), (int)(R))
988
Craig Topperc6338672018-05-31 00:51:20 +0000989#define _mm512_mask_max_round_ps(W, U, A, B, R) \
Craig Topper2da60bc2018-06-21 05:01:01 +0000990 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
991 (__v16sf)_mm512_max_round_ps((A), (B), (R)), \
992 (__v16sf)(W))
Michael Zuckermane6aa66a2016-06-01 08:34:03 +0000993
Craig Topperc6338672018-05-31 00:51:20 +0000994#define _mm512_maskz_max_round_ps(U, A, B, R) \
Craig Topper2da60bc2018-06-21 05:01:01 +0000995 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
996 (__v16sf)_mm512_max_round_ps((A), (B), (R)), \
997 (__v16sf)_mm512_setzero_ps())
Michael Zuckermane6aa66a2016-06-01 08:34:03 +0000998
Craig Topper74c10e32018-07-09 19:00:16 +0000999static __inline__ __m512 __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001000_mm512_max_ps(__m512 __A, __m512 __B)
1001{
Craig Topper2da60bc2018-06-21 05:01:01 +00001002 return (__m512) __builtin_ia32_maxps512((__v16sf) __A, (__v16sf) __B,
1003 _MM_FROUND_CUR_DIRECTION);
Adam Nemet0d5bb552014-07-28 17:14:40 +00001004}
1005
Craig Topper74c10e32018-07-09 19:00:16 +00001006static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +00001007_mm512_mask_max_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B)
1008{
Craig Topper2da60bc2018-06-21 05:01:01 +00001009 return (__m512)__builtin_ia32_selectps_512(__U,
1010 (__v16sf)_mm512_max_ps(__A, __B),
1011 (__v16sf)__W);
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +00001012}
1013
Craig Topper74c10e32018-07-09 19:00:16 +00001014static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +00001015_mm512_maskz_max_ps (__mmask16 __U, __m512 __A, __m512 __B)
1016{
Craig Topper2da60bc2018-06-21 05:01:01 +00001017 return (__m512)__builtin_ia32_selectps_512(__U,
1018 (__v16sf)_mm512_max_ps(__A, __B),
1019 (__v16sf)_mm512_setzero_ps());
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +00001020}
1021
Craig Topper74c10e32018-07-09 19:00:16 +00001022static __inline__ __m128 __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001023_mm_mask_max_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
Igor Breger9c2a0bf2016-02-08 12:36:48 +00001024 return (__m128) __builtin_ia32_maxss_round_mask ((__v4sf) __A,
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001025 (__v4sf) __B,
1026 (__v4sf) __W,
1027 (__mmask8) __U,
1028 _MM_FROUND_CUR_DIRECTION);
1029}
1030
Craig Topper74c10e32018-07-09 19:00:16 +00001031static __inline__ __m128 __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001032_mm_maskz_max_ss(__mmask8 __U,__m128 __A, __m128 __B) {
Igor Breger9c2a0bf2016-02-08 12:36:48 +00001033 return (__m128) __builtin_ia32_maxss_round_mask ((__v4sf) __A,
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001034 (__v4sf) __B,
1035 (__v4sf) _mm_setzero_ps (),
1036 (__mmask8) __U,
1037 _MM_FROUND_CUR_DIRECTION);
1038}
1039
Craig Topperc6338672018-05-31 00:51:20 +00001040#define _mm_max_round_ss(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001041 (__m128)__builtin_ia32_maxss_round_mask((__v4sf)(__m128)(A), \
1042 (__v4sf)(__m128)(B), \
1043 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00001044 (__mmask8)-1, (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001045
Craig Topperc6338672018-05-31 00:51:20 +00001046#define _mm_mask_max_round_ss(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001047 (__m128)__builtin_ia32_maxss_round_mask((__v4sf)(__m128)(A), \
1048 (__v4sf)(__m128)(B), \
1049 (__v4sf)(__m128)(W), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00001050 (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001051
Craig Topperc6338672018-05-31 00:51:20 +00001052#define _mm_maskz_max_round_ss(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001053 (__m128)__builtin_ia32_maxss_round_mask((__v4sf)(__m128)(A), \
1054 (__v4sf)(__m128)(B), \
1055 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00001056 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001057
Craig Topper74c10e32018-07-09 19:00:16 +00001058static __inline__ __m128d __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001059_mm_mask_max_sd(__m128d __W, __mmask8 __U,__m128d __A, __m128d __B) {
Igor Breger9c2a0bf2016-02-08 12:36:48 +00001060 return (__m128d) __builtin_ia32_maxsd_round_mask ((__v2df) __A,
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001061 (__v2df) __B,
1062 (__v2df) __W,
1063 (__mmask8) __U,
1064 _MM_FROUND_CUR_DIRECTION);
1065}
1066
Craig Topper74c10e32018-07-09 19:00:16 +00001067static __inline__ __m128d __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001068_mm_maskz_max_sd(__mmask8 __U,__m128d __A, __m128d __B) {
Igor Breger9c2a0bf2016-02-08 12:36:48 +00001069 return (__m128d) __builtin_ia32_maxsd_round_mask ((__v2df) __A,
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001070 (__v2df) __B,
1071 (__v2df) _mm_setzero_pd (),
1072 (__mmask8) __U,
1073 _MM_FROUND_CUR_DIRECTION);
1074}
1075
Craig Topperc6338672018-05-31 00:51:20 +00001076#define _mm_max_round_sd(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001077 (__m128d)__builtin_ia32_maxsd_round_mask((__v2df)(__m128d)(A), \
1078 (__v2df)(__m128d)(B), \
1079 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00001080 (__mmask8)-1, (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001081
Craig Topperc6338672018-05-31 00:51:20 +00001082#define _mm_mask_max_round_sd(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001083 (__m128d)__builtin_ia32_maxsd_round_mask((__v2df)(__m128d)(A), \
1084 (__v2df)(__m128d)(B), \
1085 (__v2df)(__m128d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00001086 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001087
Craig Topperc6338672018-05-31 00:51:20 +00001088#define _mm_maskz_max_round_sd(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001089 (__m128d)__builtin_ia32_maxsd_round_mask((__v2df)(__m128d)(A), \
1090 (__v2df)(__m128d)(B), \
1091 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00001092 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001093
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00001094static __inline __m512i
Craig Topper74c10e32018-07-09 19:00:16 +00001095__DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001096_mm512_max_epi32(__m512i __A, __m512i __B)
1097{
Craig Topperf2043b02018-05-23 04:51:54 +00001098 return (__m512i)__builtin_ia32_pmaxsd512((__v16si)__A, (__v16si)__B);
Adam Nemet0d5bb552014-07-28 17:14:40 +00001099}
1100
Craig Topper74c10e32018-07-09 19:00:16 +00001101static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001102_mm512_mask_max_epi32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B)
1103{
Craig Topperf2043b02018-05-23 04:51:54 +00001104 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1105 (__v16si)_mm512_max_epi32(__A, __B),
1106 (__v16si)__W);
Michael Zuckermande860e52016-05-10 11:34:19 +00001107}
1108
Craig Topper74c10e32018-07-09 19:00:16 +00001109static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001110_mm512_maskz_max_epi32 (__mmask16 __M, __m512i __A, __m512i __B)
1111{
Craig Topperf2043b02018-05-23 04:51:54 +00001112 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1113 (__v16si)_mm512_max_epi32(__A, __B),
1114 (__v16si)_mm512_setzero_si512());
Michael Zuckermande860e52016-05-10 11:34:19 +00001115}
1116
Craig Topper74c10e32018-07-09 19:00:16 +00001117static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001118_mm512_max_epu32(__m512i __A, __m512i __B)
1119{
Craig Topperf2043b02018-05-23 04:51:54 +00001120 return (__m512i)__builtin_ia32_pmaxud512((__v16si)__A, (__v16si)__B);
Adam Nemet0d5bb552014-07-28 17:14:40 +00001121}
1122
Craig Topper74c10e32018-07-09 19:00:16 +00001123static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001124_mm512_mask_max_epu32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B)
1125{
Craig Topperf2043b02018-05-23 04:51:54 +00001126 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1127 (__v16si)_mm512_max_epu32(__A, __B),
1128 (__v16si)__W);
Michael Zuckermande860e52016-05-10 11:34:19 +00001129}
1130
Craig Topper74c10e32018-07-09 19:00:16 +00001131static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001132_mm512_maskz_max_epu32 (__mmask16 __M, __m512i __A, __m512i __B)
1133{
Craig Topperf2043b02018-05-23 04:51:54 +00001134 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1135 (__v16si)_mm512_max_epu32(__A, __B),
1136 (__v16si)_mm512_setzero_si512());
Michael Zuckermande860e52016-05-10 11:34:19 +00001137}
1138
Craig Topper74c10e32018-07-09 19:00:16 +00001139static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001140_mm512_max_epi64(__m512i __A, __m512i __B)
1141{
Craig Topperf2043b02018-05-23 04:51:54 +00001142 return (__m512i)__builtin_ia32_pmaxsq512((__v8di)__A, (__v8di)__B);
Adam Nemet0d5bb552014-07-28 17:14:40 +00001143}
1144
Craig Topper74c10e32018-07-09 19:00:16 +00001145static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001146_mm512_mask_max_epi64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B)
1147{
Craig Topperf2043b02018-05-23 04:51:54 +00001148 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1149 (__v8di)_mm512_max_epi64(__A, __B),
1150 (__v8di)__W);
Michael Zuckermande860e52016-05-10 11:34:19 +00001151}
1152
Craig Topper74c10e32018-07-09 19:00:16 +00001153static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001154_mm512_maskz_max_epi64 (__mmask8 __M, __m512i __A, __m512i __B)
1155{
Craig Topperf2043b02018-05-23 04:51:54 +00001156 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1157 (__v8di)_mm512_max_epi64(__A, __B),
1158 (__v8di)_mm512_setzero_si512());
Michael Zuckermande860e52016-05-10 11:34:19 +00001159}
1160
Craig Topper74c10e32018-07-09 19:00:16 +00001161static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001162_mm512_max_epu64(__m512i __A, __m512i __B)
1163{
Craig Topperf2043b02018-05-23 04:51:54 +00001164 return (__m512i)__builtin_ia32_pmaxuq512((__v8di)__A, (__v8di)__B);
Adam Nemet0d5bb552014-07-28 17:14:40 +00001165}
1166
Craig Topper74c10e32018-07-09 19:00:16 +00001167static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001168_mm512_mask_max_epu64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B)
1169{
Craig Topperf2043b02018-05-23 04:51:54 +00001170 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1171 (__v8di)_mm512_max_epu64(__A, __B),
1172 (__v8di)__W);
Michael Zuckermande860e52016-05-10 11:34:19 +00001173}
1174
Craig Topper74c10e32018-07-09 19:00:16 +00001175static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001176_mm512_maskz_max_epu64 (__mmask8 __M, __m512i __A, __m512i __B)
1177{
Craig Topperf2043b02018-05-23 04:51:54 +00001178 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1179 (__v8di)_mm512_max_epu64(__A, __B),
1180 (__v8di)_mm512_setzero_si512());
Michael Zuckermande860e52016-05-10 11:34:19 +00001181}
1182
Craig Topper2da60bc2018-06-21 05:01:01 +00001183#define _mm512_min_round_pd(A, B, R) \
1184 (__m512d)__builtin_ia32_minpd512((__v8df)(__m512d)(A), \
1185 (__v8df)(__m512d)(B), (int)(R))
1186
Craig Topperc6338672018-05-31 00:51:20 +00001187#define _mm512_mask_min_round_pd(W, U, A, B, R) \
Craig Topper2da60bc2018-06-21 05:01:01 +00001188 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
1189 (__v8df)_mm512_min_round_pd((A), (B), (R)), \
1190 (__v8df)(W))
Michael Zuckermane6aa66a2016-06-01 08:34:03 +00001191
Craig Topperc6338672018-05-31 00:51:20 +00001192#define _mm512_maskz_min_round_pd(U, A, B, R) \
Craig Topper2da60bc2018-06-21 05:01:01 +00001193 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
1194 (__v8df)_mm512_min_round_pd((A), (B), (R)), \
1195 (__v8df)_mm512_setzero_pd())
Michael Zuckermane6aa66a2016-06-01 08:34:03 +00001196
Craig Topper74c10e32018-07-09 19:00:16 +00001197static __inline__ __m512d __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001198_mm512_min_pd(__m512d __A, __m512d __B)
1199{
Craig Topper2da60bc2018-06-21 05:01:01 +00001200 return (__m512d) __builtin_ia32_minpd512((__v8df) __A, (__v8df) __B,
1201 _MM_FROUND_CUR_DIRECTION);
Adam Nemet0d5bb552014-07-28 17:14:40 +00001202}
1203
Craig Topper74c10e32018-07-09 19:00:16 +00001204static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +00001205_mm512_mask_min_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B)
1206{
Craig Topper2da60bc2018-06-21 05:01:01 +00001207 return (__m512d)__builtin_ia32_selectpd_512(__U,
1208 (__v8df)_mm512_min_pd(__A, __B),
1209 (__v8df)__W);
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +00001210}
1211
Craig Topper74c10e32018-07-09 19:00:16 +00001212static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +00001213_mm512_maskz_min_pd (__mmask8 __U, __m512d __A, __m512d __B)
1214{
Craig Topper2da60bc2018-06-21 05:01:01 +00001215 return (__m512d)__builtin_ia32_selectpd_512(__U,
1216 (__v8df)_mm512_min_pd(__A, __B),
1217 (__v8df)_mm512_setzero_pd());
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +00001218}
1219
Craig Topper2da60bc2018-06-21 05:01:01 +00001220#define _mm512_min_round_ps(A, B, R) \
1221 (__m512)__builtin_ia32_minps512((__v16sf)(__m512)(A), \
1222 (__v16sf)(__m512)(B), (int)(R))
1223
1224#define _mm512_mask_min_round_ps(W, U, A, B, R) \
1225 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
1226 (__v16sf)_mm512_min_round_ps((A), (B), (R)), \
1227 (__v16sf)(W))
1228
1229#define _mm512_maskz_min_round_ps(U, A, B, R) \
1230 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
1231 (__v16sf)_mm512_min_round_ps((A), (B), (R)), \
1232 (__v16sf)_mm512_setzero_ps())
1233
Craig Topper74c10e32018-07-09 19:00:16 +00001234static __inline__ __m512 __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001235_mm512_min_ps(__m512 __A, __m512 __B)
1236{
Craig Topper2da60bc2018-06-21 05:01:01 +00001237 return (__m512) __builtin_ia32_minps512((__v16sf) __A, (__v16sf) __B,
1238 _MM_FROUND_CUR_DIRECTION);
Adam Nemet0d5bb552014-07-28 17:14:40 +00001239}
1240
Craig Topper74c10e32018-07-09 19:00:16 +00001241static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +00001242_mm512_mask_min_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B)
1243{
Craig Topper2da60bc2018-06-21 05:01:01 +00001244 return (__m512)__builtin_ia32_selectps_512(__U,
1245 (__v16sf)_mm512_min_ps(__A, __B),
1246 (__v16sf)__W);
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +00001247}
1248
Craig Topper74c10e32018-07-09 19:00:16 +00001249static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +00001250_mm512_maskz_min_ps (__mmask16 __U, __m512 __A, __m512 __B)
1251{
Craig Topper2da60bc2018-06-21 05:01:01 +00001252 return (__m512)__builtin_ia32_selectps_512(__U,
1253 (__v16sf)_mm512_min_ps(__A, __B),
1254 (__v16sf)_mm512_setzero_ps());
Michael Zuckermanf9be3bb2016-05-09 12:38:49 +00001255}
1256
Craig Topper74c10e32018-07-09 19:00:16 +00001257static __inline__ __m128 __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001258_mm_mask_min_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
Igor Breger9c2a0bf2016-02-08 12:36:48 +00001259 return (__m128) __builtin_ia32_minss_round_mask ((__v4sf) __A,
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001260 (__v4sf) __B,
1261 (__v4sf) __W,
1262 (__mmask8) __U,
1263 _MM_FROUND_CUR_DIRECTION);
1264}
1265
Craig Topper74c10e32018-07-09 19:00:16 +00001266static __inline__ __m128 __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001267_mm_maskz_min_ss(__mmask8 __U,__m128 __A, __m128 __B) {
Igor Breger9c2a0bf2016-02-08 12:36:48 +00001268 return (__m128) __builtin_ia32_minss_round_mask ((__v4sf) __A,
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001269 (__v4sf) __B,
1270 (__v4sf) _mm_setzero_ps (),
1271 (__mmask8) __U,
1272 _MM_FROUND_CUR_DIRECTION);
1273}
1274
Craig Topperc6338672018-05-31 00:51:20 +00001275#define _mm_min_round_ss(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001276 (__m128)__builtin_ia32_minss_round_mask((__v4sf)(__m128)(A), \
1277 (__v4sf)(__m128)(B), \
1278 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00001279 (__mmask8)-1, (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001280
Craig Topperc6338672018-05-31 00:51:20 +00001281#define _mm_mask_min_round_ss(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001282 (__m128)__builtin_ia32_minss_round_mask((__v4sf)(__m128)(A), \
1283 (__v4sf)(__m128)(B), \
1284 (__v4sf)(__m128)(W), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00001285 (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001286
Craig Topperc6338672018-05-31 00:51:20 +00001287#define _mm_maskz_min_round_ss(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001288 (__m128)__builtin_ia32_minss_round_mask((__v4sf)(__m128)(A), \
1289 (__v4sf)(__m128)(B), \
1290 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00001291 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001292
Craig Topper74c10e32018-07-09 19:00:16 +00001293static __inline__ __m128d __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001294_mm_mask_min_sd(__m128d __W, __mmask8 __U,__m128d __A, __m128d __B) {
Igor Breger9c2a0bf2016-02-08 12:36:48 +00001295 return (__m128d) __builtin_ia32_minsd_round_mask ((__v2df) __A,
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001296 (__v2df) __B,
1297 (__v2df) __W,
1298 (__mmask8) __U,
1299 _MM_FROUND_CUR_DIRECTION);
1300}
1301
Craig Topper74c10e32018-07-09 19:00:16 +00001302static __inline__ __m128d __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001303_mm_maskz_min_sd(__mmask8 __U,__m128d __A, __m128d __B) {
Igor Breger9c2a0bf2016-02-08 12:36:48 +00001304 return (__m128d) __builtin_ia32_minsd_round_mask ((__v2df) __A,
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001305 (__v2df) __B,
1306 (__v2df) _mm_setzero_pd (),
1307 (__mmask8) __U,
1308 _MM_FROUND_CUR_DIRECTION);
1309}
1310
Craig Topperc6338672018-05-31 00:51:20 +00001311#define _mm_min_round_sd(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001312 (__m128d)__builtin_ia32_minsd_round_mask((__v2df)(__m128d)(A), \
1313 (__v2df)(__m128d)(B), \
1314 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00001315 (__mmask8)-1, (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001316
Craig Topperc6338672018-05-31 00:51:20 +00001317#define _mm_mask_min_round_sd(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001318 (__m128d)__builtin_ia32_minsd_round_mask((__v2df)(__m128d)(A), \
1319 (__v2df)(__m128d)(B), \
1320 (__v2df)(__m128d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00001321 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001322
Craig Topperc6338672018-05-31 00:51:20 +00001323#define _mm_maskz_min_round_sd(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001324 (__m128d)__builtin_ia32_minsd_round_mask((__v2df)(__m128d)(A), \
1325 (__v2df)(__m128d)(B), \
1326 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00001327 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001328
Adam Nemet0d5bb552014-07-28 17:14:40 +00001329static __inline __m512i
Craig Topper74c10e32018-07-09 19:00:16 +00001330__DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001331_mm512_min_epi32(__m512i __A, __m512i __B)
1332{
Craig Topperf2043b02018-05-23 04:51:54 +00001333 return (__m512i)__builtin_ia32_pminsd512((__v16si)__A, (__v16si)__B);
Adam Nemet0d5bb552014-07-28 17:14:40 +00001334}
1335
Craig Topper74c10e32018-07-09 19:00:16 +00001336static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001337_mm512_mask_min_epi32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B)
1338{
Craig Topperf2043b02018-05-23 04:51:54 +00001339 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1340 (__v16si)_mm512_min_epi32(__A, __B),
1341 (__v16si)__W);
Michael Zuckermande860e52016-05-10 11:34:19 +00001342}
1343
Craig Topper74c10e32018-07-09 19:00:16 +00001344static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001345_mm512_maskz_min_epi32 (__mmask16 __M, __m512i __A, __m512i __B)
1346{
Craig Topperf2043b02018-05-23 04:51:54 +00001347 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1348 (__v16si)_mm512_min_epi32(__A, __B),
1349 (__v16si)_mm512_setzero_si512());
Michael Zuckermande860e52016-05-10 11:34:19 +00001350}
1351
Craig Topper74c10e32018-07-09 19:00:16 +00001352static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001353_mm512_min_epu32(__m512i __A, __m512i __B)
1354{
Craig Topperf2043b02018-05-23 04:51:54 +00001355 return (__m512i)__builtin_ia32_pminud512((__v16si)__A, (__v16si)__B);
Adam Nemet0d5bb552014-07-28 17:14:40 +00001356}
1357
Craig Topper74c10e32018-07-09 19:00:16 +00001358static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001359_mm512_mask_min_epu32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B)
1360{
Craig Topperf2043b02018-05-23 04:51:54 +00001361 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1362 (__v16si)_mm512_min_epu32(__A, __B),
1363 (__v16si)__W);
Michael Zuckermande860e52016-05-10 11:34:19 +00001364}
1365
Craig Topper74c10e32018-07-09 19:00:16 +00001366static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001367_mm512_maskz_min_epu32 (__mmask16 __M, __m512i __A, __m512i __B)
1368{
Craig Topperf2043b02018-05-23 04:51:54 +00001369 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1370 (__v16si)_mm512_min_epu32(__A, __B),
1371 (__v16si)_mm512_setzero_si512());
Michael Zuckermande860e52016-05-10 11:34:19 +00001372}
1373
Craig Topper74c10e32018-07-09 19:00:16 +00001374static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001375_mm512_min_epi64(__m512i __A, __m512i __B)
1376{
Craig Topperf2043b02018-05-23 04:51:54 +00001377 return (__m512i)__builtin_ia32_pminsq512((__v8di)__A, (__v8di)__B);
Adam Nemet0d5bb552014-07-28 17:14:40 +00001378}
1379
Craig Topper74c10e32018-07-09 19:00:16 +00001380static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001381_mm512_mask_min_epi64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B)
1382{
Craig Topperf2043b02018-05-23 04:51:54 +00001383 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1384 (__v8di)_mm512_min_epi64(__A, __B),
1385 (__v8di)__W);
Michael Zuckermande860e52016-05-10 11:34:19 +00001386}
1387
Craig Topper74c10e32018-07-09 19:00:16 +00001388static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001389_mm512_maskz_min_epi64 (__mmask8 __M, __m512i __A, __m512i __B)
1390{
Craig Topperf2043b02018-05-23 04:51:54 +00001391 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1392 (__v8di)_mm512_min_epi64(__A, __B),
1393 (__v8di)_mm512_setzero_si512());
Michael Zuckermande860e52016-05-10 11:34:19 +00001394}
1395
Craig Topper74c10e32018-07-09 19:00:16 +00001396static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001397_mm512_min_epu64(__m512i __A, __m512i __B)
1398{
Craig Topperf2043b02018-05-23 04:51:54 +00001399 return (__m512i)__builtin_ia32_pminuq512((__v8di)__A, (__v8di)__B);
Adam Nemet0d5bb552014-07-28 17:14:40 +00001400}
1401
Craig Topper74c10e32018-07-09 19:00:16 +00001402static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001403_mm512_mask_min_epu64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B)
1404{
Craig Topperf2043b02018-05-23 04:51:54 +00001405 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1406 (__v8di)_mm512_min_epu64(__A, __B),
1407 (__v8di)__W);
Michael Zuckermande860e52016-05-10 11:34:19 +00001408}
1409
Craig Topper74c10e32018-07-09 19:00:16 +00001410static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermande860e52016-05-10 11:34:19 +00001411_mm512_maskz_min_epu64 (__mmask8 __M, __m512i __A, __m512i __B)
1412{
Craig Topperf2043b02018-05-23 04:51:54 +00001413 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1414 (__v8di)_mm512_min_epu64(__A, __B),
1415 (__v8di)_mm512_setzero_si512());
Michael Zuckermande860e52016-05-10 11:34:19 +00001416}
1417
Craig Topper74c10e32018-07-09 19:00:16 +00001418static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001419_mm512_mul_epi32(__m512i __X, __m512i __Y)
1420{
Craig Topper70536f42016-12-27 04:04:57 +00001421 return (__m512i)__builtin_ia32_pmuldq512((__v16si)__X, (__v16si) __Y);
Adam Nemet0d5bb552014-07-28 17:14:40 +00001422}
1423
Craig Topper74c10e32018-07-09 19:00:16 +00001424static __inline __m512i __DEFAULT_FN_ATTRS512
Craig Topper70536f42016-12-27 04:04:57 +00001425_mm512_mask_mul_epi32(__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +00001426{
Craig Topper70536f42016-12-27 04:04:57 +00001427 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1428 (__v8di)_mm512_mul_epi32(__X, __Y),
1429 (__v8di)__W);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +00001430}
1431
Craig Topper74c10e32018-07-09 19:00:16 +00001432static __inline __m512i __DEFAULT_FN_ATTRS512
Craig Topper70536f42016-12-27 04:04:57 +00001433_mm512_maskz_mul_epi32(__mmask8 __M, __m512i __X, __m512i __Y)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +00001434{
Craig Topper70536f42016-12-27 04:04:57 +00001435 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1436 (__v8di)_mm512_mul_epi32(__X, __Y),
1437 (__v8di)_mm512_setzero_si512 ());
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +00001438}
1439
Craig Topper74c10e32018-07-09 19:00:16 +00001440static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001441_mm512_mul_epu32(__m512i __X, __m512i __Y)
1442{
Craig Topper70536f42016-12-27 04:04:57 +00001443 return (__m512i)__builtin_ia32_pmuludq512((__v16si)__X, (__v16si)__Y);
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001444}
1445
Craig Topper74c10e32018-07-09 19:00:16 +00001446static __inline __m512i __DEFAULT_FN_ATTRS512
Craig Topper70536f42016-12-27 04:04:57 +00001447_mm512_mask_mul_epu32(__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +00001448{
Craig Topper70536f42016-12-27 04:04:57 +00001449 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1450 (__v8di)_mm512_mul_epu32(__X, __Y),
1451 (__v8di)__W);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +00001452}
1453
Craig Topper74c10e32018-07-09 19:00:16 +00001454static __inline __m512i __DEFAULT_FN_ATTRS512
Craig Topper70536f42016-12-27 04:04:57 +00001455_mm512_maskz_mul_epu32(__mmask8 __M, __m512i __X, __m512i __Y)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +00001456{
Craig Topper70536f42016-12-27 04:04:57 +00001457 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1458 (__v8di)_mm512_mul_epu32(__X, __Y),
1459 (__v8di)_mm512_setzero_si512 ());
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +00001460}
1461
Craig Topper74c10e32018-07-09 19:00:16 +00001462static __inline __m512i __DEFAULT_FN_ATTRS512
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +00001463_mm512_mullo_epi32 (__m512i __A, __m512i __B)
1464{
Craig Topper6a77b622016-06-04 05:43:41 +00001465 return (__m512i) ((__v16su) __A * (__v16su) __B);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +00001466}
1467
Craig Topper74c10e32018-07-09 19:00:16 +00001468static __inline __m512i __DEFAULT_FN_ATTRS512
Craig Topperf43e4a12016-09-03 19:19:49 +00001469_mm512_maskz_mullo_epi32(__mmask16 __M, __m512i __A, __m512i __B)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +00001470{
Craig Topperf43e4a12016-09-03 19:19:49 +00001471 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1472 (__v16si)_mm512_mullo_epi32(__A, __B),
1473 (__v16si)_mm512_setzero_si512());
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +00001474}
1475
Craig Topper74c10e32018-07-09 19:00:16 +00001476static __inline __m512i __DEFAULT_FN_ATTRS512
Craig Topperf43e4a12016-09-03 19:19:49 +00001477_mm512_mask_mullo_epi32(__m512i __W, __mmask16 __M, __m512i __A, __m512i __B)
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +00001478{
Craig Topperf43e4a12016-09-03 19:19:49 +00001479 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1480 (__v16si)_mm512_mullo_epi32(__A, __B),
1481 (__v16si)__W);
Elena Demikhovsky35dc8c02015-04-28 13:28:01 +00001482}
1483
Craig Topper74c10e32018-07-09 19:00:16 +00001484static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Toppere95bde32018-04-26 05:38:39 +00001485_mm512_mullox_epi64 (__m512i __A, __m512i __B) {
1486 return (__m512i) ((__v8du) __A * (__v8du) __B);
1487}
1488
Craig Topper74c10e32018-07-09 19:00:16 +00001489static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Toppere95bde32018-04-26 05:38:39 +00001490_mm512_mask_mullox_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) {
1491 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
1492 (__v8di)_mm512_mullox_epi64(__A, __B),
1493 (__v8di)__W);
1494}
1495
Craig Topper8bf793f2018-06-29 05:43:33 +00001496#define _mm512_sqrt_round_pd(A, R) \
1497 (__m512d)__builtin_ia32_sqrtpd512((__v8df)(__m512d)(A), (int)(R))
1498
Craig Topperc6338672018-05-31 00:51:20 +00001499#define _mm512_mask_sqrt_round_pd(W, U, A, R) \
Craig Topper8bf793f2018-06-29 05:43:33 +00001500 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
1501 (__v8df)_mm512_sqrt_round_pd((A), (R)), \
1502 (__v8df)(__m512d)(W))
Michael Zuckermane6aa66a2016-06-01 08:34:03 +00001503
Craig Topperc6338672018-05-31 00:51:20 +00001504#define _mm512_maskz_sqrt_round_pd(U, A, R) \
Craig Topper8bf793f2018-06-29 05:43:33 +00001505 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
1506 (__v8df)_mm512_sqrt_round_pd((A), (R)), \
1507 (__v8df)_mm512_setzero_pd())
Michael Zuckermane6aa66a2016-06-01 08:34:03 +00001508
Craig Topper74c10e32018-07-09 19:00:16 +00001509static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper8bf793f2018-06-29 05:43:33 +00001510_mm512_sqrt_pd(__m512d __A)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001511{
Craig Topper8bf793f2018-06-29 05:43:33 +00001512 return (__m512d)__builtin_ia32_sqrtpd512((__v8df)__A,
1513 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001514}
1515
Craig Topper74c10e32018-07-09 19:00:16 +00001516static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermancb856772016-05-16 11:42:01 +00001517_mm512_mask_sqrt_pd (__m512d __W, __mmask8 __U, __m512d __A)
1518{
Craig Topper8bf793f2018-06-29 05:43:33 +00001519 return (__m512d)__builtin_ia32_selectpd_512(__U,
1520 (__v8df)_mm512_sqrt_pd(__A),
1521 (__v8df)__W);
Michael Zuckermancb856772016-05-16 11:42:01 +00001522}
1523
Craig Topper74c10e32018-07-09 19:00:16 +00001524static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermancb856772016-05-16 11:42:01 +00001525_mm512_maskz_sqrt_pd (__mmask8 __U, __m512d __A)
1526{
Craig Topper8bf793f2018-06-29 05:43:33 +00001527 return (__m512d)__builtin_ia32_selectpd_512(__U,
1528 (__v8df)_mm512_sqrt_pd(__A),
1529 (__v8df)_mm512_setzero_pd());
Michael Zuckermancb856772016-05-16 11:42:01 +00001530}
1531
Craig Topper8bf793f2018-06-29 05:43:33 +00001532#define _mm512_sqrt_round_ps(A, R) \
1533 (__m512)__builtin_ia32_sqrtps512((__v16sf)(__m512)(A), (int)(R))
1534
Craig Topperc6338672018-05-31 00:51:20 +00001535#define _mm512_mask_sqrt_round_ps(W, U, A, R) \
Craig Topper8bf793f2018-06-29 05:43:33 +00001536 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
1537 (__v16sf)_mm512_sqrt_round_ps((A), (R)), \
1538 (__v16sf)(__m512)(W))
Michael Zuckermane6aa66a2016-06-01 08:34:03 +00001539
Craig Topperc6338672018-05-31 00:51:20 +00001540#define _mm512_maskz_sqrt_round_ps(U, A, R) \
Craig Topper8bf793f2018-06-29 05:43:33 +00001541 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
1542 (__v16sf)_mm512_sqrt_round_ps((A), (R)), \
1543 (__v16sf)_mm512_setzero_ps())
Michael Zuckermane6aa66a2016-06-01 08:34:03 +00001544
Craig Topper74c10e32018-07-09 19:00:16 +00001545static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper8bf793f2018-06-29 05:43:33 +00001546_mm512_sqrt_ps(__m512 __A)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001547{
Craig Topper8bf793f2018-06-29 05:43:33 +00001548 return (__m512)__builtin_ia32_sqrtps512((__v16sf)__A,
1549 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001550}
1551
Craig Topper74c10e32018-07-09 19:00:16 +00001552static __inline__ __m512 __DEFAULT_FN_ATTRS512
Asaf Badouh880f0c22016-06-13 15:15:57 +00001553_mm512_mask_sqrt_ps(__m512 __W, __mmask16 __U, __m512 __A)
Michael Zuckermancb856772016-05-16 11:42:01 +00001554{
Craig Topper8bf793f2018-06-29 05:43:33 +00001555 return (__m512)__builtin_ia32_selectps_512(__U,
1556 (__v16sf)_mm512_sqrt_ps(__A),
1557 (__v16sf)__W);
Michael Zuckermancb856772016-05-16 11:42:01 +00001558}
1559
Craig Topper74c10e32018-07-09 19:00:16 +00001560static __inline__ __m512 __DEFAULT_FN_ATTRS512
Asaf Badouh880f0c22016-06-13 15:15:57 +00001561_mm512_maskz_sqrt_ps( __mmask16 __U, __m512 __A)
Michael Zuckermancb856772016-05-16 11:42:01 +00001562{
Craig Topper8bf793f2018-06-29 05:43:33 +00001563 return (__m512)__builtin_ia32_selectps_512(__U,
1564 (__v16sf)_mm512_sqrt_ps(__A),
1565 (__v16sf)_mm512_setzero_ps());
Michael Zuckermancb856772016-05-16 11:42:01 +00001566}
1567
Craig Topper74c10e32018-07-09 19:00:16 +00001568static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001569_mm512_rsqrt14_pd(__m512d __A)
1570{
1571 return (__m512d) __builtin_ia32_rsqrt14pd512_mask ((__v8df) __A,
1572 (__v8df)
1573 _mm512_setzero_pd (),
1574 (__mmask8) -1);}
1575
Craig Topper74c10e32018-07-09 19:00:16 +00001576static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermancb856772016-05-16 11:42:01 +00001577_mm512_mask_rsqrt14_pd (__m512d __W, __mmask8 __U, __m512d __A)
1578{
1579 return (__m512d) __builtin_ia32_rsqrt14pd512_mask ((__v8df) __A,
1580 (__v8df) __W,
1581 (__mmask8) __U);
1582}
1583
Craig Topper74c10e32018-07-09 19:00:16 +00001584static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermancb856772016-05-16 11:42:01 +00001585_mm512_maskz_rsqrt14_pd (__mmask8 __U, __m512d __A)
1586{
1587 return (__m512d) __builtin_ia32_rsqrt14pd512_mask ((__v8df) __A,
1588 (__v8df)
1589 _mm512_setzero_pd (),
1590 (__mmask8) __U);
1591}
1592
Craig Topper74c10e32018-07-09 19:00:16 +00001593static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001594_mm512_rsqrt14_ps(__m512 __A)
1595{
1596 return (__m512) __builtin_ia32_rsqrt14ps512_mask ((__v16sf) __A,
1597 (__v16sf)
1598 _mm512_setzero_ps (),
1599 (__mmask16) -1);
1600}
1601
Craig Topper74c10e32018-07-09 19:00:16 +00001602static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermancb856772016-05-16 11:42:01 +00001603_mm512_mask_rsqrt14_ps (__m512 __W, __mmask16 __U, __m512 __A)
1604{
1605 return (__m512) __builtin_ia32_rsqrt14ps512_mask ((__v16sf) __A,
1606 (__v16sf) __W,
1607 (__mmask16) __U);
1608}
1609
Craig Topper74c10e32018-07-09 19:00:16 +00001610static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermancb856772016-05-16 11:42:01 +00001611_mm512_maskz_rsqrt14_ps (__mmask16 __U, __m512 __A)
1612{
1613 return (__m512) __builtin_ia32_rsqrt14ps512_mask ((__v16sf) __A,
1614 (__v16sf)
1615 _mm512_setzero_ps (),
1616 (__mmask16) __U);
1617}
1618
Craig Topper74c10e32018-07-09 19:00:16 +00001619static __inline__ __m128 __DEFAULT_FN_ATTRS128
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001620_mm_rsqrt14_ss(__m128 __A, __m128 __B)
1621{
Igor Breger9c2a0bf2016-02-08 12:36:48 +00001622 return (__m128) __builtin_ia32_rsqrt14ss_mask ((__v4sf) __A,
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001623 (__v4sf) __B,
1624 (__v4sf)
1625 _mm_setzero_ps (),
1626 (__mmask8) -1);
1627}
1628
Craig Topper74c10e32018-07-09 19:00:16 +00001629static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00001630_mm_mask_rsqrt14_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00001631{
1632 return (__m128) __builtin_ia32_rsqrt14ss_mask ((__v4sf) __A,
1633 (__v4sf) __B,
1634 (__v4sf) __W,
1635 (__mmask8) __U);
1636}
1637
Craig Topper74c10e32018-07-09 19:00:16 +00001638static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00001639_mm_maskz_rsqrt14_ss (__mmask8 __U, __m128 __A, __m128 __B)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00001640{
1641 return (__m128) __builtin_ia32_rsqrt14ss_mask ((__v4sf) __A,
1642 (__v4sf) __B,
1643 (__v4sf) _mm_setzero_ps (),
1644 (__mmask8) __U);
1645}
1646
Craig Topper74c10e32018-07-09 19:00:16 +00001647static __inline__ __m128d __DEFAULT_FN_ATTRS128
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001648_mm_rsqrt14_sd(__m128d __A, __m128d __B)
1649{
Igor Breger9c2a0bf2016-02-08 12:36:48 +00001650 return (__m128d) __builtin_ia32_rsqrt14sd_mask ((__v2df) __A,
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001651 (__v2df) __B,
1652 (__v2df)
1653 _mm_setzero_pd (),
1654 (__mmask8) -1);
1655}
1656
Craig Topper74c10e32018-07-09 19:00:16 +00001657static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00001658_mm_mask_rsqrt14_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00001659{
1660 return (__m128d) __builtin_ia32_rsqrt14sd_mask ( (__v2df) __A,
1661 (__v2df) __B,
1662 (__v2df) __W,
1663 (__mmask8) __U);
1664}
1665
Craig Topper74c10e32018-07-09 19:00:16 +00001666static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00001667_mm_maskz_rsqrt14_sd (__mmask8 __U, __m128d __A, __m128d __B)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00001668{
1669 return (__m128d) __builtin_ia32_rsqrt14sd_mask ( (__v2df) __A,
1670 (__v2df) __B,
1671 (__v2df) _mm_setzero_pd (),
1672 (__mmask8) __U);
1673}
1674
Craig Topper74c10e32018-07-09 19:00:16 +00001675static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001676_mm512_rcp14_pd(__m512d __A)
1677{
1678 return (__m512d) __builtin_ia32_rcp14pd512_mask ((__v8df) __A,
1679 (__v8df)
1680 _mm512_setzero_pd (),
1681 (__mmask8) -1);
1682}
1683
Craig Topper74c10e32018-07-09 19:00:16 +00001684static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermancb856772016-05-16 11:42:01 +00001685_mm512_mask_rcp14_pd (__m512d __W, __mmask8 __U, __m512d __A)
1686{
1687 return (__m512d) __builtin_ia32_rcp14pd512_mask ((__v8df) __A,
1688 (__v8df) __W,
1689 (__mmask8) __U);
1690}
1691
Craig Topper74c10e32018-07-09 19:00:16 +00001692static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermancb856772016-05-16 11:42:01 +00001693_mm512_maskz_rcp14_pd (__mmask8 __U, __m512d __A)
1694{
1695 return (__m512d) __builtin_ia32_rcp14pd512_mask ((__v8df) __A,
1696 (__v8df)
1697 _mm512_setzero_pd (),
1698 (__mmask8) __U);
1699}
1700
Craig Topper74c10e32018-07-09 19:00:16 +00001701static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001702_mm512_rcp14_ps(__m512 __A)
1703{
1704 return (__m512) __builtin_ia32_rcp14ps512_mask ((__v16sf) __A,
1705 (__v16sf)
1706 _mm512_setzero_ps (),
1707 (__mmask16) -1);
1708}
Michael Zuckermancb856772016-05-16 11:42:01 +00001709
Craig Topper74c10e32018-07-09 19:00:16 +00001710static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermancb856772016-05-16 11:42:01 +00001711_mm512_mask_rcp14_ps (__m512 __W, __mmask16 __U, __m512 __A)
1712{
1713 return (__m512) __builtin_ia32_rcp14ps512_mask ((__v16sf) __A,
1714 (__v16sf) __W,
1715 (__mmask16) __U);
1716}
1717
Craig Topper74c10e32018-07-09 19:00:16 +00001718static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermancb856772016-05-16 11:42:01 +00001719_mm512_maskz_rcp14_ps (__mmask16 __U, __m512 __A)
1720{
1721 return (__m512) __builtin_ia32_rcp14ps512_mask ((__v16sf) __A,
1722 (__v16sf)
1723 _mm512_setzero_ps (),
1724 (__mmask16) __U);
1725}
1726
Craig Topper74c10e32018-07-09 19:00:16 +00001727static __inline__ __m128 __DEFAULT_FN_ATTRS128
Adam Nemet9a3ea602014-07-28 17:14:38 +00001728_mm_rcp14_ss(__m128 __A, __m128 __B)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001729{
Igor Breger9c2a0bf2016-02-08 12:36:48 +00001730 return (__m128) __builtin_ia32_rcp14ss_mask ((__v4sf) __A,
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001731 (__v4sf) __B,
1732 (__v4sf)
1733 _mm_setzero_ps (),
1734 (__mmask8) -1);
1735}
1736
Craig Topper74c10e32018-07-09 19:00:16 +00001737static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00001738_mm_mask_rcp14_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00001739{
1740 return (__m128) __builtin_ia32_rcp14ss_mask ((__v4sf) __A,
1741 (__v4sf) __B,
1742 (__v4sf) __W,
1743 (__mmask8) __U);
1744}
1745
Craig Topper74c10e32018-07-09 19:00:16 +00001746static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00001747_mm_maskz_rcp14_ss (__mmask8 __U, __m128 __A, __m128 __B)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00001748{
1749 return (__m128) __builtin_ia32_rcp14ss_mask ((__v4sf) __A,
1750 (__v4sf) __B,
1751 (__v4sf) _mm_setzero_ps (),
1752 (__mmask8) __U);
1753}
1754
Craig Topper74c10e32018-07-09 19:00:16 +00001755static __inline__ __m128d __DEFAULT_FN_ATTRS128
Adam Nemet9a3ea602014-07-28 17:14:38 +00001756_mm_rcp14_sd(__m128d __A, __m128d __B)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001757{
Igor Breger9c2a0bf2016-02-08 12:36:48 +00001758 return (__m128d) __builtin_ia32_rcp14sd_mask ((__v2df) __A,
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001759 (__v2df) __B,
1760 (__v2df)
1761 _mm_setzero_pd (),
1762 (__mmask8) -1);
1763}
1764
Craig Topper74c10e32018-07-09 19:00:16 +00001765static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00001766_mm_mask_rcp14_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00001767{
1768 return (__m128d) __builtin_ia32_rcp14sd_mask ( (__v2df) __A,
1769 (__v2df) __B,
1770 (__v2df) __W,
1771 (__mmask8) __U);
1772}
1773
Craig Topper74c10e32018-07-09 19:00:16 +00001774static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00001775_mm_maskz_rcp14_sd (__mmask8 __U, __m128d __A, __m128d __B)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00001776{
1777 return (__m128d) __builtin_ia32_rcp14sd_mask ( (__v2df) __A,
1778 (__v2df) __B,
1779 (__v2df) _mm_setzero_pd (),
1780 (__mmask8) __U);
1781}
1782
Craig Topper74c10e32018-07-09 19:00:16 +00001783static __inline __m512 __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001784_mm512_floor_ps(__m512 __A)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001785{
Adam Nemet0d5bb552014-07-28 17:14:40 +00001786 return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A,
1787 _MM_FROUND_FLOOR,
1788 (__v16sf) __A, -1,
1789 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001790}
1791
Craig Topper74c10e32018-07-09 19:00:16 +00001792static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman7360d8a2016-05-10 07:30:58 +00001793_mm512_mask_floor_ps (__m512 __W, __mmask16 __U, __m512 __A)
1794{
1795 return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A,
1796 _MM_FROUND_FLOOR,
1797 (__v16sf) __W, __U,
1798 _MM_FROUND_CUR_DIRECTION);
1799}
1800
Craig Topper74c10e32018-07-09 19:00:16 +00001801static __inline __m512d __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001802_mm512_floor_pd(__m512d __A)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001803{
Adam Nemet0d5bb552014-07-28 17:14:40 +00001804 return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A,
1805 _MM_FROUND_FLOOR,
1806 (__v8df) __A, -1,
1807 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001808}
1809
Craig Topper74c10e32018-07-09 19:00:16 +00001810static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman7360d8a2016-05-10 07:30:58 +00001811_mm512_mask_floor_pd (__m512d __W, __mmask8 __U, __m512d __A)
1812{
1813 return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A,
1814 _MM_FROUND_FLOOR,
1815 (__v8df) __W, __U,
1816 _MM_FROUND_CUR_DIRECTION);
1817}
1818
Craig Topper74c10e32018-07-09 19:00:16 +00001819static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman7360d8a2016-05-10 07:30:58 +00001820_mm512_mask_ceil_ps (__m512 __W, __mmask16 __U, __m512 __A)
1821{
1822 return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A,
1823 _MM_FROUND_CEIL,
1824 (__v16sf) __W, __U,
1825 _MM_FROUND_CUR_DIRECTION);
1826}
1827
Craig Topper74c10e32018-07-09 19:00:16 +00001828static __inline __m512 __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001829_mm512_ceil_ps(__m512 __A)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00001830{
Adam Nemet0d5bb552014-07-28 17:14:40 +00001831 return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A,
1832 _MM_FROUND_CEIL,
1833 (__v16sf) __A, -1,
1834 _MM_FROUND_CUR_DIRECTION);
1835}
1836
Craig Topper74c10e32018-07-09 19:00:16 +00001837static __inline __m512d __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001838_mm512_ceil_pd(__m512d __A)
1839{
1840 return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A,
1841 _MM_FROUND_CEIL,
1842 (__v8df) __A, -1,
1843 _MM_FROUND_CUR_DIRECTION);
1844}
1845
Craig Topper74c10e32018-07-09 19:00:16 +00001846static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman7360d8a2016-05-10 07:30:58 +00001847_mm512_mask_ceil_pd (__m512d __W, __mmask8 __U, __m512d __A)
1848{
1849 return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A,
1850 _MM_FROUND_CEIL,
1851 (__v8df) __W, __U,
1852 _MM_FROUND_CUR_DIRECTION);
1853}
1854
Craig Topper74c10e32018-07-09 19:00:16 +00001855static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001856_mm512_abs_epi64(__m512i __A)
1857{
Craig Topperf2043b02018-05-23 04:51:54 +00001858 return (__m512i)__builtin_ia32_pabsq512((__v8di)__A);
Adam Nemet0d5bb552014-07-28 17:14:40 +00001859}
1860
Craig Topper74c10e32018-07-09 19:00:16 +00001861static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanbf05a452016-05-16 18:57:24 +00001862_mm512_mask_abs_epi64 (__m512i __W, __mmask8 __U, __m512i __A)
1863{
Craig Topperf2043b02018-05-23 04:51:54 +00001864 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
1865 (__v8di)_mm512_abs_epi64(__A),
1866 (__v8di)__W);
Michael Zuckermanbf05a452016-05-16 18:57:24 +00001867}
1868
Craig Topper74c10e32018-07-09 19:00:16 +00001869static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanbf05a452016-05-16 18:57:24 +00001870_mm512_maskz_abs_epi64 (__mmask8 __U, __m512i __A)
1871{
Craig Topperf2043b02018-05-23 04:51:54 +00001872 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
1873 (__v8di)_mm512_abs_epi64(__A),
1874 (__v8di)_mm512_setzero_si512());
Michael Zuckermanbf05a452016-05-16 18:57:24 +00001875}
1876
Craig Topper74c10e32018-07-09 19:00:16 +00001877static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00001878_mm512_abs_epi32(__m512i __A)
1879{
Craig Topperf2043b02018-05-23 04:51:54 +00001880 return (__m512i)__builtin_ia32_pabsd512((__v16si) __A);
Adam Nemet0d5bb552014-07-28 17:14:40 +00001881}
1882
Craig Topper74c10e32018-07-09 19:00:16 +00001883static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanbf05a452016-05-16 18:57:24 +00001884_mm512_mask_abs_epi32 (__m512i __W, __mmask16 __U, __m512i __A)
1885{
Craig Topper26df8c42018-05-24 17:32:49 +00001886 return (__m512i)__builtin_ia32_selectd_512(__U,
Craig Topperf2043b02018-05-23 04:51:54 +00001887 (__v16si)_mm512_abs_epi32(__A),
1888 (__v16si)__W);
Michael Zuckermanbf05a452016-05-16 18:57:24 +00001889}
1890
Craig Topper74c10e32018-07-09 19:00:16 +00001891static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanbf05a452016-05-16 18:57:24 +00001892_mm512_maskz_abs_epi32 (__mmask16 __U, __m512i __A)
1893{
Craig Topper26df8c42018-05-24 17:32:49 +00001894 return (__m512i)__builtin_ia32_selectd_512(__U,
Craig Topperf2043b02018-05-23 04:51:54 +00001895 (__v16si)_mm512_abs_epi32(__A),
1896 (__v16si)_mm512_setzero_si512());
Michael Zuckermanbf05a452016-05-16 18:57:24 +00001897}
1898
Craig Topper74c10e32018-07-09 19:00:16 +00001899static __inline__ __m128 __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001900_mm_mask_add_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
Tomasz Krupa82aa42a2018-06-14 17:36:23 +00001901 __A = _mm_add_ss(__A, __B);
1902 __A[0] = (__U & 1) ? __A[0] : __W[0];
1903 return __A;
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001904}
1905
Craig Topper74c10e32018-07-09 19:00:16 +00001906static __inline__ __m128 __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001907_mm_maskz_add_ss(__mmask8 __U,__m128 __A, __m128 __B) {
Tomasz Krupa82aa42a2018-06-14 17:36:23 +00001908 __A = _mm_add_ss(__A, __B);
1909 __A[0] = (__U & 1) ? __A[0] : 0;
1910 return __A;
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001911}
1912
Craig Topperc6338672018-05-31 00:51:20 +00001913#define _mm_add_round_ss(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001914 (__m128)__builtin_ia32_addss_round_mask((__v4sf)(__m128)(A), \
1915 (__v4sf)(__m128)(B), \
1916 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00001917 (__mmask8)-1, (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001918
Craig Topperc6338672018-05-31 00:51:20 +00001919#define _mm_mask_add_round_ss(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001920 (__m128)__builtin_ia32_addss_round_mask((__v4sf)(__m128)(A), \
1921 (__v4sf)(__m128)(B), \
1922 (__v4sf)(__m128)(W), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00001923 (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001924
Craig Topperc6338672018-05-31 00:51:20 +00001925#define _mm_maskz_add_round_ss(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001926 (__m128)__builtin_ia32_addss_round_mask((__v4sf)(__m128)(A), \
1927 (__v4sf)(__m128)(B), \
1928 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00001929 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001930
Craig Topper74c10e32018-07-09 19:00:16 +00001931static __inline__ __m128d __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001932_mm_mask_add_sd(__m128d __W, __mmask8 __U,__m128d __A, __m128d __B) {
Tomasz Krupa82aa42a2018-06-14 17:36:23 +00001933 __A = _mm_add_sd(__A, __B);
1934 __A[0] = (__U & 1) ? __A[0] : __W[0];
1935 return __A;
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001936}
1937
Craig Topper74c10e32018-07-09 19:00:16 +00001938static __inline__ __m128d __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001939_mm_maskz_add_sd(__mmask8 __U,__m128d __A, __m128d __B) {
Tomasz Krupa82aa42a2018-06-14 17:36:23 +00001940 __A = _mm_add_sd(__A, __B);
1941 __A[0] = (__U & 1) ? __A[0] : 0;
1942 return __A;
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001943}
Craig Topperc6338672018-05-31 00:51:20 +00001944#define _mm_add_round_sd(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001945 (__m128d)__builtin_ia32_addsd_round_mask((__v2df)(__m128d)(A), \
1946 (__v2df)(__m128d)(B), \
1947 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00001948 (__mmask8)-1, (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001949
Craig Topperc6338672018-05-31 00:51:20 +00001950#define _mm_mask_add_round_sd(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001951 (__m128d)__builtin_ia32_addsd_round_mask((__v2df)(__m128d)(A), \
1952 (__v2df)(__m128d)(B), \
1953 (__v2df)(__m128d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00001954 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001955
Craig Topperc6338672018-05-31 00:51:20 +00001956#define _mm_maskz_add_round_sd(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00001957 (__m128d)__builtin_ia32_addsd_round_mask((__v2df)(__m128d)(A), \
1958 (__v2df)(__m128d)(B), \
1959 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00001960 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00001961
Craig Topper74c10e32018-07-09 19:00:16 +00001962static __inline__ __m512d __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00001963_mm512_mask_add_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00001964 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
1965 (__v8df)_mm512_add_pd(__A, __B),
1966 (__v8df)__W);
Asaf Badouhffeb6242015-07-21 15:27:28 +00001967}
1968
Craig Topper74c10e32018-07-09 19:00:16 +00001969static __inline__ __m512d __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00001970_mm512_maskz_add_pd(__mmask8 __U, __m512d __A, __m512d __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00001971 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
1972 (__v8df)_mm512_add_pd(__A, __B),
1973 (__v8df)_mm512_setzero_pd());
Asaf Badouhffeb6242015-07-21 15:27:28 +00001974}
1975
Craig Topper74c10e32018-07-09 19:00:16 +00001976static __inline__ __m512 __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00001977_mm512_mask_add_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00001978 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
1979 (__v16sf)_mm512_add_ps(__A, __B),
1980 (__v16sf)__W);
Asaf Badouhffeb6242015-07-21 15:27:28 +00001981}
1982
Craig Topper74c10e32018-07-09 19:00:16 +00001983static __inline__ __m512 __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00001984_mm512_maskz_add_ps(__mmask16 __U, __m512 __A, __m512 __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00001985 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
1986 (__v16sf)_mm512_add_ps(__A, __B),
1987 (__v16sf)_mm512_setzero_ps());
Asaf Badouhffeb6242015-07-21 15:27:28 +00001988}
1989
Craig Topperc6338672018-05-31 00:51:20 +00001990#define _mm512_add_round_pd(A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00001991 (__m512d)__builtin_ia32_addpd512((__v8df)(__m512d)(A), \
1992 (__v8df)(__m512d)(B), (int)(R))
Asaf Badouhffeb6242015-07-21 15:27:28 +00001993
Craig Topperc6338672018-05-31 00:51:20 +00001994#define _mm512_mask_add_round_pd(W, U, A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00001995 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
1996 (__v8df)_mm512_add_round_pd((A), (B), (R)), \
1997 (__v8df)(__m512d)(W));
Asaf Badouhffeb6242015-07-21 15:27:28 +00001998
Craig Topperc6338672018-05-31 00:51:20 +00001999#define _mm512_maskz_add_round_pd(U, A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002000 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
2001 (__v8df)_mm512_add_round_pd((A), (B), (R)), \
2002 (__v8df)_mm512_setzero_pd());
Asaf Badouhffeb6242015-07-21 15:27:28 +00002003
Craig Topperc6338672018-05-31 00:51:20 +00002004#define _mm512_add_round_ps(A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002005 (__m512)__builtin_ia32_addps512((__v16sf)(__m512)(A), \
2006 (__v16sf)(__m512)(B), (int)(R))
Asaf Badouhffeb6242015-07-21 15:27:28 +00002007
Craig Topperc6338672018-05-31 00:51:20 +00002008#define _mm512_mask_add_round_ps(W, U, A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002009 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2010 (__v16sf)_mm512_add_round_ps((A), (B), (R)), \
2011 (__v16sf)(__m512)(W));
Asaf Badouhffeb6242015-07-21 15:27:28 +00002012
Craig Topperc6338672018-05-31 00:51:20 +00002013#define _mm512_maskz_add_round_ps(U, A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002014 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2015 (__v16sf)_mm512_add_round_ps((A), (B), (R)), \
2016 (__v16sf)_mm512_setzero_ps());
Asaf Badouhffeb6242015-07-21 15:27:28 +00002017
Craig Topper74c10e32018-07-09 19:00:16 +00002018static __inline__ __m128 __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002019_mm_mask_sub_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
Tomasz Krupa82aa42a2018-06-14 17:36:23 +00002020 __A = _mm_sub_ss(__A, __B);
2021 __A[0] = (__U & 1) ? __A[0] : __W[0];
2022 return __A;
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002023}
2024
Craig Topper74c10e32018-07-09 19:00:16 +00002025static __inline__ __m128 __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002026_mm_maskz_sub_ss(__mmask8 __U,__m128 __A, __m128 __B) {
Tomasz Krupa82aa42a2018-06-14 17:36:23 +00002027 __A = _mm_sub_ss(__A, __B);
2028 __A[0] = (__U & 1) ? __A[0] : 0;
2029 return __A;
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002030}
Craig Topperc6338672018-05-31 00:51:20 +00002031#define _mm_sub_round_ss(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002032 (__m128)__builtin_ia32_subss_round_mask((__v4sf)(__m128)(A), \
2033 (__v4sf)(__m128)(B), \
2034 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00002035 (__mmask8)-1, (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002036
Craig Topperc6338672018-05-31 00:51:20 +00002037#define _mm_mask_sub_round_ss(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002038 (__m128)__builtin_ia32_subss_round_mask((__v4sf)(__m128)(A), \
2039 (__v4sf)(__m128)(B), \
2040 (__v4sf)(__m128)(W), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00002041 (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002042
Craig Topperc6338672018-05-31 00:51:20 +00002043#define _mm_maskz_sub_round_ss(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002044 (__m128)__builtin_ia32_subss_round_mask((__v4sf)(__m128)(A), \
2045 (__v4sf)(__m128)(B), \
2046 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00002047 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002048
Craig Topper74c10e32018-07-09 19:00:16 +00002049static __inline__ __m128d __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002050_mm_mask_sub_sd(__m128d __W, __mmask8 __U,__m128d __A, __m128d __B) {
Tomasz Krupa82aa42a2018-06-14 17:36:23 +00002051 __A = _mm_sub_sd(__A, __B);
2052 __A[0] = (__U & 1) ? __A[0] : __W[0];
2053 return __A;
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002054}
2055
Craig Topper74c10e32018-07-09 19:00:16 +00002056static __inline__ __m128d __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002057_mm_maskz_sub_sd(__mmask8 __U,__m128d __A, __m128d __B) {
Tomasz Krupa82aa42a2018-06-14 17:36:23 +00002058 __A = _mm_sub_sd(__A, __B);
2059 __A[0] = (__U & 1) ? __A[0] : 0;
2060 return __A;
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002061}
2062
Craig Topperc6338672018-05-31 00:51:20 +00002063#define _mm_sub_round_sd(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002064 (__m128d)__builtin_ia32_subsd_round_mask((__v2df)(__m128d)(A), \
2065 (__v2df)(__m128d)(B), \
2066 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00002067 (__mmask8)-1, (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002068
Craig Topperc6338672018-05-31 00:51:20 +00002069#define _mm_mask_sub_round_sd(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002070 (__m128d)__builtin_ia32_subsd_round_mask((__v2df)(__m128d)(A), \
2071 (__v2df)(__m128d)(B), \
2072 (__v2df)(__m128d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00002073 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002074
Craig Topperc6338672018-05-31 00:51:20 +00002075#define _mm_maskz_sub_round_sd(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002076 (__m128d)__builtin_ia32_subsd_round_mask((__v2df)(__m128d)(A), \
2077 (__v2df)(__m128d)(B), \
2078 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00002079 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002080
Craig Topper74c10e32018-07-09 19:00:16 +00002081static __inline__ __m512d __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00002082_mm512_mask_sub_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00002083 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
2084 (__v8df)_mm512_sub_pd(__A, __B),
2085 (__v8df)__W);
Asaf Badouhffeb6242015-07-21 15:27:28 +00002086}
2087
Craig Topper74c10e32018-07-09 19:00:16 +00002088static __inline__ __m512d __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00002089_mm512_maskz_sub_pd(__mmask8 __U, __m512d __A, __m512d __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00002090 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
2091 (__v8df)_mm512_sub_pd(__A, __B),
2092 (__v8df)_mm512_setzero_pd());
Asaf Badouhffeb6242015-07-21 15:27:28 +00002093}
2094
Craig Topper74c10e32018-07-09 19:00:16 +00002095static __inline__ __m512 __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00002096_mm512_mask_sub_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00002097 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
2098 (__v16sf)_mm512_sub_ps(__A, __B),
2099 (__v16sf)__W);
Asaf Badouhffeb6242015-07-21 15:27:28 +00002100}
2101
Craig Topper74c10e32018-07-09 19:00:16 +00002102static __inline__ __m512 __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00002103_mm512_maskz_sub_ps(__mmask16 __U, __m512 __A, __m512 __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00002104 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
2105 (__v16sf)_mm512_sub_ps(__A, __B),
2106 (__v16sf)_mm512_setzero_ps());
Asaf Badouhffeb6242015-07-21 15:27:28 +00002107}
2108
Craig Topperc6338672018-05-31 00:51:20 +00002109#define _mm512_sub_round_pd(A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002110 (__m512d)__builtin_ia32_subpd512((__v8df)(__m512d)(A), \
2111 (__v8df)(__m512d)(B), (int)(R))
Asaf Badouhffeb6242015-07-21 15:27:28 +00002112
Craig Topperc6338672018-05-31 00:51:20 +00002113#define _mm512_mask_sub_round_pd(W, U, A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002114 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
2115 (__v8df)_mm512_sub_round_pd((A), (B), (R)), \
2116 (__v8df)(__m512d)(W));
Asaf Badouhffeb6242015-07-21 15:27:28 +00002117
Craig Topperc6338672018-05-31 00:51:20 +00002118#define _mm512_maskz_sub_round_pd(U, A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002119 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
2120 (__v8df)_mm512_sub_round_pd((A), (B), (R)), \
2121 (__v8df)_mm512_setzero_pd());
Asaf Badouhffeb6242015-07-21 15:27:28 +00002122
Craig Topperc6338672018-05-31 00:51:20 +00002123#define _mm512_sub_round_ps(A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002124 (__m512)__builtin_ia32_subps512((__v16sf)(__m512)(A), \
2125 (__v16sf)(__m512)(B), (int)(R))
Asaf Badouhffeb6242015-07-21 15:27:28 +00002126
Craig Topper3614b412018-06-10 06:01:42 +00002127#define _mm512_mask_sub_round_ps(W, U, A, B, R) \
2128 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2129 (__v16sf)_mm512_sub_round_ps((A), (B), (R)), \
2130 (__v16sf)(__m512)(W));
Asaf Badouhffeb6242015-07-21 15:27:28 +00002131
Craig Topper3614b412018-06-10 06:01:42 +00002132#define _mm512_maskz_sub_round_ps(U, A, B, R) \
2133 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2134 (__v16sf)_mm512_sub_round_ps((A), (B), (R)), \
2135 (__v16sf)_mm512_setzero_ps());
Asaf Badouhffeb6242015-07-21 15:27:28 +00002136
Craig Topper74c10e32018-07-09 19:00:16 +00002137static __inline__ __m128 __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002138_mm_mask_mul_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
Tomasz Krupa82aa42a2018-06-14 17:36:23 +00002139 __A = _mm_mul_ss(__A, __B);
2140 __A[0] = (__U & 1) ? __A[0] : __W[0];
2141 return __A;
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002142}
2143
Craig Topper74c10e32018-07-09 19:00:16 +00002144static __inline__ __m128 __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002145_mm_maskz_mul_ss(__mmask8 __U,__m128 __A, __m128 __B) {
Tomasz Krupa82aa42a2018-06-14 17:36:23 +00002146 __A = _mm_mul_ss(__A, __B);
2147 __A[0] = (__U & 1) ? __A[0] : 0;
2148 return __A;
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002149}
Craig Topperc6338672018-05-31 00:51:20 +00002150#define _mm_mul_round_ss(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002151 (__m128)__builtin_ia32_mulss_round_mask((__v4sf)(__m128)(A), \
2152 (__v4sf)(__m128)(B), \
2153 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00002154 (__mmask8)-1, (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002155
Craig Topperc6338672018-05-31 00:51:20 +00002156#define _mm_mask_mul_round_ss(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002157 (__m128)__builtin_ia32_mulss_round_mask((__v4sf)(__m128)(A), \
2158 (__v4sf)(__m128)(B), \
2159 (__v4sf)(__m128)(W), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00002160 (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002161
Craig Topperc6338672018-05-31 00:51:20 +00002162#define _mm_maskz_mul_round_ss(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002163 (__m128)__builtin_ia32_mulss_round_mask((__v4sf)(__m128)(A), \
2164 (__v4sf)(__m128)(B), \
2165 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00002166 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002167
Craig Topper74c10e32018-07-09 19:00:16 +00002168static __inline__ __m128d __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002169_mm_mask_mul_sd(__m128d __W, __mmask8 __U,__m128d __A, __m128d __B) {
Tomasz Krupa82aa42a2018-06-14 17:36:23 +00002170 __A = _mm_mul_sd(__A, __B);
2171 __A[0] = (__U & 1) ? __A[0] : __W[0];
2172 return __A;
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002173}
2174
Craig Topper74c10e32018-07-09 19:00:16 +00002175static __inline__ __m128d __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002176_mm_maskz_mul_sd(__mmask8 __U,__m128d __A, __m128d __B) {
Tomasz Krupa82aa42a2018-06-14 17:36:23 +00002177 __A = _mm_mul_sd(__A, __B);
2178 __A[0] = (__U & 1) ? __A[0] : 0;
2179 return __A;
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002180}
2181
Craig Topperc6338672018-05-31 00:51:20 +00002182#define _mm_mul_round_sd(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002183 (__m128d)__builtin_ia32_mulsd_round_mask((__v2df)(__m128d)(A), \
2184 (__v2df)(__m128d)(B), \
2185 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00002186 (__mmask8)-1, (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002187
Craig Topperc6338672018-05-31 00:51:20 +00002188#define _mm_mask_mul_round_sd(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002189 (__m128d)__builtin_ia32_mulsd_round_mask((__v2df)(__m128d)(A), \
2190 (__v2df)(__m128d)(B), \
2191 (__v2df)(__m128d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00002192 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002193
Craig Topperc6338672018-05-31 00:51:20 +00002194#define _mm_maskz_mul_round_sd(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002195 (__m128d)__builtin_ia32_mulsd_round_mask((__v2df)(__m128d)(A), \
2196 (__v2df)(__m128d)(B), \
2197 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00002198 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002199
Craig Topper74c10e32018-07-09 19:00:16 +00002200static __inline__ __m512d __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00002201_mm512_mask_mul_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00002202 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
2203 (__v8df)_mm512_mul_pd(__A, __B),
2204 (__v8df)__W);
Asaf Badouhffeb6242015-07-21 15:27:28 +00002205}
2206
Craig Topper74c10e32018-07-09 19:00:16 +00002207static __inline__ __m512d __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00002208_mm512_maskz_mul_pd(__mmask8 __U, __m512d __A, __m512d __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00002209 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
2210 (__v8df)_mm512_mul_pd(__A, __B),
2211 (__v8df)_mm512_setzero_pd());
Asaf Badouhffeb6242015-07-21 15:27:28 +00002212}
2213
Craig Topper74c10e32018-07-09 19:00:16 +00002214static __inline__ __m512 __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00002215_mm512_mask_mul_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00002216 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
2217 (__v16sf)_mm512_mul_ps(__A, __B),
2218 (__v16sf)__W);
Asaf Badouhffeb6242015-07-21 15:27:28 +00002219}
2220
Craig Topper74c10e32018-07-09 19:00:16 +00002221static __inline__ __m512 __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00002222_mm512_maskz_mul_ps(__mmask16 __U, __m512 __A, __m512 __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00002223 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
2224 (__v16sf)_mm512_mul_ps(__A, __B),
2225 (__v16sf)_mm512_setzero_ps());
Asaf Badouhffeb6242015-07-21 15:27:28 +00002226}
2227
Craig Topperc6338672018-05-31 00:51:20 +00002228#define _mm512_mul_round_pd(A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002229 (__m512d)__builtin_ia32_mulpd512((__v8df)(__m512d)(A), \
2230 (__v8df)(__m512d)(B), (int)(R))
Asaf Badouhffeb6242015-07-21 15:27:28 +00002231
Craig Topperc6338672018-05-31 00:51:20 +00002232#define _mm512_mask_mul_round_pd(W, U, A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002233 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
2234 (__v8df)_mm512_mul_round_pd((A), (B), (R)), \
2235 (__v8df)(__m512d)(W));
Asaf Badouhffeb6242015-07-21 15:27:28 +00002236
Craig Topperc6338672018-05-31 00:51:20 +00002237#define _mm512_maskz_mul_round_pd(U, A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002238 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
2239 (__v8df)_mm512_mul_round_pd((A), (B), (R)), \
2240 (__v8df)_mm512_setzero_pd());
Asaf Badouhffeb6242015-07-21 15:27:28 +00002241
Craig Topperc6338672018-05-31 00:51:20 +00002242#define _mm512_mul_round_ps(A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002243 (__m512)__builtin_ia32_mulps512((__v16sf)(__m512)(A), \
2244 (__v16sf)(__m512)(B), (int)(R))
Asaf Badouhffeb6242015-07-21 15:27:28 +00002245
Craig Topper3614b412018-06-10 06:01:42 +00002246#define _mm512_mask_mul_round_ps(W, U, A, B, R) \
2247 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2248 (__v16sf)_mm512_mul_round_ps((A), (B), (R)), \
2249 (__v16sf)(__m512)(W));
Asaf Badouhffeb6242015-07-21 15:27:28 +00002250
Craig Topper3614b412018-06-10 06:01:42 +00002251#define _mm512_maskz_mul_round_ps(U, A, B, R) \
2252 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2253 (__v16sf)_mm512_mul_round_ps((A), (B), (R)), \
2254 (__v16sf)_mm512_setzero_ps());
Asaf Badouhffeb6242015-07-21 15:27:28 +00002255
Craig Topper74c10e32018-07-09 19:00:16 +00002256static __inline__ __m128 __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002257_mm_mask_div_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
Igor Breger9c2a0bf2016-02-08 12:36:48 +00002258 return (__m128) __builtin_ia32_divss_round_mask ((__v4sf) __A,
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002259 (__v4sf) __B,
2260 (__v4sf) __W,
2261 (__mmask8) __U,
2262 _MM_FROUND_CUR_DIRECTION);
2263}
2264
Craig Topper74c10e32018-07-09 19:00:16 +00002265static __inline__ __m128 __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002266_mm_maskz_div_ss(__mmask8 __U,__m128 __A, __m128 __B) {
Igor Breger9c2a0bf2016-02-08 12:36:48 +00002267 return (__m128) __builtin_ia32_divss_round_mask ((__v4sf) __A,
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002268 (__v4sf) __B,
2269 (__v4sf) _mm_setzero_ps (),
2270 (__mmask8) __U,
2271 _MM_FROUND_CUR_DIRECTION);
2272}
2273
Craig Topperc6338672018-05-31 00:51:20 +00002274#define _mm_div_round_ss(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002275 (__m128)__builtin_ia32_divss_round_mask((__v4sf)(__m128)(A), \
2276 (__v4sf)(__m128)(B), \
2277 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00002278 (__mmask8)-1, (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002279
Craig Topperc6338672018-05-31 00:51:20 +00002280#define _mm_mask_div_round_ss(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002281 (__m128)__builtin_ia32_divss_round_mask((__v4sf)(__m128)(A), \
2282 (__v4sf)(__m128)(B), \
2283 (__v4sf)(__m128)(W), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00002284 (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002285
Craig Topperc6338672018-05-31 00:51:20 +00002286#define _mm_maskz_div_round_ss(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002287 (__m128)__builtin_ia32_divss_round_mask((__v4sf)(__m128)(A), \
2288 (__v4sf)(__m128)(B), \
2289 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00002290 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002291
Craig Topper74c10e32018-07-09 19:00:16 +00002292static __inline__ __m128d __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002293_mm_mask_div_sd(__m128d __W, __mmask8 __U,__m128d __A, __m128d __B) {
Igor Breger9c2a0bf2016-02-08 12:36:48 +00002294 return (__m128d) __builtin_ia32_divsd_round_mask ((__v2df) __A,
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002295 (__v2df) __B,
2296 (__v2df) __W,
2297 (__mmask8) __U,
2298 _MM_FROUND_CUR_DIRECTION);
2299}
2300
Craig Topper74c10e32018-07-09 19:00:16 +00002301static __inline__ __m128d __DEFAULT_FN_ATTRS128
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002302_mm_maskz_div_sd(__mmask8 __U,__m128d __A, __m128d __B) {
Igor Breger9c2a0bf2016-02-08 12:36:48 +00002303 return (__m128d) __builtin_ia32_divsd_round_mask ((__v2df) __A,
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002304 (__v2df) __B,
2305 (__v2df) _mm_setzero_pd (),
2306 (__mmask8) __U,
2307 _MM_FROUND_CUR_DIRECTION);
2308}
2309
Craig Topperc6338672018-05-31 00:51:20 +00002310#define _mm_div_round_sd(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002311 (__m128d)__builtin_ia32_divsd_round_mask((__v2df)(__m128d)(A), \
2312 (__v2df)(__m128d)(B), \
2313 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00002314 (__mmask8)-1, (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002315
Craig Topperc6338672018-05-31 00:51:20 +00002316#define _mm_mask_div_round_sd(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002317 (__m128d)__builtin_ia32_divsd_round_mask((__v2df)(__m128d)(A), \
2318 (__v2df)(__m128d)(B), \
2319 (__v2df)(__m128d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00002320 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002321
Craig Topperc6338672018-05-31 00:51:20 +00002322#define _mm_maskz_div_round_sd(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00002323 (__m128d)__builtin_ia32_divsd_round_mask((__v2df)(__m128d)(A), \
2324 (__v2df)(__m128d)(B), \
2325 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00002326 (__mmask8)(U), (int)(R))
Asaf Badouhf6a58b62015-07-23 12:13:32 +00002327
Craig Topper74c10e32018-07-09 19:00:16 +00002328static __inline __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman223676d2016-06-14 12:38:58 +00002329_mm512_div_pd(__m512d __a, __m512d __b)
2330{
2331 return (__m512d)((__v8df)__a/(__v8df)__b);
2332}
2333
Craig Topper74c10e32018-07-09 19:00:16 +00002334static __inline__ __m512d __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00002335_mm512_mask_div_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00002336 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
2337 (__v8df)_mm512_div_pd(__A, __B),
2338 (__v8df)__W);
Asaf Badouhffeb6242015-07-21 15:27:28 +00002339}
2340
Craig Topper74c10e32018-07-09 19:00:16 +00002341static __inline__ __m512d __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00002342_mm512_maskz_div_pd(__mmask8 __U, __m512d __A, __m512d __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00002343 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
2344 (__v8df)_mm512_div_pd(__A, __B),
2345 (__v8df)_mm512_setzero_pd());
Asaf Badouhffeb6242015-07-21 15:27:28 +00002346}
2347
Craig Topper74c10e32018-07-09 19:00:16 +00002348static __inline __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman223676d2016-06-14 12:38:58 +00002349_mm512_div_ps(__m512 __a, __m512 __b)
2350{
2351 return (__m512)((__v16sf)__a/(__v16sf)__b);
2352}
2353
Craig Topper74c10e32018-07-09 19:00:16 +00002354static __inline__ __m512 __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00002355_mm512_mask_div_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00002356 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
2357 (__v16sf)_mm512_div_ps(__A, __B),
2358 (__v16sf)__W);
Asaf Badouhffeb6242015-07-21 15:27:28 +00002359}
2360
Craig Topper74c10e32018-07-09 19:00:16 +00002361static __inline__ __m512 __DEFAULT_FN_ATTRS512
Asaf Badouhffeb6242015-07-21 15:27:28 +00002362_mm512_maskz_div_ps(__mmask16 __U, __m512 __A, __m512 __B) {
Craig Topperc4a82282016-10-02 17:43:00 +00002363 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
2364 (__v16sf)_mm512_div_ps(__A, __B),
2365 (__v16sf)_mm512_setzero_ps());
Asaf Badouhffeb6242015-07-21 15:27:28 +00002366}
2367
Craig Topperc6338672018-05-31 00:51:20 +00002368#define _mm512_div_round_pd(A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002369 (__m512d)__builtin_ia32_divpd512((__v8df)(__m512d)(A), \
2370 (__v8df)(__m512d)(B), (int)(R))
Asaf Badouhffeb6242015-07-21 15:27:28 +00002371
Craig Topperc6338672018-05-31 00:51:20 +00002372#define _mm512_mask_div_round_pd(W, U, A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002373 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
2374 (__v8df)_mm512_div_round_pd((A), (B), (R)), \
2375 (__v8df)(__m512d)(W));
Asaf Badouhffeb6242015-07-21 15:27:28 +00002376
Craig Topperc6338672018-05-31 00:51:20 +00002377#define _mm512_maskz_div_round_pd(U, A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002378 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
2379 (__v8df)_mm512_div_round_pd((A), (B), (R)), \
2380 (__v8df)_mm512_setzero_pd());
Asaf Badouhffeb6242015-07-21 15:27:28 +00002381
Craig Topperc6338672018-05-31 00:51:20 +00002382#define _mm512_div_round_ps(A, B, R) \
Craig Topper3614b412018-06-10 06:01:42 +00002383 (__m512)__builtin_ia32_divps512((__v16sf)(__m512)(A), \
2384 (__v16sf)(__m512)(B), (int)(R))
Asaf Badouhffeb6242015-07-21 15:27:28 +00002385
Craig Topper3614b412018-06-10 06:01:42 +00002386#define _mm512_mask_div_round_ps(W, U, A, B, R) \
2387 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2388 (__v16sf)_mm512_div_round_ps((A), (B), (R)), \
2389 (__v16sf)(__m512)(W));
Asaf Badouhffeb6242015-07-21 15:27:28 +00002390
Craig Topper3614b412018-06-10 06:01:42 +00002391#define _mm512_maskz_div_round_ps(U, A, B, R) \
2392 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2393 (__v16sf)_mm512_div_round_ps((A), (B), (R)), \
2394 (__v16sf)_mm512_setzero_ps());
Asaf Badouhffeb6242015-07-21 15:27:28 +00002395
Craig Topperc6338672018-05-31 00:51:20 +00002396#define _mm512_roundscale_ps(A, B) \
Craig Topper8c18e112016-05-17 04:41:50 +00002397 (__m512)__builtin_ia32_rndscaleps_mask((__v16sf)(__m512)(A), (int)(B), \
Craig Topperae5f0a82018-06-04 02:56:18 +00002398 (__v16sf)_mm512_undefined_ps(), \
2399 (__mmask16)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00002400 _MM_FROUND_CUR_DIRECTION)
Craig Topper72c7d512015-02-01 07:35:35 +00002401
Craig Topperc6338672018-05-31 00:51:20 +00002402#define _mm512_mask_roundscale_ps(A, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00002403 (__m512)__builtin_ia32_rndscaleps_mask((__v16sf)(__m512)(C), (int)(imm), \
2404 (__v16sf)(__m512)(A), (__mmask16)(B), \
Craig Topperc6338672018-05-31 00:51:20 +00002405 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman7360d8a2016-05-10 07:30:58 +00002406
Craig Topperc6338672018-05-31 00:51:20 +00002407#define _mm512_maskz_roundscale_ps(A, B, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00002408 (__m512)__builtin_ia32_rndscaleps_mask((__v16sf)(__m512)(B), (int)(imm), \
2409 (__v16sf)_mm512_setzero_ps(), \
2410 (__mmask16)(A), \
Craig Topperc6338672018-05-31 00:51:20 +00002411 _MM_FROUND_CUR_DIRECTION)
Craig Topper8c18e112016-05-17 04:41:50 +00002412
Craig Topperc6338672018-05-31 00:51:20 +00002413#define _mm512_mask_roundscale_round_ps(A, B, C, imm, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00002414 (__m512)__builtin_ia32_rndscaleps_mask((__v16sf)(__m512)(C), (int)(imm), \
2415 (__v16sf)(__m512)(A), (__mmask16)(B), \
Craig Topperc6338672018-05-31 00:51:20 +00002416 (int)(R))
Michael Zuckermanc301c192016-06-01 07:35:44 +00002417
Craig Topperc6338672018-05-31 00:51:20 +00002418#define _mm512_maskz_roundscale_round_ps(A, B, imm, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00002419 (__m512)__builtin_ia32_rndscaleps_mask((__v16sf)(__m512)(B), (int)(imm), \
2420 (__v16sf)_mm512_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00002421 (__mmask16)(A), (int)(R))
Michael Zuckermanc301c192016-06-01 07:35:44 +00002422
Craig Topperc6338672018-05-31 00:51:20 +00002423#define _mm512_roundscale_round_ps(A, imm, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00002424 (__m512)__builtin_ia32_rndscaleps_mask((__v16sf)(__m512)(A), (int)(imm), \
2425 (__v16sf)_mm512_undefined_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00002426 (__mmask16)-1, (int)(R))
Michael Zuckermanc301c192016-06-01 07:35:44 +00002427
Craig Topperc6338672018-05-31 00:51:20 +00002428#define _mm512_roundscale_pd(A, B) \
Craig Topper8c18e112016-05-17 04:41:50 +00002429 (__m512d)__builtin_ia32_rndscalepd_mask((__v8df)(__m512d)(A), (int)(B), \
Craig Topperae5f0a82018-06-04 02:56:18 +00002430 (__v8df)_mm512_undefined_pd(), \
2431 (__mmask8)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00002432 _MM_FROUND_CUR_DIRECTION)
Adam Nemet0d5bb552014-07-28 17:14:40 +00002433
Craig Topperc6338672018-05-31 00:51:20 +00002434#define _mm512_mask_roundscale_pd(A, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00002435 (__m512d)__builtin_ia32_rndscalepd_mask((__v8df)(__m512d)(C), (int)(imm), \
2436 (__v8df)(__m512d)(A), (__mmask8)(B), \
Craig Topperc6338672018-05-31 00:51:20 +00002437 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman7360d8a2016-05-10 07:30:58 +00002438
Craig Topperc6338672018-05-31 00:51:20 +00002439#define _mm512_maskz_roundscale_pd(A, B, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00002440 (__m512d)__builtin_ia32_rndscalepd_mask((__v8df)(__m512d)(B), (int)(imm), \
2441 (__v8df)_mm512_setzero_pd(), \
2442 (__mmask8)(A), \
Craig Topperc6338672018-05-31 00:51:20 +00002443 _MM_FROUND_CUR_DIRECTION)
Craig Topper8c18e112016-05-17 04:41:50 +00002444
Craig Topperc6338672018-05-31 00:51:20 +00002445#define _mm512_mask_roundscale_round_pd(A, B, C, imm, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00002446 (__m512d)__builtin_ia32_rndscalepd_mask((__v8df)(__m512d)(C), (int)(imm), \
2447 (__v8df)(__m512d)(A), (__mmask8)(B), \
Craig Topperc6338672018-05-31 00:51:20 +00002448 (int)(R))
Michael Zuckermanc301c192016-06-01 07:35:44 +00002449
Craig Topperc6338672018-05-31 00:51:20 +00002450#define _mm512_maskz_roundscale_round_pd(A, B, imm, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00002451 (__m512d)__builtin_ia32_rndscalepd_mask((__v8df)(__m512d)(B), (int)(imm), \
2452 (__v8df)_mm512_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00002453 (__mmask8)(A), (int)(R))
Michael Zuckermanc301c192016-06-01 07:35:44 +00002454
Craig Topperc6338672018-05-31 00:51:20 +00002455#define _mm512_roundscale_round_pd(A, imm, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00002456 (__m512d)__builtin_ia32_rndscalepd_mask((__v8df)(__m512d)(A), (int)(imm), \
2457 (__v8df)_mm512_undefined_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00002458 (__mmask8)-1, (int)(R))
Michael Zuckermanc301c192016-06-01 07:35:44 +00002459
Craig Topperc6338672018-05-31 00:51:20 +00002460#define _mm512_fmadd_round_pd(A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002461 (__m512d)__builtin_ia32_vfmaddpd512_mask((__v8df)(__m512d)(A), \
2462 (__v8df)(__m512d)(B), \
2463 (__v8df)(__m512d)(C), \
2464 (__mmask8)-1, (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002465
2466
Craig Topperc6338672018-05-31 00:51:20 +00002467#define _mm512_mask_fmadd_round_pd(A, U, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002468 (__m512d)__builtin_ia32_vfmaddpd512_mask((__v8df)(__m512d)(A), \
2469 (__v8df)(__m512d)(B), \
2470 (__v8df)(__m512d)(C), \
2471 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002472
2473
Craig Topperc6338672018-05-31 00:51:20 +00002474#define _mm512_mask3_fmadd_round_pd(A, B, C, U, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002475 (__m512d)__builtin_ia32_vfmaddpd512_mask3((__v8df)(__m512d)(A), \
2476 (__v8df)(__m512d)(B), \
2477 (__v8df)(__m512d)(C), \
2478 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002479
2480
Craig Topperc6338672018-05-31 00:51:20 +00002481#define _mm512_maskz_fmadd_round_pd(U, A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002482 (__m512d)__builtin_ia32_vfmaddpd512_maskz((__v8df)(__m512d)(A), \
2483 (__v8df)(__m512d)(B), \
2484 (__v8df)(__m512d)(C), \
2485 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002486
2487
Craig Topperc6338672018-05-31 00:51:20 +00002488#define _mm512_fmsub_round_pd(A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002489 (__m512d)__builtin_ia32_vfmaddpd512_mask((__v8df)(__m512d)(A), \
2490 (__v8df)(__m512d)(B), \
2491 -(__v8df)(__m512d)(C), \
2492 (__mmask8)-1, (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002493
2494
Craig Topperc6338672018-05-31 00:51:20 +00002495#define _mm512_mask_fmsub_round_pd(A, U, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002496 (__m512d)__builtin_ia32_vfmaddpd512_mask((__v8df)(__m512d)(A), \
2497 (__v8df)(__m512d)(B), \
2498 -(__v8df)(__m512d)(C), \
2499 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002500
2501
Craig Topperc6338672018-05-31 00:51:20 +00002502#define _mm512_maskz_fmsub_round_pd(U, A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002503 (__m512d)__builtin_ia32_vfmaddpd512_maskz((__v8df)(__m512d)(A), \
2504 (__v8df)(__m512d)(B), \
2505 -(__v8df)(__m512d)(C), \
2506 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002507
2508
Craig Topperc6338672018-05-31 00:51:20 +00002509#define _mm512_fnmadd_round_pd(A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002510 (__m512d)__builtin_ia32_vfmaddpd512_mask(-(__v8df)(__m512d)(A), \
2511 (__v8df)(__m512d)(B), \
2512 (__v8df)(__m512d)(C), \
2513 (__mmask8)-1, (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002514
2515
Craig Topperc6338672018-05-31 00:51:20 +00002516#define _mm512_mask3_fnmadd_round_pd(A, B, C, U, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002517 (__m512d)__builtin_ia32_vfmaddpd512_mask3(-(__v8df)(__m512d)(A), \
2518 (__v8df)(__m512d)(B), \
2519 (__v8df)(__m512d)(C), \
2520 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002521
2522
Craig Topperc6338672018-05-31 00:51:20 +00002523#define _mm512_maskz_fnmadd_round_pd(U, A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002524 (__m512d)__builtin_ia32_vfmaddpd512_maskz(-(__v8df)(__m512d)(A), \
2525 (__v8df)(__m512d)(B), \
2526 (__v8df)(__m512d)(C), \
2527 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002528
2529
Craig Topperc6338672018-05-31 00:51:20 +00002530#define _mm512_fnmsub_round_pd(A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002531 (__m512d)__builtin_ia32_vfmaddpd512_mask(-(__v8df)(__m512d)(A), \
2532 (__v8df)(__m512d)(B), \
2533 -(__v8df)(__m512d)(C), \
2534 (__mmask8)-1, (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002535
2536
Craig Topperc6338672018-05-31 00:51:20 +00002537#define _mm512_maskz_fnmsub_round_pd(U, A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002538 (__m512d)__builtin_ia32_vfmaddpd512_maskz(-(__v8df)(__m512d)(A), \
2539 (__v8df)(__m512d)(B), \
2540 -(__v8df)(__m512d)(C), \
2541 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002542
2543
Craig Topper74c10e32018-07-09 19:00:16 +00002544static __inline__ __m512d __DEFAULT_FN_ATTRS512
Adam Nemet2278fcb2014-08-14 17:17:57 +00002545_mm512_fmadd_pd(__m512d __A, __m512d __B, __m512d __C)
2546{
Craig Topperb92c77d2018-06-07 02:46:02 +00002547 return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
2548 (__v8df) __B,
2549 (__v8df) __C,
2550 (__mmask8) -1,
2551 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002552}
2553
Craig Topper74c10e32018-07-09 19:00:16 +00002554static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002555_mm512_mask_fmadd_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
2556{
Craig Topperb92c77d2018-06-07 02:46:02 +00002557 return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
2558 (__v8df) __B,
2559 (__v8df) __C,
2560 (__mmask8) __U,
2561 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002562}
2563
Craig Topper74c10e32018-07-09 19:00:16 +00002564static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002565_mm512_mask3_fmadd_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
2566{
Craig Topperb92c77d2018-06-07 02:46:02 +00002567 return (__m512d) __builtin_ia32_vfmaddpd512_mask3 ((__v8df) __A,
2568 (__v8df) __B,
2569 (__v8df) __C,
2570 (__mmask8) __U,
2571 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002572}
2573
Craig Topper74c10e32018-07-09 19:00:16 +00002574static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002575_mm512_maskz_fmadd_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
2576{
Craig Topperb92c77d2018-06-07 02:46:02 +00002577 return (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) __A,
2578 (__v8df) __B,
2579 (__v8df) __C,
2580 (__mmask8) __U,
2581 _MM_FROUND_CUR_DIRECTION);
Adam Nemet2278fcb2014-08-14 17:17:57 +00002582}
2583
Craig Topper74c10e32018-07-09 19:00:16 +00002584static __inline__ __m512d __DEFAULT_FN_ATTRS512
Adam Nemet2278fcb2014-08-14 17:17:57 +00002585_mm512_fmsub_pd(__m512d __A, __m512d __B, __m512d __C)
2586{
Craig Topperb92c77d2018-06-07 02:46:02 +00002587 return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
2588 (__v8df) __B,
2589 -(__v8df) __C,
2590 (__mmask8) -1,
2591 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002592}
2593
Craig Topper74c10e32018-07-09 19:00:16 +00002594static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002595_mm512_mask_fmsub_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
2596{
Craig Topperb92c77d2018-06-07 02:46:02 +00002597 return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
2598 (__v8df) __B,
2599 -(__v8df) __C,
2600 (__mmask8) __U,
2601 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002602}
2603
Craig Topper74c10e32018-07-09 19:00:16 +00002604static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002605_mm512_maskz_fmsub_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
2606{
Craig Topperb92c77d2018-06-07 02:46:02 +00002607 return (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) __A,
2608 (__v8df) __B,
2609 -(__v8df) __C,
2610 (__mmask8) __U,
2611 _MM_FROUND_CUR_DIRECTION);
Adam Nemet2278fcb2014-08-14 17:17:57 +00002612}
2613
Craig Topper74c10e32018-07-09 19:00:16 +00002614static __inline__ __m512d __DEFAULT_FN_ATTRS512
Adam Nemet2278fcb2014-08-14 17:17:57 +00002615_mm512_fnmadd_pd(__m512d __A, __m512d __B, __m512d __C)
2616{
Craig Topperb92c77d2018-06-07 02:46:02 +00002617 return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
2618 -(__v8df) __B,
2619 (__v8df) __C,
2620 (__mmask8) -1,
2621 _MM_FROUND_CUR_DIRECTION);
Adam Nemet2278fcb2014-08-14 17:17:57 +00002622}
2623
Craig Topper74c10e32018-07-09 19:00:16 +00002624static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002625_mm512_mask3_fnmadd_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
2626{
Craig Topperb92c77d2018-06-07 02:46:02 +00002627 return (__m512d) __builtin_ia32_vfmaddpd512_mask3 (-(__v8df) __A,
2628 (__v8df) __B,
2629 (__v8df) __C,
2630 (__mmask8) __U,
2631 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002632}
2633
Craig Topper74c10e32018-07-09 19:00:16 +00002634static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002635_mm512_maskz_fnmadd_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
2636{
Craig Topperb92c77d2018-06-07 02:46:02 +00002637 return (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) __A,
2638 (__v8df) __B,
2639 (__v8df) __C,
2640 (__mmask8) __U,
2641 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002642}
2643
Craig Topper74c10e32018-07-09 19:00:16 +00002644static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002645_mm512_fnmsub_pd(__m512d __A, __m512d __B, __m512d __C)
2646{
Craig Topperb92c77d2018-06-07 02:46:02 +00002647 return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
2648 -(__v8df) __B,
2649 -(__v8df) __C,
2650 (__mmask8) -1,
2651 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002652}
2653
Craig Topper74c10e32018-07-09 19:00:16 +00002654static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002655_mm512_maskz_fnmsub_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
2656{
Craig Topperb92c77d2018-06-07 02:46:02 +00002657 return (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) __A,
2658 (__v8df) __B,
2659 -(__v8df) __C,
2660 (__mmask8) __U,
2661 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002662}
2663
Craig Topperc6338672018-05-31 00:51:20 +00002664#define _mm512_fmadd_round_ps(A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002665 (__m512)__builtin_ia32_vfmaddps512_mask((__v16sf)(__m512)(A), \
2666 (__v16sf)(__m512)(B), \
2667 (__v16sf)(__m512)(C), \
2668 (__mmask16)-1, (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002669
2670
Craig Topperc6338672018-05-31 00:51:20 +00002671#define _mm512_mask_fmadd_round_ps(A, U, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002672 (__m512)__builtin_ia32_vfmaddps512_mask((__v16sf)(__m512)(A), \
2673 (__v16sf)(__m512)(B), \
2674 (__v16sf)(__m512)(C), \
2675 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002676
2677
Craig Topperc6338672018-05-31 00:51:20 +00002678#define _mm512_mask3_fmadd_round_ps(A, B, C, U, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002679 (__m512)__builtin_ia32_vfmaddps512_mask3((__v16sf)(__m512)(A), \
2680 (__v16sf)(__m512)(B), \
2681 (__v16sf)(__m512)(C), \
2682 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002683
2684
Craig Topperc6338672018-05-31 00:51:20 +00002685#define _mm512_maskz_fmadd_round_ps(U, A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002686 (__m512)__builtin_ia32_vfmaddps512_maskz((__v16sf)(__m512)(A), \
2687 (__v16sf)(__m512)(B), \
2688 (__v16sf)(__m512)(C), \
2689 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002690
2691
Craig Topperc6338672018-05-31 00:51:20 +00002692#define _mm512_fmsub_round_ps(A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002693 (__m512)__builtin_ia32_vfmaddps512_mask((__v16sf)(__m512)(A), \
2694 (__v16sf)(__m512)(B), \
2695 -(__v16sf)(__m512)(C), \
2696 (__mmask16)-1, (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002697
2698
Craig Topperc6338672018-05-31 00:51:20 +00002699#define _mm512_mask_fmsub_round_ps(A, U, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002700 (__m512)__builtin_ia32_vfmaddps512_mask((__v16sf)(__m512)(A), \
2701 (__v16sf)(__m512)(B), \
2702 -(__v16sf)(__m512)(C), \
2703 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002704
2705
Craig Topperc6338672018-05-31 00:51:20 +00002706#define _mm512_maskz_fmsub_round_ps(U, A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002707 (__m512)__builtin_ia32_vfmaddps512_maskz((__v16sf)(__m512)(A), \
2708 (__v16sf)(__m512)(B), \
2709 -(__v16sf)(__m512)(C), \
2710 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002711
2712
Craig Topperc6338672018-05-31 00:51:20 +00002713#define _mm512_fnmadd_round_ps(A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002714 (__m512)__builtin_ia32_vfmaddps512_mask((__v16sf)(__m512)(A), \
2715 -(__v16sf)(__m512)(B), \
2716 (__v16sf)(__m512)(C), \
2717 (__mmask16)-1, (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002718
2719
Craig Topperc6338672018-05-31 00:51:20 +00002720#define _mm512_mask3_fnmadd_round_ps(A, B, C, U, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002721 (__m512)__builtin_ia32_vfmaddps512_mask3(-(__v16sf)(__m512)(A), \
2722 (__v16sf)(__m512)(B), \
2723 (__v16sf)(__m512)(C), \
2724 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002725
2726
Craig Topperc6338672018-05-31 00:51:20 +00002727#define _mm512_maskz_fnmadd_round_ps(U, A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002728 (__m512)__builtin_ia32_vfmaddps512_maskz(-(__v16sf)(__m512)(A), \
2729 (__v16sf)(__m512)(B), \
2730 (__v16sf)(__m512)(C), \
2731 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002732
2733
Craig Topperc6338672018-05-31 00:51:20 +00002734#define _mm512_fnmsub_round_ps(A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002735 (__m512)__builtin_ia32_vfmaddps512_mask((__v16sf)(__m512)(A), \
2736 -(__v16sf)(__m512)(B), \
2737 -(__v16sf)(__m512)(C), \
2738 (__mmask16)-1, (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002739
2740
Craig Topperc6338672018-05-31 00:51:20 +00002741#define _mm512_maskz_fnmsub_round_ps(U, A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002742 (__m512)__builtin_ia32_vfmaddps512_maskz(-(__v16sf)(__m512)(A), \
2743 (__v16sf)(__m512)(B), \
2744 -(__v16sf)(__m512)(C), \
2745 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002746
2747
Craig Topper74c10e32018-07-09 19:00:16 +00002748static __inline__ __m512 __DEFAULT_FN_ATTRS512
Adam Nemet2278fcb2014-08-14 17:17:57 +00002749_mm512_fmadd_ps(__m512 __A, __m512 __B, __m512 __C)
2750{
Craig Topperb92c77d2018-06-07 02:46:02 +00002751 return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
2752 (__v16sf) __B,
2753 (__v16sf) __C,
2754 (__mmask16) -1,
2755 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002756}
2757
Craig Topper74c10e32018-07-09 19:00:16 +00002758static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002759_mm512_mask_fmadd_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
2760{
Craig Topperb92c77d2018-06-07 02:46:02 +00002761 return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
2762 (__v16sf) __B,
2763 (__v16sf) __C,
2764 (__mmask16) __U,
2765 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002766}
2767
Craig Topper74c10e32018-07-09 19:00:16 +00002768static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002769_mm512_mask3_fmadd_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
2770{
Craig Topperb92c77d2018-06-07 02:46:02 +00002771 return (__m512) __builtin_ia32_vfmaddps512_mask3 ((__v16sf) __A,
2772 (__v16sf) __B,
2773 (__v16sf) __C,
2774 (__mmask16) __U,
2775 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002776}
2777
Craig Topper74c10e32018-07-09 19:00:16 +00002778static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002779_mm512_maskz_fmadd_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
2780{
Craig Topperb92c77d2018-06-07 02:46:02 +00002781 return (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) __A,
2782 (__v16sf) __B,
2783 (__v16sf) __C,
2784 (__mmask16) __U,
2785 _MM_FROUND_CUR_DIRECTION);
Adam Nemet2278fcb2014-08-14 17:17:57 +00002786}
2787
Craig Topper74c10e32018-07-09 19:00:16 +00002788static __inline__ __m512 __DEFAULT_FN_ATTRS512
Adam Nemet2278fcb2014-08-14 17:17:57 +00002789_mm512_fmsub_ps(__m512 __A, __m512 __B, __m512 __C)
2790{
Craig Topperb92c77d2018-06-07 02:46:02 +00002791 return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
2792 (__v16sf) __B,
2793 -(__v16sf) __C,
2794 (__mmask16) -1,
2795 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002796}
2797
Craig Topper74c10e32018-07-09 19:00:16 +00002798static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002799_mm512_mask_fmsub_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
2800{
Craig Topperb92c77d2018-06-07 02:46:02 +00002801 return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
2802 (__v16sf) __B,
2803 -(__v16sf) __C,
2804 (__mmask16) __U,
2805 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002806}
2807
Craig Topper74c10e32018-07-09 19:00:16 +00002808static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002809_mm512_maskz_fmsub_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
2810{
Craig Topperb92c77d2018-06-07 02:46:02 +00002811 return (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) __A,
2812 (__v16sf) __B,
2813 -(__v16sf) __C,
2814 (__mmask16) __U,
2815 _MM_FROUND_CUR_DIRECTION);
Adam Nemet2278fcb2014-08-14 17:17:57 +00002816}
2817
Craig Topper74c10e32018-07-09 19:00:16 +00002818static __inline__ __m512 __DEFAULT_FN_ATTRS512
Adam Nemet2278fcb2014-08-14 17:17:57 +00002819_mm512_fnmadd_ps(__m512 __A, __m512 __B, __m512 __C)
2820{
Craig Topperb92c77d2018-06-07 02:46:02 +00002821 return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
2822 -(__v16sf) __B,
2823 (__v16sf) __C,
2824 (__mmask16) -1,
2825 _MM_FROUND_CUR_DIRECTION);
Adam Nemet2278fcb2014-08-14 17:17:57 +00002826}
2827
Craig Topper74c10e32018-07-09 19:00:16 +00002828static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002829_mm512_mask3_fnmadd_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
2830{
Craig Topperb92c77d2018-06-07 02:46:02 +00002831 return (__m512) __builtin_ia32_vfmaddps512_mask3 (-(__v16sf) __A,
2832 (__v16sf) __B,
2833 (__v16sf) __C,
2834 (__mmask16) __U,
2835 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002836}
2837
Craig Topper74c10e32018-07-09 19:00:16 +00002838static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002839_mm512_maskz_fnmadd_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
2840{
Craig Topperb92c77d2018-06-07 02:46:02 +00002841 return (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) __A,
2842 (__v16sf) __B,
2843 (__v16sf) __C,
2844 (__mmask16) __U,
2845 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002846}
2847
Craig Topper74c10e32018-07-09 19:00:16 +00002848static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002849_mm512_fnmsub_ps(__m512 __A, __m512 __B, __m512 __C)
2850{
Craig Topperb92c77d2018-06-07 02:46:02 +00002851 return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
2852 -(__v16sf) __B,
2853 -(__v16sf) __C,
2854 (__mmask16) -1,
2855 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002856}
2857
Craig Topper74c10e32018-07-09 19:00:16 +00002858static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002859_mm512_maskz_fnmsub_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
2860{
Craig Topperb92c77d2018-06-07 02:46:02 +00002861 return (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) __A,
2862 (__v16sf) __B,
2863 -(__v16sf) __C,
2864 (__mmask16) __U,
2865 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002866}
2867
Craig Topperc6338672018-05-31 00:51:20 +00002868#define _mm512_fmaddsub_round_pd(A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002869 (__m512d)__builtin_ia32_vfmaddsubpd512_mask((__v8df)(__m512d)(A), \
2870 (__v8df)(__m512d)(B), \
2871 (__v8df)(__m512d)(C), \
2872 (__mmask8)-1, (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002873
2874
Craig Topperc6338672018-05-31 00:51:20 +00002875#define _mm512_mask_fmaddsub_round_pd(A, U, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002876 (__m512d)__builtin_ia32_vfmaddsubpd512_mask((__v8df)(__m512d)(A), \
2877 (__v8df)(__m512d)(B), \
2878 (__v8df)(__m512d)(C), \
2879 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002880
2881
Craig Topperc6338672018-05-31 00:51:20 +00002882#define _mm512_mask3_fmaddsub_round_pd(A, B, C, U, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002883 (__m512d)__builtin_ia32_vfmaddsubpd512_mask3((__v8df)(__m512d)(A), \
2884 (__v8df)(__m512d)(B), \
2885 (__v8df)(__m512d)(C), \
2886 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002887
2888
Craig Topperc6338672018-05-31 00:51:20 +00002889#define _mm512_maskz_fmaddsub_round_pd(U, A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002890 (__m512d)__builtin_ia32_vfmaddsubpd512_maskz((__v8df)(__m512d)(A), \
2891 (__v8df)(__m512d)(B), \
2892 (__v8df)(__m512d)(C), \
2893 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002894
2895
Craig Topperc6338672018-05-31 00:51:20 +00002896#define _mm512_fmsubadd_round_pd(A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002897 (__m512d)__builtin_ia32_vfmaddsubpd512_mask((__v8df)(__m512d)(A), \
2898 (__v8df)(__m512d)(B), \
2899 -(__v8df)(__m512d)(C), \
2900 (__mmask8)-1, (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002901
2902
Craig Topperc6338672018-05-31 00:51:20 +00002903#define _mm512_mask_fmsubadd_round_pd(A, U, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002904 (__m512d)__builtin_ia32_vfmaddsubpd512_mask((__v8df)(__m512d)(A), \
2905 (__v8df)(__m512d)(B), \
2906 -(__v8df)(__m512d)(C), \
2907 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002908
2909
Craig Topperc6338672018-05-31 00:51:20 +00002910#define _mm512_maskz_fmsubadd_round_pd(U, A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002911 (__m512d)__builtin_ia32_vfmaddsubpd512_maskz((__v8df)(__m512d)(A), \
2912 (__v8df)(__m512d)(B), \
2913 -(__v8df)(__m512d)(C), \
2914 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002915
2916
Craig Topper74c10e32018-07-09 19:00:16 +00002917static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002918_mm512_fmaddsub_pd(__m512d __A, __m512d __B, __m512d __C)
2919{
Craig Topperb92c77d2018-06-07 02:46:02 +00002920 return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A,
2921 (__v8df) __B,
2922 (__v8df) __C,
2923 (__mmask8) -1,
2924 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002925}
2926
Craig Topper74c10e32018-07-09 19:00:16 +00002927static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002928_mm512_mask_fmaddsub_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
2929{
Craig Topperb92c77d2018-06-07 02:46:02 +00002930 return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A,
2931 (__v8df) __B,
2932 (__v8df) __C,
2933 (__mmask8) __U,
2934 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002935}
2936
Craig Topper74c10e32018-07-09 19:00:16 +00002937static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002938_mm512_mask3_fmaddsub_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
2939{
Craig Topperb92c77d2018-06-07 02:46:02 +00002940 return (__m512d) __builtin_ia32_vfmaddsubpd512_mask3 ((__v8df) __A,
2941 (__v8df) __B,
2942 (__v8df) __C,
2943 (__mmask8) __U,
2944 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002945}
2946
Craig Topper74c10e32018-07-09 19:00:16 +00002947static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002948_mm512_maskz_fmaddsub_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
2949{
Craig Topperb92c77d2018-06-07 02:46:02 +00002950 return (__m512d) __builtin_ia32_vfmaddsubpd512_maskz ((__v8df) __A,
2951 (__v8df) __B,
2952 (__v8df) __C,
2953 (__mmask8) __U,
2954 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002955}
2956
Craig Topper74c10e32018-07-09 19:00:16 +00002957static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002958_mm512_fmsubadd_pd(__m512d __A, __m512d __B, __m512d __C)
2959{
Craig Topperb92c77d2018-06-07 02:46:02 +00002960 return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A,
2961 (__v8df) __B,
2962 -(__v8df) __C,
2963 (__mmask8) -1,
2964 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002965}
2966
Craig Topper74c10e32018-07-09 19:00:16 +00002967static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002968_mm512_mask_fmsubadd_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
2969{
Craig Topperb92c77d2018-06-07 02:46:02 +00002970 return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A,
2971 (__v8df) __B,
2972 -(__v8df) __C,
2973 (__mmask8) __U,
2974 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002975}
2976
Craig Topper74c10e32018-07-09 19:00:16 +00002977static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002978_mm512_maskz_fmsubadd_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
2979{
Craig Topperb92c77d2018-06-07 02:46:02 +00002980 return (__m512d) __builtin_ia32_vfmaddsubpd512_maskz ((__v8df) __A,
2981 (__v8df) __B,
2982 -(__v8df) __C,
2983 (__mmask8) __U,
2984 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002985}
2986
Craig Topperc6338672018-05-31 00:51:20 +00002987#define _mm512_fmaddsub_round_ps(A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002988 (__m512)__builtin_ia32_vfmaddsubps512_mask((__v16sf)(__m512)(A), \
2989 (__v16sf)(__m512)(B), \
2990 (__v16sf)(__m512)(C), \
2991 (__mmask16)-1, (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002992
2993
Craig Topperc6338672018-05-31 00:51:20 +00002994#define _mm512_mask_fmaddsub_round_ps(A, U, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00002995 (__m512)__builtin_ia32_vfmaddsubps512_mask((__v16sf)(__m512)(A), \
2996 (__v16sf)(__m512)(B), \
2997 (__v16sf)(__m512)(C), \
2998 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00002999
3000
Craig Topperc6338672018-05-31 00:51:20 +00003001#define _mm512_mask3_fmaddsub_round_ps(A, B, C, U, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003002 (__m512)__builtin_ia32_vfmaddsubps512_mask3((__v16sf)(__m512)(A), \
3003 (__v16sf)(__m512)(B), \
3004 (__v16sf)(__m512)(C), \
3005 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003006
3007
Craig Topperc6338672018-05-31 00:51:20 +00003008#define _mm512_maskz_fmaddsub_round_ps(U, A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003009 (__m512)__builtin_ia32_vfmaddsubps512_maskz((__v16sf)(__m512)(A), \
3010 (__v16sf)(__m512)(B), \
3011 (__v16sf)(__m512)(C), \
3012 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003013
3014
Craig Topperc6338672018-05-31 00:51:20 +00003015#define _mm512_fmsubadd_round_ps(A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003016 (__m512)__builtin_ia32_vfmaddsubps512_mask((__v16sf)(__m512)(A), \
3017 (__v16sf)(__m512)(B), \
3018 -(__v16sf)(__m512)(C), \
3019 (__mmask16)-1, (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003020
3021
Craig Topperc6338672018-05-31 00:51:20 +00003022#define _mm512_mask_fmsubadd_round_ps(A, U, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003023 (__m512)__builtin_ia32_vfmaddsubps512_mask((__v16sf)(__m512)(A), \
3024 (__v16sf)(__m512)(B), \
3025 -(__v16sf)(__m512)(C), \
3026 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003027
3028
Craig Topperc6338672018-05-31 00:51:20 +00003029#define _mm512_maskz_fmsubadd_round_ps(U, A, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003030 (__m512)__builtin_ia32_vfmaddsubps512_maskz((__v16sf)(__m512)(A), \
3031 (__v16sf)(__m512)(B), \
3032 -(__v16sf)(__m512)(C), \
3033 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003034
3035
Craig Topper74c10e32018-07-09 19:00:16 +00003036static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003037_mm512_fmaddsub_ps(__m512 __A, __m512 __B, __m512 __C)
3038{
Craig Topperb92c77d2018-06-07 02:46:02 +00003039 return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A,
3040 (__v16sf) __B,
3041 (__v16sf) __C,
3042 (__mmask16) -1,
3043 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003044}
3045
Craig Topper74c10e32018-07-09 19:00:16 +00003046static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003047_mm512_mask_fmaddsub_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
3048{
Craig Topperb92c77d2018-06-07 02:46:02 +00003049 return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A,
3050 (__v16sf) __B,
3051 (__v16sf) __C,
3052 (__mmask16) __U,
3053 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003054}
3055
Craig Topper74c10e32018-07-09 19:00:16 +00003056static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003057_mm512_mask3_fmaddsub_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
3058{
Craig Topperb92c77d2018-06-07 02:46:02 +00003059 return (__m512) __builtin_ia32_vfmaddsubps512_mask3 ((__v16sf) __A,
3060 (__v16sf) __B,
3061 (__v16sf) __C,
3062 (__mmask16) __U,
3063 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003064}
3065
Craig Topper74c10e32018-07-09 19:00:16 +00003066static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003067_mm512_maskz_fmaddsub_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
3068{
Craig Topperb92c77d2018-06-07 02:46:02 +00003069 return (__m512) __builtin_ia32_vfmaddsubps512_maskz ((__v16sf) __A,
3070 (__v16sf) __B,
3071 (__v16sf) __C,
3072 (__mmask16) __U,
3073 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003074}
3075
Craig Topper74c10e32018-07-09 19:00:16 +00003076static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003077_mm512_fmsubadd_ps(__m512 __A, __m512 __B, __m512 __C)
3078{
Craig Topperb92c77d2018-06-07 02:46:02 +00003079 return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A,
3080 (__v16sf) __B,
3081 -(__v16sf) __C,
3082 (__mmask16) -1,
3083 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003084}
3085
Craig Topper74c10e32018-07-09 19:00:16 +00003086static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003087_mm512_mask_fmsubadd_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
3088{
Craig Topperb92c77d2018-06-07 02:46:02 +00003089 return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A,
3090 (__v16sf) __B,
3091 -(__v16sf) __C,
3092 (__mmask16) __U,
3093 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003094}
3095
Craig Topper74c10e32018-07-09 19:00:16 +00003096static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003097_mm512_maskz_fmsubadd_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
3098{
Craig Topperb92c77d2018-06-07 02:46:02 +00003099 return (__m512) __builtin_ia32_vfmaddsubps512_maskz ((__v16sf) __A,
3100 (__v16sf) __B,
3101 -(__v16sf) __C,
3102 (__mmask16) __U,
3103 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003104}
3105
Craig Topperc6338672018-05-31 00:51:20 +00003106#define _mm512_mask3_fmsub_round_pd(A, B, C, U, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003107 (__m512d)__builtin_ia32_vfmsubpd512_mask3((__v8df)(__m512d)(A), \
3108 (__v8df)(__m512d)(B), \
3109 (__v8df)(__m512d)(C), \
3110 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003111
3112
Craig Topper74c10e32018-07-09 19:00:16 +00003113static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003114_mm512_mask3_fmsub_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
3115{
Craig Topperb92c77d2018-06-07 02:46:02 +00003116 return (__m512d)__builtin_ia32_vfmsubpd512_mask3 ((__v8df) __A,
3117 (__v8df) __B,
3118 (__v8df) __C,
3119 (__mmask8) __U,
3120 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003121}
3122
Craig Topperc6338672018-05-31 00:51:20 +00003123#define _mm512_mask3_fmsub_round_ps(A, B, C, U, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003124 (__m512)__builtin_ia32_vfmsubps512_mask3((__v16sf)(__m512)(A), \
3125 (__v16sf)(__m512)(B), \
3126 (__v16sf)(__m512)(C), \
3127 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003128
Craig Topper74c10e32018-07-09 19:00:16 +00003129static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003130_mm512_mask3_fmsub_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
3131{
Craig Topperb92c77d2018-06-07 02:46:02 +00003132 return (__m512)__builtin_ia32_vfmsubps512_mask3 ((__v16sf) __A,
3133 (__v16sf) __B,
3134 (__v16sf) __C,
3135 (__mmask16) __U,
3136 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003137}
3138
Craig Topperc6338672018-05-31 00:51:20 +00003139#define _mm512_mask3_fmsubadd_round_pd(A, B, C, U, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003140 (__m512d)__builtin_ia32_vfmsubaddpd512_mask3((__v8df)(__m512d)(A), \
3141 (__v8df)(__m512d)(B), \
3142 (__v8df)(__m512d)(C), \
3143 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003144
3145
Craig Topper74c10e32018-07-09 19:00:16 +00003146static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003147_mm512_mask3_fmsubadd_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
3148{
Craig Topperb92c77d2018-06-07 02:46:02 +00003149 return (__m512d)__builtin_ia32_vfmsubaddpd512_mask3 ((__v8df) __A,
3150 (__v8df) __B,
3151 (__v8df) __C,
3152 (__mmask8) __U,
3153 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003154}
3155
Craig Topperc6338672018-05-31 00:51:20 +00003156#define _mm512_mask3_fmsubadd_round_ps(A, B, C, U, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003157 (__m512)__builtin_ia32_vfmsubaddps512_mask3((__v16sf)(__m512)(A), \
3158 (__v16sf)(__m512)(B), \
3159 (__v16sf)(__m512)(C), \
3160 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003161
3162
Craig Topper74c10e32018-07-09 19:00:16 +00003163static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003164_mm512_mask3_fmsubadd_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
3165{
Craig Topperb92c77d2018-06-07 02:46:02 +00003166 return (__m512)__builtin_ia32_vfmsubaddps512_mask3 ((__v16sf) __A,
3167 (__v16sf) __B,
3168 (__v16sf) __C,
3169 (__mmask16) __U,
3170 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003171}
3172
Craig Topperc6338672018-05-31 00:51:20 +00003173#define _mm512_mask_fnmadd_round_pd(A, U, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003174 (__m512d)__builtin_ia32_vfmaddpd512_mask((__v8df)(__m512d)(A), \
3175 -(__v8df)(__m512d)(B), \
3176 (__v8df)(__m512d)(C), \
3177 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003178
3179
Craig Topper74c10e32018-07-09 19:00:16 +00003180static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003181_mm512_mask_fnmadd_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
3182{
Craig Topperb92c77d2018-06-07 02:46:02 +00003183 return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
3184 -(__v8df) __B,
3185 (__v8df) __C,
3186 (__mmask8) __U,
3187 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003188}
3189
Craig Topperc6338672018-05-31 00:51:20 +00003190#define _mm512_mask_fnmadd_round_ps(A, U, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003191 (__m512)__builtin_ia32_vfmaddps512_mask((__v16sf)(__m512)(A), \
3192 -(__v16sf)(__m512)(B), \
3193 (__v16sf)(__m512)(C), \
3194 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003195
3196
Craig Topper74c10e32018-07-09 19:00:16 +00003197static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003198_mm512_mask_fnmadd_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
3199{
Craig Topperb92c77d2018-06-07 02:46:02 +00003200 return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
3201 -(__v16sf) __B,
3202 (__v16sf) __C,
3203 (__mmask16) __U,
3204 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003205}
3206
Craig Topperc6338672018-05-31 00:51:20 +00003207#define _mm512_mask_fnmsub_round_pd(A, U, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003208 (__m512d)__builtin_ia32_vfmaddpd512_mask((__v8df)(__m512d)(A), \
3209 -(__v8df)(__m512d)(B), \
3210 -(__v8df)(__m512d)(C), \
3211 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003212
3213
Craig Topperc6338672018-05-31 00:51:20 +00003214#define _mm512_mask3_fnmsub_round_pd(A, B, C, U, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003215 (__m512d)__builtin_ia32_vfmsubpd512_mask3(-(__v8df)(__m512d)(A), \
3216 (__v8df)(__m512d)(B), \
3217 (__v8df)(__m512d)(C), \
3218 (__mmask8)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003219
3220
Craig Topper74c10e32018-07-09 19:00:16 +00003221static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003222_mm512_mask_fnmsub_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
3223{
Craig Topperb92c77d2018-06-07 02:46:02 +00003224 return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
3225 -(__v8df) __B,
3226 -(__v8df) __C,
3227 (__mmask8) __U,
3228 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003229}
3230
Craig Topper74c10e32018-07-09 19:00:16 +00003231static __inline__ __m512d __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003232_mm512_mask3_fnmsub_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
3233{
Craig Topperb92c77d2018-06-07 02:46:02 +00003234 return (__m512d) __builtin_ia32_vfmsubpd512_mask3 (-(__v8df) __A,
3235 (__v8df) __B,
3236 (__v8df) __C,
3237 (__mmask8) __U,
3238 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003239}
3240
Craig Topperc6338672018-05-31 00:51:20 +00003241#define _mm512_mask_fnmsub_round_ps(A, U, B, C, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003242 (__m512)__builtin_ia32_vfmaddps512_mask((__v16sf)(__m512)(A), \
3243 -(__v16sf)(__m512)(B), \
3244 -(__v16sf)(__m512)(C), \
3245 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003246
3247
Craig Topperc6338672018-05-31 00:51:20 +00003248#define _mm512_mask3_fnmsub_round_ps(A, B, C, U, R) \
Craig Topperb92c77d2018-06-07 02:46:02 +00003249 (__m512)__builtin_ia32_vfmsubps512_mask3(-(__v16sf)(__m512)(A), \
3250 (__v16sf)(__m512)(B), \
3251 (__v16sf)(__m512)(C), \
3252 (__mmask16)(U), (int)(R))
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003253
3254
Craig Topper74c10e32018-07-09 19:00:16 +00003255static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003256_mm512_mask_fnmsub_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
3257{
Craig Topperb92c77d2018-06-07 02:46:02 +00003258 return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
3259 -(__v16sf) __B,
3260 -(__v16sf) __C,
3261 (__mmask16) __U,
3262 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003263}
3264
Craig Topper74c10e32018-07-09 19:00:16 +00003265static __inline__ __m512 __DEFAULT_FN_ATTRS512
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003266_mm512_mask3_fnmsub_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
3267{
Craig Topperb92c77d2018-06-07 02:46:02 +00003268 return (__m512) __builtin_ia32_vfmsubps512_mask3 (-(__v16sf) __A,
3269 (__v16sf) __B,
3270 (__v16sf) __C,
3271 (__mmask16) __U,
3272 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyc563c2c2015-06-29 09:20:57 +00003273}
3274
3275
3276
Adam Nemet0d5bb552014-07-28 17:14:40 +00003277/* Vector permutations */
3278
Craig Topper74c10e32018-07-09 19:00:16 +00003279static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00003280_mm512_permutex2var_epi32(__m512i __A, __m512i __I, __m512i __B)
3281{
Craig Topper68a272d2018-05-29 03:26:38 +00003282 return (__m512i)__builtin_ia32_vpermi2vard512((__v16si)__A, (__v16si) __I,
3283 (__v16si) __B);
Adam Nemet0d5bb552014-07-28 17:14:40 +00003284}
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00003285
Craig Topper74c10e32018-07-09 19:00:16 +00003286static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper68a272d2018-05-29 03:26:38 +00003287_mm512_mask_permutex2var_epi32(__m512i __A, __mmask16 __U, __m512i __I,
3288 __m512i __B)
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00003289{
Craig Topper68a272d2018-05-29 03:26:38 +00003290 return (__m512i)__builtin_ia32_selectd_512(__U,
3291 (__v16si)_mm512_permutex2var_epi32(__A, __I, __B),
3292 (__v16si)__A);
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00003293}
3294
Craig Topper74c10e32018-07-09 19:00:16 +00003295static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper68a272d2018-05-29 03:26:38 +00003296_mm512_mask2_permutex2var_epi32(__m512i __A, __m512i __I, __mmask16 __U,
3297 __m512i __B)
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00003298{
Craig Topper68a272d2018-05-29 03:26:38 +00003299 return (__m512i)__builtin_ia32_selectd_512(__U,
3300 (__v16si)_mm512_permutex2var_epi32(__A, __I, __B),
3301 (__v16si)__I);
3302}
3303
Craig Topper74c10e32018-07-09 19:00:16 +00003304static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper68a272d2018-05-29 03:26:38 +00003305_mm512_maskz_permutex2var_epi32(__mmask16 __U, __m512i __A, __m512i __I,
3306 __m512i __B)
3307{
3308 return (__m512i)__builtin_ia32_selectd_512(__U,
3309 (__v16si)_mm512_permutex2var_epi32(__A, __I, __B),
3310 (__v16si)_mm512_setzero_si512());
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00003311}
3312
Craig Topper74c10e32018-07-09 19:00:16 +00003313static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00003314_mm512_permutex2var_epi64(__m512i __A, __m512i __I, __m512i __B)
3315{
Craig Topper68a272d2018-05-29 03:26:38 +00003316 return (__m512i)__builtin_ia32_vpermi2varq512((__v8di)__A, (__v8di) __I,
3317 (__v8di) __B);
Adam Nemet0d5bb552014-07-28 17:14:40 +00003318}
3319
Craig Topper74c10e32018-07-09 19:00:16 +00003320static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper68a272d2018-05-29 03:26:38 +00003321_mm512_mask_permutex2var_epi64(__m512i __A, __mmask8 __U, __m512i __I,
3322 __m512i __B)
3323{
3324 return (__m512i)__builtin_ia32_selectq_512(__U,
3325 (__v8di)_mm512_permutex2var_epi64(__A, __I, __B),
3326 (__v8di)__A);
3327}
3328
Craig Topper74c10e32018-07-09 19:00:16 +00003329static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper68a272d2018-05-29 03:26:38 +00003330_mm512_mask2_permutex2var_epi64(__m512i __A, __m512i __I, __mmask8 __U,
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00003331 __m512i __B)
Adam Nemet0d5bb552014-07-28 17:14:40 +00003332{
Craig Topper68a272d2018-05-29 03:26:38 +00003333 return (__m512i)__builtin_ia32_selectq_512(__U,
3334 (__v8di)_mm512_permutex2var_epi64(__A, __I, __B),
3335 (__v8di)__I);
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00003336}
3337
Craig Topper74c10e32018-07-09 19:00:16 +00003338static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper68a272d2018-05-29 03:26:38 +00003339_mm512_maskz_permutex2var_epi64(__mmask8 __U, __m512i __A, __m512i __I,
3340 __m512i __B)
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00003341{
Craig Topper68a272d2018-05-29 03:26:38 +00003342 return (__m512i)__builtin_ia32_selectq_512(__U,
3343 (__v8di)_mm512_permutex2var_epi64(__A, __I, __B),
3344 (__v8di)_mm512_setzero_si512());
Adam Nemet0d5bb552014-07-28 17:14:40 +00003345}
3346
Craig Topperc6338672018-05-31 00:51:20 +00003347#define _mm512_alignr_epi64(A, B, I) \
Craig Toppere56819e2018-06-07 21:27:41 +00003348 (__m512i)__builtin_ia32_alignq512((__v8di)(__m512i)(A), \
3349 (__v8di)(__m512i)(B), (int)(I))
Adam Nemet5bf7baa2014-08-05 17:28:23 +00003350
Craig Topperc6338672018-05-31 00:51:20 +00003351#define _mm512_mask_alignr_epi64(W, U, A, B, imm) \
Craig Topper6aefe002016-11-23 01:47:12 +00003352 (__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
3353 (__v8di)_mm512_alignr_epi64((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00003354 (__v8di)(__m512i)(W))
Michael Zuckerman533e0652016-04-28 12:47:30 +00003355
Craig Topperc6338672018-05-31 00:51:20 +00003356#define _mm512_maskz_alignr_epi64(U, A, B, imm) \
Craig Topper6aefe002016-11-23 01:47:12 +00003357 (__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
3358 (__v8di)_mm512_alignr_epi64((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00003359 (__v8di)_mm512_setzero_si512())
Michael Zuckerman533e0652016-04-28 12:47:30 +00003360
Craig Topperc6338672018-05-31 00:51:20 +00003361#define _mm512_alignr_epi32(A, B, I) \
Craig Toppere56819e2018-06-07 21:27:41 +00003362 (__m512i)__builtin_ia32_alignd512((__v16si)(__m512i)(A), \
3363 (__v16si)(__m512i)(B), (int)(I))
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00003364
Craig Topperc6338672018-05-31 00:51:20 +00003365#define _mm512_mask_alignr_epi32(W, U, A, B, imm) \
Craig Topper6aefe002016-11-23 01:47:12 +00003366 (__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
3367 (__v16si)_mm512_alignr_epi32((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00003368 (__v16si)(__m512i)(W))
Adam Nemet5bf7baa2014-08-05 17:28:23 +00003369
Craig Topperc6338672018-05-31 00:51:20 +00003370#define _mm512_maskz_alignr_epi32(U, A, B, imm) \
Craig Topper6aefe002016-11-23 01:47:12 +00003371 (__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
3372 (__v16si)_mm512_alignr_epi32((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00003373 (__v16si)_mm512_setzero_si512())
Adam Nemetf893ede2015-01-19 20:12:05 +00003374/* Vector Extract */
3375
Craig Topperc6338672018-05-31 00:51:20 +00003376#define _mm512_extractf64x4_pd(A, I) \
Craig Topper5f50f3382018-06-08 21:50:07 +00003377 (__m256d)__builtin_ia32_extractf64x4_mask((__v8df)(__m512d)(A), (int)(I), \
Craig Topper5cbeeed2018-07-07 17:03:32 +00003378 (__v4df)_mm256_undefined_pd(), \
Craig Topper5f50f3382018-06-08 21:50:07 +00003379 (__mmask8)-1)
Adam Nemetf893ede2015-01-19 20:12:05 +00003380
Craig Topperc6338672018-05-31 00:51:20 +00003381#define _mm512_mask_extractf64x4_pd(W, U, A, imm) \
Craig Topper5f50f3382018-06-08 21:50:07 +00003382 (__m256d)__builtin_ia32_extractf64x4_mask((__v8df)(__m512d)(A), (int)(imm), \
3383 (__v4df)(__m256d)(W), \
3384 (__mmask8)(U))
Michael Zuckerman2564d2f2016-05-10 10:14:50 +00003385
Craig Topperc6338672018-05-31 00:51:20 +00003386#define _mm512_maskz_extractf64x4_pd(U, A, imm) \
Craig Topper5f50f3382018-06-08 21:50:07 +00003387 (__m256d)__builtin_ia32_extractf64x4_mask((__v8df)(__m512d)(A), (int)(imm), \
3388 (__v4df)_mm256_setzero_pd(), \
3389 (__mmask8)(U))
Michael Zuckerman2564d2f2016-05-10 10:14:50 +00003390
Craig Topperc6338672018-05-31 00:51:20 +00003391#define _mm512_extractf32x4_ps(A, I) \
Craig Topper5f50f3382018-06-08 21:50:07 +00003392 (__m128)__builtin_ia32_extractf32x4_mask((__v16sf)(__m512)(A), (int)(I), \
3393 (__v4sf)_mm_undefined_ps(), \
3394 (__mmask8)-1)
Adam Nemetf893ede2015-01-19 20:12:05 +00003395
Craig Topperc6338672018-05-31 00:51:20 +00003396#define _mm512_mask_extractf32x4_ps(W, U, A, imm) \
Craig Topper5f50f3382018-06-08 21:50:07 +00003397 (__m128)__builtin_ia32_extractf32x4_mask((__v16sf)(__m512)(A), (int)(imm), \
3398 (__v4sf)(__m128)(W), \
3399 (__mmask8)(U))
Michael Zuckerman2564d2f2016-05-10 10:14:50 +00003400
Craig Topperc6338672018-05-31 00:51:20 +00003401#define _mm512_maskz_extractf32x4_ps(U, A, imm) \
Craig Topper5f50f3382018-06-08 21:50:07 +00003402 (__m128)__builtin_ia32_extractf32x4_mask((__v16sf)(__m512)(A), (int)(imm), \
3403 (__v4sf)_mm_setzero_ps(), \
3404 (__mmask8)(U))
Craig Topper93ffabd2016-10-31 04:30:56 +00003405
Adam Nemet0d5bb552014-07-28 17:14:40 +00003406/* Vector Blend */
3407
Craig Topper74c10e32018-07-09 19:00:16 +00003408static __inline __m512d __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00003409_mm512_mask_blend_pd(__mmask8 __U, __m512d __A, __m512d __W)
3410{
Igor Bregeraadb8762016-06-08 13:59:20 +00003411 return (__m512d) __builtin_ia32_selectpd_512 ((__mmask8) __U,
Adam Nemet0d5bb552014-07-28 17:14:40 +00003412 (__v8df) __W,
Igor Bregeraadb8762016-06-08 13:59:20 +00003413 (__v8df) __A);
Adam Nemet0d5bb552014-07-28 17:14:40 +00003414}
3415
Craig Topper74c10e32018-07-09 19:00:16 +00003416static __inline __m512 __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00003417_mm512_mask_blend_ps(__mmask16 __U, __m512 __A, __m512 __W)
3418{
Igor Bregeraadb8762016-06-08 13:59:20 +00003419 return (__m512) __builtin_ia32_selectps_512 ((__mmask16) __U,
Adam Nemet0d5bb552014-07-28 17:14:40 +00003420 (__v16sf) __W,
Igor Bregeraadb8762016-06-08 13:59:20 +00003421 (__v16sf) __A);
Adam Nemet0d5bb552014-07-28 17:14:40 +00003422}
3423
Craig Topper74c10e32018-07-09 19:00:16 +00003424static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00003425_mm512_mask_blend_epi64(__mmask8 __U, __m512i __A, __m512i __W)
3426{
Igor Bregeraadb8762016-06-08 13:59:20 +00003427 return (__m512i) __builtin_ia32_selectq_512 ((__mmask8) __U,
Adam Nemet0d5bb552014-07-28 17:14:40 +00003428 (__v8di) __W,
Igor Bregeraadb8762016-06-08 13:59:20 +00003429 (__v8di) __A);
Adam Nemet0d5bb552014-07-28 17:14:40 +00003430}
3431
Craig Topper74c10e32018-07-09 19:00:16 +00003432static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00003433_mm512_mask_blend_epi32(__mmask16 __U, __m512i __A, __m512i __W)
3434{
Igor Bregeraadb8762016-06-08 13:59:20 +00003435 return (__m512i) __builtin_ia32_selectd_512 ((__mmask16) __U,
Adam Nemet0d5bb552014-07-28 17:14:40 +00003436 (__v16si) __W,
Igor Bregeraadb8762016-06-08 13:59:20 +00003437 (__v16si) __A);
Adam Nemet0d5bb552014-07-28 17:14:40 +00003438}
3439
3440/* Compare */
3441
Craig Topperc6338672018-05-31 00:51:20 +00003442#define _mm512_cmp_round_ps_mask(A, B, P, R) \
Craig Topper53565c62015-02-01 22:27:40 +00003443 (__mmask16)__builtin_ia32_cmpps512_mask((__v16sf)(__m512)(A), \
Craig Topper8c18e112016-05-17 04:41:50 +00003444 (__v16sf)(__m512)(B), (int)(P), \
Craig Topperc6338672018-05-31 00:51:20 +00003445 (__mmask16)-1, (int)(R))
Adam Nemet0d5bb552014-07-28 17:14:40 +00003446
Craig Topperc6338672018-05-31 00:51:20 +00003447#define _mm512_mask_cmp_round_ps_mask(U, A, B, P, R) \
Craig Topper53565c62015-02-01 22:27:40 +00003448 (__mmask16)__builtin_ia32_cmpps512_mask((__v16sf)(__m512)(A), \
Craig Topper8c18e112016-05-17 04:41:50 +00003449 (__v16sf)(__m512)(B), (int)(P), \
Craig Topperc6338672018-05-31 00:51:20 +00003450 (__mmask16)(U), (int)(R))
Craig Topper53565c62015-02-01 22:27:40 +00003451
3452#define _mm512_cmp_ps_mask(A, B, P) \
3453 _mm512_cmp_round_ps_mask((A), (B), (P), _MM_FROUND_CUR_DIRECTION)
Craig Topper53565c62015-02-01 22:27:40 +00003454#define _mm512_mask_cmp_ps_mask(U, A, B, P) \
3455 _mm512_mask_cmp_round_ps_mask((U), (A), (B), (P), _MM_FROUND_CUR_DIRECTION)
3456
Ayman Musa2e250e82016-09-27 14:06:32 +00003457#define _mm512_cmpeq_ps_mask(A, B) \
3458 _mm512_cmp_ps_mask((A), (B), _CMP_EQ_OQ)
3459#define _mm512_mask_cmpeq_ps_mask(k, A, B) \
3460 _mm512_mask_cmp_ps_mask((k), (A), (B), _CMP_EQ_OQ)
3461
3462#define _mm512_cmplt_ps_mask(A, B) \
3463 _mm512_cmp_ps_mask((A), (B), _CMP_LT_OS)
3464#define _mm512_mask_cmplt_ps_mask(k, A, B) \
3465 _mm512_mask_cmp_ps_mask((k), (A), (B), _CMP_LT_OS)
3466
3467#define _mm512_cmple_ps_mask(A, B) \
3468 _mm512_cmp_ps_mask((A), (B), _CMP_LE_OS)
3469#define _mm512_mask_cmple_ps_mask(k, A, B) \
3470 _mm512_mask_cmp_ps_mask((k), (A), (B), _CMP_LE_OS)
3471
3472#define _mm512_cmpunord_ps_mask(A, B) \
3473 _mm512_cmp_ps_mask((A), (B), _CMP_UNORD_Q)
3474#define _mm512_mask_cmpunord_ps_mask(k, A, B) \
3475 _mm512_mask_cmp_ps_mask((k), (A), (B), _CMP_UNORD_Q)
3476
3477#define _mm512_cmpneq_ps_mask(A, B) \
3478 _mm512_cmp_ps_mask((A), (B), _CMP_NEQ_UQ)
3479#define _mm512_mask_cmpneq_ps_mask(k, A, B) \
3480 _mm512_mask_cmp_ps_mask((k), (A), (B), _CMP_NEQ_UQ)
3481
3482#define _mm512_cmpnlt_ps_mask(A, B) \
3483 _mm512_cmp_ps_mask((A), (B), _CMP_NLT_US)
3484#define _mm512_mask_cmpnlt_ps_mask(k, A, B) \
3485 _mm512_mask_cmp_ps_mask((k), (A), (B), _CMP_NLT_US)
3486
3487#define _mm512_cmpnle_ps_mask(A, B) \
3488 _mm512_cmp_ps_mask((A), (B), _CMP_NLE_US)
3489#define _mm512_mask_cmpnle_ps_mask(k, A, B) \
3490 _mm512_mask_cmp_ps_mask((k), (A), (B), _CMP_NLE_US)
3491
3492#define _mm512_cmpord_ps_mask(A, B) \
3493 _mm512_cmp_ps_mask((A), (B), _CMP_ORD_Q)
3494#define _mm512_mask_cmpord_ps_mask(k, A, B) \
3495 _mm512_mask_cmp_ps_mask((k), (A), (B), _CMP_ORD_Q)
3496
Craig Topperc6338672018-05-31 00:51:20 +00003497#define _mm512_cmp_round_pd_mask(A, B, P, R) \
Craig Topper53565c62015-02-01 22:27:40 +00003498 (__mmask8)__builtin_ia32_cmppd512_mask((__v8df)(__m512d)(A), \
Craig Topper8c18e112016-05-17 04:41:50 +00003499 (__v8df)(__m512d)(B), (int)(P), \
Craig Topperc6338672018-05-31 00:51:20 +00003500 (__mmask8)-1, (int)(R))
Craig Topper53565c62015-02-01 22:27:40 +00003501
Craig Topperc6338672018-05-31 00:51:20 +00003502#define _mm512_mask_cmp_round_pd_mask(U, A, B, P, R) \
Craig Topper53565c62015-02-01 22:27:40 +00003503 (__mmask8)__builtin_ia32_cmppd512_mask((__v8df)(__m512d)(A), \
Craig Topper8c18e112016-05-17 04:41:50 +00003504 (__v8df)(__m512d)(B), (int)(P), \
Craig Topperc6338672018-05-31 00:51:20 +00003505 (__mmask8)(U), (int)(R))
Craig Topper53565c62015-02-01 22:27:40 +00003506
3507#define _mm512_cmp_pd_mask(A, B, P) \
3508 _mm512_cmp_round_pd_mask((A), (B), (P), _MM_FROUND_CUR_DIRECTION)
Craig Topper53565c62015-02-01 22:27:40 +00003509#define _mm512_mask_cmp_pd_mask(U, A, B, P) \
3510 _mm512_mask_cmp_round_pd_mask((U), (A), (B), (P), _MM_FROUND_CUR_DIRECTION)
Adam Nemet0d5bb552014-07-28 17:14:40 +00003511
Ayman Musa2e250e82016-09-27 14:06:32 +00003512#define _mm512_cmpeq_pd_mask(A, B) \
3513 _mm512_cmp_pd_mask((A), (B), _CMP_EQ_OQ)
3514#define _mm512_mask_cmpeq_pd_mask(k, A, B) \
3515 _mm512_mask_cmp_pd_mask((k), (A), (B), _CMP_EQ_OQ)
3516
3517#define _mm512_cmplt_pd_mask(A, B) \
3518 _mm512_cmp_pd_mask((A), (B), _CMP_LT_OS)
3519#define _mm512_mask_cmplt_pd_mask(k, A, B) \
3520 _mm512_mask_cmp_pd_mask((k), (A), (B), _CMP_LT_OS)
3521
3522#define _mm512_cmple_pd_mask(A, B) \
3523 _mm512_cmp_pd_mask((A), (B), _CMP_LE_OS)
3524#define _mm512_mask_cmple_pd_mask(k, A, B) \
3525 _mm512_mask_cmp_pd_mask((k), (A), (B), _CMP_LE_OS)
3526
3527#define _mm512_cmpunord_pd_mask(A, B) \
3528 _mm512_cmp_pd_mask((A), (B), _CMP_UNORD_Q)
3529#define _mm512_mask_cmpunord_pd_mask(k, A, B) \
3530 _mm512_mask_cmp_pd_mask((k), (A), (B), _CMP_UNORD_Q)
3531
3532#define _mm512_cmpneq_pd_mask(A, B) \
3533 _mm512_cmp_pd_mask((A), (B), _CMP_NEQ_UQ)
3534#define _mm512_mask_cmpneq_pd_mask(k, A, B) \
3535 _mm512_mask_cmp_pd_mask((k), (A), (B), _CMP_NEQ_UQ)
3536
3537#define _mm512_cmpnlt_pd_mask(A, B) \
3538 _mm512_cmp_pd_mask((A), (B), _CMP_NLT_US)
3539#define _mm512_mask_cmpnlt_pd_mask(k, A, B) \
3540 _mm512_mask_cmp_pd_mask((k), (A), (B), _CMP_NLT_US)
3541
3542#define _mm512_cmpnle_pd_mask(A, B) \
3543 _mm512_cmp_pd_mask((A), (B), _CMP_NLE_US)
3544#define _mm512_mask_cmpnle_pd_mask(k, A, B) \
3545 _mm512_mask_cmp_pd_mask((k), (A), (B), _CMP_NLE_US)
3546
3547#define _mm512_cmpord_pd_mask(A, B) \
3548 _mm512_cmp_pd_mask((A), (B), _CMP_ORD_Q)
3549#define _mm512_mask_cmpord_pd_mask(k, A, B) \
3550 _mm512_mask_cmp_pd_mask((k), (A), (B), _CMP_ORD_Q)
3551
Adam Nemet0d5bb552014-07-28 17:14:40 +00003552/* Conversion */
3553
Craig Topperc6338672018-05-31 00:51:20 +00003554#define _mm512_cvtt_roundps_epu32(A, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00003555 (__m512i)__builtin_ia32_cvttps2udq512_mask((__v16sf)(__m512)(A), \
3556 (__v16si)_mm512_undefined_epi32(), \
Craig Topperc6338672018-05-31 00:51:20 +00003557 (__mmask16)-1, (int)(R))
Michael Zuckerman6170c152016-06-01 14:41:41 +00003558
Craig Topperc6338672018-05-31 00:51:20 +00003559#define _mm512_mask_cvtt_roundps_epu32(W, U, A, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00003560 (__m512i)__builtin_ia32_cvttps2udq512_mask((__v16sf)(__m512)(A), \
3561 (__v16si)(__m512i)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00003562 (__mmask16)(U), (int)(R))
Michael Zuckerman6170c152016-06-01 14:41:41 +00003563
Craig Topperc6338672018-05-31 00:51:20 +00003564#define _mm512_maskz_cvtt_roundps_epu32(U, A, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00003565 (__m512i)__builtin_ia32_cvttps2udq512_mask((__v16sf)(__m512)(A), \
3566 (__v16si)_mm512_setzero_si512(), \
Craig Topperc6338672018-05-31 00:51:20 +00003567 (__mmask16)(U), (int)(R))
Michael Zuckerman6170c152016-06-01 14:41:41 +00003568
3569
Craig Topper74c10e32018-07-09 19:00:16 +00003570static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00003571_mm512_cvttps_epu32(__m512 __A)
3572{
3573 return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A,
3574 (__v16si)
3575 _mm512_setzero_si512 (),
3576 (__mmask16) -1,
3577 _MM_FROUND_CUR_DIRECTION);
3578}
3579
Craig Topper74c10e32018-07-09 19:00:16 +00003580static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003581_mm512_mask_cvttps_epu32 (__m512i __W, __mmask16 __U, __m512 __A)
3582{
3583 return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A,
3584 (__v16si) __W,
3585 (__mmask16) __U,
3586 _MM_FROUND_CUR_DIRECTION);
3587}
3588
Craig Topper74c10e32018-07-09 19:00:16 +00003589static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003590_mm512_maskz_cvttps_epu32 (__mmask16 __U, __m512 __A)
3591{
3592 return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A,
3593 (__v16si) _mm512_setzero_si512 (),
3594 (__mmask16) __U,
3595 _MM_FROUND_CUR_DIRECTION);
3596}
3597
Craig Topperc6338672018-05-31 00:51:20 +00003598#define _mm512_cvt_roundepi32_ps(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00003599 (__m512)__builtin_ia32_cvtdq2ps512_mask((__v16si)(__m512i)(A), \
Craig Topper72c7d512015-02-01 07:35:35 +00003600 (__v16sf)_mm512_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00003601 (__mmask16)-1, (int)(R))
Adam Nemet0d5bb552014-07-28 17:14:40 +00003602
Craig Topperc6338672018-05-31 00:51:20 +00003603#define _mm512_mask_cvt_roundepi32_ps(W, U, A, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00003604 (__m512)__builtin_ia32_cvtdq2ps512_mask((__v16si)(__m512i)(A), \
3605 (__v16sf)(__m512)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00003606 (__mmask16)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00003607
Craig Topperc6338672018-05-31 00:51:20 +00003608#define _mm512_maskz_cvt_roundepi32_ps(U, A, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00003609 (__m512)__builtin_ia32_cvtdq2ps512_mask((__v16si)(__m512i)(A), \
3610 (__v16sf)_mm512_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00003611 (__mmask16)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00003612
Craig Topperc6338672018-05-31 00:51:20 +00003613#define _mm512_cvt_roundepu32_ps(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00003614 (__m512)__builtin_ia32_cvtudq2ps512_mask((__v16si)(__m512i)(A), \
Craig Topper72c7d512015-02-01 07:35:35 +00003615 (__v16sf)_mm512_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00003616 (__mmask16)-1, (int)(R))
Adam Nemet0d5bb552014-07-28 17:14:40 +00003617
Craig Topperc6338672018-05-31 00:51:20 +00003618#define _mm512_mask_cvt_roundepu32_ps(W, U, A, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00003619 (__m512)__builtin_ia32_cvtudq2ps512_mask((__v16si)(__m512i)(A), \
3620 (__v16sf)(__m512)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00003621 (__mmask16)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00003622
Craig Topperc6338672018-05-31 00:51:20 +00003623#define _mm512_maskz_cvt_roundepu32_ps(U, A, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00003624 (__m512)__builtin_ia32_cvtudq2ps512_mask((__v16si)(__m512i)(A), \
3625 (__v16sf)_mm512_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00003626 (__mmask16)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00003627
Craig Topper74c10e32018-07-09 19:00:16 +00003628static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003629_mm512_cvtepu32_ps (__m512i __A)
3630{
Craig Topper842171d2018-05-21 20:19:17 +00003631 return (__m512)__builtin_convertvector((__v16su)__A, __v16sf);
Michael Zuckermanf1544752016-05-09 10:32:51 +00003632}
3633
Craig Topper74c10e32018-07-09 19:00:16 +00003634static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003635_mm512_mask_cvtepu32_ps (__m512 __W, __mmask16 __U, __m512i __A)
3636{
Craig Topper842171d2018-05-21 20:19:17 +00003637 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
3638 (__v16sf)_mm512_cvtepu32_ps(__A),
3639 (__v16sf)__W);
Michael Zuckermanf1544752016-05-09 10:32:51 +00003640}
3641
Craig Topper74c10e32018-07-09 19:00:16 +00003642static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003643_mm512_maskz_cvtepu32_ps (__mmask16 __U, __m512i __A)
3644{
Craig Topper842171d2018-05-21 20:19:17 +00003645 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
3646 (__v16sf)_mm512_cvtepu32_ps(__A),
3647 (__v16sf)_mm512_setzero_ps());
Michael Zuckermanf1544752016-05-09 10:32:51 +00003648}
3649
Craig Topper74c10e32018-07-09 19:00:16 +00003650static __inline __m512d __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00003651_mm512_cvtepi32_pd(__m256i __A)
3652{
Simon Pilgrim698528d2016-11-16 09:27:40 +00003653 return (__m512d)__builtin_convertvector((__v8si)__A, __v8df);
Adam Nemet0d5bb552014-07-28 17:14:40 +00003654}
3655
Craig Topper74c10e32018-07-09 19:00:16 +00003656static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003657_mm512_mask_cvtepi32_pd (__m512d __W, __mmask8 __U, __m256i __A)
3658{
Simon Pilgrim698528d2016-11-16 09:27:40 +00003659 return (__m512d)__builtin_ia32_selectpd_512((__mmask8) __U,
3660 (__v8df)_mm512_cvtepi32_pd(__A),
3661 (__v8df)__W);
Michael Zuckermanf1544752016-05-09 10:32:51 +00003662}
3663
Craig Topper74c10e32018-07-09 19:00:16 +00003664static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003665_mm512_maskz_cvtepi32_pd (__mmask8 __U, __m256i __A)
3666{
Simon Pilgrim698528d2016-11-16 09:27:40 +00003667 return (__m512d)__builtin_ia32_selectpd_512((__mmask8) __U,
3668 (__v8df)_mm512_cvtepi32_pd(__A),
3669 (__v8df)_mm512_setzero_pd());
Michael Zuckermanf1544752016-05-09 10:32:51 +00003670}
3671
Craig Topper74c10e32018-07-09 19:00:16 +00003672static __inline__ __m512d __DEFAULT_FN_ATTRS512
Ayman Musa2e250e82016-09-27 14:06:32 +00003673_mm512_cvtepi32lo_pd(__m512i __A)
3674{
3675 return (__m512d) _mm512_cvtepi32_pd(_mm512_castsi512_si256(__A));
3676}
3677
Craig Topper74c10e32018-07-09 19:00:16 +00003678static __inline__ __m512d __DEFAULT_FN_ATTRS512
Ayman Musa2e250e82016-09-27 14:06:32 +00003679_mm512_mask_cvtepi32lo_pd(__m512d __W, __mmask8 __U,__m512i __A)
3680{
3681 return (__m512d) _mm512_mask_cvtepi32_pd(__W, __U, _mm512_castsi512_si256(__A));
3682}
3683
Craig Topper74c10e32018-07-09 19:00:16 +00003684static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003685_mm512_cvtepi32_ps (__m512i __A)
3686{
Craig Topper842171d2018-05-21 20:19:17 +00003687 return (__m512)__builtin_convertvector((__v16si)__A, __v16sf);
Michael Zuckermanf1544752016-05-09 10:32:51 +00003688}
3689
Craig Topper74c10e32018-07-09 19:00:16 +00003690static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003691_mm512_mask_cvtepi32_ps (__m512 __W, __mmask16 __U, __m512i __A)
3692{
Craig Topper842171d2018-05-21 20:19:17 +00003693 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
3694 (__v16sf)_mm512_cvtepi32_ps(__A),
3695 (__v16sf)__W);
Michael Zuckermanf1544752016-05-09 10:32:51 +00003696}
3697
Craig Topper74c10e32018-07-09 19:00:16 +00003698static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003699_mm512_maskz_cvtepi32_ps (__mmask16 __U, __m512i __A)
3700{
Craig Topper842171d2018-05-21 20:19:17 +00003701 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
3702 (__v16sf)_mm512_cvtepi32_ps(__A),
3703 (__v16sf)_mm512_setzero_ps());
Michael Zuckermanf1544752016-05-09 10:32:51 +00003704}
3705
Craig Topper74c10e32018-07-09 19:00:16 +00003706static __inline __m512d __DEFAULT_FN_ATTRS512
Adam Nemet0d5bb552014-07-28 17:14:40 +00003707_mm512_cvtepu32_pd(__m256i __A)
3708{
Simon Pilgrim698528d2016-11-16 09:27:40 +00003709 return (__m512d)__builtin_convertvector((__v8su)__A, __v8df);
Adam Nemet0d5bb552014-07-28 17:14:40 +00003710}
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00003711
Craig Topper74c10e32018-07-09 19:00:16 +00003712static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003713_mm512_mask_cvtepu32_pd (__m512d __W, __mmask8 __U, __m256i __A)
3714{
Simon Pilgrim698528d2016-11-16 09:27:40 +00003715 return (__m512d)__builtin_ia32_selectpd_512((__mmask8) __U,
3716 (__v8df)_mm512_cvtepu32_pd(__A),
3717 (__v8df)__W);
Michael Zuckermanf1544752016-05-09 10:32:51 +00003718}
3719
Craig Topper74c10e32018-07-09 19:00:16 +00003720static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003721_mm512_maskz_cvtepu32_pd (__mmask8 __U, __m256i __A)
3722{
Simon Pilgrim698528d2016-11-16 09:27:40 +00003723 return (__m512d)__builtin_ia32_selectpd_512((__mmask8) __U,
3724 (__v8df)_mm512_cvtepu32_pd(__A),
3725 (__v8df)_mm512_setzero_pd());
Michael Zuckermanf1544752016-05-09 10:32:51 +00003726}
3727
Craig Topper74c10e32018-07-09 19:00:16 +00003728static __inline__ __m512d __DEFAULT_FN_ATTRS512
Ayman Musa2e250e82016-09-27 14:06:32 +00003729_mm512_cvtepu32lo_pd(__m512i __A)
3730{
3731 return (__m512d) _mm512_cvtepu32_pd(_mm512_castsi512_si256(__A));
3732}
3733
Craig Topper74c10e32018-07-09 19:00:16 +00003734static __inline__ __m512d __DEFAULT_FN_ATTRS512
Ayman Musa2e250e82016-09-27 14:06:32 +00003735_mm512_mask_cvtepu32lo_pd(__m512d __W, __mmask8 __U,__m512i __A)
3736{
3737 return (__m512d) _mm512_mask_cvtepu32_pd(__W, __U, _mm512_castsi512_si256(__A));
3738}
3739
Craig Topperc6338672018-05-31 00:51:20 +00003740#define _mm512_cvt_roundpd_ps(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00003741 (__m256)__builtin_ia32_cvtpd2ps512_mask((__v8df)(__m512d)(A), \
Craig Topper72c7d512015-02-01 07:35:35 +00003742 (__v8sf)_mm256_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00003743 (__mmask8)-1, (int)(R))
Craig Topper72c7d512015-02-01 07:35:35 +00003744
Craig Topperc6338672018-05-31 00:51:20 +00003745#define _mm512_mask_cvt_roundpd_ps(W, U, A, R) \
Michael Zuckerman186d8672016-05-31 11:27:34 +00003746 (__m256)__builtin_ia32_cvtpd2ps512_mask((__v8df)(__m512d)(A), \
Craig Topperf3efec62016-06-08 06:08:07 +00003747 (__v8sf)(__m256)(W), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00003748 (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00003749
Craig Topperc6338672018-05-31 00:51:20 +00003750#define _mm512_maskz_cvt_roundpd_ps(U, A, R) \
Michael Zuckerman186d8672016-05-31 11:27:34 +00003751 (__m256)__builtin_ia32_cvtpd2ps512_mask((__v8df)(__m512d)(A), \
3752 (__v8sf)_mm256_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00003753 (__mmask8)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00003754
Craig Topper74c10e32018-07-09 19:00:16 +00003755static __inline__ __m256 __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003756_mm512_cvtpd_ps (__m512d __A)
3757{
3758 return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A,
3759 (__v8sf) _mm256_undefined_ps (),
3760 (__mmask8) -1,
3761 _MM_FROUND_CUR_DIRECTION);
3762}
3763
Craig Topper74c10e32018-07-09 19:00:16 +00003764static __inline__ __m256 __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003765_mm512_mask_cvtpd_ps (__m256 __W, __mmask8 __U, __m512d __A)
3766{
3767 return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A,
3768 (__v8sf) __W,
3769 (__mmask8) __U,
3770 _MM_FROUND_CUR_DIRECTION);
3771}
3772
Craig Topper74c10e32018-07-09 19:00:16 +00003773static __inline__ __m256 __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003774_mm512_maskz_cvtpd_ps (__mmask8 __U, __m512d __A)
3775{
3776 return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A,
3777 (__v8sf) _mm256_setzero_ps (),
3778 (__mmask8) __U,
3779 _MM_FROUND_CUR_DIRECTION);
3780}
Michael Zuckerman2564d2f2016-05-10 10:14:50 +00003781
Craig Topper74c10e32018-07-09 19:00:16 +00003782static __inline__ __m512 __DEFAULT_FN_ATTRS512
Ayman Musa2e250e82016-09-27 14:06:32 +00003783_mm512_cvtpd_pslo (__m512d __A)
3784{
3785 return (__m512) __builtin_shufflevector((__v8sf) _mm512_cvtpd_ps(__A),
3786 (__v8sf) _mm256_setzero_ps (),
3787 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
3788}
3789
Craig Topper74c10e32018-07-09 19:00:16 +00003790static __inline__ __m512 __DEFAULT_FN_ATTRS512
Ayman Musa2e250e82016-09-27 14:06:32 +00003791_mm512_mask_cvtpd_pslo (__m512 __W, __mmask8 __U,__m512d __A)
3792{
3793 return (__m512) __builtin_shufflevector (
3794 (__v8sf) _mm512_mask_cvtpd_ps (_mm512_castps512_ps256(__W),
3795 __U, __A),
3796 (__v8sf) _mm256_setzero_ps (),
3797 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
3798}
3799
Craig Topperc6338672018-05-31 00:51:20 +00003800#define _mm512_cvt_roundps_ph(A, I) \
Craig Topperf3efec62016-06-08 06:08:07 +00003801 (__m256i)__builtin_ia32_vcvtps2ph512_mask((__v16sf)(__m512)(A), (int)(I), \
3802 (__v16hi)_mm256_undefined_si256(), \
Craig Topperc6338672018-05-31 00:51:20 +00003803 (__mmask16)-1)
Michael Zuckerman6170c152016-06-01 14:41:41 +00003804
Craig Topperc6338672018-05-31 00:51:20 +00003805#define _mm512_mask_cvt_roundps_ph(U, W, A, I) \
Craig Topperf3efec62016-06-08 06:08:07 +00003806 (__m256i)__builtin_ia32_vcvtps2ph512_mask((__v16sf)(__m512)(A), (int)(I), \
3807 (__v16hi)(__m256i)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00003808 (__mmask16)(W))
Michael Zuckerman6170c152016-06-01 14:41:41 +00003809
Craig Topperc6338672018-05-31 00:51:20 +00003810#define _mm512_maskz_cvt_roundps_ph(W, A, I) \
Craig Topperf3efec62016-06-08 06:08:07 +00003811 (__m256i)__builtin_ia32_vcvtps2ph512_mask((__v16sf)(__m512)(A), (int)(I), \
3812 (__v16hi)_mm256_setzero_si256(), \
Craig Topperc6338672018-05-31 00:51:20 +00003813 (__mmask16)(W))
Michael Zuckerman6170c152016-06-01 14:41:41 +00003814
Craig Topperc6338672018-05-31 00:51:20 +00003815#define _mm512_cvtps_ph(A, I) \
Craig Topper8c18e112016-05-17 04:41:50 +00003816 (__m256i)__builtin_ia32_vcvtps2ph512_mask((__v16sf)(__m512)(A), (int)(I), \
Craig Topper72c7d512015-02-01 07:35:35 +00003817 (__v16hi)_mm256_setzero_si256(), \
Craig Topperc6338672018-05-31 00:51:20 +00003818 (__mmask16)-1)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00003819
Craig Topperc6338672018-05-31 00:51:20 +00003820#define _mm512_mask_cvtps_ph(U, W, A, I) \
Craig Topper8c18e112016-05-17 04:41:50 +00003821 (__m256i)__builtin_ia32_vcvtps2ph512_mask((__v16sf)(__m512)(A), (int)(I), \
3822 (__v16hi)(__m256i)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00003823 (__mmask16)(W))
Michael Zuckermanf1544752016-05-09 10:32:51 +00003824
Craig Topperc6338672018-05-31 00:51:20 +00003825#define _mm512_maskz_cvtps_ph(W, A, I) \
Craig Topper8c18e112016-05-17 04:41:50 +00003826 (__m256i)__builtin_ia32_vcvtps2ph512_mask((__v16sf)(__m512)(A), (int)(I), \
3827 (__v16hi)_mm256_setzero_si256(), \
Craig Topperc6338672018-05-31 00:51:20 +00003828 (__mmask16)(W))
Michael Zuckerman6170c152016-06-01 14:41:41 +00003829
Craig Topperc6338672018-05-31 00:51:20 +00003830#define _mm512_cvt_roundph_ps(A, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00003831 (__m512)__builtin_ia32_vcvtph2ps512_mask((__v16hi)(__m256i)(A), \
3832 (__v16sf)_mm512_undefined_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00003833 (__mmask16)-1, (int)(R))
Michael Zuckerman6170c152016-06-01 14:41:41 +00003834
Craig Topperc6338672018-05-31 00:51:20 +00003835#define _mm512_mask_cvt_roundph_ps(W, U, A, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00003836 (__m512)__builtin_ia32_vcvtph2ps512_mask((__v16hi)(__m256i)(A), \
3837 (__v16sf)(__m512)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00003838 (__mmask16)(U), (int)(R))
Michael Zuckerman6170c152016-06-01 14:41:41 +00003839
Craig Topperc6338672018-05-31 00:51:20 +00003840#define _mm512_maskz_cvt_roundph_ps(U, A, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00003841 (__m512)__builtin_ia32_vcvtph2ps512_mask((__v16hi)(__m256i)(A), \
3842 (__v16sf)_mm512_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00003843 (__mmask16)(U), (int)(R))
Craig Topperf3efec62016-06-08 06:08:07 +00003844
3845
Craig Topper74c10e32018-07-09 19:00:16 +00003846static __inline __m512 __DEFAULT_FN_ATTRS512
Adam Nemet9a3ea602014-07-28 17:14:38 +00003847_mm512_cvtph_ps(__m256i __A)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00003848{
3849 return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A,
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00003850 (__v16sf)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00003851 _mm512_setzero_ps (),
3852 (__mmask16) -1,
3853 _MM_FROUND_CUR_DIRECTION);
3854}
3855
Craig Topper74c10e32018-07-09 19:00:16 +00003856static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003857_mm512_mask_cvtph_ps (__m512 __W, __mmask16 __U, __m256i __A)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00003858{
Michael Zuckermanf1544752016-05-09 10:32:51 +00003859 return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A,
3860 (__v16sf) __W,
3861 (__mmask16) __U,
3862 _MM_FROUND_CUR_DIRECTION);
3863}
3864
Craig Topper74c10e32018-07-09 19:00:16 +00003865static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003866_mm512_maskz_cvtph_ps (__mmask16 __U, __m256i __A)
3867{
3868 return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A,
3869 (__v16sf) _mm512_setzero_ps (),
3870 (__mmask16) __U,
3871 _MM_FROUND_CUR_DIRECTION);
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00003872}
3873
Craig Topperc6338672018-05-31 00:51:20 +00003874#define _mm512_cvtt_roundpd_epi32(A, R) \
Michael Zuckerman186d8672016-05-31 11:27:34 +00003875 (__m256i)__builtin_ia32_cvttpd2dq512_mask((__v8df)(__m512d)(A), \
3876 (__v8si)_mm256_setzero_si256(), \
Craig Topperc6338672018-05-31 00:51:20 +00003877 (__mmask8)-1, (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00003878
Craig Topperc6338672018-05-31 00:51:20 +00003879#define _mm512_mask_cvtt_roundpd_epi32(W, U, A, R) \
Michael Zuckerman186d8672016-05-31 11:27:34 +00003880 (__m256i)__builtin_ia32_cvttpd2dq512_mask((__v8df)(__m512d)(A), \
Craig Topperf3efec62016-06-08 06:08:07 +00003881 (__v8si)(__m256i)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00003882 (__mmask8)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00003883
Craig Topperc6338672018-05-31 00:51:20 +00003884#define _mm512_maskz_cvtt_roundpd_epi32(U, A, R) \
Michael Zuckerman186d8672016-05-31 11:27:34 +00003885 (__m256i)__builtin_ia32_cvttpd2dq512_mask((__v8df)(__m512d)(A), \
3886 (__v8si)_mm256_setzero_si256(), \
Craig Topperc6338672018-05-31 00:51:20 +00003887 (__mmask8)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00003888
Craig Topper74c10e32018-07-09 19:00:16 +00003889static __inline __m256i __DEFAULT_FN_ATTRS512
Michael Kuperstein5c2cb0e2015-09-21 11:45:27 +00003890_mm512_cvttpd_epi32(__m512d __a)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00003891{
Michael Kuperstein5c2cb0e2015-09-21 11:45:27 +00003892 return (__m256i)__builtin_ia32_cvttpd2dq512_mask((__v8df) __a,
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00003893 (__v8si)_mm256_setzero_si256(),
3894 (__mmask8) -1,
3895 _MM_FROUND_CUR_DIRECTION);
3896}
3897
Craig Topper74c10e32018-07-09 19:00:16 +00003898static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003899_mm512_mask_cvttpd_epi32 (__m256i __W, __mmask8 __U, __m512d __A)
3900{
3901 return (__m256i) __builtin_ia32_cvttpd2dq512_mask ((__v8df) __A,
3902 (__v8si) __W,
3903 (__mmask8) __U,
3904 _MM_FROUND_CUR_DIRECTION);
3905}
3906
Craig Topper74c10e32018-07-09 19:00:16 +00003907static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003908_mm512_maskz_cvttpd_epi32 (__mmask8 __U, __m512d __A)
3909{
3910 return (__m256i) __builtin_ia32_cvttpd2dq512_mask ((__v8df) __A,
3911 (__v8si) _mm256_setzero_si256 (),
3912 (__mmask8) __U,
3913 _MM_FROUND_CUR_DIRECTION);
3914}
3915
Craig Topperc6338672018-05-31 00:51:20 +00003916#define _mm512_cvtt_roundps_epi32(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00003917 (__m512i)__builtin_ia32_cvttps2dq512_mask((__v16sf)(__m512)(A), \
Craig Topper72c7d512015-02-01 07:35:35 +00003918 (__v16si)_mm512_setzero_si512(), \
Craig Topperc6338672018-05-31 00:51:20 +00003919 (__mmask16)-1, (int)(R))
Michael Zuckerman2564d2f2016-05-10 10:14:50 +00003920
Craig Topperc6338672018-05-31 00:51:20 +00003921#define _mm512_mask_cvtt_roundps_epi32(W, U, A, R) \
Michael Zuckerman186d8672016-05-31 11:27:34 +00003922 (__m512i)__builtin_ia32_cvttps2dq512_mask((__v16sf)(__m512)(A), \
Craig Topperf3efec62016-06-08 06:08:07 +00003923 (__v16si)(__m512i)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00003924 (__mmask16)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00003925
Craig Topperc6338672018-05-31 00:51:20 +00003926#define _mm512_maskz_cvtt_roundps_epi32(U, A, R) \
Michael Zuckerman186d8672016-05-31 11:27:34 +00003927 (__m512i)__builtin_ia32_cvttps2dq512_mask((__v16sf)(__m512)(A), \
3928 (__v16si)_mm512_setzero_si512(), \
Craig Topperc6338672018-05-31 00:51:20 +00003929 (__mmask16)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00003930
Craig Topper74c10e32018-07-09 19:00:16 +00003931static __inline __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003932_mm512_cvttps_epi32(__m512 __a)
3933{
3934 return (__m512i)
3935 __builtin_ia32_cvttps2dq512_mask((__v16sf) __a,
3936 (__v16si) _mm512_setzero_si512 (),
3937 (__mmask16) -1, _MM_FROUND_CUR_DIRECTION);
3938}
3939
Craig Topper74c10e32018-07-09 19:00:16 +00003940static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003941_mm512_mask_cvttps_epi32 (__m512i __W, __mmask16 __U, __m512 __A)
3942{
3943 return (__m512i) __builtin_ia32_cvttps2dq512_mask ((__v16sf) __A,
3944 (__v16si) __W,
3945 (__mmask16) __U,
3946 _MM_FROUND_CUR_DIRECTION);
3947}
3948
Craig Topper74c10e32018-07-09 19:00:16 +00003949static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003950_mm512_maskz_cvttps_epi32 (__mmask16 __U, __m512 __A)
3951{
3952 return (__m512i) __builtin_ia32_cvttps2dq512_mask ((__v16sf) __A,
3953 (__v16si) _mm512_setzero_si512 (),
3954 (__mmask16) __U,
3955 _MM_FROUND_CUR_DIRECTION);
3956}
Craig Topper72c7d512015-02-01 07:35:35 +00003957
Craig Topperc6338672018-05-31 00:51:20 +00003958#define _mm512_cvt_roundps_epi32(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00003959 (__m512i)__builtin_ia32_cvtps2dq512_mask((__v16sf)(__m512)(A), \
Craig Topper72c7d512015-02-01 07:35:35 +00003960 (__v16si)_mm512_setzero_si512(), \
Craig Topperc6338672018-05-31 00:51:20 +00003961 (__mmask16)-1, (int)(R))
Craig Topper72c7d512015-02-01 07:35:35 +00003962
Craig Topperc6338672018-05-31 00:51:20 +00003963#define _mm512_mask_cvt_roundps_epi32(W, U, A, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00003964 (__m512i)__builtin_ia32_cvtps2dq512_mask((__v16sf)(__m512)(A), \
3965 (__v16si)(__m512i)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00003966 (__mmask16)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00003967
Craig Topperc6338672018-05-31 00:51:20 +00003968#define _mm512_maskz_cvt_roundps_epi32(U, A, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00003969 (__m512i)__builtin_ia32_cvtps2dq512_mask((__v16sf)(__m512)(A), \
3970 (__v16si)_mm512_setzero_si512(), \
Craig Topperc6338672018-05-31 00:51:20 +00003971 (__mmask16)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00003972
Craig Topper74c10e32018-07-09 19:00:16 +00003973static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003974_mm512_cvtps_epi32 (__m512 __A)
3975{
3976 return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A,
3977 (__v16si) _mm512_undefined_epi32 (),
3978 (__mmask16) -1,
3979 _MM_FROUND_CUR_DIRECTION);
3980}
3981
Craig Topper74c10e32018-07-09 19:00:16 +00003982static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003983_mm512_mask_cvtps_epi32 (__m512i __W, __mmask16 __U, __m512 __A)
3984{
3985 return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A,
3986 (__v16si) __W,
3987 (__mmask16) __U,
3988 _MM_FROUND_CUR_DIRECTION);
3989}
3990
Craig Topper74c10e32018-07-09 19:00:16 +00003991static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00003992_mm512_maskz_cvtps_epi32 (__mmask16 __U, __m512 __A)
3993{
3994 return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A,
3995 (__v16si)
3996 _mm512_setzero_si512 (),
3997 (__mmask16) __U,
3998 _MM_FROUND_CUR_DIRECTION);
3999}
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00004000
Craig Topperc6338672018-05-31 00:51:20 +00004001#define _mm512_cvt_roundpd_epi32(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00004002 (__m256i)__builtin_ia32_cvtpd2dq512_mask((__v8df)(__m512d)(A), \
Craig Topper72c7d512015-02-01 07:35:35 +00004003 (__v8si)_mm256_setzero_si256(), \
Craig Topperc6338672018-05-31 00:51:20 +00004004 (__mmask8)-1, (int)(R))
Craig Topper72c7d512015-02-01 07:35:35 +00004005
Craig Topperc6338672018-05-31 00:51:20 +00004006#define _mm512_mask_cvt_roundpd_epi32(W, U, A, R) \
Michael Zuckerman186d8672016-05-31 11:27:34 +00004007 (__m256i)__builtin_ia32_cvtpd2dq512_mask((__v8df)(__m512d)(A), \
Craig Topperf3efec62016-06-08 06:08:07 +00004008 (__v8si)(__m256i)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00004009 (__mmask8)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00004010
Craig Topperc6338672018-05-31 00:51:20 +00004011#define _mm512_maskz_cvt_roundpd_epi32(U, A, R) \
Michael Zuckerman186d8672016-05-31 11:27:34 +00004012 (__m256i)__builtin_ia32_cvtpd2dq512_mask((__v8df)(__m512d)(A), \
4013 (__v8si)_mm256_setzero_si256(), \
Craig Topperc6338672018-05-31 00:51:20 +00004014 (__mmask8)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00004015
Craig Topper74c10e32018-07-09 19:00:16 +00004016static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00004017_mm512_cvtpd_epi32 (__m512d __A)
4018{
4019 return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A,
4020 (__v8si)
4021 _mm256_undefined_si256 (),
4022 (__mmask8) -1,
4023 _MM_FROUND_CUR_DIRECTION);
4024}
4025
Craig Topper74c10e32018-07-09 19:00:16 +00004026static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00004027_mm512_mask_cvtpd_epi32 (__m256i __W, __mmask8 __U, __m512d __A)
4028{
4029 return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A,
4030 (__v8si) __W,
4031 (__mmask8) __U,
4032 _MM_FROUND_CUR_DIRECTION);
4033}
4034
Craig Topper74c10e32018-07-09 19:00:16 +00004035static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00004036_mm512_maskz_cvtpd_epi32 (__mmask8 __U, __m512d __A)
4037{
4038 return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A,
4039 (__v8si)
4040 _mm256_setzero_si256 (),
4041 (__mmask8) __U,
4042 _MM_FROUND_CUR_DIRECTION);
4043}
4044
Craig Topperc6338672018-05-31 00:51:20 +00004045#define _mm512_cvt_roundps_epu32(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00004046 (__m512i)__builtin_ia32_cvtps2udq512_mask((__v16sf)(__m512)(A), \
Craig Topper72c7d512015-02-01 07:35:35 +00004047 (__v16si)_mm512_setzero_si512(), \
Craig Topperc6338672018-05-31 00:51:20 +00004048 (__mmask16)-1, (int)(R))
Craig Topper72c7d512015-02-01 07:35:35 +00004049
Craig Topperc6338672018-05-31 00:51:20 +00004050#define _mm512_mask_cvt_roundps_epu32(W, U, A, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00004051 (__m512i)__builtin_ia32_cvtps2udq512_mask((__v16sf)(__m512)(A), \
4052 (__v16si)(__m512i)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00004053 (__mmask16)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00004054
Craig Topperc6338672018-05-31 00:51:20 +00004055#define _mm512_maskz_cvt_roundps_epu32(U, A, R) \
Craig Topperf3efec62016-06-08 06:08:07 +00004056 (__m512i)__builtin_ia32_cvtps2udq512_mask((__v16sf)(__m512)(A), \
4057 (__v16si)_mm512_setzero_si512(), \
Craig Topperc6338672018-05-31 00:51:20 +00004058 (__mmask16)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00004059
Craig Topper74c10e32018-07-09 19:00:16 +00004060static __inline__ __m512i __DEFAULT_FN_ATTRS512
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00004061_mm512_cvtps_epu32 ( __m512 __A)
Michael Zuckermanf1544752016-05-09 10:32:51 +00004062{
4063 return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A,\
4064 (__v16si)\
Craig Topper0a485d12018-07-07 22:03:19 +00004065 _mm512_undefined_epi32 (),
Michael Zuckermanf1544752016-05-09 10:32:51 +00004066 (__mmask16) -1,\
Craig Topper0a485d12018-07-07 22:03:19 +00004067 _MM_FROUND_CUR_DIRECTION);
Michael Zuckermanf1544752016-05-09 10:32:51 +00004068}
4069
Craig Topper74c10e32018-07-09 19:00:16 +00004070static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00004071_mm512_mask_cvtps_epu32 (__m512i __W, __mmask16 __U, __m512 __A)
4072{
4073 return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A,
4074 (__v16si) __W,
4075 (__mmask16) __U,
4076 _MM_FROUND_CUR_DIRECTION);
4077}
4078
Craig Topper74c10e32018-07-09 19:00:16 +00004079static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman9fcf3552016-05-30 13:22:12 +00004080_mm512_maskz_cvtps_epu32 ( __mmask16 __U, __m512 __A)
4081{
4082 return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A,
Simon Pilgrim0b37ffb2017-07-28 14:01:51 +00004083 (__v16si)
Michael Zuckerman9fcf3552016-05-30 13:22:12 +00004084 _mm512_setzero_si512 (),
4085 (__mmask16) __U ,
4086 _MM_FROUND_CUR_DIRECTION);
4087}
4088
Craig Topperc6338672018-05-31 00:51:20 +00004089#define _mm512_cvt_roundpd_epu32(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00004090 (__m256i)__builtin_ia32_cvtpd2udq512_mask((__v8df)(__m512d)(A), \
Craig Topper72c7d512015-02-01 07:35:35 +00004091 (__v8si)_mm256_setzero_si256(), \
Craig Topperc6338672018-05-31 00:51:20 +00004092 (__mmask8)-1, (int)(R))
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004093
Craig Topperc6338672018-05-31 00:51:20 +00004094#define _mm512_mask_cvt_roundpd_epu32(W, U, A, R) \
Michael Zuckerman186d8672016-05-31 11:27:34 +00004095 (__m256i)__builtin_ia32_cvtpd2udq512_mask((__v8df)(__m512d)(A), \
Craig Toppercbf39292018-05-31 01:24:40 +00004096 (__v8si)(__m256i)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00004097 (__mmask8)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00004098
Craig Topperc6338672018-05-31 00:51:20 +00004099#define _mm512_maskz_cvt_roundpd_epu32(U, A, R) \
Michael Zuckerman186d8672016-05-31 11:27:34 +00004100 (__m256i)__builtin_ia32_cvtpd2udq512_mask((__v8df)(__m512d)(A), \
4101 (__v8si)_mm256_setzero_si256(), \
Craig Topperc6338672018-05-31 00:51:20 +00004102 (__mmask8)(U), (int)(R))
Michael Zuckerman186d8672016-05-31 11:27:34 +00004103
Craig Topper74c10e32018-07-09 19:00:16 +00004104static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00004105_mm512_cvtpd_epu32 (__m512d __A)
4106{
4107 return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A,
4108 (__v8si)
4109 _mm256_undefined_si256 (),
4110 (__mmask8) -1,
4111 _MM_FROUND_CUR_DIRECTION);
4112}
4113
Craig Topper74c10e32018-07-09 19:00:16 +00004114static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00004115_mm512_mask_cvtpd_epu32 (__m256i __W, __mmask8 __U, __m512d __A)
4116{
4117 return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A,
4118 (__v8si) __W,
4119 (__mmask8) __U,
4120 _MM_FROUND_CUR_DIRECTION);
4121}
4122
Craig Topper74c10e32018-07-09 19:00:16 +00004123static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermanf1544752016-05-09 10:32:51 +00004124_mm512_maskz_cvtpd_epu32 (__mmask8 __U, __m512d __A)
4125{
4126 return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A,
4127 (__v8si)
4128 _mm256_setzero_si256 (),
4129 (__mmask8) __U,
4130 _MM_FROUND_CUR_DIRECTION);
4131}
Craig Topper79f53ca2016-06-23 06:36:42 +00004132
Craig Topper74c10e32018-07-09 19:00:16 +00004133static __inline__ double __DEFAULT_FN_ATTRS512
Simon Pilgrim60e92492017-03-21 12:46:13 +00004134_mm512_cvtsd_f64(__m512d __a)
4135{
4136 return __a[0];
4137}
4138
Craig Topper74c10e32018-07-09 19:00:16 +00004139static __inline__ float __DEFAULT_FN_ATTRS512
Simon Pilgrim60e92492017-03-21 12:46:13 +00004140_mm512_cvtss_f32(__m512 __a)
4141{
4142 return __a[0];
4143}
4144
Adam Nemet63a951e2015-01-14 01:31:17 +00004145/* Unpack and Interleave */
Craig Topper79f53ca2016-06-23 06:36:42 +00004146
Craig Topper74c10e32018-07-09 19:00:16 +00004147static __inline __m512d __DEFAULT_FN_ATTRS512
Adam Nemet63a951e2015-01-14 01:31:17 +00004148_mm512_unpackhi_pd(__m512d __a, __m512d __b)
4149{
Craig Topper79f53ca2016-06-23 06:36:42 +00004150 return (__m512d)__builtin_shufflevector((__v8df)__a, (__v8df)__b,
4151 1, 9, 1+2, 9+2, 1+4, 9+4, 1+6, 9+6);
4152}
4153
Craig Topper74c10e32018-07-09 19:00:16 +00004154static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004155_mm512_mask_unpackhi_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B)
4156{
4157 return (__m512d)__builtin_ia32_selectpd_512((__mmask8) __U,
4158 (__v8df)_mm512_unpackhi_pd(__A, __B),
4159 (__v8df)__W);
4160}
4161
Craig Topper74c10e32018-07-09 19:00:16 +00004162static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004163_mm512_maskz_unpackhi_pd(__mmask8 __U, __m512d __A, __m512d __B)
4164{
4165 return (__m512d)__builtin_ia32_selectpd_512((__mmask8) __U,
4166 (__v8df)_mm512_unpackhi_pd(__A, __B),
4167 (__v8df)_mm512_setzero_pd());
Adam Nemet63a951e2015-01-14 01:31:17 +00004168}
4169
Craig Topper74c10e32018-07-09 19:00:16 +00004170static __inline __m512d __DEFAULT_FN_ATTRS512
Adam Nemet63a951e2015-01-14 01:31:17 +00004171_mm512_unpacklo_pd(__m512d __a, __m512d __b)
4172{
Craig Topper79f53ca2016-06-23 06:36:42 +00004173 return (__m512d)__builtin_shufflevector((__v8df)__a, (__v8df)__b,
4174 0, 8, 0+2, 8+2, 0+4, 8+4, 0+6, 8+6);
4175}
4176
Craig Topper74c10e32018-07-09 19:00:16 +00004177static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004178_mm512_mask_unpacklo_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B)
4179{
4180 return (__m512d)__builtin_ia32_selectpd_512((__mmask8) __U,
4181 (__v8df)_mm512_unpacklo_pd(__A, __B),
4182 (__v8df)__W);
4183}
4184
Craig Topper74c10e32018-07-09 19:00:16 +00004185static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004186_mm512_maskz_unpacklo_pd (__mmask8 __U, __m512d __A, __m512d __B)
4187{
4188 return (__m512d)__builtin_ia32_selectpd_512((__mmask8) __U,
4189 (__v8df)_mm512_unpacklo_pd(__A, __B),
4190 (__v8df)_mm512_setzero_pd());
Adam Nemet63a951e2015-01-14 01:31:17 +00004191}
4192
Craig Topper74c10e32018-07-09 19:00:16 +00004193static __inline __m512 __DEFAULT_FN_ATTRS512
Adam Nemet63a951e2015-01-14 01:31:17 +00004194_mm512_unpackhi_ps(__m512 __a, __m512 __b)
4195{
Craig Topper79f53ca2016-06-23 06:36:42 +00004196 return (__m512)__builtin_shufflevector((__v16sf)__a, (__v16sf)__b,
4197 2, 18, 3, 19,
4198 2+4, 18+4, 3+4, 19+4,
4199 2+8, 18+8, 3+8, 19+8,
4200 2+12, 18+12, 3+12, 19+12);
4201}
4202
Craig Topper74c10e32018-07-09 19:00:16 +00004203static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004204_mm512_mask_unpackhi_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B)
4205{
4206 return (__m512)__builtin_ia32_selectps_512((__mmask16) __U,
4207 (__v16sf)_mm512_unpackhi_ps(__A, __B),
4208 (__v16sf)__W);
4209}
4210
Craig Topper74c10e32018-07-09 19:00:16 +00004211static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004212_mm512_maskz_unpackhi_ps (__mmask16 __U, __m512 __A, __m512 __B)
4213{
4214 return (__m512)__builtin_ia32_selectps_512((__mmask16) __U,
4215 (__v16sf)_mm512_unpackhi_ps(__A, __B),
4216 (__v16sf)_mm512_setzero_ps());
Adam Nemet63a951e2015-01-14 01:31:17 +00004217}
4218
Craig Topper74c10e32018-07-09 19:00:16 +00004219static __inline __m512 __DEFAULT_FN_ATTRS512
Adam Nemet63a951e2015-01-14 01:31:17 +00004220_mm512_unpacklo_ps(__m512 __a, __m512 __b)
4221{
Craig Topper79f53ca2016-06-23 06:36:42 +00004222 return (__m512)__builtin_shufflevector((__v16sf)__a, (__v16sf)__b,
4223 0, 16, 1, 17,
4224 0+4, 16+4, 1+4, 17+4,
4225 0+8, 16+8, 1+8, 17+8,
4226 0+12, 16+12, 1+12, 17+12);
4227}
4228
Craig Topper74c10e32018-07-09 19:00:16 +00004229static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004230_mm512_mask_unpacklo_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B)
4231{
4232 return (__m512)__builtin_ia32_selectps_512((__mmask16) __U,
4233 (__v16sf)_mm512_unpacklo_ps(__A, __B),
4234 (__v16sf)__W);
4235}
4236
Craig Topper74c10e32018-07-09 19:00:16 +00004237static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004238_mm512_maskz_unpacklo_ps (__mmask16 __U, __m512 __A, __m512 __B)
4239{
4240 return (__m512)__builtin_ia32_selectps_512((__mmask16) __U,
4241 (__v16sf)_mm512_unpacklo_ps(__A, __B),
4242 (__v16sf)_mm512_setzero_ps());
4243}
4244
Craig Topper74c10e32018-07-09 19:00:16 +00004245static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004246_mm512_unpackhi_epi32(__m512i __A, __m512i __B)
4247{
4248 return (__m512i)__builtin_shufflevector((__v16si)__A, (__v16si)__B,
4249 2, 18, 3, 19,
4250 2+4, 18+4, 3+4, 19+4,
4251 2+8, 18+8, 3+8, 19+8,
4252 2+12, 18+12, 3+12, 19+12);
4253}
4254
Craig Topper74c10e32018-07-09 19:00:16 +00004255static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004256_mm512_mask_unpackhi_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
4257{
4258 return (__m512i)__builtin_ia32_selectd_512((__mmask16) __U,
4259 (__v16si)_mm512_unpackhi_epi32(__A, __B),
4260 (__v16si)__W);
4261}
4262
Craig Topper74c10e32018-07-09 19:00:16 +00004263static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004264_mm512_maskz_unpackhi_epi32(__mmask16 __U, __m512i __A, __m512i __B)
4265{
4266 return (__m512i)__builtin_ia32_selectd_512((__mmask16) __U,
4267 (__v16si)_mm512_unpackhi_epi32(__A, __B),
4268 (__v16si)_mm512_setzero_si512());
4269}
4270
Craig Topper74c10e32018-07-09 19:00:16 +00004271static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004272_mm512_unpacklo_epi32(__m512i __A, __m512i __B)
4273{
4274 return (__m512i)__builtin_shufflevector((__v16si)__A, (__v16si)__B,
4275 0, 16, 1, 17,
4276 0+4, 16+4, 1+4, 17+4,
4277 0+8, 16+8, 1+8, 17+8,
4278 0+12, 16+12, 1+12, 17+12);
4279}
4280
Craig Topper74c10e32018-07-09 19:00:16 +00004281static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004282_mm512_mask_unpacklo_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
4283{
4284 return (__m512i)__builtin_ia32_selectd_512((__mmask16) __U,
4285 (__v16si)_mm512_unpacklo_epi32(__A, __B),
4286 (__v16si)__W);
4287}
4288
Craig Topper74c10e32018-07-09 19:00:16 +00004289static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004290_mm512_maskz_unpacklo_epi32(__mmask16 __U, __m512i __A, __m512i __B)
4291{
4292 return (__m512i)__builtin_ia32_selectd_512((__mmask16) __U,
4293 (__v16si)_mm512_unpacklo_epi32(__A, __B),
4294 (__v16si)_mm512_setzero_si512());
4295}
4296
Craig Topper74c10e32018-07-09 19:00:16 +00004297static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004298_mm512_unpackhi_epi64(__m512i __A, __m512i __B)
4299{
4300 return (__m512i)__builtin_shufflevector((__v8di)__A, (__v8di)__B,
4301 1, 9, 1+2, 9+2, 1+4, 9+4, 1+6, 9+6);
4302}
4303
Craig Topper74c10e32018-07-09 19:00:16 +00004304static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004305_mm512_mask_unpackhi_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
4306{
4307 return (__m512i)__builtin_ia32_selectq_512((__mmask8) __U,
4308 (__v8di)_mm512_unpackhi_epi64(__A, __B),
4309 (__v8di)__W);
4310}
4311
Craig Topper74c10e32018-07-09 19:00:16 +00004312static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004313_mm512_maskz_unpackhi_epi64(__mmask8 __U, __m512i __A, __m512i __B)
4314{
4315 return (__m512i)__builtin_ia32_selectq_512((__mmask8) __U,
4316 (__v8di)_mm512_unpackhi_epi64(__A, __B),
4317 (__v8di)_mm512_setzero_si512());
4318}
4319
Craig Topper74c10e32018-07-09 19:00:16 +00004320static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004321_mm512_unpacklo_epi64 (__m512i __A, __m512i __B)
4322{
4323 return (__m512i)__builtin_shufflevector((__v8di)__A, (__v8di)__B,
4324 0, 8, 0+2, 8+2, 0+4, 8+4, 0+6, 8+6);
4325}
4326
Craig Topper74c10e32018-07-09 19:00:16 +00004327static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004328_mm512_mask_unpacklo_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
4329{
4330 return (__m512i)__builtin_ia32_selectq_512((__mmask8) __U,
4331 (__v8di)_mm512_unpacklo_epi64(__A, __B),
4332 (__v8di)__W);
4333}
4334
Craig Topper74c10e32018-07-09 19:00:16 +00004335static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper79f53ca2016-06-23 06:36:42 +00004336_mm512_maskz_unpacklo_epi64 (__mmask8 __U, __m512i __A, __m512i __B)
4337{
4338 return (__m512i)__builtin_ia32_selectq_512((__mmask8) __U,
4339 (__v8di)_mm512_unpacklo_epi64(__A, __B),
4340 (__v8di)_mm512_setzero_si512());
Adam Nemet63a951e2015-01-14 01:31:17 +00004341}
4342
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00004343
Adam Nemet0d5bb552014-07-28 17:14:40 +00004344/* SIMD load ops */
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004345
Craig Topper74c10e32018-07-09 19:00:16 +00004346static __inline __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004347_mm512_loadu_si512 (void const *__P)
4348{
Craig Toppera6dd2fa2018-05-31 05:02:08 +00004349 struct __loadu_si512 {
4350 __m512i __v;
4351 } __attribute__((__packed__, __may_alias__));
4352 return ((struct __loadu_si512*)__P)->__v;
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004353}
4354
Craig Topper74c10e32018-07-09 19:00:16 +00004355static __inline __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004356_mm512_mask_loadu_epi32 (__m512i __W, __mmask16 __U, void const *__P)
4357{
Craig Topper4537ea72016-05-14 06:03:13 +00004358 return (__m512i) __builtin_ia32_loaddqusi512_mask ((const int *) __P,
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004359 (__v16si) __W,
4360 (__mmask16) __U);
4361}
4362
4363
Craig Topper74c10e32018-07-09 19:00:16 +00004364static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet9a3ea602014-07-28 17:14:38 +00004365_mm512_maskz_loadu_epi32(__mmask16 __U, void const *__P)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004366{
Craig Topper4537ea72016-05-14 06:03:13 +00004367 return (__m512i) __builtin_ia32_loaddqusi512_mask ((const int *)__P,
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004368 (__v16si)
4369 _mm512_setzero_si512 (),
4370 (__mmask16) __U);
4371}
4372
Craig Topper74c10e32018-07-09 19:00:16 +00004373static __inline __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004374_mm512_mask_loadu_epi64 (__m512i __W, __mmask8 __U, void const *__P)
4375{
Craig Topper4537ea72016-05-14 06:03:13 +00004376 return (__m512i) __builtin_ia32_loaddqudi512_mask ((const long long *) __P,
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004377 (__v8di) __W,
4378 (__mmask8) __U);
4379}
4380
Craig Topper74c10e32018-07-09 19:00:16 +00004381static __inline __m512i __DEFAULT_FN_ATTRS512
Adam Nemet9a3ea602014-07-28 17:14:38 +00004382_mm512_maskz_loadu_epi64(__mmask8 __U, void const *__P)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004383{
Craig Topper4537ea72016-05-14 06:03:13 +00004384 return (__m512i) __builtin_ia32_loaddqudi512_mask ((const long long *)__P,
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004385 (__v8di)
4386 _mm512_setzero_si512 (),
4387 (__mmask8) __U);
4388}
4389
Craig Topper74c10e32018-07-09 19:00:16 +00004390static __inline __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004391_mm512_mask_loadu_ps (__m512 __W, __mmask16 __U, void const *__P)
4392{
Craig Topper4537ea72016-05-14 06:03:13 +00004393 return (__m512) __builtin_ia32_loadups512_mask ((const float *) __P,
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004394 (__v16sf) __W,
4395 (__mmask16) __U);
4396}
4397
Craig Topper74c10e32018-07-09 19:00:16 +00004398static __inline __m512 __DEFAULT_FN_ATTRS512
Adam Nemet9a3ea602014-07-28 17:14:38 +00004399_mm512_maskz_loadu_ps(__mmask16 __U, void const *__P)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004400{
Craig Topper4537ea72016-05-14 06:03:13 +00004401 return (__m512) __builtin_ia32_loadups512_mask ((const float *)__P,
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004402 (__v16sf)
4403 _mm512_setzero_ps (),
4404 (__mmask16) __U);
4405}
4406
Craig Topper74c10e32018-07-09 19:00:16 +00004407static __inline __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004408_mm512_mask_loadu_pd (__m512d __W, __mmask8 __U, void const *__P)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004409{
Craig Topper4537ea72016-05-14 06:03:13 +00004410 return (__m512d) __builtin_ia32_loadupd512_mask ((const double *) __P,
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004411 (__v8df) __W,
4412 (__mmask8) __U);
Adam Nemetc0cff242015-01-16 18:51:50 +00004413}
4414
Craig Topper74c10e32018-07-09 19:00:16 +00004415static __inline __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004416_mm512_maskz_loadu_pd(__mmask8 __U, void const *__P)
Adam Nemetc0cff242015-01-16 18:51:50 +00004417{
Craig Topper4537ea72016-05-14 06:03:13 +00004418 return (__m512d) __builtin_ia32_loadupd512_mask ((const double *)__P,
Adam Nemetc0cff242015-01-16 18:51:50 +00004419 (__v8df)
4420 _mm512_setzero_pd (),
4421 (__mmask8) __U);
4422}
4423
Craig Topper74c10e32018-07-09 19:00:16 +00004424static __inline __m512d __DEFAULT_FN_ATTRS512
Craig Topper6afc4362017-03-17 05:59:25 +00004425_mm512_loadu_pd(void const *__p)
Adam Nemetda82bcc2014-07-31 04:00:39 +00004426{
4427 struct __loadu_pd {
4428 __m512d __v;
David Majnemer1cf22e62015-02-04 00:26:10 +00004429 } __attribute__((__packed__, __may_alias__));
Adam Nemetda82bcc2014-07-31 04:00:39 +00004430 return ((struct __loadu_pd*)__p)->__v;
4431}
4432
Craig Topper74c10e32018-07-09 19:00:16 +00004433static __inline __m512 __DEFAULT_FN_ATTRS512
Craig Topper6afc4362017-03-17 05:59:25 +00004434_mm512_loadu_ps(void const *__p)
Adam Nemetda82bcc2014-07-31 04:00:39 +00004435{
4436 struct __loadu_ps {
4437 __m512 __v;
David Majnemer1cf22e62015-02-04 00:26:10 +00004438 } __attribute__((__packed__, __may_alias__));
Adam Nemetda82bcc2014-07-31 04:00:39 +00004439 return ((struct __loadu_ps*)__p)->__v;
4440}
4441
Craig Topper74c10e32018-07-09 19:00:16 +00004442static __inline __m512 __DEFAULT_FN_ATTRS512
Craig Topper6afc4362017-03-17 05:59:25 +00004443_mm512_load_ps(void const *__p)
Adam Nemetc0cff242015-01-16 18:51:50 +00004444{
Craig Toppera6dd2fa2018-05-31 05:02:08 +00004445 return *(__m512*)__p;
Adam Nemetc0cff242015-01-16 18:51:50 +00004446}
4447
Craig Topper74c10e32018-07-09 19:00:16 +00004448static __inline __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004449_mm512_mask_load_ps (__m512 __W, __mmask16 __U, void const *__P)
4450{
4451 return (__m512) __builtin_ia32_loadaps512_mask ((const __v16sf *) __P,
4452 (__v16sf) __W,
4453 (__mmask16) __U);
4454}
4455
Craig Topper74c10e32018-07-09 19:00:16 +00004456static __inline __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004457_mm512_maskz_load_ps(__mmask16 __U, void const *__P)
4458{
4459 return (__m512) __builtin_ia32_loadaps512_mask ((const __v16sf *)__P,
4460 (__v16sf)
4461 _mm512_setzero_ps (),
4462 (__mmask16) __U);
4463}
4464
Craig Topper74c10e32018-07-09 19:00:16 +00004465static __inline __m512d __DEFAULT_FN_ATTRS512
Craig Topper6afc4362017-03-17 05:59:25 +00004466_mm512_load_pd(void const *__p)
Adam Nemetc0cff242015-01-16 18:51:50 +00004467{
Craig Toppera6dd2fa2018-05-31 05:02:08 +00004468 return *(__m512d*)__p;
Adam Nemetc0cff242015-01-16 18:51:50 +00004469}
4470
Craig Topper74c10e32018-07-09 19:00:16 +00004471static __inline __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004472_mm512_mask_load_pd (__m512d __W, __mmask8 __U, void const *__P)
4473{
4474 return (__m512d) __builtin_ia32_loadapd512_mask ((const __v8df *) __P,
4475 (__v8df) __W,
4476 (__mmask8) __U);
4477}
4478
Craig Topper74c10e32018-07-09 19:00:16 +00004479static __inline __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004480_mm512_maskz_load_pd(__mmask8 __U, void const *__P)
4481{
4482 return (__m512d) __builtin_ia32_loadapd512_mask ((const __v8df *)__P,
4483 (__v8df)
4484 _mm512_setzero_pd (),
4485 (__mmask8) __U);
4486}
4487
Craig Topper74c10e32018-07-09 19:00:16 +00004488static __inline __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004489_mm512_load_si512 (void const *__P)
4490{
4491 return *(__m512i *) __P;
4492}
4493
Craig Topper74c10e32018-07-09 19:00:16 +00004494static __inline __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004495_mm512_load_epi32 (void const *__P)
4496{
4497 return *(__m512i *) __P;
4498}
4499
Craig Topper74c10e32018-07-09 19:00:16 +00004500static __inline __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004501_mm512_load_epi64 (void const *__P)
4502{
4503 return *(__m512i *) __P;
4504}
4505
Adam Nemet0d5bb552014-07-28 17:14:40 +00004506/* SIMD store ops */
4507
Craig Topper74c10e32018-07-09 19:00:16 +00004508static __inline void __DEFAULT_FN_ATTRS512
Adam Nemet9a3ea602014-07-28 17:14:38 +00004509_mm512_mask_storeu_epi64(void *__P, __mmask8 __U, __m512i __A)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004510{
Craig Topper4537ea72016-05-14 06:03:13 +00004511 __builtin_ia32_storedqudi512_mask ((long long *)__P, (__v8di) __A,
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004512 (__mmask8) __U);
4513}
4514
Craig Topper74c10e32018-07-09 19:00:16 +00004515static __inline void __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004516_mm512_storeu_si512 (void *__P, __m512i __A)
4517{
Craig Toppera6dd2fa2018-05-31 05:02:08 +00004518 struct __storeu_si512 {
4519 __m512i __v;
4520 } __attribute__((__packed__, __may_alias__));
4521 ((struct __storeu_si512*)__P)->__v = __A;
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004522}
4523
Craig Topper74c10e32018-07-09 19:00:16 +00004524static __inline void __DEFAULT_FN_ATTRS512
Adam Nemet9a3ea602014-07-28 17:14:38 +00004525_mm512_mask_storeu_epi32(void *__P, __mmask16 __U, __m512i __A)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004526{
Craig Topper4537ea72016-05-14 06:03:13 +00004527 __builtin_ia32_storedqusi512_mask ((int *)__P, (__v16si) __A,
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004528 (__mmask16) __U);
4529}
4530
Craig Topper74c10e32018-07-09 19:00:16 +00004531static __inline void __DEFAULT_FN_ATTRS512
Adam Nemet9a3ea602014-07-28 17:14:38 +00004532_mm512_mask_storeu_pd(void *__P, __mmask8 __U, __m512d __A)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004533{
Craig Topper4537ea72016-05-14 06:03:13 +00004534 __builtin_ia32_storeupd512_mask ((double *)__P, (__v8df) __A, (__mmask8) __U);
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004535}
4536
Craig Topper74c10e32018-07-09 19:00:16 +00004537static __inline void __DEFAULT_FN_ATTRS512
Adam Nemetfce1ad02014-07-28 17:14:45 +00004538_mm512_storeu_pd(void *__P, __m512d __A)
4539{
Craig Toppera6dd2fa2018-05-31 05:02:08 +00004540 struct __storeu_pd {
4541 __m512d __v;
4542 } __attribute__((__packed__, __may_alias__));
4543 ((struct __storeu_pd*)__P)->__v = __A;
Adam Nemetfce1ad02014-07-28 17:14:45 +00004544}
4545
Craig Topper74c10e32018-07-09 19:00:16 +00004546static __inline void __DEFAULT_FN_ATTRS512
Adam Nemet9a3ea602014-07-28 17:14:38 +00004547_mm512_mask_storeu_ps(void *__P, __mmask16 __U, __m512 __A)
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004548{
Craig Topper4537ea72016-05-14 06:03:13 +00004549 __builtin_ia32_storeups512_mask ((float *)__P, (__v16sf) __A,
Elena Demikhovskyfcc6df32014-07-22 11:31:39 +00004550 (__mmask16) __U);
4551}
4552
Craig Topper74c10e32018-07-09 19:00:16 +00004553static __inline void __DEFAULT_FN_ATTRS512
Adam Nemetfce1ad02014-07-28 17:14:45 +00004554_mm512_storeu_ps(void *__P, __m512 __A)
4555{
Craig Toppera6dd2fa2018-05-31 05:02:08 +00004556 struct __storeu_ps {
4557 __m512 __v;
4558 } __attribute__((__packed__, __may_alias__));
4559 ((struct __storeu_ps*)__P)->__v = __A;
Adam Nemetfce1ad02014-07-28 17:14:45 +00004560}
4561
Craig Topper74c10e32018-07-09 19:00:16 +00004562static __inline void __DEFAULT_FN_ATTRS512
Adam Nemetc0cff242015-01-16 18:51:50 +00004563_mm512_mask_store_pd(void *__P, __mmask8 __U, __m512d __A)
Adam Nemetfce1ad02014-07-28 17:14:45 +00004564{
Adam Nemetc0cff242015-01-16 18:51:50 +00004565 __builtin_ia32_storeapd512_mask ((__v8df *)__P, (__v8df) __A, (__mmask8) __U);
Adam Nemetfce1ad02014-07-28 17:14:45 +00004566}
4567
Craig Topper74c10e32018-07-09 19:00:16 +00004568static __inline void __DEFAULT_FN_ATTRS512
Adam Nemetfce1ad02014-07-28 17:14:45 +00004569_mm512_store_pd(void *__P, __m512d __A)
4570{
Reid Kleckner89fbd552018-06-04 21:39:20 +00004571 *(__m512d*)__P = __A;
Adam Nemetfce1ad02014-07-28 17:14:45 +00004572}
4573
Craig Topper74c10e32018-07-09 19:00:16 +00004574static __inline void __DEFAULT_FN_ATTRS512
Adam Nemetc0cff242015-01-16 18:51:50 +00004575_mm512_mask_store_ps(void *__P, __mmask16 __U, __m512 __A)
4576{
4577 __builtin_ia32_storeaps512_mask ((__v16sf *)__P, (__v16sf) __A,
4578 (__mmask16) __U);
4579}
4580
Craig Topper74c10e32018-07-09 19:00:16 +00004581static __inline void __DEFAULT_FN_ATTRS512
Adam Nemetc0cff242015-01-16 18:51:50 +00004582_mm512_store_ps(void *__P, __m512 __A)
4583{
Reid Kleckner89fbd552018-06-04 21:39:20 +00004584 *(__m512*)__P = __A;
Adam Nemetc0cff242015-01-16 18:51:50 +00004585}
4586
Craig Topper74c10e32018-07-09 19:00:16 +00004587static __inline void __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004588_mm512_store_si512 (void *__P, __m512i __A)
4589{
4590 *(__m512i *) __P = __A;
4591}
4592
Craig Topper74c10e32018-07-09 19:00:16 +00004593static __inline void __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004594_mm512_store_epi32 (void *__P, __m512i __A)
4595{
4596 *(__m512i *) __P = __A;
4597}
4598
Craig Topper74c10e32018-07-09 19:00:16 +00004599static __inline void __DEFAULT_FN_ATTRS512
Michael Zuckermane9e8e572016-05-10 13:13:54 +00004600_mm512_store_epi64 (void *__P, __m512i __A)
4601{
4602 *(__m512i *) __P = __A;
4603}
4604
Adam Nemet2db1d2f2014-07-30 16:51:27 +00004605/* Mask ops */
4606
Craig Topper74c10e32018-07-09 19:00:16 +00004607static __inline __mmask16 __DEFAULT_FN_ATTRS512
Adam Nemet2db1d2f2014-07-30 16:51:27 +00004608_mm512_knot(__mmask16 __M)
4609{
4610 return __builtin_ia32_knothi(__M);
4611}
4612
Robert Khasanovb9f3a912014-10-08 17:18:13 +00004613/* Integer compare */
4614
Craig Topper57f96ac2017-11-06 21:00:49 +00004615#define _mm512_cmpeq_epi32_mask(A, B) \
4616 _mm512_cmp_epi32_mask((A), (B), _MM_CMPINT_EQ)
4617#define _mm512_mask_cmpeq_epi32_mask(k, A, B) \
4618 _mm512_mask_cmp_epi32_mask((k), (A), (B), _MM_CMPINT_EQ)
4619#define _mm512_cmpge_epi32_mask(A, B) \
4620 _mm512_cmp_epi32_mask((A), (B), _MM_CMPINT_GE)
4621#define _mm512_mask_cmpge_epi32_mask(k, A, B) \
4622 _mm512_mask_cmp_epi32_mask((k), (A), (B), _MM_CMPINT_GE)
4623#define _mm512_cmpgt_epi32_mask(A, B) \
4624 _mm512_cmp_epi32_mask((A), (B), _MM_CMPINT_GT)
4625#define _mm512_mask_cmpgt_epi32_mask(k, A, B) \
4626 _mm512_mask_cmp_epi32_mask((k), (A), (B), _MM_CMPINT_GT)
4627#define _mm512_cmple_epi32_mask(A, B) \
4628 _mm512_cmp_epi32_mask((A), (B), _MM_CMPINT_LE)
4629#define _mm512_mask_cmple_epi32_mask(k, A, B) \
4630 _mm512_mask_cmp_epi32_mask((k), (A), (B), _MM_CMPINT_LE)
4631#define _mm512_cmplt_epi32_mask(A, B) \
4632 _mm512_cmp_epi32_mask((A), (B), _MM_CMPINT_LT)
4633#define _mm512_mask_cmplt_epi32_mask(k, A, B) \
4634 _mm512_mask_cmp_epi32_mask((k), (A), (B), _MM_CMPINT_LT)
4635#define _mm512_cmpneq_epi32_mask(A, B) \
4636 _mm512_cmp_epi32_mask((A), (B), _MM_CMPINT_NE)
4637#define _mm512_mask_cmpneq_epi32_mask(k, A, B) \
4638 _mm512_mask_cmp_epi32_mask((k), (A), (B), _MM_CMPINT_NE)
Robert Khasanovb9f3a912014-10-08 17:18:13 +00004639
Craig Topper57f96ac2017-11-06 21:00:49 +00004640#define _mm512_cmpeq_epu32_mask(A, B) \
4641 _mm512_cmp_epu32_mask((A), (B), _MM_CMPINT_EQ)
4642#define _mm512_mask_cmpeq_epu32_mask(k, A, B) \
4643 _mm512_mask_cmp_epu32_mask((k), (A), (B), _MM_CMPINT_EQ)
4644#define _mm512_cmpge_epu32_mask(A, B) \
4645 _mm512_cmp_epu32_mask((A), (B), _MM_CMPINT_GE)
4646#define _mm512_mask_cmpge_epu32_mask(k, A, B) \
4647 _mm512_mask_cmp_epu32_mask((k), (A), (B), _MM_CMPINT_GE)
4648#define _mm512_cmpgt_epu32_mask(A, B) \
4649 _mm512_cmp_epu32_mask((A), (B), _MM_CMPINT_GT)
4650#define _mm512_mask_cmpgt_epu32_mask(k, A, B) \
4651 _mm512_mask_cmp_epu32_mask((k), (A), (B), _MM_CMPINT_GT)
4652#define _mm512_cmple_epu32_mask(A, B) \
4653 _mm512_cmp_epu32_mask((A), (B), _MM_CMPINT_LE)
4654#define _mm512_mask_cmple_epu32_mask(k, A, B) \
4655 _mm512_mask_cmp_epu32_mask((k), (A), (B), _MM_CMPINT_LE)
4656#define _mm512_cmplt_epu32_mask(A, B) \
4657 _mm512_cmp_epu32_mask((A), (B), _MM_CMPINT_LT)
4658#define _mm512_mask_cmplt_epu32_mask(k, A, B) \
4659 _mm512_mask_cmp_epu32_mask((k), (A), (B), _MM_CMPINT_LT)
4660#define _mm512_cmpneq_epu32_mask(A, B) \
4661 _mm512_cmp_epu32_mask((A), (B), _MM_CMPINT_NE)
4662#define _mm512_mask_cmpneq_epu32_mask(k, A, B) \
4663 _mm512_mask_cmp_epu32_mask((k), (A), (B), _MM_CMPINT_NE)
Robert Khasanovb9f3a912014-10-08 17:18:13 +00004664
Craig Topper57f96ac2017-11-06 21:00:49 +00004665#define _mm512_cmpeq_epi64_mask(A, B) \
4666 _mm512_cmp_epi64_mask((A), (B), _MM_CMPINT_EQ)
4667#define _mm512_mask_cmpeq_epi64_mask(k, A, B) \
4668 _mm512_mask_cmp_epi64_mask((k), (A), (B), _MM_CMPINT_EQ)
4669#define _mm512_cmpge_epi64_mask(A, B) \
4670 _mm512_cmp_epi64_mask((A), (B), _MM_CMPINT_GE)
4671#define _mm512_mask_cmpge_epi64_mask(k, A, B) \
4672 _mm512_mask_cmp_epi64_mask((k), (A), (B), _MM_CMPINT_GE)
4673#define _mm512_cmpgt_epi64_mask(A, B) \
4674 _mm512_cmp_epi64_mask((A), (B), _MM_CMPINT_GT)
4675#define _mm512_mask_cmpgt_epi64_mask(k, A, B) \
4676 _mm512_mask_cmp_epi64_mask((k), (A), (B), _MM_CMPINT_GT)
4677#define _mm512_cmple_epi64_mask(A, B) \
4678 _mm512_cmp_epi64_mask((A), (B), _MM_CMPINT_LE)
4679#define _mm512_mask_cmple_epi64_mask(k, A, B) \
4680 _mm512_mask_cmp_epi64_mask((k), (A), (B), _MM_CMPINT_LE)
4681#define _mm512_cmplt_epi64_mask(A, B) \
4682 _mm512_cmp_epi64_mask((A), (B), _MM_CMPINT_LT)
4683#define _mm512_mask_cmplt_epi64_mask(k, A, B) \
4684 _mm512_mask_cmp_epi64_mask((k), (A), (B), _MM_CMPINT_LT)
4685#define _mm512_cmpneq_epi64_mask(A, B) \
4686 _mm512_cmp_epi64_mask((A), (B), _MM_CMPINT_NE)
4687#define _mm512_mask_cmpneq_epi64_mask(k, A, B) \
4688 _mm512_mask_cmp_epi64_mask((k), (A), (B), _MM_CMPINT_NE)
Craig Topper4cac1c22015-01-25 23:30:07 +00004689
Craig Topper57f96ac2017-11-06 21:00:49 +00004690#define _mm512_cmpeq_epu64_mask(A, B) \
4691 _mm512_cmp_epu64_mask((A), (B), _MM_CMPINT_EQ)
4692#define _mm512_mask_cmpeq_epu64_mask(k, A, B) \
4693 _mm512_mask_cmp_epu64_mask((k), (A), (B), _MM_CMPINT_EQ)
4694#define _mm512_cmpge_epu64_mask(A, B) \
4695 _mm512_cmp_epu64_mask((A), (B), _MM_CMPINT_GE)
4696#define _mm512_mask_cmpge_epu64_mask(k, A, B) \
4697 _mm512_mask_cmp_epu64_mask((k), (A), (B), _MM_CMPINT_GE)
4698#define _mm512_cmpgt_epu64_mask(A, B) \
4699 _mm512_cmp_epu64_mask((A), (B), _MM_CMPINT_GT)
4700#define _mm512_mask_cmpgt_epu64_mask(k, A, B) \
4701 _mm512_mask_cmp_epu64_mask((k), (A), (B), _MM_CMPINT_GT)
4702#define _mm512_cmple_epu64_mask(A, B) \
4703 _mm512_cmp_epu64_mask((A), (B), _MM_CMPINT_LE)
4704#define _mm512_mask_cmple_epu64_mask(k, A, B) \
4705 _mm512_mask_cmp_epu64_mask((k), (A), (B), _MM_CMPINT_LE)
4706#define _mm512_cmplt_epu64_mask(A, B) \
4707 _mm512_cmp_epu64_mask((A), (B), _MM_CMPINT_LT)
4708#define _mm512_mask_cmplt_epu64_mask(k, A, B) \
4709 _mm512_mask_cmp_epu64_mask((k), (A), (B), _MM_CMPINT_LT)
4710#define _mm512_cmpneq_epu64_mask(A, B) \
4711 _mm512_cmp_epu64_mask((A), (B), _MM_CMPINT_NE)
4712#define _mm512_mask_cmpneq_epu64_mask(k, A, B) \
4713 _mm512_mask_cmp_epu64_mask((k), (A), (B), _MM_CMPINT_NE)
Craig Topper4cac1c22015-01-25 23:30:07 +00004714
Craig Topper74c10e32018-07-09 19:00:16 +00004715static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004716_mm512_cvtepi8_epi32(__m128i __A)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004717{
Craig Topper0c5da262016-10-23 07:35:47 +00004718 /* This function always performs a signed extension, but __v16qi is a char
4719 which may be signed or unsigned, so use __v16qs. */
4720 return (__m512i)__builtin_convertvector((__v16qs)__A, __v16si);
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004721}
4722
Craig Topper74c10e32018-07-09 19:00:16 +00004723static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004724_mm512_mask_cvtepi8_epi32(__m512i __W, __mmask16 __U, __m128i __A)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004725{
Craig Topper0c5da262016-10-23 07:35:47 +00004726 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4727 (__v16si)_mm512_cvtepi8_epi32(__A),
4728 (__v16si)__W);
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004729}
4730
Craig Topper74c10e32018-07-09 19:00:16 +00004731static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004732_mm512_maskz_cvtepi8_epi32(__mmask16 __U, __m128i __A)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004733{
Craig Topper0c5da262016-10-23 07:35:47 +00004734 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4735 (__v16si)_mm512_cvtepi8_epi32(__A),
4736 (__v16si)_mm512_setzero_si512());
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004737}
4738
Craig Topper74c10e32018-07-09 19:00:16 +00004739static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004740_mm512_cvtepi8_epi64(__m128i __A)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004741{
Craig Topper0c5da262016-10-23 07:35:47 +00004742 /* This function always performs a signed extension, but __v16qi is a char
4743 which may be signed or unsigned, so use __v16qs. */
4744 return (__m512i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__A, (__v16qs)__A, 0, 1, 2, 3, 4, 5, 6, 7), __v8di);
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004745}
4746
Craig Topper74c10e32018-07-09 19:00:16 +00004747static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004748_mm512_mask_cvtepi8_epi64(__m512i __W, __mmask8 __U, __m128i __A)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004749{
Craig Topper0c5da262016-10-23 07:35:47 +00004750 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4751 (__v8di)_mm512_cvtepi8_epi64(__A),
4752 (__v8di)__W);
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004753}
4754
Craig Topper74c10e32018-07-09 19:00:16 +00004755static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004756_mm512_maskz_cvtepi8_epi64(__mmask8 __U, __m128i __A)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004757{
Craig Topper0c5da262016-10-23 07:35:47 +00004758 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4759 (__v8di)_mm512_cvtepi8_epi64(__A),
4760 (__v8di)_mm512_setzero_si512 ());
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004761}
4762
Craig Topper74c10e32018-07-09 19:00:16 +00004763static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004764_mm512_cvtepi32_epi64(__m256i __X)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004765{
Craig Topper0c5da262016-10-23 07:35:47 +00004766 return (__m512i)__builtin_convertvector((__v8si)__X, __v8di);
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004767}
4768
Craig Topper74c10e32018-07-09 19:00:16 +00004769static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004770_mm512_mask_cvtepi32_epi64(__m512i __W, __mmask8 __U, __m256i __X)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004771{
Craig Topper0c5da262016-10-23 07:35:47 +00004772 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4773 (__v8di)_mm512_cvtepi32_epi64(__X),
4774 (__v8di)__W);
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004775}
4776
Craig Topper74c10e32018-07-09 19:00:16 +00004777static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004778_mm512_maskz_cvtepi32_epi64(__mmask8 __U, __m256i __X)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004779{
Craig Topper0c5da262016-10-23 07:35:47 +00004780 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4781 (__v8di)_mm512_cvtepi32_epi64(__X),
4782 (__v8di)_mm512_setzero_si512());
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004783}
4784
Craig Topper74c10e32018-07-09 19:00:16 +00004785static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004786_mm512_cvtepi16_epi32(__m256i __A)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004787{
Craig Topper0c5da262016-10-23 07:35:47 +00004788 return (__m512i)__builtin_convertvector((__v16hi)__A, __v16si);
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004789}
4790
Craig Topper74c10e32018-07-09 19:00:16 +00004791static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004792_mm512_mask_cvtepi16_epi32(__m512i __W, __mmask16 __U, __m256i __A)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004793{
Craig Topper0c5da262016-10-23 07:35:47 +00004794 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4795 (__v16si)_mm512_cvtepi16_epi32(__A),
4796 (__v16si)__W);
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004797}
4798
Craig Topper74c10e32018-07-09 19:00:16 +00004799static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004800_mm512_maskz_cvtepi16_epi32(__mmask16 __U, __m256i __A)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004801{
Craig Topper0c5da262016-10-23 07:35:47 +00004802 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4803 (__v16si)_mm512_cvtepi16_epi32(__A),
4804 (__v16si)_mm512_setzero_si512 ());
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004805}
4806
Craig Topper74c10e32018-07-09 19:00:16 +00004807static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004808_mm512_cvtepi16_epi64(__m128i __A)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004809{
Craig Topper0c5da262016-10-23 07:35:47 +00004810 return (__m512i)__builtin_convertvector((__v8hi)__A, __v8di);
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004811}
4812
Craig Topper74c10e32018-07-09 19:00:16 +00004813static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004814_mm512_mask_cvtepi16_epi64(__m512i __W, __mmask8 __U, __m128i __A)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004815{
Craig Topper0c5da262016-10-23 07:35:47 +00004816 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4817 (__v8di)_mm512_cvtepi16_epi64(__A),
4818 (__v8di)__W);
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004819}
4820
Craig Topper74c10e32018-07-09 19:00:16 +00004821static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004822_mm512_maskz_cvtepi16_epi64(__mmask8 __U, __m128i __A)
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004823{
Craig Topper0c5da262016-10-23 07:35:47 +00004824 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4825 (__v8di)_mm512_cvtepi16_epi64(__A),
4826 (__v8di)_mm512_setzero_si512());
Michael Zuckerman7cdb72f2016-02-18 09:09:34 +00004827}
4828
Craig Topper74c10e32018-07-09 19:00:16 +00004829static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004830_mm512_cvtepu8_epi32(__m128i __A)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004831{
Craig Topper0c5da262016-10-23 07:35:47 +00004832 return (__m512i)__builtin_convertvector((__v16qu)__A, __v16si);
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004833}
4834
Craig Topper74c10e32018-07-09 19:00:16 +00004835static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004836_mm512_mask_cvtepu8_epi32(__m512i __W, __mmask16 __U, __m128i __A)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004837{
Craig Topper0c5da262016-10-23 07:35:47 +00004838 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4839 (__v16si)_mm512_cvtepu8_epi32(__A),
4840 (__v16si)__W);
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004841}
4842
Craig Topper74c10e32018-07-09 19:00:16 +00004843static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004844_mm512_maskz_cvtepu8_epi32(__mmask16 __U, __m128i __A)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004845{
Craig Topper0c5da262016-10-23 07:35:47 +00004846 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4847 (__v16si)_mm512_cvtepu8_epi32(__A),
4848 (__v16si)_mm512_setzero_si512());
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004849}
4850
Craig Topper74c10e32018-07-09 19:00:16 +00004851static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004852_mm512_cvtepu8_epi64(__m128i __A)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004853{
Craig Topper0c5da262016-10-23 07:35:47 +00004854 return (__m512i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__A, (__v16qu)__A, 0, 1, 2, 3, 4, 5, 6, 7), __v8di);
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004855}
4856
Craig Topper74c10e32018-07-09 19:00:16 +00004857static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004858_mm512_mask_cvtepu8_epi64(__m512i __W, __mmask8 __U, __m128i __A)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004859{
Craig Topper0c5da262016-10-23 07:35:47 +00004860 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4861 (__v8di)_mm512_cvtepu8_epi64(__A),
4862 (__v8di)__W);
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004863}
4864
Craig Topper74c10e32018-07-09 19:00:16 +00004865static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004866_mm512_maskz_cvtepu8_epi64(__mmask8 __U, __m128i __A)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004867{
Craig Topper0c5da262016-10-23 07:35:47 +00004868 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4869 (__v8di)_mm512_cvtepu8_epi64(__A),
4870 (__v8di)_mm512_setzero_si512());
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004871}
4872
Craig Topper74c10e32018-07-09 19:00:16 +00004873static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004874_mm512_cvtepu32_epi64(__m256i __X)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004875{
Craig Topper0c5da262016-10-23 07:35:47 +00004876 return (__m512i)__builtin_convertvector((__v8su)__X, __v8di);
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004877}
4878
Craig Topper74c10e32018-07-09 19:00:16 +00004879static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004880_mm512_mask_cvtepu32_epi64(__m512i __W, __mmask8 __U, __m256i __X)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004881{
Craig Topper0c5da262016-10-23 07:35:47 +00004882 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4883 (__v8di)_mm512_cvtepu32_epi64(__X),
4884 (__v8di)__W);
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004885}
4886
Craig Topper74c10e32018-07-09 19:00:16 +00004887static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004888_mm512_maskz_cvtepu32_epi64(__mmask8 __U, __m256i __X)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004889{
Craig Topper0c5da262016-10-23 07:35:47 +00004890 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4891 (__v8di)_mm512_cvtepu32_epi64(__X),
4892 (__v8di)_mm512_setzero_si512());
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004893}
4894
Craig Topper74c10e32018-07-09 19:00:16 +00004895static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004896_mm512_cvtepu16_epi32(__m256i __A)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004897{
Craig Topper0c5da262016-10-23 07:35:47 +00004898 return (__m512i)__builtin_convertvector((__v16hu)__A, __v16si);
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004899}
4900
Craig Topper74c10e32018-07-09 19:00:16 +00004901static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004902_mm512_mask_cvtepu16_epi32(__m512i __W, __mmask16 __U, __m256i __A)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004903{
Craig Topper0c5da262016-10-23 07:35:47 +00004904 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4905 (__v16si)_mm512_cvtepu16_epi32(__A),
4906 (__v16si)__W);
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004907}
4908
Craig Topper74c10e32018-07-09 19:00:16 +00004909static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004910_mm512_maskz_cvtepu16_epi32(__mmask16 __U, __m256i __A)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004911{
Craig Topper0c5da262016-10-23 07:35:47 +00004912 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4913 (__v16si)_mm512_cvtepu16_epi32(__A),
4914 (__v16si)_mm512_setzero_si512());
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004915}
4916
Craig Topper74c10e32018-07-09 19:00:16 +00004917static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004918_mm512_cvtepu16_epi64(__m128i __A)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004919{
Craig Topper0c5da262016-10-23 07:35:47 +00004920 return (__m512i)__builtin_convertvector((__v8hu)__A, __v8di);
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004921}
4922
Craig Topper74c10e32018-07-09 19:00:16 +00004923static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004924_mm512_mask_cvtepu16_epi64(__m512i __W, __mmask8 __U, __m128i __A)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004925{
Craig Topper0c5da262016-10-23 07:35:47 +00004926 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4927 (__v8di)_mm512_cvtepu16_epi64(__A),
4928 (__v8di)__W);
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004929}
4930
Craig Topper74c10e32018-07-09 19:00:16 +00004931static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper0c5da262016-10-23 07:35:47 +00004932_mm512_maskz_cvtepu16_epi64(__mmask8 __U, __m128i __A)
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004933{
Craig Topper0c5da262016-10-23 07:35:47 +00004934 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4935 (__v8di)_mm512_cvtepu16_epi64(__A),
4936 (__v8di)_mm512_setzero_si512());
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004937}
4938
Craig Topper74c10e32018-07-09 19:00:16 +00004939static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermane98cc742016-02-23 15:59:47 +00004940_mm512_rorv_epi32 (__m512i __A, __m512i __B)
4941{
Craig Topper0e9de762018-06-30 01:32:14 +00004942 return (__m512i)__builtin_ia32_prorvd512((__v16si)__A, (__v16si)__B);
Michael Zuckermane98cc742016-02-23 15:59:47 +00004943}
4944
Craig Topper74c10e32018-07-09 19:00:16 +00004945static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermane98cc742016-02-23 15:59:47 +00004946_mm512_mask_rorv_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
4947{
Craig Topper0e9de762018-06-30 01:32:14 +00004948 return (__m512i)__builtin_ia32_selectd_512(__U,
4949 (__v16si)_mm512_rorv_epi32(__A, __B),
4950 (__v16si)__W);
Michael Zuckermane98cc742016-02-23 15:59:47 +00004951}
4952
Craig Topper74c10e32018-07-09 19:00:16 +00004953static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermane98cc742016-02-23 15:59:47 +00004954_mm512_maskz_rorv_epi32 (__mmask16 __U, __m512i __A, __m512i __B)
4955{
Craig Topper0e9de762018-06-30 01:32:14 +00004956 return (__m512i)__builtin_ia32_selectd_512(__U,
4957 (__v16si)_mm512_rorv_epi32(__A, __B),
4958 (__v16si)_mm512_setzero_si512());
Michael Zuckermane98cc742016-02-23 15:59:47 +00004959}
4960
Craig Topper74c10e32018-07-09 19:00:16 +00004961static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermane98cc742016-02-23 15:59:47 +00004962_mm512_rorv_epi64 (__m512i __A, __m512i __B)
4963{
Craig Topper0e9de762018-06-30 01:32:14 +00004964 return (__m512i)__builtin_ia32_prorvq512((__v8di)__A, (__v8di)__B);
Michael Zuckermane98cc742016-02-23 15:59:47 +00004965}
4966
Craig Topper74c10e32018-07-09 19:00:16 +00004967static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermane98cc742016-02-23 15:59:47 +00004968_mm512_mask_rorv_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
4969{
Craig Topper0e9de762018-06-30 01:32:14 +00004970 return (__m512i)__builtin_ia32_selectq_512(__U,
4971 (__v8di)_mm512_rorv_epi64(__A, __B),
4972 (__v8di)__W);
Michael Zuckermane98cc742016-02-23 15:59:47 +00004973}
4974
Craig Topper74c10e32018-07-09 19:00:16 +00004975static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermane98cc742016-02-23 15:59:47 +00004976_mm512_maskz_rorv_epi64 (__mmask8 __U, __m512i __A, __m512i __B)
4977{
Craig Topper0e9de762018-06-30 01:32:14 +00004978 return (__m512i)__builtin_ia32_selectq_512(__U,
4979 (__v8di)_mm512_rorv_epi64(__A, __B),
4980 (__v8di)_mm512_setzero_si512());
Michael Zuckermane98cc742016-02-23 15:59:47 +00004981}
4982
4983
Michael Zuckerman7a33dce2016-02-21 14:00:11 +00004984
Craig Topperc6338672018-05-31 00:51:20 +00004985#define _mm512_cmp_epi32_mask(a, b, p) \
Craig Topper3a71f352015-11-29 06:50:33 +00004986 (__mmask16)__builtin_ia32_cmpd512_mask((__v16si)(__m512i)(a), \
Craig Topper8c18e112016-05-17 04:41:50 +00004987 (__v16si)(__m512i)(b), (int)(p), \
Craig Topperc6338672018-05-31 00:51:20 +00004988 (__mmask16)-1)
Craig Topper4cac1c22015-01-25 23:30:07 +00004989
Craig Topperc6338672018-05-31 00:51:20 +00004990#define _mm512_cmp_epu32_mask(a, b, p) \
Craig Topper3a71f352015-11-29 06:50:33 +00004991 (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)(__m512i)(a), \
Craig Topper8c18e112016-05-17 04:41:50 +00004992 (__v16si)(__m512i)(b), (int)(p), \
Craig Topperc6338672018-05-31 00:51:20 +00004993 (__mmask16)-1)
Craig Topper4cac1c22015-01-25 23:30:07 +00004994
Craig Topperc6338672018-05-31 00:51:20 +00004995#define _mm512_cmp_epi64_mask(a, b, p) \
Craig Topper3a71f352015-11-29 06:50:33 +00004996 (__mmask8)__builtin_ia32_cmpq512_mask((__v8di)(__m512i)(a), \
Craig Topper8c18e112016-05-17 04:41:50 +00004997 (__v8di)(__m512i)(b), (int)(p), \
Craig Topperc6338672018-05-31 00:51:20 +00004998 (__mmask8)-1)
Craig Topper4cac1c22015-01-25 23:30:07 +00004999
Craig Topperc6338672018-05-31 00:51:20 +00005000#define _mm512_cmp_epu64_mask(a, b, p) \
Craig Topper3a71f352015-11-29 06:50:33 +00005001 (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)(__m512i)(a), \
Craig Topper8c18e112016-05-17 04:41:50 +00005002 (__v8di)(__m512i)(b), (int)(p), \
Craig Topperc6338672018-05-31 00:51:20 +00005003 (__mmask8)-1)
Craig Topper4cac1c22015-01-25 23:30:07 +00005004
Craig Topperc6338672018-05-31 00:51:20 +00005005#define _mm512_mask_cmp_epi32_mask(m, a, b, p) \
Craig Topper3a71f352015-11-29 06:50:33 +00005006 (__mmask16)__builtin_ia32_cmpd512_mask((__v16si)(__m512i)(a), \
Craig Topper8c18e112016-05-17 04:41:50 +00005007 (__v16si)(__m512i)(b), (int)(p), \
Craig Topperc6338672018-05-31 00:51:20 +00005008 (__mmask16)(m))
Craig Topper4cac1c22015-01-25 23:30:07 +00005009
Craig Topperc6338672018-05-31 00:51:20 +00005010#define _mm512_mask_cmp_epu32_mask(m, a, b, p) \
Craig Topper3a71f352015-11-29 06:50:33 +00005011 (__mmask16)__builtin_ia32_ucmpd512_mask((__v16si)(__m512i)(a), \
Craig Topper8c18e112016-05-17 04:41:50 +00005012 (__v16si)(__m512i)(b), (int)(p), \
Craig Topperc6338672018-05-31 00:51:20 +00005013 (__mmask16)(m))
Craig Topper4cac1c22015-01-25 23:30:07 +00005014
Craig Topperc6338672018-05-31 00:51:20 +00005015#define _mm512_mask_cmp_epi64_mask(m, a, b, p) \
Craig Topper3a71f352015-11-29 06:50:33 +00005016 (__mmask8)__builtin_ia32_cmpq512_mask((__v8di)(__m512i)(a), \
Craig Topper8c18e112016-05-17 04:41:50 +00005017 (__v8di)(__m512i)(b), (int)(p), \
Craig Topperc6338672018-05-31 00:51:20 +00005018 (__mmask8)(m))
Craig Topper4cac1c22015-01-25 23:30:07 +00005019
Craig Topperc6338672018-05-31 00:51:20 +00005020#define _mm512_mask_cmp_epu64_mask(m, a, b, p) \
Craig Topper3a71f352015-11-29 06:50:33 +00005021 (__mmask8)__builtin_ia32_ucmpq512_mask((__v8di)(__m512i)(a), \
Craig Topper8c18e112016-05-17 04:41:50 +00005022 (__v8di)(__m512i)(b), (int)(p), \
Craig Topperc6338672018-05-31 00:51:20 +00005023 (__mmask8)(m))
Eric Christopher4d1851682015-06-17 07:09:20 +00005024
Craig Topperc6338672018-05-31 00:51:20 +00005025#define _mm512_rol_epi32(a, b) \
Craig Topper0e9de762018-06-30 01:32:14 +00005026 (__m512i)__builtin_ia32_prold512((__v16si)(__m512i)(a), (int)(b))
Michael Zuckerman38a27272016-02-22 09:05:41 +00005027
Craig Topperc6338672018-05-31 00:51:20 +00005028#define _mm512_mask_rol_epi32(W, U, a, b) \
Craig Topper0e9de762018-06-30 01:32:14 +00005029 (__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
5030 (__v16si)_mm512_rol_epi32((a), (b)), \
5031 (__v16si)(__m512i)(W))
Michael Zuckerman38a27272016-02-22 09:05:41 +00005032
Craig Topperc6338672018-05-31 00:51:20 +00005033#define _mm512_maskz_rol_epi32(U, a, b) \
Craig Topper0e9de762018-06-30 01:32:14 +00005034 (__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
5035 (__v16si)_mm512_rol_epi32((a), (b)), \
5036 (__v16si)_mm512_setzero_si512())
Michael Zuckerman38a27272016-02-22 09:05:41 +00005037
Craig Topperc6338672018-05-31 00:51:20 +00005038#define _mm512_rol_epi64(a, b) \
Craig Topper0e9de762018-06-30 01:32:14 +00005039 (__m512i)__builtin_ia32_prolq512((__v8di)(__m512i)(a), (int)(b))
Michael Zuckerman38a27272016-02-22 09:05:41 +00005040
Craig Topperc6338672018-05-31 00:51:20 +00005041#define _mm512_mask_rol_epi64(W, U, a, b) \
Craig Topper0e9de762018-06-30 01:32:14 +00005042 (__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
5043 (__v8di)_mm512_rol_epi64((a), (b)), \
5044 (__v8di)(__m512i)(W))
Michael Zuckerman38a27272016-02-22 09:05:41 +00005045
Craig Topperc6338672018-05-31 00:51:20 +00005046#define _mm512_maskz_rol_epi64(U, a, b) \
Craig Topper0e9de762018-06-30 01:32:14 +00005047 (__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
5048 (__v8di)_mm512_rol_epi64((a), (b)), \
5049 (__v8di)_mm512_setzero_si512())
5050
Craig Topper74c10e32018-07-09 19:00:16 +00005051static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman0231f162016-02-23 13:41:13 +00005052_mm512_rolv_epi32 (__m512i __A, __m512i __B)
5053{
Craig Topper0e9de762018-06-30 01:32:14 +00005054 return (__m512i)__builtin_ia32_prolvd512((__v16si)__A, (__v16si)__B);
Michael Zuckerman0231f162016-02-23 13:41:13 +00005055}
5056
Craig Topper74c10e32018-07-09 19:00:16 +00005057static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman0231f162016-02-23 13:41:13 +00005058_mm512_mask_rolv_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
5059{
Craig Topper0e9de762018-06-30 01:32:14 +00005060 return (__m512i)__builtin_ia32_selectd_512(__U,
5061 (__v16si)_mm512_rolv_epi32(__A, __B),
5062 (__v16si)__W);
Michael Zuckerman0231f162016-02-23 13:41:13 +00005063}
5064
Craig Topper74c10e32018-07-09 19:00:16 +00005065static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman0231f162016-02-23 13:41:13 +00005066_mm512_maskz_rolv_epi32 (__mmask16 __U, __m512i __A, __m512i __B)
5067{
Craig Topper0e9de762018-06-30 01:32:14 +00005068 return (__m512i)__builtin_ia32_selectd_512(__U,
5069 (__v16si)_mm512_rolv_epi32(__A, __B),
5070 (__v16si)_mm512_setzero_si512());
Michael Zuckerman0231f162016-02-23 13:41:13 +00005071}
5072
Craig Topper74c10e32018-07-09 19:00:16 +00005073static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman0231f162016-02-23 13:41:13 +00005074_mm512_rolv_epi64 (__m512i __A, __m512i __B)
5075{
Craig Topper0e9de762018-06-30 01:32:14 +00005076 return (__m512i)__builtin_ia32_prolvq512((__v8di)__A, (__v8di)__B);
Michael Zuckerman0231f162016-02-23 13:41:13 +00005077}
5078
Craig Topper74c10e32018-07-09 19:00:16 +00005079static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman0231f162016-02-23 13:41:13 +00005080_mm512_mask_rolv_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
5081{
Craig Topper0e9de762018-06-30 01:32:14 +00005082 return (__m512i)__builtin_ia32_selectq_512(__U,
5083 (__v8di)_mm512_rolv_epi64(__A, __B),
5084 (__v8di)__W);
Michael Zuckerman0231f162016-02-23 13:41:13 +00005085}
5086
Craig Topper74c10e32018-07-09 19:00:16 +00005087static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman0231f162016-02-23 13:41:13 +00005088_mm512_maskz_rolv_epi64 (__mmask8 __U, __m512i __A, __m512i __B)
5089{
Craig Topper0e9de762018-06-30 01:32:14 +00005090 return (__m512i)__builtin_ia32_selectq_512(__U,
5091 (__v8di)_mm512_rolv_epi64(__A, __B),
5092 (__v8di)_mm512_setzero_si512());
Michael Zuckerman0231f162016-02-23 13:41:13 +00005093}
5094
Craig Topperc6338672018-05-31 00:51:20 +00005095#define _mm512_ror_epi32(A, B) \
Craig Topper0e9de762018-06-30 01:32:14 +00005096 (__m512i)__builtin_ia32_prord512((__v16si)(__m512i)(A), (int)(B))
Michael Zuckerman0231f162016-02-23 13:41:13 +00005097
Craig Topperc6338672018-05-31 00:51:20 +00005098#define _mm512_mask_ror_epi32(W, U, A, B) \
Craig Topper0e9de762018-06-30 01:32:14 +00005099 (__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
5100 (__v16si)_mm512_ror_epi32((A), (B)), \
5101 (__v16si)(__m512i)(W))
Michael Zuckerman0231f162016-02-23 13:41:13 +00005102
Craig Topperc6338672018-05-31 00:51:20 +00005103#define _mm512_maskz_ror_epi32(U, A, B) \
Craig Topper0e9de762018-06-30 01:32:14 +00005104 (__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
5105 (__v16si)_mm512_ror_epi32((A), (B)), \
5106 (__v16si)_mm512_setzero_si512())
Michael Zuckerman0231f162016-02-23 13:41:13 +00005107
Craig Topperc6338672018-05-31 00:51:20 +00005108#define _mm512_ror_epi64(A, B) \
Craig Topper0e9de762018-06-30 01:32:14 +00005109 (__m512i)__builtin_ia32_prorq512((__v8di)(__m512i)(A), (int)(B))
Michael Zuckerman0231f162016-02-23 13:41:13 +00005110
Craig Topperc6338672018-05-31 00:51:20 +00005111#define _mm512_mask_ror_epi64(W, U, A, B) \
Craig Topper0e9de762018-06-30 01:32:14 +00005112 (__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
5113 (__v8di)_mm512_ror_epi64((A), (B)), \
5114 (__v8di)(__m512i)(W))
Michael Zuckerman0231f162016-02-23 13:41:13 +00005115
Craig Topperc6338672018-05-31 00:51:20 +00005116#define _mm512_maskz_ror_epi64(U, A, B) \
Craig Topper0e9de762018-06-30 01:32:14 +00005117 (__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
5118 (__v8di)_mm512_ror_epi64((A), (B)), \
5119 (__v8di)_mm512_setzero_si512())
Michael Zuckerman38a27272016-02-22 09:05:41 +00005120
Craig Topper74c10e32018-07-09 19:00:16 +00005121static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005122_mm512_slli_epi32(__m512i __A, int __B)
5123{
5124 return (__m512i)__builtin_ia32_pslldi512((__v16si)__A, __B);
5125}
Michael Zuckerman1ac360c2016-03-01 11:38:16 +00005126
Craig Topper74c10e32018-07-09 19:00:16 +00005127static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005128_mm512_mask_slli_epi32(__m512i __W, __mmask16 __U, __m512i __A, int __B)
5129{
Craig Topperd7e5b212016-11-13 07:26:31 +00005130 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5131 (__v16si)_mm512_slli_epi32(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005132 (__v16si)__W);
5133}
Michael Zuckerman1ac360c2016-03-01 11:38:16 +00005134
Craig Topper74c10e32018-07-09 19:00:16 +00005135static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005136_mm512_maskz_slli_epi32(__mmask16 __U, __m512i __A, int __B) {
Craig Topperd7e5b212016-11-13 07:26:31 +00005137 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5138 (__v16si)_mm512_slli_epi32(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005139 (__v16si)_mm512_setzero_si512());
5140}
Michael Zuckerman1ac360c2016-03-01 11:38:16 +00005141
Craig Topper74c10e32018-07-09 19:00:16 +00005142static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005143_mm512_slli_epi64(__m512i __A, int __B)
5144{
5145 return (__m512i)__builtin_ia32_psllqi512((__v8di)__A, __B);
5146}
Michael Zuckerman1ac360c2016-03-01 11:38:16 +00005147
Craig Topper74c10e32018-07-09 19:00:16 +00005148static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005149_mm512_mask_slli_epi64(__m512i __W, __mmask8 __U, __m512i __A, int __B)
5150{
Craig Topperd7e5b212016-11-13 07:26:31 +00005151 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5152 (__v8di)_mm512_slli_epi64(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005153 (__v8di)__W);
5154}
Michael Zuckerman1ac360c2016-03-01 11:38:16 +00005155
Craig Topper74c10e32018-07-09 19:00:16 +00005156static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005157_mm512_maskz_slli_epi64(__mmask8 __U, __m512i __A, int __B)
5158{
Craig Topperd7e5b212016-11-13 07:26:31 +00005159 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5160 (__v8di)_mm512_slli_epi64(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005161 (__v8di)_mm512_setzero_si512());
5162}
Michael Zuckerman1ac360c2016-03-01 11:38:16 +00005163
Craig Topper74c10e32018-07-09 19:00:16 +00005164static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005165_mm512_srli_epi32(__m512i __A, int __B)
5166{
5167 return (__m512i)__builtin_ia32_psrldi512((__v16si)__A, __B);
5168}
Michael Zuckerman38a27272016-02-22 09:05:41 +00005169
Craig Topper74c10e32018-07-09 19:00:16 +00005170static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005171_mm512_mask_srli_epi32(__m512i __W, __mmask16 __U, __m512i __A, int __B)
5172{
Craig Topperd7e5b212016-11-13 07:26:31 +00005173 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5174 (__v16si)_mm512_srli_epi32(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005175 (__v16si)__W);
5176}
Michael Zuckermand176d742016-03-01 17:49:03 +00005177
Craig Topper74c10e32018-07-09 19:00:16 +00005178static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005179_mm512_maskz_srli_epi32(__mmask16 __U, __m512i __A, int __B) {
Craig Topperd7e5b212016-11-13 07:26:31 +00005180 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5181 (__v16si)_mm512_srli_epi32(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005182 (__v16si)_mm512_setzero_si512());
5183}
Michael Zuckermand176d742016-03-01 17:49:03 +00005184
Craig Topper74c10e32018-07-09 19:00:16 +00005185static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005186_mm512_srli_epi64(__m512i __A, int __B)
5187{
5188 return (__m512i)__builtin_ia32_psrlqi512((__v8di)__A, __B);
5189}
Michael Zuckermand176d742016-03-01 17:49:03 +00005190
Craig Topper74c10e32018-07-09 19:00:16 +00005191static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005192_mm512_mask_srli_epi64(__m512i __W, __mmask8 __U, __m512i __A, int __B)
5193{
Craig Topperd7e5b212016-11-13 07:26:31 +00005194 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5195 (__v8di)_mm512_srli_epi64(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005196 (__v8di)__W);
5197}
Michael Zuckermand176d742016-03-01 17:49:03 +00005198
Craig Topper74c10e32018-07-09 19:00:16 +00005199static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005200_mm512_maskz_srli_epi64(__mmask8 __U, __m512i __A, int __B)
5201{
Craig Topperd7e5b212016-11-13 07:26:31 +00005202 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5203 (__v8di)_mm512_srli_epi64(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005204 (__v8di)_mm512_setzero_si512());
5205}
Michael Zuckermand176d742016-03-01 17:49:03 +00005206
Craig Topper74c10e32018-07-09 19:00:16 +00005207static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanffbb67a2016-03-03 09:26:01 +00005208_mm512_mask_load_epi32 (__m512i __W, __mmask16 __U, void const *__P)
5209{
5210 return (__m512i) __builtin_ia32_movdqa32load512_mask ((const __v16si *) __P,
5211 (__v16si) __W,
5212 (__mmask16) __U);
5213}
5214
Craig Topper74c10e32018-07-09 19:00:16 +00005215static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanffbb67a2016-03-03 09:26:01 +00005216_mm512_maskz_load_epi32 (__mmask16 __U, void const *__P)
5217{
5218 return (__m512i) __builtin_ia32_movdqa32load512_mask ((const __v16si *) __P,
5219 (__v16si)
5220 _mm512_setzero_si512 (),
5221 (__mmask16) __U);
5222}
5223
Craig Topper74c10e32018-07-09 19:00:16 +00005224static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckermanffbb67a2016-03-03 09:26:01 +00005225_mm512_mask_store_epi32 (void *__P, __mmask16 __U, __m512i __A)
5226{
5227 __builtin_ia32_movdqa32store512_mask ((__v16si *) __P, (__v16si) __A,
5228 (__mmask16) __U);
5229}
5230
Craig Topper74c10e32018-07-09 19:00:16 +00005231static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermane6542002016-05-23 08:01:48 +00005232_mm512_mask_mov_epi32 (__m512i __W, __mmask16 __U, __m512i __A)
5233{
Igor Bregeraadb8762016-06-08 13:59:20 +00005234 return (__m512i) __builtin_ia32_selectd_512 ((__mmask16) __U,
5235 (__v16si) __A,
5236 (__v16si) __W);
Michael Zuckermane6542002016-05-23 08:01:48 +00005237}
5238
Craig Topper74c10e32018-07-09 19:00:16 +00005239static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermane6542002016-05-23 08:01:48 +00005240_mm512_maskz_mov_epi32 (__mmask16 __U, __m512i __A)
5241{
Igor Bregeraadb8762016-06-08 13:59:20 +00005242 return (__m512i) __builtin_ia32_selectd_512 ((__mmask16) __U,
5243 (__v16si) __A,
5244 (__v16si) _mm512_setzero_si512 ());
Michael Zuckermane6542002016-05-23 08:01:48 +00005245}
5246
Craig Topper74c10e32018-07-09 19:00:16 +00005247static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanffbb67a2016-03-03 09:26:01 +00005248_mm512_mask_mov_epi64 (__m512i __W, __mmask8 __U, __m512i __A)
5249{
Igor Bregeraadb8762016-06-08 13:59:20 +00005250 return (__m512i) __builtin_ia32_selectq_512 ((__mmask8) __U,
5251 (__v8di) __A,
5252 (__v8di) __W);
Michael Zuckermanffbb67a2016-03-03 09:26:01 +00005253}
5254
Craig Topper74c10e32018-07-09 19:00:16 +00005255static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanffbb67a2016-03-03 09:26:01 +00005256_mm512_maskz_mov_epi64 (__mmask8 __U, __m512i __A)
5257{
Igor Bregeraadb8762016-06-08 13:59:20 +00005258 return (__m512i) __builtin_ia32_selectq_512 ((__mmask8) __U,
5259 (__v8di) __A,
5260 (__v8di) _mm512_setzero_si512 ());
Michael Zuckermanffbb67a2016-03-03 09:26:01 +00005261}
5262
Craig Topper74c10e32018-07-09 19:00:16 +00005263static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanffbb67a2016-03-03 09:26:01 +00005264_mm512_mask_load_epi64 (__m512i __W, __mmask8 __U, void const *__P)
5265{
5266 return (__m512i) __builtin_ia32_movdqa64load512_mask ((const __v8di *) __P,
5267 (__v8di) __W,
5268 (__mmask8) __U);
5269}
5270
Craig Topper74c10e32018-07-09 19:00:16 +00005271static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckermanffbb67a2016-03-03 09:26:01 +00005272_mm512_maskz_load_epi64 (__mmask8 __U, void const *__P)
5273{
5274 return (__m512i) __builtin_ia32_movdqa64load512_mask ((const __v8di *) __P,
5275 (__v8di)
5276 _mm512_setzero_si512 (),
5277 (__mmask8) __U);
5278}
5279
Craig Topper74c10e32018-07-09 19:00:16 +00005280static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckermanffbb67a2016-03-03 09:26:01 +00005281_mm512_mask_store_epi64 (void *__P, __mmask8 __U, __m512i __A)
5282{
5283 __builtin_ia32_movdqa64store512_mask ((__v8di *) __P, (__v8di) __A,
5284 (__mmask8) __U);
5285}
5286
Craig Topper74c10e32018-07-09 19:00:16 +00005287static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman0d67e4b2016-03-03 13:43:05 +00005288_mm512_movedup_pd (__m512d __A)
5289{
Simon Pilgrim275d7212016-07-02 17:16:25 +00005290 return (__m512d)__builtin_shufflevector((__v8df)__A, (__v8df)__A,
5291 0, 0, 2, 2, 4, 4, 6, 6);
Michael Zuckerman0d67e4b2016-03-03 13:43:05 +00005292}
5293
Craig Topper74c10e32018-07-09 19:00:16 +00005294static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman0d67e4b2016-03-03 13:43:05 +00005295_mm512_mask_movedup_pd (__m512d __W, __mmask8 __U, __m512d __A)
5296{
Simon Pilgrim275d7212016-07-02 17:16:25 +00005297 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
5298 (__v8df)_mm512_movedup_pd(__A),
5299 (__v8df)__W);
Michael Zuckerman0d67e4b2016-03-03 13:43:05 +00005300}
5301
Craig Topper74c10e32018-07-09 19:00:16 +00005302static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman0d67e4b2016-03-03 13:43:05 +00005303_mm512_maskz_movedup_pd (__mmask8 __U, __m512d __A)
5304{
Simon Pilgrim275d7212016-07-02 17:16:25 +00005305 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
5306 (__v8df)_mm512_movedup_pd(__A),
5307 (__v8df)_mm512_setzero_pd());
Michael Zuckerman0d67e4b2016-03-03 13:43:05 +00005308}
5309
Craig Topperc6338672018-05-31 00:51:20 +00005310#define _mm512_fixupimm_round_pd(A, B, C, imm, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005311 (__m512d)__builtin_ia32_fixupimmpd512_mask((__v8df)(__m512d)(A), \
5312 (__v8df)(__m512d)(B), \
5313 (__v8di)(__m512i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005314 (__mmask8)-1, (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005315
Craig Topperc6338672018-05-31 00:51:20 +00005316#define _mm512_mask_fixupimm_round_pd(A, U, B, C, imm, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005317 (__m512d)__builtin_ia32_fixupimmpd512_mask((__v8df)(__m512d)(A), \
5318 (__v8df)(__m512d)(B), \
5319 (__v8di)(__m512i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005320 (__mmask8)(U), (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005321
Craig Topperc6338672018-05-31 00:51:20 +00005322#define _mm512_fixupimm_pd(A, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005323 (__m512d)__builtin_ia32_fixupimmpd512_mask((__v8df)(__m512d)(A), \
5324 (__v8df)(__m512d)(B), \
5325 (__v8di)(__m512i)(C), (int)(imm), \
5326 (__mmask8)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00005327 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermandef78752016-03-28 12:23:09 +00005328
Craig Topperc6338672018-05-31 00:51:20 +00005329#define _mm512_mask_fixupimm_pd(A, U, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005330 (__m512d)__builtin_ia32_fixupimmpd512_mask((__v8df)(__m512d)(A), \
5331 (__v8df)(__m512d)(B), \
5332 (__v8di)(__m512i)(C), (int)(imm), \
5333 (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00005334 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermandef78752016-03-28 12:23:09 +00005335
Craig Topperc6338672018-05-31 00:51:20 +00005336#define _mm512_maskz_fixupimm_round_pd(U, A, B, C, imm, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005337 (__m512d)__builtin_ia32_fixupimmpd512_maskz((__v8df)(__m512d)(A), \
5338 (__v8df)(__m512d)(B), \
5339 (__v8di)(__m512i)(C), \
5340 (int)(imm), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00005341 (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005342
Craig Topperc6338672018-05-31 00:51:20 +00005343#define _mm512_maskz_fixupimm_pd(U, A, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005344 (__m512d)__builtin_ia32_fixupimmpd512_maskz((__v8df)(__m512d)(A), \
5345 (__v8df)(__m512d)(B), \
5346 (__v8di)(__m512i)(C), \
5347 (int)(imm), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00005348 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermandef78752016-03-28 12:23:09 +00005349
Craig Topperc6338672018-05-31 00:51:20 +00005350#define _mm512_fixupimm_round_ps(A, B, C, imm, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005351 (__m512)__builtin_ia32_fixupimmps512_mask((__v16sf)(__m512)(A), \
5352 (__v16sf)(__m512)(B), \
5353 (__v16si)(__m512i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005354 (__mmask16)-1, (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005355
Craig Topperc6338672018-05-31 00:51:20 +00005356#define _mm512_mask_fixupimm_round_ps(A, U, B, C, imm, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005357 (__m512)__builtin_ia32_fixupimmps512_mask((__v16sf)(__m512)(A), \
5358 (__v16sf)(__m512)(B), \
5359 (__v16si)(__m512i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005360 (__mmask16)(U), (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005361
Craig Topperc6338672018-05-31 00:51:20 +00005362#define _mm512_fixupimm_ps(A, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005363 (__m512)__builtin_ia32_fixupimmps512_mask((__v16sf)(__m512)(A), \
5364 (__v16sf)(__m512)(B), \
5365 (__v16si)(__m512i)(C), (int)(imm), \
5366 (__mmask16)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00005367 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermandef78752016-03-28 12:23:09 +00005368
Craig Topperc6338672018-05-31 00:51:20 +00005369#define _mm512_mask_fixupimm_ps(A, U, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005370 (__m512)__builtin_ia32_fixupimmps512_mask((__v16sf)(__m512)(A), \
5371 (__v16sf)(__m512)(B), \
5372 (__v16si)(__m512i)(C), (int)(imm), \
5373 (__mmask16)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00005374 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermandef78752016-03-28 12:23:09 +00005375
Craig Topperc6338672018-05-31 00:51:20 +00005376#define _mm512_maskz_fixupimm_round_ps(U, A, B, C, imm, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005377 (__m512)__builtin_ia32_fixupimmps512_maskz((__v16sf)(__m512)(A), \
5378 (__v16sf)(__m512)(B), \
5379 (__v16si)(__m512i)(C), \
5380 (int)(imm), (__mmask16)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00005381 (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005382
Craig Topperc6338672018-05-31 00:51:20 +00005383#define _mm512_maskz_fixupimm_ps(U, A, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005384 (__m512)__builtin_ia32_fixupimmps512_maskz((__v16sf)(__m512)(A), \
5385 (__v16sf)(__m512)(B), \
5386 (__v16si)(__m512i)(C), \
5387 (int)(imm), (__mmask16)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00005388 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermandef78752016-03-28 12:23:09 +00005389
Craig Topperc6338672018-05-31 00:51:20 +00005390#define _mm_fixupimm_round_sd(A, B, C, imm, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005391 (__m128d)__builtin_ia32_fixupimmsd_mask((__v2df)(__m128d)(A), \
5392 (__v2df)(__m128d)(B), \
5393 (__v2di)(__m128i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005394 (__mmask8)-1, (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005395
Craig Topperc6338672018-05-31 00:51:20 +00005396#define _mm_mask_fixupimm_round_sd(A, U, B, C, imm, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005397 (__m128d)__builtin_ia32_fixupimmsd_mask((__v2df)(__m128d)(A), \
5398 (__v2df)(__m128d)(B), \
5399 (__v2di)(__m128i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005400 (__mmask8)(U), (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005401
Craig Topperc6338672018-05-31 00:51:20 +00005402#define _mm_fixupimm_sd(A, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005403 (__m128d)__builtin_ia32_fixupimmsd_mask((__v2df)(__m128d)(A), \
5404 (__v2df)(__m128d)(B), \
5405 (__v2di)(__m128i)(C), (int)(imm), \
5406 (__mmask8)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00005407 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermandef78752016-03-28 12:23:09 +00005408
Craig Topperc6338672018-05-31 00:51:20 +00005409#define _mm_mask_fixupimm_sd(A, U, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005410 (__m128d)__builtin_ia32_fixupimmsd_mask((__v2df)(__m128d)(A), \
5411 (__v2df)(__m128d)(B), \
5412 (__v2di)(__m128i)(C), (int)(imm), \
5413 (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00005414 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermandef78752016-03-28 12:23:09 +00005415
Craig Topperc6338672018-05-31 00:51:20 +00005416#define _mm_maskz_fixupimm_round_sd(U, A, B, C, imm, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005417 (__m128d)__builtin_ia32_fixupimmsd_maskz((__v2df)(__m128d)(A), \
5418 (__v2df)(__m128d)(B), \
5419 (__v2di)(__m128i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005420 (__mmask8)(U), (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005421
Craig Topperc6338672018-05-31 00:51:20 +00005422#define _mm_maskz_fixupimm_sd(U, A, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005423 (__m128d)__builtin_ia32_fixupimmsd_maskz((__v2df)(__m128d)(A), \
5424 (__v2df)(__m128d)(B), \
5425 (__v2di)(__m128i)(C), (int)(imm), \
5426 (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00005427 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermandef78752016-03-28 12:23:09 +00005428
Craig Topperc6338672018-05-31 00:51:20 +00005429#define _mm_fixupimm_round_ss(A, B, C, imm, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005430 (__m128)__builtin_ia32_fixupimmss_mask((__v4sf)(__m128)(A), \
5431 (__v4sf)(__m128)(B), \
5432 (__v4si)(__m128i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005433 (__mmask8)-1, (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005434
Craig Topperc6338672018-05-31 00:51:20 +00005435#define _mm_mask_fixupimm_round_ss(A, U, B, C, imm, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005436 (__m128)__builtin_ia32_fixupimmss_mask((__v4sf)(__m128)(A), \
5437 (__v4sf)(__m128)(B), \
5438 (__v4si)(__m128i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005439 (__mmask8)(U), (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005440
Craig Topperc6338672018-05-31 00:51:20 +00005441#define _mm_fixupimm_ss(A, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005442 (__m128)__builtin_ia32_fixupimmss_mask((__v4sf)(__m128)(A), \
5443 (__v4sf)(__m128)(B), \
5444 (__v4si)(__m128i)(C), (int)(imm), \
5445 (__mmask8)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00005446 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermandef78752016-03-28 12:23:09 +00005447
Craig Topperc6338672018-05-31 00:51:20 +00005448#define _mm_mask_fixupimm_ss(A, U, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005449 (__m128)__builtin_ia32_fixupimmss_mask((__v4sf)(__m128)(A), \
5450 (__v4sf)(__m128)(B), \
5451 (__v4si)(__m128i)(C), (int)(imm), \
5452 (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00005453 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermandef78752016-03-28 12:23:09 +00005454
Craig Topperc6338672018-05-31 00:51:20 +00005455#define _mm_maskz_fixupimm_round_ss(U, A, B, C, imm, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005456 (__m128)__builtin_ia32_fixupimmss_maskz((__v4sf)(__m128)(A), \
5457 (__v4sf)(__m128)(B), \
5458 (__v4si)(__m128i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005459 (__mmask8)(U), (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005460
Craig Topperc6338672018-05-31 00:51:20 +00005461#define _mm_maskz_fixupimm_ss(U, A, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005462 (__m128)__builtin_ia32_fixupimmss_maskz((__v4sf)(__m128)(A), \
5463 (__v4sf)(__m128)(B), \
5464 (__v4si)(__m128i)(C), (int)(imm), \
5465 (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00005466 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermandef78752016-03-28 12:23:09 +00005467
Craig Topperc6338672018-05-31 00:51:20 +00005468#define _mm_getexp_round_sd(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005469 (__m128d)__builtin_ia32_getexpsd128_round_mask((__v2df)(__m128d)(A), \
5470 (__v2df)(__m128d)(B), \
5471 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00005472 (__mmask8)-1, (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005473
5474
Craig Topper74c10e32018-07-09 19:00:16 +00005475static __inline__ __m128d __DEFAULT_FN_ATTRS128
Michael Zuckermandef78752016-03-28 12:23:09 +00005476_mm_getexp_sd (__m128d __A, __m128d __B)
5477{
5478 return (__m128d) __builtin_ia32_getexpsd128_round_mask ((__v2df) __A,
5479 (__v2df) __B, (__v2df) _mm_setzero_pd(), (__mmask8) -1, _MM_FROUND_CUR_DIRECTION);
5480}
5481
Craig Topper74c10e32018-07-09 19:00:16 +00005482static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00005483_mm_mask_getexp_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005484{
5485 return (__m128d) __builtin_ia32_getexpsd128_round_mask ( (__v2df) __A,
5486 (__v2df) __B,
5487 (__v2df) __W,
5488 (__mmask8) __U,
5489 _MM_FROUND_CUR_DIRECTION);
5490}
5491
Craig Topperc6338672018-05-31 00:51:20 +00005492#define _mm_mask_getexp_round_sd(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005493 (__m128d)__builtin_ia32_getexpsd128_round_mask((__v2df)(__m128d)(A), \
5494 (__v2df)(__m128d)(B), \
5495 (__v2df)(__m128d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00005496 (__mmask8)(U), (int)(R))
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005497
Craig Topper74c10e32018-07-09 19:00:16 +00005498static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00005499_mm_maskz_getexp_sd (__mmask8 __U, __m128d __A, __m128d __B)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005500{
5501 return (__m128d) __builtin_ia32_getexpsd128_round_mask ( (__v2df) __A,
5502 (__v2df) __B,
5503 (__v2df) _mm_setzero_pd (),
5504 (__mmask8) __U,
5505 _MM_FROUND_CUR_DIRECTION);
5506}
5507
Craig Topperc6338672018-05-31 00:51:20 +00005508#define _mm_maskz_getexp_round_sd(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005509 (__m128d)__builtin_ia32_getexpsd128_round_mask((__v2df)(__m128d)(A), \
5510 (__v2df)(__m128d)(B), \
5511 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00005512 (__mmask8)(U), (int)(R))
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005513
Craig Topperc6338672018-05-31 00:51:20 +00005514#define _mm_getexp_round_ss(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005515 (__m128)__builtin_ia32_getexpss128_round_mask((__v4sf)(__m128)(A), \
5516 (__v4sf)(__m128)(B), \
5517 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00005518 (__mmask8)-1, (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005519
Craig Topper74c10e32018-07-09 19:00:16 +00005520static __inline__ __m128 __DEFAULT_FN_ATTRS128
Michael Zuckermandef78752016-03-28 12:23:09 +00005521_mm_getexp_ss (__m128 __A, __m128 __B)
5522{
5523 return (__m128) __builtin_ia32_getexpss128_round_mask ((__v4sf) __A,
5524 (__v4sf) __B, (__v4sf) _mm_setzero_ps(), (__mmask8) -1, _MM_FROUND_CUR_DIRECTION);
5525}
5526
Craig Topper74c10e32018-07-09 19:00:16 +00005527static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00005528_mm_mask_getexp_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005529{
Craig Topper58187d32016-05-17 04:41:29 +00005530 return (__m128) __builtin_ia32_getexpss128_round_mask ((__v4sf) __A,
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005531 (__v4sf) __B,
5532 (__v4sf) __W,
5533 (__mmask8) __U,
5534 _MM_FROUND_CUR_DIRECTION);
5535}
5536
Craig Topperc6338672018-05-31 00:51:20 +00005537#define _mm_mask_getexp_round_ss(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005538 (__m128)__builtin_ia32_getexpss128_round_mask((__v4sf)(__m128)(A), \
5539 (__v4sf)(__m128)(B), \
5540 (__v4sf)(__m128)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00005541 (__mmask8)(U), (int)(R))
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005542
Craig Topper74c10e32018-07-09 19:00:16 +00005543static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00005544_mm_maskz_getexp_ss (__mmask8 __U, __m128 __A, __m128 __B)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005545{
Craig Topper58187d32016-05-17 04:41:29 +00005546 return (__m128) __builtin_ia32_getexpss128_round_mask ((__v4sf) __A,
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005547 (__v4sf) __B,
Craig Topper5cbeeed2018-07-07 17:03:32 +00005548 (__v4sf) _mm_setzero_ps (),
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005549 (__mmask8) __U,
5550 _MM_FROUND_CUR_DIRECTION);
5551}
5552
Craig Topperc6338672018-05-31 00:51:20 +00005553#define _mm_maskz_getexp_round_ss(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005554 (__m128)__builtin_ia32_getexpss128_round_mask((__v4sf)(__m128)(A), \
5555 (__v4sf)(__m128)(B), \
5556 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00005557 (__mmask8)(U), (int)(R))
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005558
Craig Topperc6338672018-05-31 00:51:20 +00005559#define _mm_getmant_round_sd(A, B, C, D, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005560 (__m128d)__builtin_ia32_getmantsd_round_mask((__v2df)(__m128d)(A), \
5561 (__v2df)(__m128d)(B), \
5562 (int)(((D)<<2) | (C)), \
5563 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00005564 (__mmask8)-1, (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005565
Craig Topperc6338672018-05-31 00:51:20 +00005566#define _mm_getmant_sd(A, B, C, D) \
Craig Topper8c18e112016-05-17 04:41:50 +00005567 (__m128d)__builtin_ia32_getmantsd_round_mask((__v2df)(__m128d)(A), \
5568 (__v2df)(__m128d)(B), \
5569 (int)(((D)<<2) | (C)), \
5570 (__v2df)_mm_setzero_pd(), \
5571 (__mmask8)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00005572 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermandef78752016-03-28 12:23:09 +00005573
Craig Topperc6338672018-05-31 00:51:20 +00005574#define _mm_mask_getmant_sd(W, U, A, B, C, D) \
Craig Topper8c18e112016-05-17 04:41:50 +00005575 (__m128d)__builtin_ia32_getmantsd_round_mask((__v2df)(__m128d)(A), \
5576 (__v2df)(__m128d)(B), \
5577 (int)(((D)<<2) | (C)), \
5578 (__v2df)(__m128d)(W), \
5579 (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00005580 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005581
Craig Topperc6338672018-05-31 00:51:20 +00005582#define _mm_mask_getmant_round_sd(W, U, A, B, C, D, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005583 (__m128d)__builtin_ia32_getmantsd_round_mask((__v2df)(__m128d)(A), \
5584 (__v2df)(__m128d)(B), \
5585 (int)(((D)<<2) | (C)), \
5586 (__v2df)(__m128d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00005587 (__mmask8)(U), (int)(R))
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005588
Craig Topperc6338672018-05-31 00:51:20 +00005589#define _mm_maskz_getmant_sd(U, A, B, C, D) \
Craig Topper8c18e112016-05-17 04:41:50 +00005590 (__m128d)__builtin_ia32_getmantsd_round_mask((__v2df)(__m128d)(A), \
5591 (__v2df)(__m128d)(B), \
5592 (int)(((D)<<2) | (C)), \
5593 (__v2df)_mm_setzero_pd(), \
5594 (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00005595 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005596
Craig Topperc6338672018-05-31 00:51:20 +00005597#define _mm_maskz_getmant_round_sd(U, A, B, C, D, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005598 (__m128d)__builtin_ia32_getmantsd_round_mask((__v2df)(__m128d)(A), \
5599 (__v2df)(__m128d)(B), \
5600 (int)(((D)<<2) | (C)), \
5601 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00005602 (__mmask8)(U), (int)(R))
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005603
Craig Topperc6338672018-05-31 00:51:20 +00005604#define _mm_getmant_round_ss(A, B, C, D, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005605 (__m128)__builtin_ia32_getmantss_round_mask((__v4sf)(__m128)(A), \
5606 (__v4sf)(__m128)(B), \
5607 (int)(((D)<<2) | (C)), \
5608 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00005609 (__mmask8)-1, (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005610
Craig Topperc6338672018-05-31 00:51:20 +00005611#define _mm_getmant_ss(A, B, C, D) \
Craig Topper8c18e112016-05-17 04:41:50 +00005612 (__m128)__builtin_ia32_getmantss_round_mask((__v4sf)(__m128)(A), \
5613 (__v4sf)(__m128)(B), \
5614 (int)(((D)<<2) | (C)), \
5615 (__v4sf)_mm_setzero_ps(), \
5616 (__mmask8)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00005617 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermandef78752016-03-28 12:23:09 +00005618
Craig Topperc6338672018-05-31 00:51:20 +00005619#define _mm_mask_getmant_ss(W, U, A, B, C, D) \
Craig Topper8c18e112016-05-17 04:41:50 +00005620 (__m128)__builtin_ia32_getmantss_round_mask((__v4sf)(__m128)(A), \
5621 (__v4sf)(__m128)(B), \
5622 (int)(((D)<<2) | (C)), \
5623 (__v4sf)(__m128)(W), \
5624 (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00005625 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005626
Craig Topperc6338672018-05-31 00:51:20 +00005627#define _mm_mask_getmant_round_ss(W, U, A, B, C, D, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005628 (__m128)__builtin_ia32_getmantss_round_mask((__v4sf)(__m128)(A), \
5629 (__v4sf)(__m128)(B), \
5630 (int)(((D)<<2) | (C)), \
5631 (__v4sf)(__m128)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00005632 (__mmask8)(U), (int)(R))
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005633
Craig Topperc6338672018-05-31 00:51:20 +00005634#define _mm_maskz_getmant_ss(U, A, B, C, D) \
Craig Topper8c18e112016-05-17 04:41:50 +00005635 (__m128)__builtin_ia32_getmantss_round_mask((__v4sf)(__m128)(A), \
5636 (__v4sf)(__m128)(B), \
5637 (int)(((D)<<2) | (C)), \
Craig Topper5cbeeed2018-07-07 17:03:32 +00005638 (__v4sf)_mm_setzero_ps(), \
Craig Topper8c18e112016-05-17 04:41:50 +00005639 (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00005640 _MM_FROUND_CUR_DIRECTION)
Michael Zuckermana1ceca22016-04-22 10:06:10 +00005641
Craig Topperc6338672018-05-31 00:51:20 +00005642#define _mm_maskz_getmant_round_ss(U, A, B, C, D, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005643 (__m128)__builtin_ia32_getmantss_round_mask((__v4sf)(__m128)(A), \
5644 (__v4sf)(__m128)(B), \
5645 (int)(((D)<<2) | (C)), \
5646 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00005647 (__mmask8)(U), (int)(R))
Michael Zuckermandef78752016-03-28 12:23:09 +00005648
Craig Topper74c10e32018-07-09 19:00:16 +00005649static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
Michael Zuckermandef78752016-03-28 12:23:09 +00005650_mm512_kmov (__mmask16 __A)
5651{
5652 return __A;
5653}
5654
Craig Topperc6338672018-05-31 00:51:20 +00005655#define _mm_comi_round_sd(A, B, P, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005656 (int)__builtin_ia32_vcomisd((__v2df)(__m128d)(A), (__v2df)(__m128d)(B), \
Craig Topperc6338672018-05-31 00:51:20 +00005657 (int)(P), (int)(R))
Michael Zuckermane71d59f2016-03-07 19:15:00 +00005658
Craig Topperc6338672018-05-31 00:51:20 +00005659#define _mm_comi_round_ss(A, B, P, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005660 (int)__builtin_ia32_vcomiss((__v4sf)(__m128)(A), (__v4sf)(__m128)(B), \
Craig Topperc6338672018-05-31 00:51:20 +00005661 (int)(P), (int)(R))
Michael Zuckermane71d59f2016-03-07 19:15:00 +00005662
Craig Topper45db56c2016-07-21 07:38:39 +00005663#ifdef __x86_64__
Craig Topperc6338672018-05-31 00:51:20 +00005664#define _mm_cvt_roundsd_si64(A, R) \
5665 (long long)__builtin_ia32_vcvtsd2si64((__v2df)(__m128d)(A), (int)(R))
Craig Topper45db56c2016-07-21 07:38:39 +00005666#endif
Simon Pilgrim427154d2016-07-04 21:30:47 +00005667
Craig Topper74c10e32018-07-09 19:00:16 +00005668static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005669_mm512_sll_epi32(__m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005670{
Craig Topper1a441932016-11-12 07:16:59 +00005671 return (__m512i)__builtin_ia32_pslld512((__v16si) __A, (__v4si)__B);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005672}
5673
Craig Topper74c10e32018-07-09 19:00:16 +00005674static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005675_mm512_mask_sll_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005676{
Craig Topperd7e5b212016-11-13 07:26:31 +00005677 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5678 (__v16si)_mm512_sll_epi32(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005679 (__v16si)__W);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005680}
5681
Craig Topper74c10e32018-07-09 19:00:16 +00005682static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005683_mm512_maskz_sll_epi32(__mmask16 __U, __m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005684{
Craig Topperd7e5b212016-11-13 07:26:31 +00005685 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5686 (__v16si)_mm512_sll_epi32(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005687 (__v16si)_mm512_setzero_si512());
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005688}
5689
Craig Topper74c10e32018-07-09 19:00:16 +00005690static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005691_mm512_sll_epi64(__m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005692{
Craig Topper1a441932016-11-12 07:16:59 +00005693 return (__m512i)__builtin_ia32_psllq512((__v8di)__A, (__v2di)__B);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005694}
5695
Craig Topper74c10e32018-07-09 19:00:16 +00005696static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005697_mm512_mask_sll_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005698{
Craig Topperd7e5b212016-11-13 07:26:31 +00005699 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5700 (__v8di)_mm512_sll_epi64(__A, __B),
5701 (__v8di)__W);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005702}
5703
Craig Topper74c10e32018-07-09 19:00:16 +00005704static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005705_mm512_maskz_sll_epi64(__mmask8 __U, __m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005706{
Craig Topperd7e5b212016-11-13 07:26:31 +00005707 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5708 (__v8di)_mm512_sll_epi64(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005709 (__v8di)_mm512_setzero_si512());
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005710}
5711
Craig Topper74c10e32018-07-09 19:00:16 +00005712static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005713_mm512_sllv_epi32(__m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005714{
Craig Topper5e0709d2016-11-13 07:26:34 +00005715 return (__m512i)__builtin_ia32_psllv16si((__v16si)__X, (__v16si)__Y);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005716}
5717
Craig Topper74c10e32018-07-09 19:00:16 +00005718static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005719_mm512_mask_sllv_epi32(__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005720{
Craig Topper5e0709d2016-11-13 07:26:34 +00005721 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5722 (__v16si)_mm512_sllv_epi32(__X, __Y),
5723 (__v16si)__W);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005724}
5725
Craig Topper74c10e32018-07-09 19:00:16 +00005726static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005727_mm512_maskz_sllv_epi32(__mmask16 __U, __m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005728{
Craig Topper5e0709d2016-11-13 07:26:34 +00005729 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5730 (__v16si)_mm512_sllv_epi32(__X, __Y),
5731 (__v16si)_mm512_setzero_si512());
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005732}
5733
Craig Topper74c10e32018-07-09 19:00:16 +00005734static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005735_mm512_sllv_epi64(__m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005736{
Craig Topper5e0709d2016-11-13 07:26:34 +00005737 return (__m512i)__builtin_ia32_psllv8di((__v8di)__X, (__v8di)__Y);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005738}
5739
Craig Topper74c10e32018-07-09 19:00:16 +00005740static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005741_mm512_mask_sllv_epi64(__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005742{
Craig Topper5e0709d2016-11-13 07:26:34 +00005743 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5744 (__v8di)_mm512_sllv_epi64(__X, __Y),
5745 (__v8di)__W);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005746}
5747
Craig Topper74c10e32018-07-09 19:00:16 +00005748static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005749_mm512_maskz_sllv_epi64(__mmask8 __U, __m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005750{
Craig Topper5e0709d2016-11-13 07:26:34 +00005751 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5752 (__v8di)_mm512_sllv_epi64(__X, __Y),
5753 (__v8di)_mm512_setzero_si512());
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005754}
5755
Craig Topper74c10e32018-07-09 19:00:16 +00005756static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005757_mm512_sra_epi32(__m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005758{
Craig Topper1a441932016-11-12 07:16:59 +00005759 return (__m512i)__builtin_ia32_psrad512((__v16si) __A, (__v4si)__B);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005760}
5761
Craig Topper74c10e32018-07-09 19:00:16 +00005762static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005763_mm512_mask_sra_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005764{
Craig Topperd7e5b212016-11-13 07:26:31 +00005765 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5766 (__v16si)_mm512_sra_epi32(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005767 (__v16si)__W);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005768}
5769
Craig Topper74c10e32018-07-09 19:00:16 +00005770static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005771_mm512_maskz_sra_epi32(__mmask16 __U, __m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005772{
Craig Topperd7e5b212016-11-13 07:26:31 +00005773 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5774 (__v16si)_mm512_sra_epi32(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005775 (__v16si)_mm512_setzero_si512());
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005776}
5777
Craig Topper74c10e32018-07-09 19:00:16 +00005778static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005779_mm512_sra_epi64(__m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005780{
Craig Topper1a441932016-11-12 07:16:59 +00005781 return (__m512i)__builtin_ia32_psraq512((__v8di)__A, (__v2di)__B);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005782}
5783
Craig Topper74c10e32018-07-09 19:00:16 +00005784static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005785_mm512_mask_sra_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005786{
Craig Topperd7e5b212016-11-13 07:26:31 +00005787 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5788 (__v8di)_mm512_sra_epi64(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005789 (__v8di)__W);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005790}
5791
Craig Topper74c10e32018-07-09 19:00:16 +00005792static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005793_mm512_maskz_sra_epi64(__mmask8 __U, __m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005794{
Craig Topperd7e5b212016-11-13 07:26:31 +00005795 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5796 (__v8di)_mm512_sra_epi64(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005797 (__v8di)_mm512_setzero_si512());
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005798}
5799
Craig Topper74c10e32018-07-09 19:00:16 +00005800static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005801_mm512_srav_epi32(__m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005802{
Craig Topper5e0709d2016-11-13 07:26:34 +00005803 return (__m512i)__builtin_ia32_psrav16si((__v16si)__X, (__v16si)__Y);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005804}
5805
Craig Topper74c10e32018-07-09 19:00:16 +00005806static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005807_mm512_mask_srav_epi32(__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005808{
Craig Topper5e0709d2016-11-13 07:26:34 +00005809 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5810 (__v16si)_mm512_srav_epi32(__X, __Y),
5811 (__v16si)__W);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005812}
5813
Craig Topper74c10e32018-07-09 19:00:16 +00005814static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005815_mm512_maskz_srav_epi32(__mmask16 __U, __m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005816{
Craig Topper5e0709d2016-11-13 07:26:34 +00005817 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5818 (__v16si)_mm512_srav_epi32(__X, __Y),
5819 (__v16si)_mm512_setzero_si512());
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005820}
5821
Craig Topper74c10e32018-07-09 19:00:16 +00005822static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005823_mm512_srav_epi64(__m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005824{
Craig Topper5e0709d2016-11-13 07:26:34 +00005825 return (__m512i)__builtin_ia32_psrav8di((__v8di)__X, (__v8di)__Y);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005826}
5827
Craig Topper74c10e32018-07-09 19:00:16 +00005828static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005829_mm512_mask_srav_epi64(__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005830{
Craig Topper5e0709d2016-11-13 07:26:34 +00005831 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5832 (__v8di)_mm512_srav_epi64(__X, __Y),
5833 (__v8di)__W);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005834}
5835
Craig Topper74c10e32018-07-09 19:00:16 +00005836static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005837_mm512_maskz_srav_epi64(__mmask8 __U, __m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005838{
Craig Topper5e0709d2016-11-13 07:26:34 +00005839 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5840 (__v8di)_mm512_srav_epi64(__X, __Y),
5841 (__v8di)_mm512_setzero_si512());
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005842}
5843
Craig Topper74c10e32018-07-09 19:00:16 +00005844static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005845_mm512_srl_epi32(__m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005846{
Craig Topper1a441932016-11-12 07:16:59 +00005847 return (__m512i)__builtin_ia32_psrld512((__v16si) __A, (__v4si)__B);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005848}
5849
Craig Topper74c10e32018-07-09 19:00:16 +00005850static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005851_mm512_mask_srl_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005852{
Craig Topperd7e5b212016-11-13 07:26:31 +00005853 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5854 (__v16si)_mm512_srl_epi32(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005855 (__v16si)__W);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005856}
5857
Craig Topper74c10e32018-07-09 19:00:16 +00005858static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005859_mm512_maskz_srl_epi32(__mmask16 __U, __m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005860{
Craig Topperd7e5b212016-11-13 07:26:31 +00005861 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5862 (__v16si)_mm512_srl_epi32(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005863 (__v16si)_mm512_setzero_si512());
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005864}
5865
Craig Topper74c10e32018-07-09 19:00:16 +00005866static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005867_mm512_srl_epi64(__m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005868{
Craig Topper1a441932016-11-12 07:16:59 +00005869 return (__m512i)__builtin_ia32_psrlq512((__v8di)__A, (__v2di)__B);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005870}
5871
Craig Topper74c10e32018-07-09 19:00:16 +00005872static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005873_mm512_mask_srl_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005874{
Craig Topperd7e5b212016-11-13 07:26:31 +00005875 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5876 (__v8di)_mm512_srl_epi64(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005877 (__v8di)__W);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005878}
5879
Craig Topper74c10e32018-07-09 19:00:16 +00005880static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00005881_mm512_maskz_srl_epi64(__mmask8 __U, __m512i __A, __m128i __B)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005882{
Craig Topperd7e5b212016-11-13 07:26:31 +00005883 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5884 (__v8di)_mm512_srl_epi64(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00005885 (__v8di)_mm512_setzero_si512());
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005886}
5887
Craig Topper74c10e32018-07-09 19:00:16 +00005888static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005889_mm512_srlv_epi32(__m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005890{
Craig Topper5e0709d2016-11-13 07:26:34 +00005891 return (__m512i)__builtin_ia32_psrlv16si((__v16si)__X, (__v16si)__Y);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005892}
5893
Craig Topper74c10e32018-07-09 19:00:16 +00005894static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005895_mm512_mask_srlv_epi32(__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005896{
Craig Topper5e0709d2016-11-13 07:26:34 +00005897 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5898 (__v16si)_mm512_srlv_epi32(__X, __Y),
5899 (__v16si)__W);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005900}
5901
Craig Topper74c10e32018-07-09 19:00:16 +00005902static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005903_mm512_maskz_srlv_epi32(__mmask16 __U, __m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005904{
Craig Topper5e0709d2016-11-13 07:26:34 +00005905 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5906 (__v16si)_mm512_srlv_epi32(__X, __Y),
5907 (__v16si)_mm512_setzero_si512());
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005908}
5909
Craig Topper74c10e32018-07-09 19:00:16 +00005910static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005911_mm512_srlv_epi64 (__m512i __X, __m512i __Y)
5912{
Craig Topper5e0709d2016-11-13 07:26:34 +00005913 return (__m512i)__builtin_ia32_psrlv8di((__v8di)__X, (__v8di)__Y);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005914}
5915
Craig Topper74c10e32018-07-09 19:00:16 +00005916static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005917_mm512_mask_srlv_epi64(__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005918{
Craig Topper5e0709d2016-11-13 07:26:34 +00005919 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5920 (__v8di)_mm512_srlv_epi64(__X, __Y),
5921 (__v8di)__W);
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005922}
5923
Craig Topper74c10e32018-07-09 19:00:16 +00005924static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper5e0709d2016-11-13 07:26:34 +00005925_mm512_maskz_srlv_epi64(__mmask8 __U, __m512i __X, __m512i __Y)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005926{
Craig Topper5e0709d2016-11-13 07:26:34 +00005927 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5928 (__v8di)_mm512_srlv_epi64(__X, __Y),
5929 (__v8di)_mm512_setzero_si512());
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005930}
5931
Craig Topperc6338672018-05-31 00:51:20 +00005932#define _mm512_ternarylogic_epi32(A, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005933 (__m512i)__builtin_ia32_pternlogd512_mask((__v16si)(__m512i)(A), \
5934 (__v16si)(__m512i)(B), \
5935 (__v16si)(__m512i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005936 (__mmask16)-1)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005937
Craig Topperc6338672018-05-31 00:51:20 +00005938#define _mm512_mask_ternarylogic_epi32(A, U, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005939 (__m512i)__builtin_ia32_pternlogd512_mask((__v16si)(__m512i)(A), \
5940 (__v16si)(__m512i)(B), \
5941 (__v16si)(__m512i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005942 (__mmask16)(U))
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005943
Craig Topperc6338672018-05-31 00:51:20 +00005944#define _mm512_maskz_ternarylogic_epi32(U, A, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005945 (__m512i)__builtin_ia32_pternlogd512_maskz((__v16si)(__m512i)(A), \
5946 (__v16si)(__m512i)(B), \
5947 (__v16si)(__m512i)(C), \
Craig Topperc6338672018-05-31 00:51:20 +00005948 (int)(imm), (__mmask16)(U))
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005949
Craig Topperc6338672018-05-31 00:51:20 +00005950#define _mm512_ternarylogic_epi64(A, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005951 (__m512i)__builtin_ia32_pternlogq512_mask((__v8di)(__m512i)(A), \
5952 (__v8di)(__m512i)(B), \
5953 (__v8di)(__m512i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005954 (__mmask8)-1)
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005955
Craig Topperc6338672018-05-31 00:51:20 +00005956#define _mm512_mask_ternarylogic_epi64(A, U, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005957 (__m512i)__builtin_ia32_pternlogq512_mask((__v8di)(__m512i)(A), \
5958 (__v8di)(__m512i)(B), \
5959 (__v8di)(__m512i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005960 (__mmask8)(U))
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005961
Craig Topperc6338672018-05-31 00:51:20 +00005962#define _mm512_maskz_ternarylogic_epi64(U, A, B, C, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00005963 (__m512i)__builtin_ia32_pternlogq512_maskz((__v8di)(__m512i)(A), \
5964 (__v8di)(__m512i)(B), \
5965 (__v8di)(__m512i)(C), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00005966 (__mmask8)(U))
Michael Zuckerman81f468c2016-04-11 17:04:21 +00005967
Craig Topper45db56c2016-07-21 07:38:39 +00005968#ifdef __x86_64__
Craig Topperc6338672018-05-31 00:51:20 +00005969#define _mm_cvt_roundsd_i64(A, R) \
5970 (long long)__builtin_ia32_vcvtsd2si64((__v2df)(__m128d)(A), (int)(R))
Craig Topper45db56c2016-07-21 07:38:39 +00005971#endif
Michael Zuckerman8d161992016-04-10 17:24:03 +00005972
Craig Topperc6338672018-05-31 00:51:20 +00005973#define _mm_cvt_roundsd_si32(A, R) \
5974 (int)__builtin_ia32_vcvtsd2si32((__v2df)(__m128d)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00005975
Craig Topperc6338672018-05-31 00:51:20 +00005976#define _mm_cvt_roundsd_i32(A, R) \
5977 (int)__builtin_ia32_vcvtsd2si32((__v2df)(__m128d)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00005978
Craig Topperc6338672018-05-31 00:51:20 +00005979#define _mm_cvt_roundsd_u32(A, R) \
5980 (unsigned int)__builtin_ia32_vcvtsd2usi32((__v2df)(__m128d)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00005981
Craig Topper74c10e32018-07-09 19:00:16 +00005982static __inline__ unsigned __DEFAULT_FN_ATTRS128
Michael Zuckerman8d161992016-04-10 17:24:03 +00005983_mm_cvtsd_u32 (__m128d __A)
5984{
5985 return (unsigned) __builtin_ia32_vcvtsd2usi32 ((__v2df) __A,
5986 _MM_FROUND_CUR_DIRECTION);
5987}
5988
Craig Topper45db56c2016-07-21 07:38:39 +00005989#ifdef __x86_64__
Craig Topperc6338672018-05-31 00:51:20 +00005990#define _mm_cvt_roundsd_u64(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00005991 (unsigned long long)__builtin_ia32_vcvtsd2usi64((__v2df)(__m128d)(A), \
Craig Topperc6338672018-05-31 00:51:20 +00005992 (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00005993
Craig Topper74c10e32018-07-09 19:00:16 +00005994static __inline__ unsigned long long __DEFAULT_FN_ATTRS128
Michael Zuckerman8d161992016-04-10 17:24:03 +00005995_mm_cvtsd_u64 (__m128d __A)
5996{
5997 return (unsigned long long) __builtin_ia32_vcvtsd2usi64 ((__v2df)
5998 __A,
5999 _MM_FROUND_CUR_DIRECTION);
6000}
Craig Topper45db56c2016-07-21 07:38:39 +00006001#endif
Michael Zuckerman8d161992016-04-10 17:24:03 +00006002
Craig Topperc6338672018-05-31 00:51:20 +00006003#define _mm_cvt_roundss_si32(A, R) \
6004 (int)__builtin_ia32_vcvtss2si32((__v4sf)(__m128)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006005
Craig Topperc6338672018-05-31 00:51:20 +00006006#define _mm_cvt_roundss_i32(A, R) \
6007 (int)__builtin_ia32_vcvtss2si32((__v4sf)(__m128)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006008
Craig Topper45db56c2016-07-21 07:38:39 +00006009#ifdef __x86_64__
Craig Topperc6338672018-05-31 00:51:20 +00006010#define _mm_cvt_roundss_si64(A, R) \
6011 (long long)__builtin_ia32_vcvtss2si64((__v4sf)(__m128)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006012
Craig Topperc6338672018-05-31 00:51:20 +00006013#define _mm_cvt_roundss_i64(A, R) \
6014 (long long)__builtin_ia32_vcvtss2si64((__v4sf)(__m128)(A), (int)(R))
Craig Topper45db56c2016-07-21 07:38:39 +00006015#endif
Michael Zuckerman8d161992016-04-10 17:24:03 +00006016
Craig Topperc6338672018-05-31 00:51:20 +00006017#define _mm_cvt_roundss_u32(A, R) \
6018 (unsigned int)__builtin_ia32_vcvtss2usi32((__v4sf)(__m128)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006019
Craig Topper74c10e32018-07-09 19:00:16 +00006020static __inline__ unsigned __DEFAULT_FN_ATTRS128
Michael Zuckerman8d161992016-04-10 17:24:03 +00006021_mm_cvtss_u32 (__m128 __A)
6022{
6023 return (unsigned) __builtin_ia32_vcvtss2usi32 ((__v4sf) __A,
6024 _MM_FROUND_CUR_DIRECTION);
6025}
6026
Craig Topper45db56c2016-07-21 07:38:39 +00006027#ifdef __x86_64__
Craig Topperc6338672018-05-31 00:51:20 +00006028#define _mm_cvt_roundss_u64(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006029 (unsigned long long)__builtin_ia32_vcvtss2usi64((__v4sf)(__m128)(A), \
Craig Topperc6338672018-05-31 00:51:20 +00006030 (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006031
Craig Topper74c10e32018-07-09 19:00:16 +00006032static __inline__ unsigned long long __DEFAULT_FN_ATTRS128
Michael Zuckerman8d161992016-04-10 17:24:03 +00006033_mm_cvtss_u64 (__m128 __A)
6034{
6035 return (unsigned long long) __builtin_ia32_vcvtss2usi64 ((__v4sf)
6036 __A,
6037 _MM_FROUND_CUR_DIRECTION);
6038}
Craig Topper45db56c2016-07-21 07:38:39 +00006039#endif
Michael Zuckerman8d161992016-04-10 17:24:03 +00006040
Craig Topperc6338672018-05-31 00:51:20 +00006041#define _mm_cvtt_roundsd_i32(A, R) \
6042 (int)__builtin_ia32_vcvttsd2si32((__v2df)(__m128d)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006043
Craig Topperc6338672018-05-31 00:51:20 +00006044#define _mm_cvtt_roundsd_si32(A, R) \
6045 (int)__builtin_ia32_vcvttsd2si32((__v2df)(__m128d)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006046
Craig Topper74c10e32018-07-09 19:00:16 +00006047static __inline__ int __DEFAULT_FN_ATTRS128
Michael Zuckerman8d161992016-04-10 17:24:03 +00006048_mm_cvttsd_i32 (__m128d __A)
6049{
6050 return (int) __builtin_ia32_vcvttsd2si32 ((__v2df) __A,
6051 _MM_FROUND_CUR_DIRECTION);
6052}
6053
Craig Topper45db56c2016-07-21 07:38:39 +00006054#ifdef __x86_64__
Craig Topperc6338672018-05-31 00:51:20 +00006055#define _mm_cvtt_roundsd_si64(A, R) \
6056 (long long)__builtin_ia32_vcvttsd2si64((__v2df)(__m128d)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006057
Craig Topperc6338672018-05-31 00:51:20 +00006058#define _mm_cvtt_roundsd_i64(A, R) \
6059 (long long)__builtin_ia32_vcvttsd2si64((__v2df)(__m128d)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006060
Craig Topper74c10e32018-07-09 19:00:16 +00006061static __inline__ long long __DEFAULT_FN_ATTRS128
Michael Zuckerman8d161992016-04-10 17:24:03 +00006062_mm_cvttsd_i64 (__m128d __A)
6063{
6064 return (long long) __builtin_ia32_vcvttsd2si64 ((__v2df) __A,
6065 _MM_FROUND_CUR_DIRECTION);
6066}
Craig Topper45db56c2016-07-21 07:38:39 +00006067#endif
Michael Zuckerman8d161992016-04-10 17:24:03 +00006068
Craig Topperc6338672018-05-31 00:51:20 +00006069#define _mm_cvtt_roundsd_u32(A, R) \
6070 (unsigned int)__builtin_ia32_vcvttsd2usi32((__v2df)(__m128d)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006071
Craig Topper74c10e32018-07-09 19:00:16 +00006072static __inline__ unsigned __DEFAULT_FN_ATTRS128
Michael Zuckerman8d161992016-04-10 17:24:03 +00006073_mm_cvttsd_u32 (__m128d __A)
6074{
6075 return (unsigned) __builtin_ia32_vcvttsd2usi32 ((__v2df) __A,
6076 _MM_FROUND_CUR_DIRECTION);
6077}
6078
Craig Topper45db56c2016-07-21 07:38:39 +00006079#ifdef __x86_64__
Craig Topperc6338672018-05-31 00:51:20 +00006080#define _mm_cvtt_roundsd_u64(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006081 (unsigned long long)__builtin_ia32_vcvttsd2usi64((__v2df)(__m128d)(A), \
Craig Topperc6338672018-05-31 00:51:20 +00006082 (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006083
Craig Topper74c10e32018-07-09 19:00:16 +00006084static __inline__ unsigned long long __DEFAULT_FN_ATTRS128
Michael Zuckerman8d161992016-04-10 17:24:03 +00006085_mm_cvttsd_u64 (__m128d __A)
6086{
6087 return (unsigned long long) __builtin_ia32_vcvttsd2usi64 ((__v2df)
6088 __A,
6089 _MM_FROUND_CUR_DIRECTION);
6090}
Craig Topper45db56c2016-07-21 07:38:39 +00006091#endif
Michael Zuckerman8d161992016-04-10 17:24:03 +00006092
Craig Topperc6338672018-05-31 00:51:20 +00006093#define _mm_cvtt_roundss_i32(A, R) \
6094 (int)__builtin_ia32_vcvttss2si32((__v4sf)(__m128)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006095
Craig Topperc6338672018-05-31 00:51:20 +00006096#define _mm_cvtt_roundss_si32(A, R) \
6097 (int)__builtin_ia32_vcvttss2si32((__v4sf)(__m128)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006098
Craig Topper74c10e32018-07-09 19:00:16 +00006099static __inline__ int __DEFAULT_FN_ATTRS128
Michael Zuckerman8d161992016-04-10 17:24:03 +00006100_mm_cvttss_i32 (__m128 __A)
6101{
6102 return (int) __builtin_ia32_vcvttss2si32 ((__v4sf) __A,
6103 _MM_FROUND_CUR_DIRECTION);
6104}
6105
Craig Topper45db56c2016-07-21 07:38:39 +00006106#ifdef __x86_64__
Craig Topperc6338672018-05-31 00:51:20 +00006107#define _mm_cvtt_roundss_i64(A, R) \
6108 (long long)__builtin_ia32_vcvttss2si64((__v4sf)(__m128)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006109
Craig Topperc6338672018-05-31 00:51:20 +00006110#define _mm_cvtt_roundss_si64(A, R) \
6111 (long long)__builtin_ia32_vcvttss2si64((__v4sf)(__m128)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006112
Craig Topper74c10e32018-07-09 19:00:16 +00006113static __inline__ long long __DEFAULT_FN_ATTRS128
Michael Zuckerman8d161992016-04-10 17:24:03 +00006114_mm_cvttss_i64 (__m128 __A)
6115{
6116 return (long long) __builtin_ia32_vcvttss2si64 ((__v4sf) __A,
6117 _MM_FROUND_CUR_DIRECTION);
6118}
Craig Topper45db56c2016-07-21 07:38:39 +00006119#endif
Michael Zuckerman8d161992016-04-10 17:24:03 +00006120
Craig Topperc6338672018-05-31 00:51:20 +00006121#define _mm_cvtt_roundss_u32(A, R) \
6122 (unsigned int)__builtin_ia32_vcvttss2usi32((__v4sf)(__m128)(A), (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006123
Craig Topper74c10e32018-07-09 19:00:16 +00006124static __inline__ unsigned __DEFAULT_FN_ATTRS128
Michael Zuckerman8d161992016-04-10 17:24:03 +00006125_mm_cvttss_u32 (__m128 __A)
6126{
6127 return (unsigned) __builtin_ia32_vcvttss2usi32 ((__v4sf) __A,
6128 _MM_FROUND_CUR_DIRECTION);
6129}
6130
Craig Topper45db56c2016-07-21 07:38:39 +00006131#ifdef __x86_64__
Craig Topperc6338672018-05-31 00:51:20 +00006132#define _mm_cvtt_roundss_u64(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006133 (unsigned long long)__builtin_ia32_vcvttss2usi64((__v4sf)(__m128)(A), \
Craig Topperc6338672018-05-31 00:51:20 +00006134 (int)(R))
Michael Zuckerman8d161992016-04-10 17:24:03 +00006135
Craig Topper74c10e32018-07-09 19:00:16 +00006136static __inline__ unsigned long long __DEFAULT_FN_ATTRS128
Michael Zuckerman8d161992016-04-10 17:24:03 +00006137_mm_cvttss_u64 (__m128 __A)
6138{
6139 return (unsigned long long) __builtin_ia32_vcvttss2usi64 ((__v4sf)
6140 __A,
6141 _MM_FROUND_CUR_DIRECTION);
6142}
Craig Topper45db56c2016-07-21 07:38:39 +00006143#endif
Michael Zuckerman8d161992016-04-10 17:24:03 +00006144
Craig Topperc6338672018-05-31 00:51:20 +00006145#define _mm512_permute_pd(X, C) \
Craig Topperacf56012018-06-08 00:59:27 +00006146 (__m512d)__builtin_ia32_vpermilpd512((__v8df)(__m512d)(X), (int)(C))
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006147
Craig Topperc6338672018-05-31 00:51:20 +00006148#define _mm512_mask_permute_pd(W, U, X, C) \
Simon Pilgrim17388f22016-07-04 11:06:15 +00006149 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
6150 (__v8df)_mm512_permute_pd((X), (C)), \
Craig Topperc6338672018-05-31 00:51:20 +00006151 (__v8df)(__m512d)(W))
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006152
Craig Topperc6338672018-05-31 00:51:20 +00006153#define _mm512_maskz_permute_pd(U, X, C) \
Simon Pilgrim17388f22016-07-04 11:06:15 +00006154 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
6155 (__v8df)_mm512_permute_pd((X), (C)), \
Craig Topperc6338672018-05-31 00:51:20 +00006156 (__v8df)_mm512_setzero_pd())
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006157
Craig Topperc6338672018-05-31 00:51:20 +00006158#define _mm512_permute_ps(X, C) \
Craig Topperacf56012018-06-08 00:59:27 +00006159 (__m512)__builtin_ia32_vpermilps512((__v16sf)(__m512)(X), (int)(C))
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006160
Craig Topperc6338672018-05-31 00:51:20 +00006161#define _mm512_mask_permute_ps(W, U, X, C) \
Simon Pilgrim17388f22016-07-04 11:06:15 +00006162 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
6163 (__v16sf)_mm512_permute_ps((X), (C)), \
Craig Topperc6338672018-05-31 00:51:20 +00006164 (__v16sf)(__m512)(W))
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006165
Craig Topperc6338672018-05-31 00:51:20 +00006166#define _mm512_maskz_permute_ps(U, X, C) \
Simon Pilgrim17388f22016-07-04 11:06:15 +00006167 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
6168 (__v16sf)_mm512_permute_ps((X), (C)), \
Craig Topperc6338672018-05-31 00:51:20 +00006169 (__v16sf)_mm512_setzero_ps())
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006170
Craig Topper74c10e32018-07-09 19:00:16 +00006171static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper678b07f2016-12-11 01:26:52 +00006172_mm512_permutevar_pd(__m512d __A, __m512i __C)
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006173{
Craig Topper678b07f2016-12-11 01:26:52 +00006174 return (__m512d)__builtin_ia32_vpermilvarpd512((__v8df)__A, (__v8di)__C);
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006175}
6176
Craig Topper74c10e32018-07-09 19:00:16 +00006177static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper678b07f2016-12-11 01:26:52 +00006178_mm512_mask_permutevar_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512i __C)
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006179{
Craig Topper678b07f2016-12-11 01:26:52 +00006180 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
6181 (__v8df)_mm512_permutevar_pd(__A, __C),
6182 (__v8df)__W);
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006183}
6184
Craig Topper74c10e32018-07-09 19:00:16 +00006185static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper678b07f2016-12-11 01:26:52 +00006186_mm512_maskz_permutevar_pd(__mmask8 __U, __m512d __A, __m512i __C)
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006187{
Craig Topper678b07f2016-12-11 01:26:52 +00006188 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
6189 (__v8df)_mm512_permutevar_pd(__A, __C),
6190 (__v8df)_mm512_setzero_pd());
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006191}
6192
Craig Topper74c10e32018-07-09 19:00:16 +00006193static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper678b07f2016-12-11 01:26:52 +00006194_mm512_permutevar_ps(__m512 __A, __m512i __C)
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006195{
Craig Topper678b07f2016-12-11 01:26:52 +00006196 return (__m512)__builtin_ia32_vpermilvarps512((__v16sf)__A, (__v16si)__C);
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006197}
6198
Craig Topper74c10e32018-07-09 19:00:16 +00006199static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper678b07f2016-12-11 01:26:52 +00006200_mm512_mask_permutevar_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512i __C)
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006201{
Craig Topper678b07f2016-12-11 01:26:52 +00006202 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
6203 (__v16sf)_mm512_permutevar_ps(__A, __C),
6204 (__v16sf)__W);
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006205}
6206
Craig Topper74c10e32018-07-09 19:00:16 +00006207static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper678b07f2016-12-11 01:26:52 +00006208_mm512_maskz_permutevar_ps(__mmask16 __U, __m512 __A, __m512i __C)
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006209{
Craig Topper678b07f2016-12-11 01:26:52 +00006210 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
6211 (__v16sf)_mm512_permutevar_ps(__A, __C),
6212 (__v16sf)_mm512_setzero_ps());
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006213}
6214
Craig Topper74c10e32018-07-09 19:00:16 +00006215static __inline __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00006216_mm512_permutex2var_pd(__m512d __A, __m512i __I, __m512d __B)
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006217{
Craig Topper68a272d2018-05-29 03:26:38 +00006218 return (__m512d)__builtin_ia32_vpermi2varpd512((__v8df)__A, (__v8di)__I,
6219 (__v8df)__B);
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00006220}
6221
Craig Topper74c10e32018-07-09 19:00:16 +00006222static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper68a272d2018-05-29 03:26:38 +00006223_mm512_mask_permutex2var_pd(__m512d __A, __mmask8 __U, __m512i __I, __m512d __B)
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00006224{
Craig Topper68a272d2018-05-29 03:26:38 +00006225 return (__m512d)__builtin_ia32_selectpd_512(__U,
6226 (__v8df)_mm512_permutex2var_pd(__A, __I, __B),
6227 (__v8df)__A);
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006228}
6229
Craig Topper74c10e32018-07-09 19:00:16 +00006230static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper68a272d2018-05-29 03:26:38 +00006231_mm512_mask2_permutex2var_pd(__m512d __A, __m512i __I, __mmask8 __U,
6232 __m512d __B)
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006233{
Craig Topper68a272d2018-05-29 03:26:38 +00006234 return (__m512d)__builtin_ia32_selectpd_512(__U,
6235 (__v8df)_mm512_permutex2var_pd(__A, __I, __B),
6236 (__v8df)(__m512d)__I);
6237}
6238
Craig Topper74c10e32018-07-09 19:00:16 +00006239static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper68a272d2018-05-29 03:26:38 +00006240_mm512_maskz_permutex2var_pd(__mmask8 __U, __m512d __A, __m512i __I,
6241 __m512d __B)
6242{
6243 return (__m512d)__builtin_ia32_selectpd_512(__U,
6244 (__v8df)_mm512_permutex2var_pd(__A, __I, __B),
6245 (__v8df)_mm512_setzero_pd());
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00006246}
6247
Craig Topper74c10e32018-07-09 19:00:16 +00006248static __inline __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00006249_mm512_permutex2var_ps(__m512 __A, __m512i __I, __m512 __B)
6250{
Craig Topper68a272d2018-05-29 03:26:38 +00006251 return (__m512)__builtin_ia32_vpermi2varps512((__v16sf)__A, (__v16si)__I,
6252 (__v16sf) __B);
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00006253}
6254
Craig Topper74c10e32018-07-09 19:00:16 +00006255static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper68a272d2018-05-29 03:26:38 +00006256_mm512_mask_permutex2var_ps(__m512 __A, __mmask16 __U, __m512i __I, __m512 __B)
Michael Zuckerman5e2c6b62016-05-11 11:21:18 +00006257{
Craig Topper68a272d2018-05-29 03:26:38 +00006258 return (__m512)__builtin_ia32_selectps_512(__U,
6259 (__v16sf)_mm512_permutex2var_ps(__A, __I, __B),
6260 (__v16sf)__A);
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006261}
6262
Craig Topper74c10e32018-07-09 19:00:16 +00006263static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper68a272d2018-05-29 03:26:38 +00006264_mm512_mask2_permutex2var_ps(__m512 __A, __m512i __I, __mmask16 __U, __m512 __B)
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006265{
Craig Topper68a272d2018-05-29 03:26:38 +00006266 return (__m512)__builtin_ia32_selectps_512(__U,
6267 (__v16sf)_mm512_permutex2var_ps(__A, __I, __B),
6268 (__v16sf)(__m512)__I);
6269}
6270
Craig Topper74c10e32018-07-09 19:00:16 +00006271static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper68a272d2018-05-29 03:26:38 +00006272_mm512_maskz_permutex2var_ps(__mmask16 __U, __m512 __A, __m512i __I, __m512 __B)
6273{
6274 return (__m512)__builtin_ia32_selectps_512(__U,
6275 (__v16sf)_mm512_permutex2var_ps(__A, __I, __B),
6276 (__v16sf)_mm512_setzero_ps());
Michael Zuckermand8d2f622016-04-11 07:15:34 +00006277}
6278
Michael Zuckerman07525092016-04-11 10:22:07 +00006279
Craig Topperc6338672018-05-31 00:51:20 +00006280#define _mm512_cvtt_roundpd_epu32(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006281 (__m256i)__builtin_ia32_cvttpd2udq512_mask((__v8df)(__m512d)(A), \
6282 (__v8si)_mm256_undefined_si256(), \
Craig Topperc6338672018-05-31 00:51:20 +00006283 (__mmask8)-1, (int)(R))
Michael Zuckerman138fc5b2016-05-03 11:05:24 +00006284
Craig Topperc6338672018-05-31 00:51:20 +00006285#define _mm512_mask_cvtt_roundpd_epu32(W, U, A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006286 (__m256i)__builtin_ia32_cvttpd2udq512_mask((__v8df)(__m512d)(A), \
6287 (__v8si)(__m256i)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00006288 (__mmask8)(U), (int)(R))
Michael Zuckerman138fc5b2016-05-03 11:05:24 +00006289
Craig Topperc6338672018-05-31 00:51:20 +00006290#define _mm512_maskz_cvtt_roundpd_epu32(U, A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006291 (__m256i)__builtin_ia32_cvttpd2udq512_mask((__v8df)(__m512d)(A), \
6292 (__v8si)_mm256_setzero_si256(), \
Craig Topperc6338672018-05-31 00:51:20 +00006293 (__mmask8)(U), (int)(R))
Michael Zuckerman138fc5b2016-05-03 11:05:24 +00006294
Craig Topper74c10e32018-07-09 19:00:16 +00006295static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckerman138fc5b2016-05-03 11:05:24 +00006296_mm512_cvttpd_epu32 (__m512d __A)
6297{
6298 return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A,
6299 (__v8si)
6300 _mm256_undefined_si256 (),
6301 (__mmask8) -1,
6302 _MM_FROUND_CUR_DIRECTION);
6303}
6304
Craig Topper74c10e32018-07-09 19:00:16 +00006305static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckerman138fc5b2016-05-03 11:05:24 +00006306_mm512_mask_cvttpd_epu32 (__m256i __W, __mmask8 __U, __m512d __A)
6307{
6308 return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A,
6309 (__v8si) __W,
6310 (__mmask8) __U,
6311 _MM_FROUND_CUR_DIRECTION);
6312}
6313
Craig Topper74c10e32018-07-09 19:00:16 +00006314static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckerman138fc5b2016-05-03 11:05:24 +00006315_mm512_maskz_cvttpd_epu32 (__mmask8 __U, __m512d __A)
6316{
6317 return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A,
6318 (__v8si)
6319 _mm256_setzero_si256 (),
6320 (__mmask8) __U,
6321 _MM_FROUND_CUR_DIRECTION);
6322}
Michael Zuckerman07525092016-04-11 10:22:07 +00006323
Craig Topperc6338672018-05-31 00:51:20 +00006324#define _mm_roundscale_round_sd(A, B, imm, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006325 (__m128d)__builtin_ia32_rndscalesd_round_mask((__v2df)(__m128d)(A), \
6326 (__v2df)(__m128d)(B), \
6327 (__v2df)_mm_setzero_pd(), \
6328 (__mmask8)-1, (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00006329 (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006330
Craig Topperc6338672018-05-31 00:51:20 +00006331#define _mm_roundscale_sd(A, B, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00006332 (__m128d)__builtin_ia32_rndscalesd_round_mask((__v2df)(__m128d)(A), \
6333 (__v2df)(__m128d)(B), \
6334 (__v2df)_mm_setzero_pd(), \
6335 (__mmask8)-1, (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00006336 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006337
Craig Topperc6338672018-05-31 00:51:20 +00006338#define _mm_mask_roundscale_sd(W, U, A, B, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00006339 (__m128d)__builtin_ia32_rndscalesd_round_mask((__v2df)(__m128d)(A), \
6340 (__v2df)(__m128d)(B), \
6341 (__v2df)(__m128d)(W), \
6342 (__mmask8)(U), (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00006343 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006344
Craig Topperc6338672018-05-31 00:51:20 +00006345#define _mm_mask_roundscale_round_sd(W, U, A, B, I, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006346 (__m128d)__builtin_ia32_rndscalesd_round_mask((__v2df)(__m128d)(A), \
6347 (__v2df)(__m128d)(B), \
6348 (__v2df)(__m128d)(W), \
6349 (__mmask8)(U), (int)(I), \
Craig Topperc6338672018-05-31 00:51:20 +00006350 (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006351
Craig Topperc6338672018-05-31 00:51:20 +00006352#define _mm_maskz_roundscale_sd(U, A, B, I) \
Craig Topper8c18e112016-05-17 04:41:50 +00006353 (__m128d)__builtin_ia32_rndscalesd_round_mask((__v2df)(__m128d)(A), \
6354 (__v2df)(__m128d)(B), \
6355 (__v2df)_mm_setzero_pd(), \
6356 (__mmask8)(U), (int)(I), \
Craig Topperc6338672018-05-31 00:51:20 +00006357 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006358
Craig Topperc6338672018-05-31 00:51:20 +00006359#define _mm_maskz_roundscale_round_sd(U, A, B, I, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006360 (__m128d)__builtin_ia32_rndscalesd_round_mask((__v2df)(__m128d)(A), \
6361 (__v2df)(__m128d)(B), \
6362 (__v2df)_mm_setzero_pd(), \
6363 (__mmask8)(U), (int)(I), \
Craig Topperc6338672018-05-31 00:51:20 +00006364 (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006365
Craig Topperc6338672018-05-31 00:51:20 +00006366#define _mm_roundscale_round_ss(A, B, imm, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006367 (__m128)__builtin_ia32_rndscaless_round_mask((__v4sf)(__m128)(A), \
6368 (__v4sf)(__m128)(B), \
6369 (__v4sf)_mm_setzero_ps(), \
6370 (__mmask8)-1, (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00006371 (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006372
Craig Topperc6338672018-05-31 00:51:20 +00006373#define _mm_roundscale_ss(A, B, imm) \
Craig Topper8c18e112016-05-17 04:41:50 +00006374 (__m128)__builtin_ia32_rndscaless_round_mask((__v4sf)(__m128)(A), \
6375 (__v4sf)(__m128)(B), \
6376 (__v4sf)_mm_setzero_ps(), \
6377 (__mmask8)-1, (int)(imm), \
Craig Topperc6338672018-05-31 00:51:20 +00006378 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006379
Craig Topperc6338672018-05-31 00:51:20 +00006380#define _mm_mask_roundscale_ss(W, U, A, B, I) \
Craig Topper8c18e112016-05-17 04:41:50 +00006381 (__m128)__builtin_ia32_rndscaless_round_mask((__v4sf)(__m128)(A), \
6382 (__v4sf)(__m128)(B), \
6383 (__v4sf)(__m128)(W), \
6384 (__mmask8)(U), (int)(I), \
Craig Topperc6338672018-05-31 00:51:20 +00006385 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006386
Craig Topperc6338672018-05-31 00:51:20 +00006387#define _mm_mask_roundscale_round_ss(W, U, A, B, I, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006388 (__m128)__builtin_ia32_rndscaless_round_mask((__v4sf)(__m128)(A), \
6389 (__v4sf)(__m128)(B), \
6390 (__v4sf)(__m128)(W), \
6391 (__mmask8)(U), (int)(I), \
Craig Topperc6338672018-05-31 00:51:20 +00006392 (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006393
Craig Topperc6338672018-05-31 00:51:20 +00006394#define _mm_maskz_roundscale_ss(U, A, B, I) \
Craig Topper8c18e112016-05-17 04:41:50 +00006395 (__m128)__builtin_ia32_rndscaless_round_mask((__v4sf)(__m128)(A), \
6396 (__v4sf)(__m128)(B), \
6397 (__v4sf)_mm_setzero_ps(), \
6398 (__mmask8)(U), (int)(I), \
Craig Topperc6338672018-05-31 00:51:20 +00006399 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006400
Craig Topperc6338672018-05-31 00:51:20 +00006401#define _mm_maskz_roundscale_round_ss(U, A, B, I, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006402 (__m128)__builtin_ia32_rndscaless_round_mask((__v4sf)(__m128)(A), \
6403 (__v4sf)(__m128)(B), \
6404 (__v4sf)_mm_setzero_ps(), \
6405 (__mmask8)(U), (int)(I), \
Craig Topperc6338672018-05-31 00:51:20 +00006406 (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006407
Craig Topperc6338672018-05-31 00:51:20 +00006408#define _mm512_scalef_round_pd(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006409 (__m512d)__builtin_ia32_scalefpd512_mask((__v8df)(__m512d)(A), \
6410 (__v8df)(__m512d)(B), \
6411 (__v8df)_mm512_undefined_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00006412 (__mmask8)-1, (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006413
Craig Topperc6338672018-05-31 00:51:20 +00006414#define _mm512_mask_scalef_round_pd(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006415 (__m512d)__builtin_ia32_scalefpd512_mask((__v8df)(__m512d)(A), \
6416 (__v8df)(__m512d)(B), \
6417 (__v8df)(__m512d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00006418 (__mmask8)(U), (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006419
Craig Topperc6338672018-05-31 00:51:20 +00006420#define _mm512_maskz_scalef_round_pd(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006421 (__m512d)__builtin_ia32_scalefpd512_mask((__v8df)(__m512d)(A), \
6422 (__v8df)(__m512d)(B), \
6423 (__v8df)_mm512_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00006424 (__mmask8)(U), (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006425
Craig Topper74c10e32018-07-09 19:00:16 +00006426static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006427_mm512_scalef_pd (__m512d __A, __m512d __B)
6428{
6429 return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A,
6430 (__v8df) __B,
6431 (__v8df)
6432 _mm512_undefined_pd (),
6433 (__mmask8) -1,
6434 _MM_FROUND_CUR_DIRECTION);
6435}
6436
Craig Topper74c10e32018-07-09 19:00:16 +00006437static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006438_mm512_mask_scalef_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B)
6439{
6440 return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A,
6441 (__v8df) __B,
6442 (__v8df) __W,
6443 (__mmask8) __U,
6444 _MM_FROUND_CUR_DIRECTION);
6445}
6446
Craig Topper74c10e32018-07-09 19:00:16 +00006447static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006448_mm512_maskz_scalef_pd (__mmask8 __U, __m512d __A, __m512d __B)
6449{
6450 return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A,
6451 (__v8df) __B,
6452 (__v8df)
6453 _mm512_setzero_pd (),
6454 (__mmask8) __U,
6455 _MM_FROUND_CUR_DIRECTION);
6456}
6457
Craig Topperc6338672018-05-31 00:51:20 +00006458#define _mm512_scalef_round_ps(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006459 (__m512)__builtin_ia32_scalefps512_mask((__v16sf)(__m512)(A), \
6460 (__v16sf)(__m512)(B), \
6461 (__v16sf)_mm512_undefined_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00006462 (__mmask16)-1, (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006463
Craig Topperc6338672018-05-31 00:51:20 +00006464#define _mm512_mask_scalef_round_ps(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006465 (__m512)__builtin_ia32_scalefps512_mask((__v16sf)(__m512)(A), \
6466 (__v16sf)(__m512)(B), \
6467 (__v16sf)(__m512)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00006468 (__mmask16)(U), (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006469
Craig Topperc6338672018-05-31 00:51:20 +00006470#define _mm512_maskz_scalef_round_ps(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006471 (__m512)__builtin_ia32_scalefps512_mask((__v16sf)(__m512)(A), \
6472 (__v16sf)(__m512)(B), \
6473 (__v16sf)_mm512_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00006474 (__mmask16)(U), (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006475
Craig Topper74c10e32018-07-09 19:00:16 +00006476static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006477_mm512_scalef_ps (__m512 __A, __m512 __B)
6478{
6479 return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A,
6480 (__v16sf) __B,
6481 (__v16sf)
6482 _mm512_undefined_ps (),
6483 (__mmask16) -1,
6484 _MM_FROUND_CUR_DIRECTION);
6485}
6486
Craig Topper74c10e32018-07-09 19:00:16 +00006487static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006488_mm512_mask_scalef_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B)
6489{
6490 return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A,
6491 (__v16sf) __B,
6492 (__v16sf) __W,
6493 (__mmask16) __U,
6494 _MM_FROUND_CUR_DIRECTION);
6495}
6496
Craig Topper74c10e32018-07-09 19:00:16 +00006497static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006498_mm512_maskz_scalef_ps (__mmask16 __U, __m512 __A, __m512 __B)
6499{
6500 return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A,
6501 (__v16sf) __B,
6502 (__v16sf)
6503 _mm512_setzero_ps (),
6504 (__mmask16) __U,
6505 _MM_FROUND_CUR_DIRECTION);
6506}
6507
Craig Topperc6338672018-05-31 00:51:20 +00006508#define _mm_scalef_round_sd(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006509 (__m128d)__builtin_ia32_scalefsd_round_mask((__v2df)(__m128d)(A), \
6510 (__v2df)(__m128d)(B), \
6511 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00006512 (__mmask8)-1, (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006513
Craig Topper74c10e32018-07-09 19:00:16 +00006514static __inline__ __m128d __DEFAULT_FN_ATTRS128
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006515_mm_scalef_sd (__m128d __A, __m128d __B)
6516{
6517 return (__m128d) __builtin_ia32_scalefsd_round_mask ((__v2df) __A,
6518 (__v2df)( __B), (__v2df) _mm_setzero_pd(),
6519 (__mmask8) -1,
6520 _MM_FROUND_CUR_DIRECTION);
6521}
6522
Craig Topper74c10e32018-07-09 19:00:16 +00006523static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00006524_mm_mask_scalef_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B)
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006525{
6526 return (__m128d) __builtin_ia32_scalefsd_round_mask ( (__v2df) __A,
6527 (__v2df) __B,
6528 (__v2df) __W,
6529 (__mmask8) __U,
6530 _MM_FROUND_CUR_DIRECTION);
6531}
6532
Craig Topperc6338672018-05-31 00:51:20 +00006533#define _mm_mask_scalef_round_sd(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006534 (__m128d)__builtin_ia32_scalefsd_round_mask((__v2df)(__m128d)(A), \
6535 (__v2df)(__m128d)(B), \
6536 (__v2df)(__m128d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00006537 (__mmask8)(U), (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006538
Craig Topper74c10e32018-07-09 19:00:16 +00006539static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00006540_mm_maskz_scalef_sd (__mmask8 __U, __m128d __A, __m128d __B)
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006541{
6542 return (__m128d) __builtin_ia32_scalefsd_round_mask ( (__v2df) __A,
6543 (__v2df) __B,
6544 (__v2df) _mm_setzero_pd (),
6545 (__mmask8) __U,
6546 _MM_FROUND_CUR_DIRECTION);
6547}
6548
Craig Topperc6338672018-05-31 00:51:20 +00006549#define _mm_maskz_scalef_round_sd(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006550 (__m128d)__builtin_ia32_scalefsd_round_mask((__v2df)(__m128d)(A), \
6551 (__v2df)(__m128d)(B), \
6552 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00006553 (__mmask8)(U), (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006554
Craig Topperc6338672018-05-31 00:51:20 +00006555#define _mm_scalef_round_ss(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006556 (__m128)__builtin_ia32_scalefss_round_mask((__v4sf)(__m128)(A), \
6557 (__v4sf)(__m128)(B), \
6558 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00006559 (__mmask8)-1, (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006560
Craig Topper74c10e32018-07-09 19:00:16 +00006561static __inline__ __m128 __DEFAULT_FN_ATTRS128
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006562_mm_scalef_ss (__m128 __A, __m128 __B)
6563{
6564 return (__m128) __builtin_ia32_scalefss_round_mask ((__v4sf) __A,
6565 (__v4sf)( __B), (__v4sf) _mm_setzero_ps(),
6566 (__mmask8) -1,
6567 _MM_FROUND_CUR_DIRECTION);
6568}
6569
Craig Topper74c10e32018-07-09 19:00:16 +00006570static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00006571_mm_mask_scalef_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006572{
6573 return (__m128) __builtin_ia32_scalefss_round_mask ( (__v4sf) __A,
6574 (__v4sf) __B,
6575 (__v4sf) __W,
6576 (__mmask8) __U,
6577 _MM_FROUND_CUR_DIRECTION);
6578}
6579
Craig Topperc6338672018-05-31 00:51:20 +00006580#define _mm_mask_scalef_round_ss(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006581 (__m128)__builtin_ia32_scalefss_round_mask((__v4sf)(__m128)(A), \
6582 (__v4sf)(__m128)(B), \
6583 (__v4sf)(__m128)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00006584 (__mmask8)(U), (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006585
Craig Topper74c10e32018-07-09 19:00:16 +00006586static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00006587_mm_maskz_scalef_ss (__mmask8 __U, __m128 __A, __m128 __B)
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006588{
6589 return (__m128) __builtin_ia32_scalefss_round_mask ( (__v4sf) __A,
6590 (__v4sf) __B,
6591 (__v4sf) _mm_setzero_ps (),
6592 (__mmask8) __U,
6593 _MM_FROUND_CUR_DIRECTION);
6594}
6595
Craig Topperc6338672018-05-31 00:51:20 +00006596#define _mm_maskz_scalef_round_ss(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00006597 (__m128)__builtin_ia32_scalefss_round_mask((__v4sf)(__m128)(A), \
6598 (__v4sf)(__m128)(B), \
6599 (__v4sf)_mm_setzero_ps(), \
6600 (__mmask8)(U), \
Craig Topper3e720a32018-07-07 22:03:16 +00006601 (int)(R))
Michael Zuckerman1af947a2016-04-11 12:32:31 +00006602
Craig Topper74c10e32018-07-09 19:00:16 +00006603static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00006604_mm512_srai_epi32(__m512i __A, int __B)
6605{
6606 return (__m512i)__builtin_ia32_psradi512((__v16si)__A, __B);
6607}
Michael Zuckerman6b5f4d82016-04-11 15:46:39 +00006608
Craig Topper74c10e32018-07-09 19:00:16 +00006609static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00006610_mm512_mask_srai_epi32(__m512i __W, __mmask16 __U, __m512i __A, int __B)
6611{
Craig Topper0a485d12018-07-07 22:03:19 +00006612 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
6613 (__v16si)_mm512_srai_epi32(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00006614 (__v16si)__W);
6615}
Michael Zuckerman6b5f4d82016-04-11 15:46:39 +00006616
Craig Topper74c10e32018-07-09 19:00:16 +00006617static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00006618_mm512_maskz_srai_epi32(__mmask16 __U, __m512i __A, int __B) {
Craig Topper0a485d12018-07-07 22:03:19 +00006619 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
6620 (__v16si)_mm512_srai_epi32(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00006621 (__v16si)_mm512_setzero_si512());
6622}
Michael Zuckerman6b5f4d82016-04-11 15:46:39 +00006623
Craig Topper74c10e32018-07-09 19:00:16 +00006624static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00006625_mm512_srai_epi64(__m512i __A, int __B)
6626{
6627 return (__m512i)__builtin_ia32_psraqi512((__v8di)__A, __B);
6628}
Michael Zuckerman6b5f4d82016-04-11 15:46:39 +00006629
Craig Topper74c10e32018-07-09 19:00:16 +00006630static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00006631_mm512_mask_srai_epi64(__m512i __W, __mmask8 __U, __m512i __A, int __B)
6632{
Craig Topper0a485d12018-07-07 22:03:19 +00006633 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
6634 (__v8di)_mm512_srai_epi64(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00006635 (__v8di)__W);
6636}
Michael Zuckerman6b5f4d82016-04-11 15:46:39 +00006637
Craig Topper74c10e32018-07-09 19:00:16 +00006638static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper1a441932016-11-12 07:16:59 +00006639_mm512_maskz_srai_epi64(__mmask8 __U, __m512i __A, int __B)
6640{
Craig Topper0a485d12018-07-07 22:03:19 +00006641 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
6642 (__v8di)_mm512_srai_epi64(__A, __B),
Craig Topper1a441932016-11-12 07:16:59 +00006643 (__v8di)_mm512_setzero_si512());
6644}
Michael Zuckerman6b5f4d82016-04-11 15:46:39 +00006645
Craig Topperc6338672018-05-31 00:51:20 +00006646#define _mm512_shuffle_f32x4(A, B, imm) \
Craig Topper93921362018-06-07 23:03:08 +00006647 (__m512)__builtin_ia32_shuf_f32x4((__v16sf)(__m512)(A), \
6648 (__v16sf)(__m512)(B), (int)(imm))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006649
Craig Topperc6338672018-05-31 00:51:20 +00006650#define _mm512_mask_shuffle_f32x4(W, U, A, B, imm) \
Jina Nahiasdca97912017-11-13 09:15:31 +00006651 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
6652 (__v16sf)_mm512_shuffle_f32x4((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00006653 (__v16sf)(__m512)(W))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006654
Craig Topperc6338672018-05-31 00:51:20 +00006655#define _mm512_maskz_shuffle_f32x4(U, A, B, imm) \
Jina Nahiasdca97912017-11-13 09:15:31 +00006656 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
6657 (__v16sf)_mm512_shuffle_f32x4((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00006658 (__v16sf)_mm512_setzero_ps())
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006659
Craig Topperc6338672018-05-31 00:51:20 +00006660#define _mm512_shuffle_f64x2(A, B, imm) \
Craig Topper93921362018-06-07 23:03:08 +00006661 (__m512d)__builtin_ia32_shuf_f64x2((__v8df)(__m512d)(A), \
6662 (__v8df)(__m512d)(B), (int)(imm))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006663
Craig Topperc6338672018-05-31 00:51:20 +00006664#define _mm512_mask_shuffle_f64x2(W, U, A, B, imm) \
Jina Nahiasdca97912017-11-13 09:15:31 +00006665 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
6666 (__v8df)_mm512_shuffle_f64x2((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00006667 (__v8df)(__m512d)(W))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006668
Craig Topperc6338672018-05-31 00:51:20 +00006669#define _mm512_maskz_shuffle_f64x2(U, A, B, imm) \
Jina Nahiasdca97912017-11-13 09:15:31 +00006670 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
6671 (__v8df)_mm512_shuffle_f64x2((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00006672 (__v8df)_mm512_setzero_pd())
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006673
Craig Topperc6338672018-05-31 00:51:20 +00006674#define _mm512_shuffle_i32x4(A, B, imm) \
Craig Topper93921362018-06-07 23:03:08 +00006675 (__m512i)__builtin_ia32_shuf_i32x4((__v16si)(__m512i)(A), \
6676 (__v16si)(__m512i)(B), (int)(imm))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006677
Craig Topperc6338672018-05-31 00:51:20 +00006678#define _mm512_mask_shuffle_i32x4(W, U, A, B, imm) \
Jina Nahiasdca97912017-11-13 09:15:31 +00006679 (__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
6680 (__v16si)_mm512_shuffle_i32x4((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00006681 (__v16si)(__m512i)(W))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006682
Craig Topperc6338672018-05-31 00:51:20 +00006683#define _mm512_maskz_shuffle_i32x4(U, A, B, imm) \
Jina Nahiasdca97912017-11-13 09:15:31 +00006684 (__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
6685 (__v16si)_mm512_shuffle_i32x4((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00006686 (__v16si)_mm512_setzero_si512())
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006687
Craig Topperc6338672018-05-31 00:51:20 +00006688#define _mm512_shuffle_i64x2(A, B, imm) \
Craig Topper93921362018-06-07 23:03:08 +00006689 (__m512i)__builtin_ia32_shuf_i64x2((__v8di)(__m512i)(A), \
6690 (__v8di)(__m512i)(B), (int)(imm))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006691
Craig Topperc6338672018-05-31 00:51:20 +00006692#define _mm512_mask_shuffle_i64x2(W, U, A, B, imm) \
Jina Nahiasdca97912017-11-13 09:15:31 +00006693 (__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
6694 (__v8di)_mm512_shuffle_i64x2((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00006695 (__v8di)(__m512i)(W))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006696
Craig Topperc6338672018-05-31 00:51:20 +00006697#define _mm512_maskz_shuffle_i64x2(U, A, B, imm) \
Jina Nahiasdca97912017-11-13 09:15:31 +00006698 (__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
6699 (__v8di)_mm512_shuffle_i64x2((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00006700 (__v8di)_mm512_setzero_si512())
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006701
Craig Topperc6338672018-05-31 00:51:20 +00006702#define _mm512_shuffle_pd(A, B, M) \
Craig Topper218da622018-07-07 17:03:34 +00006703 (__m512d)__builtin_ia32_shufpd512((__v8df)(__m512d)(A), \
6704 (__v8df)(__m512d)(B), (int)(M))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006705
Craig Topperc6338672018-05-31 00:51:20 +00006706#define _mm512_mask_shuffle_pd(W, U, A, B, M) \
Simon Pilgrim427154d2016-07-04 21:30:47 +00006707 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
6708 (__v8df)_mm512_shuffle_pd((A), (B), (M)), \
Craig Topperc6338672018-05-31 00:51:20 +00006709 (__v8df)(__m512d)(W))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006710
Craig Topperc6338672018-05-31 00:51:20 +00006711#define _mm512_maskz_shuffle_pd(U, A, B, M) \
Simon Pilgrim427154d2016-07-04 21:30:47 +00006712 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
6713 (__v8df)_mm512_shuffle_pd((A), (B), (M)), \
Craig Topperc6338672018-05-31 00:51:20 +00006714 (__v8df)_mm512_setzero_pd())
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006715
Craig Topperc6338672018-05-31 00:51:20 +00006716#define _mm512_shuffle_ps(A, B, M) \
Craig Topper218da622018-07-07 17:03:34 +00006717 (__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(A), \
6718 (__v16sf)(__m512)(B), (int)(M))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006719
Craig Topperc6338672018-05-31 00:51:20 +00006720#define _mm512_mask_shuffle_ps(W, U, A, B, M) \
Craig Topper6e76fb62016-07-10 05:57:21 +00006721 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
6722 (__v16sf)_mm512_shuffle_ps((A), (B), (M)), \
Craig Topperc6338672018-05-31 00:51:20 +00006723 (__v16sf)(__m512)(W))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006724
Craig Topperc6338672018-05-31 00:51:20 +00006725#define _mm512_maskz_shuffle_ps(U, A, B, M) \
Craig Topper6e76fb62016-07-10 05:57:21 +00006726 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
6727 (__v16sf)_mm512_shuffle_ps((A), (B), (M)), \
Craig Topperc6338672018-05-31 00:51:20 +00006728 (__v16sf)_mm512_setzero_ps())
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006729
Craig Topperc6338672018-05-31 00:51:20 +00006730#define _mm_sqrt_round_sd(A, B, R) \
Asaf Badouhf9cdb8d2016-07-05 11:36:21 +00006731 (__m128d)__builtin_ia32_sqrtsd_round_mask((__v2df)(__m128d)(A), \
6732 (__v2df)(__m128d)(B), \
Craig Topper8c18e112016-05-17 04:41:50 +00006733 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00006734 (__mmask8)-1, (int)(R))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006735
Craig Topper74c10e32018-07-09 19:00:16 +00006736static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00006737_mm_mask_sqrt_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B)
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006738{
Asaf Badouhf9cdb8d2016-07-05 11:36:21 +00006739 return (__m128d) __builtin_ia32_sqrtsd_round_mask ( (__v2df) __A,
6740 (__v2df) __B,
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006741 (__v2df) __W,
6742 (__mmask8) __U,
6743 _MM_FROUND_CUR_DIRECTION);
6744}
6745
Craig Topperc6338672018-05-31 00:51:20 +00006746#define _mm_mask_sqrt_round_sd(W, U, A, B, R) \
Asaf Badouhf9cdb8d2016-07-05 11:36:21 +00006747 (__m128d)__builtin_ia32_sqrtsd_round_mask((__v2df)(__m128d)(A), \
6748 (__v2df)(__m128d)(B), \
Craig Topper8c18e112016-05-17 04:41:50 +00006749 (__v2df)(__m128d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00006750 (__mmask8)(U), (int)(R))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006751
Craig Topper74c10e32018-07-09 19:00:16 +00006752static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00006753_mm_maskz_sqrt_sd (__mmask8 __U, __m128d __A, __m128d __B)
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006754{
Asaf Badouhf9cdb8d2016-07-05 11:36:21 +00006755 return (__m128d) __builtin_ia32_sqrtsd_round_mask ( (__v2df) __A,
6756 (__v2df) __B,
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006757 (__v2df) _mm_setzero_pd (),
6758 (__mmask8) __U,
6759 _MM_FROUND_CUR_DIRECTION);
6760}
6761
Craig Topperc6338672018-05-31 00:51:20 +00006762#define _mm_maskz_sqrt_round_sd(U, A, B, R) \
Asaf Badouhf9cdb8d2016-07-05 11:36:21 +00006763 (__m128d)__builtin_ia32_sqrtsd_round_mask((__v2df)(__m128d)(A), \
6764 (__v2df)(__m128d)(B), \
Craig Topper8c18e112016-05-17 04:41:50 +00006765 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00006766 (__mmask8)(U), (int)(R))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006767
Craig Topperc6338672018-05-31 00:51:20 +00006768#define _mm_sqrt_round_ss(A, B, R) \
Asaf Badouhf9cdb8d2016-07-05 11:36:21 +00006769 (__m128)__builtin_ia32_sqrtss_round_mask((__v4sf)(__m128)(A), \
6770 (__v4sf)(__m128)(B), \
Craig Topper8c18e112016-05-17 04:41:50 +00006771 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00006772 (__mmask8)-1, (int)(R))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006773
Craig Topper74c10e32018-07-09 19:00:16 +00006774static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00006775_mm_mask_sqrt_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006776{
Asaf Badouhf9cdb8d2016-07-05 11:36:21 +00006777 return (__m128) __builtin_ia32_sqrtss_round_mask ( (__v4sf) __A,
6778 (__v4sf) __B,
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006779 (__v4sf) __W,
6780 (__mmask8) __U,
6781 _MM_FROUND_CUR_DIRECTION);
6782}
6783
Craig Topperc6338672018-05-31 00:51:20 +00006784#define _mm_mask_sqrt_round_ss(W, U, A, B, R) \
Asaf Badouhf9cdb8d2016-07-05 11:36:21 +00006785 (__m128)__builtin_ia32_sqrtss_round_mask((__v4sf)(__m128)(A), \
6786 (__v4sf)(__m128)(B), \
Craig Topper8c18e112016-05-17 04:41:50 +00006787 (__v4sf)(__m128)(W), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00006788 (int)(R))
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006789
Craig Topper74c10e32018-07-09 19:00:16 +00006790static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00006791_mm_maskz_sqrt_ss (__mmask8 __U, __m128 __A, __m128 __B)
Michael Zuckerman04fb3bc2016-04-12 07:59:39 +00006792{
6793 return (__m128) __builtin_ia32_sqrtss_round_mask ( (__v4sf) __A,
6794 (__v4sf) __B,
6795 (__v4sf) _mm_setzero_ps (),
6796 (__mmask8) __U,
6797 _MM_FROUND_CUR_DIRECTION);
6798}
6799
Craig Topperc6338672018-05-31 00:51:20 +00006800#define _mm_maskz_sqrt_round_ss(U, A, B, R) \
Asaf Badouhf9cdb8d2016-07-05 11:36:21 +00006801 (__m128)__builtin_ia32_sqrtss_round_mask((__v4sf)(__m128)(A), \
6802 (__v4sf)(__m128)(B), \
Craig Topper8c18e112016-05-17 04:41:50 +00006803 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00006804 (__mmask8)(U), (int)(R))
Michael Zuckerman6b5f4d82016-04-11 15:46:39 +00006805
Craig Topper74c10e32018-07-09 19:00:16 +00006806static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper367c86d2017-01-18 02:17:10 +00006807_mm512_broadcast_f32x4(__m128 __A)
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006808{
Craig Topper367c86d2017-01-18 02:17:10 +00006809 return (__m512)__builtin_shufflevector((__v4sf)__A, (__v4sf)__A,
6810 0, 1, 2, 3, 0, 1, 2, 3,
6811 0, 1, 2, 3, 0, 1, 2, 3);
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006812}
6813
Craig Topper74c10e32018-07-09 19:00:16 +00006814static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper367c86d2017-01-18 02:17:10 +00006815_mm512_mask_broadcast_f32x4(__m512 __O, __mmask16 __M, __m128 __A)
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006816{
Craig Topper367c86d2017-01-18 02:17:10 +00006817 return (__m512)__builtin_ia32_selectps_512((__mmask16)__M,
6818 (__v16sf)_mm512_broadcast_f32x4(__A),
6819 (__v16sf)__O);
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006820}
6821
Craig Topper74c10e32018-07-09 19:00:16 +00006822static __inline__ __m512 __DEFAULT_FN_ATTRS512
Craig Topper367c86d2017-01-18 02:17:10 +00006823_mm512_maskz_broadcast_f32x4(__mmask16 __M, __m128 __A)
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006824{
Craig Topper367c86d2017-01-18 02:17:10 +00006825 return (__m512)__builtin_ia32_selectps_512((__mmask16)__M,
6826 (__v16sf)_mm512_broadcast_f32x4(__A),
6827 (__v16sf)_mm512_setzero_ps());
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006828}
6829
Craig Topper74c10e32018-07-09 19:00:16 +00006830static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper367c86d2017-01-18 02:17:10 +00006831_mm512_broadcast_f64x4(__m256d __A)
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006832{
Craig Topper367c86d2017-01-18 02:17:10 +00006833 return (__m512d)__builtin_shufflevector((__v4df)__A, (__v4df)__A,
6834 0, 1, 2, 3, 0, 1, 2, 3);
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006835}
6836
Craig Topper74c10e32018-07-09 19:00:16 +00006837static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper367c86d2017-01-18 02:17:10 +00006838_mm512_mask_broadcast_f64x4(__m512d __O, __mmask8 __M, __m256d __A)
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006839{
Craig Topper367c86d2017-01-18 02:17:10 +00006840 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__M,
6841 (__v8df)_mm512_broadcast_f64x4(__A),
6842 (__v8df)__O);
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006843}
6844
Craig Topper74c10e32018-07-09 19:00:16 +00006845static __inline__ __m512d __DEFAULT_FN_ATTRS512
Craig Topper367c86d2017-01-18 02:17:10 +00006846_mm512_maskz_broadcast_f64x4(__mmask8 __M, __m256d __A)
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006847{
Craig Topper367c86d2017-01-18 02:17:10 +00006848 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__M,
6849 (__v8df)_mm512_broadcast_f64x4(__A),
6850 (__v8df)_mm512_setzero_pd());
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006851}
6852
Craig Topper74c10e32018-07-09 19:00:16 +00006853static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper367c86d2017-01-18 02:17:10 +00006854_mm512_broadcast_i32x4(__m128i __A)
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006855{
Craig Topper367c86d2017-01-18 02:17:10 +00006856 return (__m512i)__builtin_shufflevector((__v4si)__A, (__v4si)__A,
6857 0, 1, 2, 3, 0, 1, 2, 3,
6858 0, 1, 2, 3, 0, 1, 2, 3);
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006859}
6860
Craig Topper74c10e32018-07-09 19:00:16 +00006861static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper367c86d2017-01-18 02:17:10 +00006862_mm512_mask_broadcast_i32x4(__m512i __O, __mmask16 __M, __m128i __A)
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006863{
Craig Topper367c86d2017-01-18 02:17:10 +00006864 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
6865 (__v16si)_mm512_broadcast_i32x4(__A),
6866 (__v16si)__O);
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006867}
6868
Craig Topper74c10e32018-07-09 19:00:16 +00006869static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper367c86d2017-01-18 02:17:10 +00006870_mm512_maskz_broadcast_i32x4(__mmask16 __M, __m128i __A)
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006871{
Craig Topper367c86d2017-01-18 02:17:10 +00006872 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
6873 (__v16si)_mm512_broadcast_i32x4(__A),
6874 (__v16si)_mm512_setzero_si512());
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006875}
6876
Craig Topper74c10e32018-07-09 19:00:16 +00006877static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper367c86d2017-01-18 02:17:10 +00006878_mm512_broadcast_i64x4(__m256i __A)
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006879{
Craig Topper367c86d2017-01-18 02:17:10 +00006880 return (__m512i)__builtin_shufflevector((__v4di)__A, (__v4di)__A,
6881 0, 1, 2, 3, 0, 1, 2, 3);
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006882}
6883
Craig Topper74c10e32018-07-09 19:00:16 +00006884static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper367c86d2017-01-18 02:17:10 +00006885_mm512_mask_broadcast_i64x4(__m512i __O, __mmask8 __M, __m256i __A)
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006886{
Craig Topper367c86d2017-01-18 02:17:10 +00006887 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
6888 (__v8di)_mm512_broadcast_i64x4(__A),
6889 (__v8di)__O);
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006890}
6891
Craig Topper74c10e32018-07-09 19:00:16 +00006892static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper367c86d2017-01-18 02:17:10 +00006893_mm512_maskz_broadcast_i64x4(__mmask8 __M, __m256i __A)
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006894{
Craig Topper367c86d2017-01-18 02:17:10 +00006895 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
6896 (__v8di)_mm512_broadcast_i64x4(__A),
6897 (__v8di)_mm512_setzero_si512());
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006898}
6899
Craig Topper74c10e32018-07-09 19:00:16 +00006900static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006901_mm512_mask_broadcastsd_pd (__m512d __O, __mmask8 __M, __m128d __A)
6902{
Simon Pilgrimf5a88372016-07-05 12:59:33 +00006903 return (__m512d)__builtin_ia32_selectpd_512(__M,
6904 (__v8df) _mm512_broadcastsd_pd(__A),
6905 (__v8df) __O);
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006906}
6907
Craig Topper74c10e32018-07-09 19:00:16 +00006908static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006909_mm512_maskz_broadcastsd_pd (__mmask8 __M, __m128d __A)
6910{
Simon Pilgrimf5a88372016-07-05 12:59:33 +00006911 return (__m512d)__builtin_ia32_selectpd_512(__M,
6912 (__v8df) _mm512_broadcastsd_pd(__A),
6913 (__v8df) _mm512_setzero_pd());
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006914}
6915
Craig Topper74c10e32018-07-09 19:00:16 +00006916static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006917_mm512_mask_broadcastss_ps (__m512 __O, __mmask16 __M, __m128 __A)
6918{
Simon Pilgrimf5a88372016-07-05 12:59:33 +00006919 return (__m512)__builtin_ia32_selectps_512(__M,
6920 (__v16sf) _mm512_broadcastss_ps(__A),
6921 (__v16sf) __O);
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006922}
6923
Craig Topper74c10e32018-07-09 19:00:16 +00006924static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006925_mm512_maskz_broadcastss_ps (__mmask16 __M, __m128 __A)
6926{
Simon Pilgrimf5a88372016-07-05 12:59:33 +00006927 return (__m512)__builtin_ia32_selectps_512(__M,
6928 (__v16sf) _mm512_broadcastss_ps(__A),
6929 (__v16sf) _mm512_setzero_ps());
Michael Zuckerman8c2900f2016-04-27 11:43:14 +00006930}
6931
Craig Topper74c10e32018-07-09 19:00:16 +00006932static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00006933_mm512_cvtsepi32_epi8 (__m512i __A)
6934{
6935 return (__m128i) __builtin_ia32_pmovsdb512_mask ((__v16si) __A,
6936 (__v16qi) _mm_undefined_si128 (),
6937 (__mmask16) -1);
6938}
6939
Craig Topper74c10e32018-07-09 19:00:16 +00006940static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00006941_mm512_mask_cvtsepi32_epi8 (__m128i __O, __mmask16 __M, __m512i __A)
6942{
6943 return (__m128i) __builtin_ia32_pmovsdb512_mask ((__v16si) __A,
6944 (__v16qi) __O, __M);
6945}
6946
Craig Topper74c10e32018-07-09 19:00:16 +00006947static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00006948_mm512_maskz_cvtsepi32_epi8 (__mmask16 __M, __m512i __A)
6949{
6950 return (__m128i) __builtin_ia32_pmovsdb512_mask ((__v16si) __A,
6951 (__v16qi) _mm_setzero_si128 (),
6952 __M);
6953}
6954
Craig Topper74c10e32018-07-09 19:00:16 +00006955static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00006956_mm512_mask_cvtsepi32_storeu_epi8 (void * __P, __mmask16 __M, __m512i __A)
6957{
6958 __builtin_ia32_pmovsdb512mem_mask ((__v16qi *) __P, (__v16si) __A, __M);
6959}
6960
Craig Topper74c10e32018-07-09 19:00:16 +00006961static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00006962_mm512_cvtsepi32_epi16 (__m512i __A)
6963{
6964 return (__m256i) __builtin_ia32_pmovsdw512_mask ((__v16si) __A,
6965 (__v16hi) _mm256_undefined_si256 (),
6966 (__mmask16) -1);
6967}
6968
Craig Topper74c10e32018-07-09 19:00:16 +00006969static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00006970_mm512_mask_cvtsepi32_epi16 (__m256i __O, __mmask16 __M, __m512i __A)
6971{
6972 return (__m256i) __builtin_ia32_pmovsdw512_mask ((__v16si) __A,
6973 (__v16hi) __O, __M);
6974}
6975
Craig Topper74c10e32018-07-09 19:00:16 +00006976static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00006977_mm512_maskz_cvtsepi32_epi16 (__mmask16 __M, __m512i __A)
6978{
6979 return (__m256i) __builtin_ia32_pmovsdw512_mask ((__v16si) __A,
6980 (__v16hi) _mm256_setzero_si256 (),
6981 __M);
6982}
6983
Craig Topper74c10e32018-07-09 19:00:16 +00006984static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00006985_mm512_mask_cvtsepi32_storeu_epi16 (void *__P, __mmask16 __M, __m512i __A)
6986{
6987 __builtin_ia32_pmovsdw512mem_mask ((__v16hi*) __P, (__v16si) __A, __M);
6988}
6989
Craig Topper74c10e32018-07-09 19:00:16 +00006990static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00006991_mm512_cvtsepi64_epi8 (__m512i __A)
6992{
6993 return (__m128i) __builtin_ia32_pmovsqb512_mask ((__v8di) __A,
6994 (__v16qi) _mm_undefined_si128 (),
6995 (__mmask8) -1);
6996}
6997
Craig Topper74c10e32018-07-09 19:00:16 +00006998static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00006999_mm512_mask_cvtsepi64_epi8 (__m128i __O, __mmask8 __M, __m512i __A)
7000{
7001 return (__m128i) __builtin_ia32_pmovsqb512_mask ((__v8di) __A,
7002 (__v16qi) __O, __M);
7003}
7004
Craig Topper74c10e32018-07-09 19:00:16 +00007005static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00007006_mm512_maskz_cvtsepi64_epi8 (__mmask8 __M, __m512i __A)
7007{
7008 return (__m128i) __builtin_ia32_pmovsqb512_mask ((__v8di) __A,
7009 (__v16qi) _mm_setzero_si128 (),
7010 __M);
7011}
7012
Craig Topper74c10e32018-07-09 19:00:16 +00007013static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00007014_mm512_mask_cvtsepi64_storeu_epi8 (void * __P, __mmask8 __M, __m512i __A)
7015{
7016 __builtin_ia32_pmovsqb512mem_mask ((__v16qi *) __P, (__v8di) __A, __M);
7017}
7018
Craig Topper74c10e32018-07-09 19:00:16 +00007019static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00007020_mm512_cvtsepi64_epi32 (__m512i __A)
7021{
Michael Zuckermane1680612016-04-13 15:02:04 +00007022 return (__m256i) __builtin_ia32_pmovsqd512_mask ((__v8di) __A,
7023 (__v8si) _mm256_undefined_si256 (),
7024 (__mmask8) -1);
7025}
7026
Craig Topper74c10e32018-07-09 19:00:16 +00007027static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00007028_mm512_mask_cvtsepi64_epi32 (__m256i __O, __mmask8 __M, __m512i __A)
7029{
7030 return (__m256i) __builtin_ia32_pmovsqd512_mask ((__v8di) __A,
7031 (__v8si) __O, __M);
7032}
7033
Craig Topper74c10e32018-07-09 19:00:16 +00007034static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00007035_mm512_maskz_cvtsepi64_epi32 (__mmask8 __M, __m512i __A)
7036{
7037 return (__m256i) __builtin_ia32_pmovsqd512_mask ((__v8di) __A,
7038 (__v8si) _mm256_setzero_si256 (),
7039 __M);
7040}
7041
Craig Topper74c10e32018-07-09 19:00:16 +00007042static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00007043_mm512_mask_cvtsepi64_storeu_epi32 (void *__P, __mmask8 __M, __m512i __A)
7044{
7045 __builtin_ia32_pmovsqd512mem_mask ((__v8si *) __P, (__v8di) __A, __M);
7046}
7047
Craig Topper74c10e32018-07-09 19:00:16 +00007048static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00007049_mm512_cvtsepi64_epi16 (__m512i __A)
7050{
7051 return (__m128i) __builtin_ia32_pmovsqw512_mask ((__v8di) __A,
7052 (__v8hi) _mm_undefined_si128 (),
7053 (__mmask8) -1);
7054}
7055
Craig Topper74c10e32018-07-09 19:00:16 +00007056static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00007057_mm512_mask_cvtsepi64_epi16 (__m128i __O, __mmask8 __M, __m512i __A)
7058{
7059 return (__m128i) __builtin_ia32_pmovsqw512_mask ((__v8di) __A,
7060 (__v8hi) __O, __M);
7061}
7062
Craig Topper74c10e32018-07-09 19:00:16 +00007063static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00007064_mm512_maskz_cvtsepi64_epi16 (__mmask8 __M, __m512i __A)
7065{
7066 return (__m128i) __builtin_ia32_pmovsqw512_mask ((__v8di) __A,
7067 (__v8hi) _mm_setzero_si128 (),
7068 __M);
7069}
7070
Craig Topper74c10e32018-07-09 19:00:16 +00007071static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckermane1680612016-04-13 15:02:04 +00007072_mm512_mask_cvtsepi64_storeu_epi16 (void * __P, __mmask8 __M, __m512i __A)
7073{
7074 __builtin_ia32_pmovsqw512mem_mask ((__v8hi *) __P, (__v8di) __A, __M);
7075}
7076
Craig Topper74c10e32018-07-09 19:00:16 +00007077static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007078_mm512_cvtusepi32_epi8 (__m512i __A)
7079{
7080 return (__m128i) __builtin_ia32_pmovusdb512_mask ((__v16si) __A,
7081 (__v16qi) _mm_undefined_si128 (),
7082 (__mmask16) -1);
7083}
7084
Craig Topper74c10e32018-07-09 19:00:16 +00007085static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007086_mm512_mask_cvtusepi32_epi8 (__m128i __O, __mmask16 __M, __m512i __A)
7087{
7088 return (__m128i) __builtin_ia32_pmovusdb512_mask ((__v16si) __A,
7089 (__v16qi) __O,
7090 __M);
7091}
7092
Craig Topper74c10e32018-07-09 19:00:16 +00007093static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007094_mm512_maskz_cvtusepi32_epi8 (__mmask16 __M, __m512i __A)
7095{
7096 return (__m128i) __builtin_ia32_pmovusdb512_mask ((__v16si) __A,
7097 (__v16qi) _mm_setzero_si128 (),
7098 __M);
7099}
7100
Craig Topper74c10e32018-07-09 19:00:16 +00007101static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007102_mm512_mask_cvtusepi32_storeu_epi8 (void * __P, __mmask16 __M, __m512i __A)
7103{
7104 __builtin_ia32_pmovusdb512mem_mask ((__v16qi *) __P, (__v16si) __A, __M);
7105}
7106
Craig Topper74c10e32018-07-09 19:00:16 +00007107static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007108_mm512_cvtusepi32_epi16 (__m512i __A)
7109{
7110 return (__m256i) __builtin_ia32_pmovusdw512_mask ((__v16si) __A,
7111 (__v16hi) _mm256_undefined_si256 (),
7112 (__mmask16) -1);
7113}
7114
Craig Topper74c10e32018-07-09 19:00:16 +00007115static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007116_mm512_mask_cvtusepi32_epi16 (__m256i __O, __mmask16 __M, __m512i __A)
7117{
7118 return (__m256i) __builtin_ia32_pmovusdw512_mask ((__v16si) __A,
7119 (__v16hi) __O,
7120 __M);
7121}
7122
Craig Topper74c10e32018-07-09 19:00:16 +00007123static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007124_mm512_maskz_cvtusepi32_epi16 (__mmask16 __M, __m512i __A)
7125{
7126 return (__m256i) __builtin_ia32_pmovusdw512_mask ((__v16si) __A,
7127 (__v16hi) _mm256_setzero_si256 (),
7128 __M);
7129}
7130
Craig Topper74c10e32018-07-09 19:00:16 +00007131static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007132_mm512_mask_cvtusepi32_storeu_epi16 (void *__P, __mmask16 __M, __m512i __A)
7133{
7134 __builtin_ia32_pmovusdw512mem_mask ((__v16hi*) __P, (__v16si) __A, __M);
7135}
7136
Craig Topper74c10e32018-07-09 19:00:16 +00007137static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007138_mm512_cvtusepi64_epi8 (__m512i __A)
7139{
7140 return (__m128i) __builtin_ia32_pmovusqb512_mask ((__v8di) __A,
7141 (__v16qi) _mm_undefined_si128 (),
7142 (__mmask8) -1);
7143}
7144
Craig Topper74c10e32018-07-09 19:00:16 +00007145static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007146_mm512_mask_cvtusepi64_epi8 (__m128i __O, __mmask8 __M, __m512i __A)
7147{
7148 return (__m128i) __builtin_ia32_pmovusqb512_mask ((__v8di) __A,
7149 (__v16qi) __O,
7150 __M);
7151}
7152
Craig Topper74c10e32018-07-09 19:00:16 +00007153static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007154_mm512_maskz_cvtusepi64_epi8 (__mmask8 __M, __m512i __A)
7155{
7156 return (__m128i) __builtin_ia32_pmovusqb512_mask ((__v8di) __A,
7157 (__v16qi) _mm_setzero_si128 (),
7158 __M);
7159}
7160
Craig Topper74c10e32018-07-09 19:00:16 +00007161static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007162_mm512_mask_cvtusepi64_storeu_epi8 (void * __P, __mmask8 __M, __m512i __A)
7163{
7164 __builtin_ia32_pmovusqb512mem_mask ((__v16qi *) __P, (__v8di) __A, __M);
7165}
7166
Craig Topper74c10e32018-07-09 19:00:16 +00007167static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007168_mm512_cvtusepi64_epi32 (__m512i __A)
7169{
7170 return (__m256i) __builtin_ia32_pmovusqd512_mask ((__v8di) __A,
7171 (__v8si) _mm256_undefined_si256 (),
7172 (__mmask8) -1);
7173}
7174
Craig Topper74c10e32018-07-09 19:00:16 +00007175static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007176_mm512_mask_cvtusepi64_epi32 (__m256i __O, __mmask8 __M, __m512i __A)
7177{
7178 return (__m256i) __builtin_ia32_pmovusqd512_mask ((__v8di) __A,
7179 (__v8si) __O, __M);
7180}
7181
Craig Topper74c10e32018-07-09 19:00:16 +00007182static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007183_mm512_maskz_cvtusepi64_epi32 (__mmask8 __M, __m512i __A)
7184{
7185 return (__m256i) __builtin_ia32_pmovusqd512_mask ((__v8di) __A,
7186 (__v8si) _mm256_setzero_si256 (),
7187 __M);
7188}
7189
Craig Topper74c10e32018-07-09 19:00:16 +00007190static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007191_mm512_mask_cvtusepi64_storeu_epi32 (void* __P, __mmask8 __M, __m512i __A)
7192{
7193 __builtin_ia32_pmovusqd512mem_mask ((__v8si*) __P, (__v8di) __A, __M);
7194}
7195
Craig Topper74c10e32018-07-09 19:00:16 +00007196static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007197_mm512_cvtusepi64_epi16 (__m512i __A)
7198{
7199 return (__m128i) __builtin_ia32_pmovusqw512_mask ((__v8di) __A,
7200 (__v8hi) _mm_undefined_si128 (),
7201 (__mmask8) -1);
7202}
7203
Craig Topper74c10e32018-07-09 19:00:16 +00007204static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007205_mm512_mask_cvtusepi64_epi16 (__m128i __O, __mmask8 __M, __m512i __A)
7206{
7207 return (__m128i) __builtin_ia32_pmovusqw512_mask ((__v8di) __A,
7208 (__v8hi) __O, __M);
7209}
7210
Craig Topper74c10e32018-07-09 19:00:16 +00007211static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007212_mm512_maskz_cvtusepi64_epi16 (__mmask8 __M, __m512i __A)
7213{
7214 return (__m128i) __builtin_ia32_pmovusqw512_mask ((__v8di) __A,
7215 (__v8hi) _mm_setzero_si128 (),
7216 __M);
7217}
7218
Craig Topper74c10e32018-07-09 19:00:16 +00007219static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckermand8715312016-04-14 06:48:09 +00007220_mm512_mask_cvtusepi64_storeu_epi16 (void *__P, __mmask8 __M, __m512i __A)
7221{
7222 __builtin_ia32_pmovusqw512mem_mask ((__v8hi*) __P, (__v8di) __A, __M);
7223}
7224
Craig Topper74c10e32018-07-09 19:00:16 +00007225static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007226_mm512_cvtepi32_epi8 (__m512i __A)
7227{
Craig Topper88097d92018-06-08 21:50:08 +00007228 return (__m128i) __builtin_ia32_pmovdb512_mask ((__v16si) __A,
7229 (__v16qi) _mm_undefined_si128 (),
7230 (__mmask16) -1);
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007231}
7232
Craig Topper74c10e32018-07-09 19:00:16 +00007233static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007234_mm512_mask_cvtepi32_epi8 (__m128i __O, __mmask16 __M, __m512i __A)
7235{
Craig Topper9d146bb2018-05-15 03:17:52 +00007236 return (__m128i) __builtin_ia32_pmovdb512_mask ((__v16si) __A,
7237 (__v16qi) __O, __M);
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007238}
7239
Craig Topper74c10e32018-07-09 19:00:16 +00007240static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007241_mm512_maskz_cvtepi32_epi8 (__mmask16 __M, __m512i __A)
7242{
Craig Topper9d146bb2018-05-15 03:17:52 +00007243 return (__m128i) __builtin_ia32_pmovdb512_mask ((__v16si) __A,
7244 (__v16qi) _mm_setzero_si128 (),
7245 __M);
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007246}
7247
Craig Topper74c10e32018-07-09 19:00:16 +00007248static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007249_mm512_mask_cvtepi32_storeu_epi8 (void * __P, __mmask16 __M, __m512i __A)
7250{
7251 __builtin_ia32_pmovdb512mem_mask ((__v16qi *) __P, (__v16si) __A, __M);
7252}
7253
Craig Topper74c10e32018-07-09 19:00:16 +00007254static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007255_mm512_cvtepi32_epi16 (__m512i __A)
7256{
Craig Topper88097d92018-06-08 21:50:08 +00007257 return (__m256i) __builtin_ia32_pmovdw512_mask ((__v16si) __A,
7258 (__v16hi) _mm256_undefined_si256 (),
7259 (__mmask16) -1);
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007260}
7261
Craig Topper74c10e32018-07-09 19:00:16 +00007262static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007263_mm512_mask_cvtepi32_epi16 (__m256i __O, __mmask16 __M, __m512i __A)
7264{
Craig Topper9d146bb2018-05-15 03:17:52 +00007265 return (__m256i) __builtin_ia32_pmovdw512_mask ((__v16si) __A,
7266 (__v16hi) __O, __M);
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007267}
7268
Craig Topper74c10e32018-07-09 19:00:16 +00007269static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007270_mm512_maskz_cvtepi32_epi16 (__mmask16 __M, __m512i __A)
7271{
Craig Topper9d146bb2018-05-15 03:17:52 +00007272 return (__m256i) __builtin_ia32_pmovdw512_mask ((__v16si) __A,
7273 (__v16hi) _mm256_setzero_si256 (),
7274 __M);
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007275}
7276
Craig Topper74c10e32018-07-09 19:00:16 +00007277static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007278_mm512_mask_cvtepi32_storeu_epi16 (void * __P, __mmask16 __M, __m512i __A)
7279{
7280 __builtin_ia32_pmovdw512mem_mask ((__v16hi *) __P, (__v16si) __A, __M);
7281}
7282
Craig Topper74c10e32018-07-09 19:00:16 +00007283static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007284_mm512_cvtepi64_epi8 (__m512i __A)
7285{
7286 return (__m128i) __builtin_ia32_pmovqb512_mask ((__v8di) __A,
7287 (__v16qi) _mm_undefined_si128 (),
7288 (__mmask8) -1);
7289}
7290
Craig Topper74c10e32018-07-09 19:00:16 +00007291static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007292_mm512_mask_cvtepi64_epi8 (__m128i __O, __mmask8 __M, __m512i __A)
7293{
7294 return (__m128i) __builtin_ia32_pmovqb512_mask ((__v8di) __A,
7295 (__v16qi) __O, __M);
7296}
7297
Craig Topper74c10e32018-07-09 19:00:16 +00007298static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007299_mm512_maskz_cvtepi64_epi8 (__mmask8 __M, __m512i __A)
7300{
7301 return (__m128i) __builtin_ia32_pmovqb512_mask ((__v8di) __A,
7302 (__v16qi) _mm_setzero_si128 (),
7303 __M);
7304}
7305
Craig Topper74c10e32018-07-09 19:00:16 +00007306static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007307_mm512_mask_cvtepi64_storeu_epi8 (void * __P, __mmask8 __M, __m512i __A)
7308{
7309 __builtin_ia32_pmovqb512mem_mask ((__v16qi *) __P, (__v8di) __A, __M);
7310}
7311
Craig Topper74c10e32018-07-09 19:00:16 +00007312static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007313_mm512_cvtepi64_epi32 (__m512i __A)
7314{
Craig Topper88097d92018-06-08 21:50:08 +00007315 return (__m256i) __builtin_ia32_pmovqd512_mask ((__v8di) __A,
7316 (__v8si) _mm256_undefined_si256 (),
7317 (__mmask8) -1);
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007318}
7319
Craig Topper74c10e32018-07-09 19:00:16 +00007320static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007321_mm512_mask_cvtepi64_epi32 (__m256i __O, __mmask8 __M, __m512i __A)
7322{
Craig Topper88097d92018-06-08 21:50:08 +00007323 return (__m256i) __builtin_ia32_pmovqd512_mask ((__v8di) __A,
7324 (__v8si) __O, __M);
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007325}
7326
Craig Topper74c10e32018-07-09 19:00:16 +00007327static __inline__ __m256i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007328_mm512_maskz_cvtepi64_epi32 (__mmask8 __M, __m512i __A)
7329{
Craig Topper88097d92018-06-08 21:50:08 +00007330 return (__m256i) __builtin_ia32_pmovqd512_mask ((__v8di) __A,
7331 (__v8si) _mm256_setzero_si256 (),
7332 __M);
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007333}
7334
Craig Topper74c10e32018-07-09 19:00:16 +00007335static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007336_mm512_mask_cvtepi64_storeu_epi32 (void* __P, __mmask8 __M, __m512i __A)
7337{
7338 __builtin_ia32_pmovqd512mem_mask ((__v8si *) __P, (__v8di) __A, __M);
7339}
7340
Craig Topper74c10e32018-07-09 19:00:16 +00007341static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007342_mm512_cvtepi64_epi16 (__m512i __A)
7343{
Craig Topper88097d92018-06-08 21:50:08 +00007344 return (__m128i) __builtin_ia32_pmovqw512_mask ((__v8di) __A,
7345 (__v8hi) _mm_undefined_si128 (),
7346 (__mmask8) -1);
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007347}
7348
Craig Topper74c10e32018-07-09 19:00:16 +00007349static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007350_mm512_mask_cvtepi64_epi16 (__m128i __O, __mmask8 __M, __m512i __A)
7351{
Craig Topper9d146bb2018-05-15 03:17:52 +00007352 return (__m128i) __builtin_ia32_pmovqw512_mask ((__v8di) __A,
7353 (__v8hi) __O, __M);
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007354}
7355
Craig Topper74c10e32018-07-09 19:00:16 +00007356static __inline__ __m128i __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007357_mm512_maskz_cvtepi64_epi16 (__mmask8 __M, __m512i __A)
7358{
Craig Topper9d146bb2018-05-15 03:17:52 +00007359 return (__m128i) __builtin_ia32_pmovqw512_mask ((__v8di) __A,
7360 (__v8hi) _mm_setzero_si128 (),
7361 __M);
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007362}
7363
Craig Topper74c10e32018-07-09 19:00:16 +00007364static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckerman0a3508a2016-04-14 07:56:51 +00007365_mm512_mask_cvtepi64_storeu_epi16 (void *__P, __mmask8 __M, __m512i __A)
7366{
7367 __builtin_ia32_pmovqw512mem_mask ((__v8hi *) __P, (__v8di) __A, __M);
7368}
7369
Craig Topperc6338672018-05-31 00:51:20 +00007370#define _mm512_extracti32x4_epi32(A, imm) \
Craig Topper5f50f3382018-06-08 21:50:07 +00007371 (__m128i)__builtin_ia32_extracti32x4_mask((__v16si)(__m512i)(A), (int)(imm), \
7372 (__v4si)_mm_undefined_si128(), \
7373 (__mmask8)-1)
Michael Zuckermanef2979a2016-04-19 15:18:23 +00007374
Craig Topperc6338672018-05-31 00:51:20 +00007375#define _mm512_mask_extracti32x4_epi32(W, U, A, imm) \
Craig Topper5f50f3382018-06-08 21:50:07 +00007376 (__m128i)__builtin_ia32_extracti32x4_mask((__v16si)(__m512i)(A), (int)(imm), \
7377 (__v4si)(__m128i)(W), \
7378 (__mmask8)(U))
Michael Zuckermanef2979a2016-04-19 15:18:23 +00007379
Craig Topperc6338672018-05-31 00:51:20 +00007380#define _mm512_maskz_extracti32x4_epi32(U, A, imm) \
Craig Topper5f50f3382018-06-08 21:50:07 +00007381 (__m128i)__builtin_ia32_extracti32x4_mask((__v16si)(__m512i)(A), (int)(imm), \
7382 (__v4si)_mm_setzero_si128(), \
7383 (__mmask8)(U))
Michael Zuckermanef2979a2016-04-19 15:18:23 +00007384
Craig Topperc6338672018-05-31 00:51:20 +00007385#define _mm512_extracti64x4_epi64(A, imm) \
Craig Topper5f50f3382018-06-08 21:50:07 +00007386 (__m256i)__builtin_ia32_extracti64x4_mask((__v8di)(__m512i)(A), (int)(imm), \
7387 (__v4di)_mm256_undefined_si256(), \
7388 (__mmask8)-1)
Michael Zuckermanef2979a2016-04-19 15:18:23 +00007389
Craig Topperc6338672018-05-31 00:51:20 +00007390#define _mm512_mask_extracti64x4_epi64(W, U, A, imm) \
Craig Topper5f50f3382018-06-08 21:50:07 +00007391 (__m256i)__builtin_ia32_extracti64x4_mask((__v8di)(__m512i)(A), (int)(imm), \
7392 (__v4di)(__m256i)(W), \
7393 (__mmask8)(U))
Michael Zuckermanef2979a2016-04-19 15:18:23 +00007394
Craig Topperc6338672018-05-31 00:51:20 +00007395#define _mm512_maskz_extracti64x4_epi64(U, A, imm) \
Craig Topper5f50f3382018-06-08 21:50:07 +00007396 (__m256i)__builtin_ia32_extracti64x4_mask((__v8di)(__m512i)(A), (int)(imm), \
7397 (__v4di)_mm256_setzero_si256(), \
7398 (__mmask8)(U))
Michael Zuckermanef2979a2016-04-19 15:18:23 +00007399
Craig Topperc6338672018-05-31 00:51:20 +00007400#define _mm512_insertf64x4(A, B, imm) \
Craig Topper3428bee2018-06-08 03:24:47 +00007401 (__m512d)__builtin_ia32_insertf64x4((__v8df)(__m512d)(A), \
7402 (__v4df)(__m256d)(B), (int)(imm))
Michael Zuckermanef2979a2016-04-19 15:18:23 +00007403
Craig Topperc6338672018-05-31 00:51:20 +00007404#define _mm512_mask_insertf64x4(W, U, A, B, imm) \
Craig Topper08bf53f2016-11-01 05:47:56 +00007405 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
7406 (__v8df)_mm512_insertf64x4((A), (B), (imm)), \
Craig Toppercbf39292018-05-31 01:24:40 +00007407 (__v8df)(__m512d)(W))
Michael Zuckermanef2979a2016-04-19 15:18:23 +00007408
Craig Topperc6338672018-05-31 00:51:20 +00007409#define _mm512_maskz_insertf64x4(U, A, B, imm) \
Craig Topper08bf53f2016-11-01 05:47:56 +00007410 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
7411 (__v8df)_mm512_insertf64x4((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00007412 (__v8df)_mm512_setzero_pd())
Michael Zuckermanef2979a2016-04-19 15:18:23 +00007413
Craig Topperc6338672018-05-31 00:51:20 +00007414#define _mm512_inserti64x4(A, B, imm) \
Craig Topper3428bee2018-06-08 03:24:47 +00007415 (__m512i)__builtin_ia32_inserti64x4((__v8di)(__m512i)(A), \
7416 (__v4di)(__m256i)(B), (int)(imm))
Michael Zuckermanef2979a2016-04-19 15:18:23 +00007417
Craig Topperc6338672018-05-31 00:51:20 +00007418#define _mm512_mask_inserti64x4(W, U, A, B, imm) \
Craig Topper08bf53f2016-11-01 05:47:56 +00007419 (__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
7420 (__v8di)_mm512_inserti64x4((A), (B), (imm)), \
Craig Toppercbf39292018-05-31 01:24:40 +00007421 (__v8di)(__m512i)(W))
Michael Zuckermanef2979a2016-04-19 15:18:23 +00007422
Craig Topperc6338672018-05-31 00:51:20 +00007423#define _mm512_maskz_inserti64x4(U, A, B, imm) \
Craig Topper08bf53f2016-11-01 05:47:56 +00007424 (__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
7425 (__v8di)_mm512_inserti64x4((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00007426 (__v8di)_mm512_setzero_si512())
Michael Zuckermanef2979a2016-04-19 15:18:23 +00007427
Craig Topperc6338672018-05-31 00:51:20 +00007428#define _mm512_insertf32x4(A, B, imm) \
Craig Topper3428bee2018-06-08 03:24:47 +00007429 (__m512)__builtin_ia32_insertf32x4((__v16sf)(__m512)(A), \
7430 (__v4sf)(__m128)(B), (int)(imm))
Craig Topperdca1f232016-05-15 21:26:20 +00007431
Craig Topperc6338672018-05-31 00:51:20 +00007432#define _mm512_mask_insertf32x4(W, U, A, B, imm) \
Craig Topper08bf53f2016-11-01 05:47:56 +00007433 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
7434 (__v16sf)_mm512_insertf32x4((A), (B), (imm)), \
Craig Toppercbf39292018-05-31 01:24:40 +00007435 (__v16sf)(__m512)(W))
Craig Topperdca1f232016-05-15 21:26:20 +00007436
Craig Topperc6338672018-05-31 00:51:20 +00007437#define _mm512_maskz_insertf32x4(U, A, B, imm) \
Craig Topper08bf53f2016-11-01 05:47:56 +00007438 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
7439 (__v16sf)_mm512_insertf32x4((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00007440 (__v16sf)_mm512_setzero_ps())
Craig Topperdca1f232016-05-15 21:26:20 +00007441
Craig Topperc6338672018-05-31 00:51:20 +00007442#define _mm512_inserti32x4(A, B, imm) \
Craig Topper3428bee2018-06-08 03:24:47 +00007443 (__m512i)__builtin_ia32_inserti32x4((__v16si)(__m512i)(A), \
7444 (__v4si)(__m128i)(B), (int)(imm))
Craig Topperdca1f232016-05-15 21:26:20 +00007445
Craig Topperc6338672018-05-31 00:51:20 +00007446#define _mm512_mask_inserti32x4(W, U, A, B, imm) \
Craig Topper08bf53f2016-11-01 05:47:56 +00007447 (__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
7448 (__v16si)_mm512_inserti32x4((A), (B), (imm)), \
Craig Toppercbf39292018-05-31 01:24:40 +00007449 (__v16si)(__m512i)(W))
Craig Topperdca1f232016-05-15 21:26:20 +00007450
Craig Topperc6338672018-05-31 00:51:20 +00007451#define _mm512_maskz_inserti32x4(U, A, B, imm) \
Craig Topper08bf53f2016-11-01 05:47:56 +00007452 (__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
7453 (__v16si)_mm512_inserti32x4((A), (B), (imm)), \
Craig Topperc6338672018-05-31 00:51:20 +00007454 (__v16si)_mm512_setzero_si512())
Craig Topperdca1f232016-05-15 21:26:20 +00007455
Craig Topperc6338672018-05-31 00:51:20 +00007456#define _mm512_getmant_round_pd(A, B, C, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007457 (__m512d)__builtin_ia32_getmantpd512_mask((__v8df)(__m512d)(A), \
7458 (int)(((C)<<2) | (B)), \
7459 (__v8df)_mm512_undefined_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00007460 (__mmask8)-1, (int)(R))
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007461
Craig Topperc6338672018-05-31 00:51:20 +00007462#define _mm512_mask_getmant_round_pd(W, U, A, B, C, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007463 (__m512d)__builtin_ia32_getmantpd512_mask((__v8df)(__m512d)(A), \
7464 (int)(((C)<<2) | (B)), \
7465 (__v8df)(__m512d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00007466 (__mmask8)(U), (int)(R))
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007467
Craig Topperc6338672018-05-31 00:51:20 +00007468#define _mm512_maskz_getmant_round_pd(U, A, B, C, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007469 (__m512d)__builtin_ia32_getmantpd512_mask((__v8df)(__m512d)(A), \
7470 (int)(((C)<<2) | (B)), \
7471 (__v8df)_mm512_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00007472 (__mmask8)(U), (int)(R))
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007473
Craig Topperc6338672018-05-31 00:51:20 +00007474#define _mm512_getmant_pd(A, B, C) \
Craig Topper8c18e112016-05-17 04:41:50 +00007475 (__m512d)__builtin_ia32_getmantpd512_mask((__v8df)(__m512d)(A), \
7476 (int)(((C)<<2) | (B)), \
7477 (__v8df)_mm512_setzero_pd(), \
7478 (__mmask8)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00007479 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007480
Craig Topperc6338672018-05-31 00:51:20 +00007481#define _mm512_mask_getmant_pd(W, U, A, B, C) \
Craig Topper8c18e112016-05-17 04:41:50 +00007482 (__m512d)__builtin_ia32_getmantpd512_mask((__v8df)(__m512d)(A), \
7483 (int)(((C)<<2) | (B)), \
7484 (__v8df)(__m512d)(W), \
7485 (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00007486 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007487
Craig Topperc6338672018-05-31 00:51:20 +00007488#define _mm512_maskz_getmant_pd(U, A, B, C) \
Craig Topper8c18e112016-05-17 04:41:50 +00007489 (__m512d)__builtin_ia32_getmantpd512_mask((__v8df)(__m512d)(A), \
7490 (int)(((C)<<2) | (B)), \
7491 (__v8df)_mm512_setzero_pd(), \
7492 (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00007493 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007494
Craig Topperc6338672018-05-31 00:51:20 +00007495#define _mm512_getmant_round_ps(A, B, C, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007496 (__m512)__builtin_ia32_getmantps512_mask((__v16sf)(__m512)(A), \
7497 (int)(((C)<<2) | (B)), \
7498 (__v16sf)_mm512_undefined_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00007499 (__mmask16)-1, (int)(R))
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007500
Craig Topperc6338672018-05-31 00:51:20 +00007501#define _mm512_mask_getmant_round_ps(W, U, A, B, C, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007502 (__m512)__builtin_ia32_getmantps512_mask((__v16sf)(__m512)(A), \
7503 (int)(((C)<<2) | (B)), \
7504 (__v16sf)(__m512)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00007505 (__mmask16)(U), (int)(R))
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007506
Craig Topperc6338672018-05-31 00:51:20 +00007507#define _mm512_maskz_getmant_round_ps(U, A, B, C, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007508 (__m512)__builtin_ia32_getmantps512_mask((__v16sf)(__m512)(A), \
7509 (int)(((C)<<2) | (B)), \
7510 (__v16sf)_mm512_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00007511 (__mmask16)(U), (int)(R))
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007512
Craig Topperc6338672018-05-31 00:51:20 +00007513#define _mm512_getmant_ps(A, B, C) \
Craig Topper8c18e112016-05-17 04:41:50 +00007514 (__m512)__builtin_ia32_getmantps512_mask((__v16sf)(__m512)(A), \
7515 (int)(((C)<<2)|(B)), \
7516 (__v16sf)_mm512_undefined_ps(), \
7517 (__mmask16)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00007518 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007519
Craig Topperc6338672018-05-31 00:51:20 +00007520#define _mm512_mask_getmant_ps(W, U, A, B, C) \
Craig Topper8c18e112016-05-17 04:41:50 +00007521 (__m512)__builtin_ia32_getmantps512_mask((__v16sf)(__m512)(A), \
7522 (int)(((C)<<2)|(B)), \
7523 (__v16sf)(__m512)(W), \
7524 (__mmask16)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00007525 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007526
Craig Topperc6338672018-05-31 00:51:20 +00007527#define _mm512_maskz_getmant_ps(U, A, B, C) \
Craig Topper8c18e112016-05-17 04:41:50 +00007528 (__m512)__builtin_ia32_getmantps512_mask((__v16sf)(__m512)(A), \
7529 (int)(((C)<<2)|(B)), \
7530 (__v16sf)_mm512_setzero_ps(), \
7531 (__mmask16)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00007532 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007533
Craig Topperc6338672018-05-31 00:51:20 +00007534#define _mm512_getexp_round_pd(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007535 (__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \
7536 (__v8df)_mm512_undefined_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00007537 (__mmask8)-1, (int)(R))
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007538
Craig Topperc6338672018-05-31 00:51:20 +00007539#define _mm512_mask_getexp_round_pd(W, U, A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007540 (__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \
7541 (__v8df)(__m512d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00007542 (__mmask8)(U), (int)(R))
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007543
Craig Topperc6338672018-05-31 00:51:20 +00007544#define _mm512_maskz_getexp_round_pd(U, A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007545 (__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \
7546 (__v8df)_mm512_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00007547 (__mmask8)(U), (int)(R))
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007548
Craig Topper74c10e32018-07-09 19:00:16 +00007549static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007550_mm512_getexp_pd (__m512d __A)
7551{
7552 return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A,
7553 (__v8df) _mm512_undefined_pd (),
7554 (__mmask8) -1,
7555 _MM_FROUND_CUR_DIRECTION);
7556}
7557
Craig Topper74c10e32018-07-09 19:00:16 +00007558static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007559_mm512_mask_getexp_pd (__m512d __W, __mmask8 __U, __m512d __A)
7560{
7561 return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A,
7562 (__v8df) __W,
7563 (__mmask8) __U,
7564 _MM_FROUND_CUR_DIRECTION);
7565}
7566
Craig Topper74c10e32018-07-09 19:00:16 +00007567static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007568_mm512_maskz_getexp_pd (__mmask8 __U, __m512d __A)
7569{
7570 return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A,
7571 (__v8df) _mm512_setzero_pd (),
7572 (__mmask8) __U,
7573 _MM_FROUND_CUR_DIRECTION);
7574}
7575
Craig Topperc6338672018-05-31 00:51:20 +00007576#define _mm512_getexp_round_ps(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007577 (__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \
7578 (__v16sf)_mm512_undefined_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00007579 (__mmask16)-1, (int)(R))
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007580
Craig Topperc6338672018-05-31 00:51:20 +00007581#define _mm512_mask_getexp_round_ps(W, U, A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007582 (__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \
7583 (__v16sf)(__m512)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00007584 (__mmask16)(U), (int)(R))
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007585
Craig Topperc6338672018-05-31 00:51:20 +00007586#define _mm512_maskz_getexp_round_ps(U, A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007587 (__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \
7588 (__v16sf)_mm512_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00007589 (__mmask16)(U), (int)(R))
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007590
Craig Topper74c10e32018-07-09 19:00:16 +00007591static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007592_mm512_getexp_ps (__m512 __A)
7593{
7594 return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A,
7595 (__v16sf) _mm512_undefined_ps (),
7596 (__mmask16) -1,
7597 _MM_FROUND_CUR_DIRECTION);
7598}
7599
Craig Topper74c10e32018-07-09 19:00:16 +00007600static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007601_mm512_mask_getexp_ps (__m512 __W, __mmask16 __U, __m512 __A)
7602{
7603 return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A,
7604 (__v16sf) __W,
7605 (__mmask16) __U,
7606 _MM_FROUND_CUR_DIRECTION);
7607}
7608
Craig Topper74c10e32018-07-09 19:00:16 +00007609static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman6fa512c2016-04-19 17:10:29 +00007610_mm512_maskz_getexp_ps (__mmask16 __U, __m512 __A)
7611{
7612 return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A,
7613 (__v16sf) _mm512_setzero_ps (),
7614 (__mmask16) __U,
7615 _MM_FROUND_CUR_DIRECTION);
7616}
7617
Craig Topperc6338672018-05-31 00:51:20 +00007618#define _mm512_i64gather_ps(index, addr, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007619 (__m256)__builtin_ia32_gatherdiv16sf((__v8sf)_mm256_undefined_ps(), \
7620 (float const *)(addr), \
7621 (__v8di)(__m512i)(index), (__mmask8)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00007622 (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007623
Craig Topperc6338672018-05-31 00:51:20 +00007624#define _mm512_mask_i64gather_ps(v1_old, mask, index, addr, scale) \
Craig Topper2e5058c2017-03-17 05:14:37 +00007625 (__m256)__builtin_ia32_gatherdiv16sf((__v8sf)(__m256)(v1_old),\
7626 (float const *)(addr), \
7627 (__v8di)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007628 (__mmask8)(mask), (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007629
Craig Topperc6338672018-05-31 00:51:20 +00007630#define _mm512_i64gather_epi32(index, addr, scale) \
Craig Topper5cbeeed2018-07-07 17:03:32 +00007631 (__m256i)__builtin_ia32_gatherdiv16si((__v8si)_mm256_undefined_si256(), \
Craig Topper8c18e112016-05-17 04:41:50 +00007632 (int const *)(addr), \
7633 (__v8di)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007634 (__mmask8)-1, (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007635
Craig Topperc6338672018-05-31 00:51:20 +00007636#define _mm512_mask_i64gather_epi32(v1_old, mask, index, addr, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007637 (__m256i)__builtin_ia32_gatherdiv16si((__v8si)(__m256i)(v1_old), \
7638 (int const *)(addr), \
7639 (__v8di)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007640 (__mmask8)(mask), (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007641
Craig Topperc6338672018-05-31 00:51:20 +00007642#define _mm512_i64gather_pd(index, addr, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007643 (__m512d)__builtin_ia32_gatherdiv8df((__v8df)_mm512_undefined_pd(), \
7644 (double const *)(addr), \
7645 (__v8di)(__m512i)(index), (__mmask8)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00007646 (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007647
Craig Topperc6338672018-05-31 00:51:20 +00007648#define _mm512_mask_i64gather_pd(v1_old, mask, index, addr, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007649 (__m512d)__builtin_ia32_gatherdiv8df((__v8df)(__m512d)(v1_old), \
7650 (double const *)(addr), \
7651 (__v8di)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007652 (__mmask8)(mask), (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007653
Craig Topperc6338672018-05-31 00:51:20 +00007654#define _mm512_i64gather_epi64(index, addr, scale) \
Craig Topper5cbeeed2018-07-07 17:03:32 +00007655 (__m512i)__builtin_ia32_gatherdiv8di((__v8di)_mm512_undefined_epi32(), \
Craig Topper8c18e112016-05-17 04:41:50 +00007656 (long long const *)(addr), \
7657 (__v8di)(__m512i)(index), (__mmask8)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00007658 (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007659
Craig Topperc6338672018-05-31 00:51:20 +00007660#define _mm512_mask_i64gather_epi64(v1_old, mask, index, addr, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007661 (__m512i)__builtin_ia32_gatherdiv8di((__v8di)(__m512i)(v1_old), \
7662 (long long const *)(addr), \
7663 (__v8di)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007664 (__mmask8)(mask), (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007665
Craig Topperc6338672018-05-31 00:51:20 +00007666#define _mm512_i32gather_ps(index, addr, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007667 (__m512)__builtin_ia32_gathersiv16sf((__v16sf)_mm512_undefined_ps(), \
7668 (float const *)(addr), \
7669 (__v16sf)(__m512)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007670 (__mmask16)-1, (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007671
Craig Topperc6338672018-05-31 00:51:20 +00007672#define _mm512_mask_i32gather_ps(v1_old, mask, index, addr, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007673 (__m512)__builtin_ia32_gathersiv16sf((__v16sf)(__m512)(v1_old), \
7674 (float const *)(addr), \
7675 (__v16sf)(__m512)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007676 (__mmask16)(mask), (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007677
Craig Topperc6338672018-05-31 00:51:20 +00007678#define _mm512_i32gather_epi32(index, addr, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007679 (__m512i)__builtin_ia32_gathersiv16si((__v16si)_mm512_undefined_epi32(), \
7680 (int const *)(addr), \
7681 (__v16si)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007682 (__mmask16)-1, (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007683
Craig Topperc6338672018-05-31 00:51:20 +00007684#define _mm512_mask_i32gather_epi32(v1_old, mask, index, addr, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007685 (__m512i)__builtin_ia32_gathersiv16si((__v16si)(__m512i)(v1_old), \
7686 (int const *)(addr), \
7687 (__v16si)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007688 (__mmask16)(mask), (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007689
Craig Topperc6338672018-05-31 00:51:20 +00007690#define _mm512_i32gather_pd(index, addr, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007691 (__m512d)__builtin_ia32_gathersiv8df((__v8df)_mm512_undefined_pd(), \
7692 (double const *)(addr), \
7693 (__v8si)(__m256i)(index), (__mmask8)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00007694 (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007695
Craig Topperc6338672018-05-31 00:51:20 +00007696#define _mm512_mask_i32gather_pd(v1_old, mask, index, addr, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007697 (__m512d)__builtin_ia32_gathersiv8df((__v8df)(__m512d)(v1_old), \
7698 (double const *)(addr), \
7699 (__v8si)(__m256i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007700 (__mmask8)(mask), (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007701
Craig Topperc6338672018-05-31 00:51:20 +00007702#define _mm512_i32gather_epi64(index, addr, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007703 (__m512i)__builtin_ia32_gathersiv8di((__v8di)_mm512_undefined_epi32(), \
7704 (long long const *)(addr), \
7705 (__v8si)(__m256i)(index), (__mmask8)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00007706 (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007707
Craig Topperc6338672018-05-31 00:51:20 +00007708#define _mm512_mask_i32gather_epi64(v1_old, mask, index, addr, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007709 (__m512i)__builtin_ia32_gathersiv8di((__v8di)(__m512i)(v1_old), \
7710 (long long const *)(addr), \
7711 (__v8si)(__m256i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007712 (__mmask8)(mask), (int)(scale))
Michael Zuckerman4fa96af2016-04-21 12:47:27 +00007713
Craig Topperc6338672018-05-31 00:51:20 +00007714#define _mm512_i64scatter_ps(addr, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007715 __builtin_ia32_scatterdiv16sf((float *)(addr), (__mmask8)-1, \
7716 (__v8di)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007717 (__v8sf)(__m256)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007718
Craig Topperc6338672018-05-31 00:51:20 +00007719#define _mm512_mask_i64scatter_ps(addr, mask, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007720 __builtin_ia32_scatterdiv16sf((float *)(addr), (__mmask8)(mask), \
7721 (__v8di)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007722 (__v8sf)(__m256)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007723
Craig Topperc6338672018-05-31 00:51:20 +00007724#define _mm512_i64scatter_epi32(addr, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007725 __builtin_ia32_scatterdiv16si((int *)(addr), (__mmask8)-1, \
7726 (__v8di)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007727 (__v8si)(__m256i)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007728
Craig Topperc6338672018-05-31 00:51:20 +00007729#define _mm512_mask_i64scatter_epi32(addr, mask, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007730 __builtin_ia32_scatterdiv16si((int *)(addr), (__mmask8)(mask), \
7731 (__v8di)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007732 (__v8si)(__m256i)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007733
Craig Topperc6338672018-05-31 00:51:20 +00007734#define _mm512_i64scatter_pd(addr, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007735 __builtin_ia32_scatterdiv8df((double *)(addr), (__mmask8)-1, \
7736 (__v8di)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007737 (__v8df)(__m512d)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007738
Craig Topperc6338672018-05-31 00:51:20 +00007739#define _mm512_mask_i64scatter_pd(addr, mask, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007740 __builtin_ia32_scatterdiv8df((double *)(addr), (__mmask8)(mask), \
7741 (__v8di)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007742 (__v8df)(__m512d)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007743
Craig Topperc6338672018-05-31 00:51:20 +00007744#define _mm512_i64scatter_epi64(addr, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007745 __builtin_ia32_scatterdiv8di((long long *)(addr), (__mmask8)-1, \
7746 (__v8di)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007747 (__v8di)(__m512i)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007748
Craig Topperc6338672018-05-31 00:51:20 +00007749#define _mm512_mask_i64scatter_epi64(addr, mask, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007750 __builtin_ia32_scatterdiv8di((long long *)(addr), (__mmask8)(mask), \
7751 (__v8di)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007752 (__v8di)(__m512i)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007753
Craig Topperc6338672018-05-31 00:51:20 +00007754#define _mm512_i32scatter_ps(addr, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007755 __builtin_ia32_scattersiv16sf((float *)(addr), (__mmask16)-1, \
7756 (__v16si)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007757 (__v16sf)(__m512)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007758
Craig Topperc6338672018-05-31 00:51:20 +00007759#define _mm512_mask_i32scatter_ps(addr, mask, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007760 __builtin_ia32_scattersiv16sf((float *)(addr), (__mmask16)(mask), \
7761 (__v16si)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007762 (__v16sf)(__m512)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007763
Craig Topperc6338672018-05-31 00:51:20 +00007764#define _mm512_i32scatter_epi32(addr, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007765 __builtin_ia32_scattersiv16si((int *)(addr), (__mmask16)-1, \
7766 (__v16si)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007767 (__v16si)(__m512i)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007768
Craig Topperc6338672018-05-31 00:51:20 +00007769#define _mm512_mask_i32scatter_epi32(addr, mask, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007770 __builtin_ia32_scattersiv16si((int *)(addr), (__mmask16)(mask), \
7771 (__v16si)(__m512i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007772 (__v16si)(__m512i)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007773
Craig Topperc6338672018-05-31 00:51:20 +00007774#define _mm512_i32scatter_pd(addr, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007775 __builtin_ia32_scattersiv8df((double *)(addr), (__mmask8)-1, \
7776 (__v8si)(__m256i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007777 (__v8df)(__m512d)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007778
Craig Topperc6338672018-05-31 00:51:20 +00007779#define _mm512_mask_i32scatter_pd(addr, mask, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007780 __builtin_ia32_scattersiv8df((double *)(addr), (__mmask8)(mask), \
7781 (__v8si)(__m256i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007782 (__v8df)(__m512d)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007783
Craig Topperc6338672018-05-31 00:51:20 +00007784#define _mm512_i32scatter_epi64(addr, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007785 __builtin_ia32_scattersiv8di((long long *)(addr), (__mmask8)-1, \
7786 (__v8si)(__m256i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007787 (__v8di)(__m512i)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007788
Craig Topperc6338672018-05-31 00:51:20 +00007789#define _mm512_mask_i32scatter_epi64(addr, mask, index, v1, scale) \
Craig Topper8c18e112016-05-17 04:41:50 +00007790 __builtin_ia32_scattersiv8di((long long *)(addr), (__mmask8)(mask), \
7791 (__v8si)(__m256i)(index), \
Craig Topperc6338672018-05-31 00:51:20 +00007792 (__v8di)(__m512i)(v1), (int)(scale))
Michael Zuckermanfcf32c22016-04-25 13:01:40 +00007793
Craig Topper74c10e32018-07-09 19:00:16 +00007794static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00007795_mm_mask_fmadd_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007796{
Gabor Buella70d8d512018-05-30 15:27:49 +00007797 __m128 __Z = __builtin_ia32_vfmaddss3((__v4sf) __W,
7798 (__v4sf) __A,
7799 (__v4sf) __B);
7800 __W[0] = (__U & 1) ? __Z[0] : __W[0];
7801 return __W;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007802}
7803
Craig Topper10f20fc2018-07-06 22:08:43 +00007804#define _mm_fmadd_round_ss(A, B, C, R) \
Craig Topper5cbeeed2018-07-07 17:03:32 +00007805 (__m128)__builtin_ia32_vfmaddss3_mask((__v4sf)(__m128)(A), \
7806 (__v4sf)(__m128)(B), \
7807 (__v4sf)(__m128)(C), (__mmask8)-1, \
7808 (int)(R))
Craig Topper10f20fc2018-07-06 22:08:43 +00007809
Craig Topperc6338672018-05-31 00:51:20 +00007810#define _mm_mask_fmadd_round_ss(W, U, A, B, R) \
Craig Topper5cbeeed2018-07-07 17:03:32 +00007811 (__m128)__builtin_ia32_vfmaddss3_mask((__v4sf)(__m128)(W), \
7812 (__v4sf)(__m128)(A), \
7813 (__v4sf)(__m128)(B), (__mmask8)(U), \
7814 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007815
Craig Topper74c10e32018-07-09 19:00:16 +00007816static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00007817_mm_maskz_fmadd_ss (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007818{
Gabor Buella70d8d512018-05-30 15:27:49 +00007819 __m128 __Z = __builtin_ia32_vfmaddss3((__v4sf) __A,
7820 (__v4sf) __B,
7821 (__v4sf) __C);
7822 __A[0] = (__U & 1) ? __Z[0] : 0;
7823 return __A;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007824}
7825
Craig Topperc6338672018-05-31 00:51:20 +00007826#define _mm_maskz_fmadd_round_ss(U, A, B, C, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007827 (__m128)__builtin_ia32_vfmaddss3_maskz((__v4sf)(__m128)(A), \
7828 (__v4sf)(__m128)(B), \
7829 (__v4sf)(__m128)(C), (__mmask8)(U), \
Craig Topper3e720a32018-07-07 22:03:16 +00007830 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007831
Craig Topper74c10e32018-07-09 19:00:16 +00007832static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00007833_mm_mask3_fmadd_ss (__m128 __W, __m128 __X, __m128 __Y, __mmask8 __U)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007834{
Gabor Buella70d8d512018-05-30 15:27:49 +00007835 __m128 __Z = __builtin_ia32_vfmaddss3((__v4sf) __W,
7836 (__v4sf) __X,
7837 (__v4sf) __Y);
7838 __Y[0] = (__U & 1) ? __Z[0] : __Y[0];
7839 return __Y;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007840}
7841
Craig Topperc6338672018-05-31 00:51:20 +00007842#define _mm_mask3_fmadd_round_ss(W, X, Y, U, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007843 (__m128)__builtin_ia32_vfmaddss3_mask3((__v4sf)(__m128)(W), \
7844 (__v4sf)(__m128)(X), \
7845 (__v4sf)(__m128)(Y), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00007846 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007847
Craig Topper74c10e32018-07-09 19:00:16 +00007848static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00007849_mm_mask_fmsub_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007850{
Gabor Buella70d8d512018-05-30 15:27:49 +00007851 __m128 __Z = __builtin_ia32_vfmaddss3((__v4sf) __W,
7852 (__v4sf) __A,
7853 -(__v4sf) __B);
7854 __W[0] = (__U & 1) ? __Z[0] : __W[0];
7855 return __W;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007856}
7857
Craig Topper10f20fc2018-07-06 22:08:43 +00007858#define _mm_fmsub_round_ss(A, B, C, R) \
7859 (__m128)__builtin_ia32_vfmaddss3_mask((__v4sf)(__m128)(A), \
7860 (__v4sf)(__m128)(B), \
7861 -(__v4sf)(__m128)(C), (__mmask8)-1, \
7862 (int)(R))
7863
Craig Topperc6338672018-05-31 00:51:20 +00007864#define _mm_mask_fmsub_round_ss(W, U, A, B, R) \
Craig Topperf2023652016-10-26 05:35:38 +00007865 (__m128)__builtin_ia32_vfmaddss3_mask((__v4sf)(__m128)(W), \
7866 (__v4sf)(__m128)(A), \
Gabor Buella70d8d512018-05-30 15:27:49 +00007867 -(__v4sf)(__m128)(B), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00007868 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007869
Craig Topper74c10e32018-07-09 19:00:16 +00007870static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00007871_mm_maskz_fmsub_ss (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007872{
Gabor Buella70d8d512018-05-30 15:27:49 +00007873 __m128 __Z = __builtin_ia32_vfmaddss3((__v4sf) __A,
7874 (__v4sf) __B,
7875 -(__v4sf) __C);
7876 __A[0] = (__U & 1) ? __Z[0] : 0;
7877 return __A;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007878}
7879
Craig Topperc6338672018-05-31 00:51:20 +00007880#define _mm_maskz_fmsub_round_ss(U, A, B, C, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00007881 (__m128)__builtin_ia32_vfmaddss3_maskz((__v4sf)(__m128)(A), \
7882 (__v4sf)(__m128)(B), \
7883 -(__v4sf)(__m128)(C), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00007884 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007885
Craig Topper74c10e32018-07-09 19:00:16 +00007886static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00007887_mm_mask3_fmsub_ss (__m128 __W, __m128 __X, __m128 __Y, __mmask8 __U)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007888{
Gabor Buella70d8d512018-05-30 15:27:49 +00007889 __m128 __Z = __builtin_ia32_vfmaddss3((__v4sf) __W,
7890 (__v4sf) __X,
7891 -(__v4sf) __Y);
7892 __Y[0] = (__U & 1) ? __Z[0] : __Y[0];
7893 return __Y;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007894}
7895
Craig Topperc6338672018-05-31 00:51:20 +00007896#define _mm_mask3_fmsub_round_ss(W, X, Y, U, R) \
Craig Topper2c8f49e2016-11-12 23:24:34 +00007897 (__m128)__builtin_ia32_vfmsubss3_mask3((__v4sf)(__m128)(W), \
Craig Topper8c18e112016-05-17 04:41:50 +00007898 (__v4sf)(__m128)(X), \
Craig Topper2c8f49e2016-11-12 23:24:34 +00007899 (__v4sf)(__m128)(Y), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00007900 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007901
Craig Topper74c10e32018-07-09 19:00:16 +00007902static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00007903_mm_mask_fnmadd_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007904{
Gabor Buella70d8d512018-05-30 15:27:49 +00007905 __m128 __Z = __builtin_ia32_vfmaddss3((__v4sf) __W,
7906 -(__v4sf) __A,
7907 (__v4sf) __B);
7908 __W[0] = (__U & 1) ? __Z[0] : __W[0];
7909 return __W;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007910}
7911
Craig Topper10f20fc2018-07-06 22:08:43 +00007912#define _mm_fnmadd_round_ss(A, B, C, R) \
7913 (__m128)__builtin_ia32_vfmaddss3_mask((__v4sf)(__m128)(A), \
7914 -(__v4sf)(__m128)(B), \
7915 (__v4sf)(__m128)(C), (__mmask8)-1, \
7916 (int)(R))
7917
Craig Topperc6338672018-05-31 00:51:20 +00007918#define _mm_mask_fnmadd_round_ss(W, U, A, B, R) \
Craig Topperf2023652016-10-26 05:35:38 +00007919 (__m128)__builtin_ia32_vfmaddss3_mask((__v4sf)(__m128)(W), \
7920 -(__v4sf)(__m128)(A), \
7921 (__v4sf)(__m128)(B), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00007922 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007923
Craig Topper74c10e32018-07-09 19:00:16 +00007924static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00007925_mm_maskz_fnmadd_ss (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007926{
Gabor Buella70d8d512018-05-30 15:27:49 +00007927 __m128 __Z = __builtin_ia32_vfmaddss3((__v4sf) __A,
7928 -(__v4sf) __B,
7929 (__v4sf) __C);
7930 __A[0] = (__U & 1) ? __Z[0] : 0;
7931 return __A;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007932}
7933
Craig Topperc6338672018-05-31 00:51:20 +00007934#define _mm_maskz_fnmadd_round_ss(U, A, B, C, R) \
Gabor Buella70d8d512018-05-30 15:27:49 +00007935 (__m128)__builtin_ia32_vfmaddss3_maskz((__v4sf)(__m128)(A), \
7936 -(__v4sf)(__m128)(B), \
Craig Topper8c18e112016-05-17 04:41:50 +00007937 (__v4sf)(__m128)(C), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00007938 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007939
Craig Topper74c10e32018-07-09 19:00:16 +00007940static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00007941_mm_mask3_fnmadd_ss (__m128 __W, __m128 __X, __m128 __Y, __mmask8 __U)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007942{
Gabor Buella70d8d512018-05-30 15:27:49 +00007943 __m128 __Z = __builtin_ia32_vfmaddss3((__v4sf) __W,
7944 -(__v4sf) __X,
7945 (__v4sf) __Y);
7946 __Y[0] = (__U & 1) ? __Z[0] : __Y[0];
7947 return __Y;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007948}
7949
Craig Topperc6338672018-05-31 00:51:20 +00007950#define _mm_mask3_fnmadd_round_ss(W, X, Y, U, R) \
Gabor Buella70d8d512018-05-30 15:27:49 +00007951 (__m128)__builtin_ia32_vfmaddss3_mask3((__v4sf)(__m128)(W), \
7952 -(__v4sf)(__m128)(X), \
Craig Topper8c18e112016-05-17 04:41:50 +00007953 (__v4sf)(__m128)(Y), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00007954 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007955
Craig Topper74c10e32018-07-09 19:00:16 +00007956static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00007957_mm_mask_fnmsub_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007958{
Gabor Buella70d8d512018-05-30 15:27:49 +00007959 __m128 __Z = __builtin_ia32_vfmaddss3((__v4sf) __W,
7960 -(__v4sf) __A,
7961 -(__v4sf) __B);
7962 __W[0] = (__U & 1) ? __Z[0] : __W[0];
7963 return __W;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007964}
7965
Craig Topper10f20fc2018-07-06 22:08:43 +00007966#define _mm_fnmsub_round_ss(A, B, C, R) \
7967 (__m128)__builtin_ia32_vfmaddss3_mask((__v4sf)(__m128)(A), \
7968 -(__v4sf)(__m128)(B), \
7969 -(__v4sf)(__m128)(C), (__mmask8)-1, \
7970 (int)(R))
7971
Craig Topperc6338672018-05-31 00:51:20 +00007972#define _mm_mask_fnmsub_round_ss(W, U, A, B, R) \
Craig Topperf2023652016-10-26 05:35:38 +00007973 (__m128)__builtin_ia32_vfmaddss3_mask((__v4sf)(__m128)(W), \
7974 -(__v4sf)(__m128)(A), \
7975 -(__v4sf)(__m128)(B), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00007976 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007977
Craig Topper74c10e32018-07-09 19:00:16 +00007978static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00007979_mm_maskz_fnmsub_ss (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007980{
Gabor Buella70d8d512018-05-30 15:27:49 +00007981 __m128 __Z = __builtin_ia32_vfmaddss3((__v4sf) __A,
7982 -(__v4sf) __B,
7983 -(__v4sf) __C);
7984 __A[0] = (__U & 1) ? __Z[0] : 0;
7985 return __A;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007986}
7987
Craig Topperc6338672018-05-31 00:51:20 +00007988#define _mm_maskz_fnmsub_round_ss(U, A, B, C, R) \
Gabor Buella70d8d512018-05-30 15:27:49 +00007989 (__m128)__builtin_ia32_vfmaddss3_maskz((__v4sf)(__m128)(A), \
7990 -(__v4sf)(__m128)(B), \
Craig Topper8c18e112016-05-17 04:41:50 +00007991 -(__v4sf)(__m128)(C), (__mmask8)(U), \
Craig Topper3e720a32018-07-07 22:03:16 +00007992 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007993
Craig Topper74c10e32018-07-09 19:00:16 +00007994static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00007995_mm_mask3_fnmsub_ss (__m128 __W, __m128 __X, __m128 __Y, __mmask8 __U)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00007996{
Gabor Buella70d8d512018-05-30 15:27:49 +00007997 __m128 __Z = __builtin_ia32_vfmaddss3((__v4sf) __W,
7998 -(__v4sf) __X,
7999 -(__v4sf) __Y);
8000 __Y[0] = (__U & 1) ? __Z[0] : __Y[0];
8001 return __Y;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008002}
8003
Craig Topperc6338672018-05-31 00:51:20 +00008004#define _mm_mask3_fnmsub_round_ss(W, X, Y, U, R) \
Gabor Buella70d8d512018-05-30 15:27:49 +00008005 (__m128)__builtin_ia32_vfmsubss3_mask3((__v4sf)(__m128)(W), \
8006 -(__v4sf)(__m128)(X), \
Craig Topper2c8f49e2016-11-12 23:24:34 +00008007 (__v4sf)(__m128)(Y), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00008008 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008009
Craig Topper74c10e32018-07-09 19:00:16 +00008010static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00008011_mm_mask_fmadd_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008012{
Gabor Buella70d8d512018-05-30 15:27:49 +00008013 __m128d __Z = __builtin_ia32_vfmaddsd3((__v2df) __W,
8014 (__v2df) __A,
8015 (__v2df) __B);
8016 __W[0] = (__U & 1) ? __Z[0] : __W[0];
8017 return __W;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008018}
8019
Craig Topper10f20fc2018-07-06 22:08:43 +00008020#define _mm_fmadd_round_sd(A, B, C, R) \
8021 (__m128d)__builtin_ia32_vfmaddsd3_mask((__v2df)(__m128d)(A), \
8022 (__v2df)(__m128d)(B), \
8023 (__v2df)(__m128d)(C), (__mmask8)-1, \
8024 (int)(R))
8025
Craig Topperc6338672018-05-31 00:51:20 +00008026#define _mm_mask_fmadd_round_sd(W, U, A, B, R) \
Craig Topperf2023652016-10-26 05:35:38 +00008027 (__m128d)__builtin_ia32_vfmaddsd3_mask((__v2df)(__m128d)(W), \
8028 (__v2df)(__m128d)(A), \
8029 (__v2df)(__m128d)(B), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00008030 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008031
Craig Topper74c10e32018-07-09 19:00:16 +00008032static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00008033_mm_maskz_fmadd_sd (__mmask8 __U, __m128d __A, __m128d __B, __m128d __C)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008034{
Gabor Buella70d8d512018-05-30 15:27:49 +00008035 __m128d __Z = __builtin_ia32_vfmaddsd3((__v2df) __A,
8036 (__v2df) __B,
8037 (__v2df) __C);
8038 __A[0] = (__U & 1) ? __Z[0] : 0;
8039 return __A;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008040}
8041
Craig Topperc6338672018-05-31 00:51:20 +00008042#define _mm_maskz_fmadd_round_sd(U, A, B, C, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00008043 (__m128d)__builtin_ia32_vfmaddsd3_maskz((__v2df)(__m128d)(A), \
8044 (__v2df)(__m128d)(B), \
8045 (__v2df)(__m128d)(C), (__mmask8)(U), \
Craig Topper3e720a32018-07-07 22:03:16 +00008046 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008047
Craig Topper74c10e32018-07-09 19:00:16 +00008048static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00008049_mm_mask3_fmadd_sd (__m128d __W, __m128d __X, __m128d __Y, __mmask8 __U)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008050{
Gabor Buella70d8d512018-05-30 15:27:49 +00008051 __m128d __Z = __builtin_ia32_vfmaddsd3((__v2df) __W,
8052 (__v2df) __X,
8053 (__v2df) __Y);
8054 __Y[0] = (__U & 1) ? __Z[0] : __Y[0];
8055 return __Y;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008056}
8057
Craig Topperc6338672018-05-31 00:51:20 +00008058#define _mm_mask3_fmadd_round_sd(W, X, Y, U, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00008059 (__m128d)__builtin_ia32_vfmaddsd3_mask3((__v2df)(__m128d)(W), \
8060 (__v2df)(__m128d)(X), \
8061 (__v2df)(__m128d)(Y), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00008062 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008063
Craig Topper74c10e32018-07-09 19:00:16 +00008064static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00008065_mm_mask_fmsub_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008066{
Gabor Buella70d8d512018-05-30 15:27:49 +00008067 __m128d __Z = __builtin_ia32_vfmaddsd3((__v2df) __W,
8068 (__v2df) __A,
8069 -(__v2df) __B);
8070 __W[0] = (__U & 1) ? __Z[0] : __W[0];
8071 return __W;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008072}
8073
Craig Topper10f20fc2018-07-06 22:08:43 +00008074#define _mm_fmsub_round_sd(A, B, C, R) \
8075 (__m128d)__builtin_ia32_vfmaddsd3_mask((__v2df)(__m128d)(A), \
8076 (__v2df)(__m128d)(B), \
8077 -(__v2df)(__m128d)(C), (__mmask8)-1, \
8078 (int)(R))
8079
Craig Topperc6338672018-05-31 00:51:20 +00008080#define _mm_mask_fmsub_round_sd(W, U, A, B, R) \
Craig Topperf2023652016-10-26 05:35:38 +00008081 (__m128d)__builtin_ia32_vfmaddsd3_mask((__v2df)(__m128d)(W), \
8082 (__v2df)(__m128d)(A), \
8083 -(__v2df)(__m128d)(B), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00008084 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008085
Craig Topper74c10e32018-07-09 19:00:16 +00008086static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00008087_mm_maskz_fmsub_sd (__mmask8 __U, __m128d __A, __m128d __B, __m128d __C)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008088{
Gabor Buella70d8d512018-05-30 15:27:49 +00008089 __m128d __Z = __builtin_ia32_vfmaddsd3((__v2df) __A,
8090 (__v2df) __B,
8091 -(__v2df) __C);
8092 __A[0] = (__U & 1) ? __Z[0] : 0;
8093 return __A;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008094}
8095
Craig Topperc6338672018-05-31 00:51:20 +00008096#define _mm_maskz_fmsub_round_sd(U, A, B, C, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00008097 (__m128d)__builtin_ia32_vfmaddsd3_maskz((__v2df)(__m128d)(A), \
8098 (__v2df)(__m128d)(B), \
8099 -(__v2df)(__m128d)(C), \
Craig Topperc6338672018-05-31 00:51:20 +00008100 (__mmask8)(U), (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008101
Craig Topper74c10e32018-07-09 19:00:16 +00008102static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00008103_mm_mask3_fmsub_sd (__m128d __W, __m128d __X, __m128d __Y, __mmask8 __U)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008104{
Gabor Buella70d8d512018-05-30 15:27:49 +00008105 __m128d __Z = __builtin_ia32_vfmaddsd3((__v2df) __W,
8106 (__v2df) __X,
8107 -(__v2df) __Y);
8108 __Y[0] = (__U & 1) ? __Z[0] : __Y[0];
8109 return __Y;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008110}
8111
Craig Topperc6338672018-05-31 00:51:20 +00008112#define _mm_mask3_fmsub_round_sd(W, X, Y, U, R) \
Craig Topper2c8f49e2016-11-12 23:24:34 +00008113 (__m128d)__builtin_ia32_vfmsubsd3_mask3((__v2df)(__m128d)(W), \
Craig Topper8c18e112016-05-17 04:41:50 +00008114 (__v2df)(__m128d)(X), \
Craig Topper2c8f49e2016-11-12 23:24:34 +00008115 (__v2df)(__m128d)(Y), \
Craig Topperc6338672018-05-31 00:51:20 +00008116 (__mmask8)(U), (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008117
Craig Topper74c10e32018-07-09 19:00:16 +00008118static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00008119_mm_mask_fnmadd_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008120{
Gabor Buella70d8d512018-05-30 15:27:49 +00008121 __m128d __Z = __builtin_ia32_vfmaddsd3((__v2df) __W,
8122 -(__v2df) __A,
8123 (__v2df) __B);
8124 __W[0] = (__U & 1) ? __Z[0] : __W[0];
8125 return __W;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008126}
8127
Craig Topper10f20fc2018-07-06 22:08:43 +00008128#define _mm_fnmadd_round_sd(A, B, C, R) \
8129 (__m128d)__builtin_ia32_vfmaddsd3_mask((__v2df)(__m128d)(A), \
8130 -(__v2df)(__m128d)(B), \
8131 (__v2df)(__m128d)(C), (__mmask8)-1, \
8132 (int)(R))
8133
Craig Topperc6338672018-05-31 00:51:20 +00008134#define _mm_mask_fnmadd_round_sd(W, U, A, B, R) \
Craig Topperf2023652016-10-26 05:35:38 +00008135 (__m128d)__builtin_ia32_vfmaddsd3_mask((__v2df)(__m128d)(W), \
8136 -(__v2df)(__m128d)(A), \
8137 (__v2df)(__m128d)(B), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00008138 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008139
Craig Topper74c10e32018-07-09 19:00:16 +00008140static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00008141_mm_maskz_fnmadd_sd (__mmask8 __U, __m128d __A, __m128d __B, __m128d __C)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008142{
Gabor Buella70d8d512018-05-30 15:27:49 +00008143 __m128d __Z = __builtin_ia32_vfmaddsd3((__v2df) __A,
8144 -(__v2df) __B,
8145 (__v2df) __C);
8146 __A[0] = (__U & 1) ? __Z[0] : 0;
8147 return __A;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008148}
8149
Craig Topperc6338672018-05-31 00:51:20 +00008150#define _mm_maskz_fnmadd_round_sd(U, A, B, C, R) \
Gabor Buella70d8d512018-05-30 15:27:49 +00008151 (__m128d)__builtin_ia32_vfmaddsd3_maskz((__v2df)(__m128d)(A), \
8152 -(__v2df)(__m128d)(B), \
Craig Topper8c18e112016-05-17 04:41:50 +00008153 (__v2df)(__m128d)(C), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00008154 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008155
Craig Topper74c10e32018-07-09 19:00:16 +00008156static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00008157_mm_mask3_fnmadd_sd (__m128d __W, __m128d __X, __m128d __Y, __mmask8 __U)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008158{
Gabor Buella70d8d512018-05-30 15:27:49 +00008159 __m128d __Z = __builtin_ia32_vfmaddsd3((__v2df) __W,
8160 -(__v2df) __X,
8161 (__v2df) __Y);
8162 __Y[0] = (__U & 1) ? __Z[0] : __Y[0];
8163 return __Y;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008164}
8165
Craig Topperc6338672018-05-31 00:51:20 +00008166#define _mm_mask3_fnmadd_round_sd(W, X, Y, U, R) \
Gabor Buella70d8d512018-05-30 15:27:49 +00008167 (__m128d)__builtin_ia32_vfmaddsd3_mask3((__v2df)(__m128d)(W), \
8168 -(__v2df)(__m128d)(X), \
Craig Topper8c18e112016-05-17 04:41:50 +00008169 (__v2df)(__m128d)(Y), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00008170 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008171
Craig Topper74c10e32018-07-09 19:00:16 +00008172static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00008173_mm_mask_fnmsub_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008174{
Gabor Buella70d8d512018-05-30 15:27:49 +00008175 __m128d __Z = __builtin_ia32_vfmaddsd3((__v2df) __W,
8176 -(__v2df) __A,
8177 -(__v2df) __B);
8178 __W[0] = (__U & 1) ? __Z[0] : __W[0];
8179 return __W;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008180}
8181
Craig Topper10f20fc2018-07-06 22:08:43 +00008182#define _mm_fnmsub_round_sd(A, B, C, R) \
8183 (__m128d)__builtin_ia32_vfmaddsd3_mask((__v2df)(__m128d)(A), \
8184 -(__v2df)(__m128d)(B), \
8185 -(__v2df)(__m128d)(C), (__mmask8)-1, \
8186 (int)(R))
8187
Craig Topperc6338672018-05-31 00:51:20 +00008188#define _mm_mask_fnmsub_round_sd(W, U, A, B, R) \
Craig Topperf2023652016-10-26 05:35:38 +00008189 (__m128d)__builtin_ia32_vfmaddsd3_mask((__v2df)(__m128d)(W), \
8190 -(__v2df)(__m128d)(A), \
8191 -(__v2df)(__m128d)(B), (__mmask8)(U), \
Craig Topperc6338672018-05-31 00:51:20 +00008192 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008193
Craig Topper74c10e32018-07-09 19:00:16 +00008194static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00008195_mm_maskz_fnmsub_sd (__mmask8 __U, __m128d __A, __m128d __B, __m128d __C)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008196{
Gabor Buella70d8d512018-05-30 15:27:49 +00008197 __m128d __Z = __builtin_ia32_vfmaddsd3((__v2df) __A,
8198 -(__v2df) __B,
8199 -(__v2df) __C);
8200 __A[0] = (__U & 1) ? __Z[0] : 0;
8201 return __A;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008202}
8203
Craig Topperc6338672018-05-31 00:51:20 +00008204#define _mm_maskz_fnmsub_round_sd(U, A, B, C, R) \
Gabor Buella70d8d512018-05-30 15:27:49 +00008205 (__m128d)__builtin_ia32_vfmaddsd3_maskz((__v2df)(__m128d)(A), \
8206 -(__v2df)(__m128d)(B), \
Craig Topper8c18e112016-05-17 04:41:50 +00008207 -(__v2df)(__m128d)(C), \
8208 (__mmask8)(U), \
Craig Topper3e720a32018-07-07 22:03:16 +00008209 (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008210
Craig Topper74c10e32018-07-09 19:00:16 +00008211static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00008212_mm_mask3_fnmsub_sd (__m128d __W, __m128d __X, __m128d __Y, __mmask8 __U)
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008213{
Gabor Buella70d8d512018-05-30 15:27:49 +00008214 __m128d __Z = __builtin_ia32_vfmaddsd3((__v2df) __W,
8215 -(__v2df) __X,
8216 -(__v2df) __Y);
8217 __Y[0] = (__U & 1) ? __Z[0] : __Y[0];
8218 return __Y;
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008219}
8220
Craig Topperc6338672018-05-31 00:51:20 +00008221#define _mm_mask3_fnmsub_round_sd(W, X, Y, U, R) \
Gabor Buella70d8d512018-05-30 15:27:49 +00008222 (__m128d)__builtin_ia32_vfmsubsd3_mask3((__v2df)(__m128d)(W), \
8223 -(__v2df)(__m128d)(X), \
Craig Topper2c8f49e2016-11-12 23:24:34 +00008224 (__v2df)(__m128d)(Y), \
Craig Topperc6338672018-05-31 00:51:20 +00008225 (__mmask8)(U), (int)(R))
Michael Zuckerman743d68c2016-04-22 10:56:24 +00008226
Craig Topperc6338672018-05-31 00:51:20 +00008227#define _mm512_permutex_pd(X, C) \
Craig Topper03f4f042018-06-08 18:00:25 +00008228 (__m512d)__builtin_ia32_permdf512((__v8df)(__m512d)(X), (int)(C))
Michael Zuckerman8938e832016-04-25 05:32:35 +00008229
Craig Topperc6338672018-05-31 00:51:20 +00008230#define _mm512_mask_permutex_pd(W, U, X, C) \
Simon Pilgrim30db8112016-07-04 13:34:44 +00008231 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
8232 (__v8df)_mm512_permutex_pd((X), (C)), \
Craig Topperc6338672018-05-31 00:51:20 +00008233 (__v8df)(__m512d)(W))
Michael Zuckerman8938e832016-04-25 05:32:35 +00008234
Craig Topperc6338672018-05-31 00:51:20 +00008235#define _mm512_maskz_permutex_pd(U, X, C) \
Simon Pilgrim30db8112016-07-04 13:34:44 +00008236 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
8237 (__v8df)_mm512_permutex_pd((X), (C)), \
Craig Topperc6338672018-05-31 00:51:20 +00008238 (__v8df)_mm512_setzero_pd())
Michael Zuckerman8938e832016-04-25 05:32:35 +00008239
Craig Topperc6338672018-05-31 00:51:20 +00008240#define _mm512_permutex_epi64(X, C) \
Craig Topper03f4f042018-06-08 18:00:25 +00008241 (__m512i)__builtin_ia32_permdi512((__v8di)(__m512i)(X), (int)(C))
Michael Zuckerman8938e832016-04-25 05:32:35 +00008242
Craig Topperc6338672018-05-31 00:51:20 +00008243#define _mm512_mask_permutex_epi64(W, U, X, C) \
Simon Pilgrim30db8112016-07-04 13:34:44 +00008244 (__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
8245 (__v8di)_mm512_permutex_epi64((X), (C)), \
Craig Topperc6338672018-05-31 00:51:20 +00008246 (__v8di)(__m512i)(W))
Michael Zuckerman8938e832016-04-25 05:32:35 +00008247
Craig Topperc6338672018-05-31 00:51:20 +00008248#define _mm512_maskz_permutex_epi64(U, X, C) \
Simon Pilgrim30db8112016-07-04 13:34:44 +00008249 (__m512i)__builtin_ia32_selectq_512((__mmask8)(U), \
8250 (__v8di)_mm512_permutex_epi64((X), (C)), \
Craig Topperc6338672018-05-31 00:51:20 +00008251 (__v8di)_mm512_setzero_si512())
Michael Zuckerman8938e832016-04-25 05:32:35 +00008252
Craig Topper74c10e32018-07-09 19:00:16 +00008253static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman8938e832016-04-25 05:32:35 +00008254_mm512_permutexvar_pd (__m512i __X, __m512d __Y)
8255{
Craig Topper55b40672018-05-20 23:34:10 +00008256 return (__m512d)__builtin_ia32_permvardf512((__v8df) __Y, (__v8di) __X);
Michael Zuckerman8938e832016-04-25 05:32:35 +00008257}
8258
Craig Topper74c10e32018-07-09 19:00:16 +00008259static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman8938e832016-04-25 05:32:35 +00008260_mm512_mask_permutexvar_pd (__m512d __W, __mmask8 __U, __m512i __X, __m512d __Y)
8261{
Craig Topper55b40672018-05-20 23:34:10 +00008262 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
8263 (__v8df)_mm512_permutexvar_pd(__X, __Y),
8264 (__v8df)__W);
Michael Zuckerman8938e832016-04-25 05:32:35 +00008265}
8266
Craig Topper74c10e32018-07-09 19:00:16 +00008267static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman8938e832016-04-25 05:32:35 +00008268_mm512_maskz_permutexvar_pd (__mmask8 __U, __m512i __X, __m512d __Y)
8269{
Craig Topper55b40672018-05-20 23:34:10 +00008270 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
8271 (__v8df)_mm512_permutexvar_pd(__X, __Y),
8272 (__v8df)_mm512_setzero_pd());
Michael Zuckerman8938e832016-04-25 05:32:35 +00008273}
8274
Craig Topper74c10e32018-07-09 19:00:16 +00008275static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman8938e832016-04-25 05:32:35 +00008276_mm512_permutexvar_epi64 (__m512i __X, __m512i __Y)
8277{
Craig Topper55b40672018-05-20 23:34:10 +00008278 return (__m512i)__builtin_ia32_permvardi512((__v8di)__Y, (__v8di)__X);
8279}
8280
Craig Topper74c10e32018-07-09 19:00:16 +00008281static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper55b40672018-05-20 23:34:10 +00008282_mm512_maskz_permutexvar_epi64 (__mmask8 __M, __m512i __X, __m512i __Y)
8283{
8284 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
8285 (__v8di)_mm512_permutexvar_epi64(__X, __Y),
8286 (__v8di)_mm512_setzero_si512());
Michael Zuckerman8938e832016-04-25 05:32:35 +00008287}
8288
Craig Topper74c10e32018-07-09 19:00:16 +00008289static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman8938e832016-04-25 05:32:35 +00008290_mm512_mask_permutexvar_epi64 (__m512i __W, __mmask8 __M, __m512i __X,
8291 __m512i __Y)
8292{
Craig Topper55b40672018-05-20 23:34:10 +00008293 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
8294 (__v8di)_mm512_permutexvar_epi64(__X, __Y),
8295 (__v8di)__W);
Michael Zuckerman8938e832016-04-25 05:32:35 +00008296}
8297
Craig Topper74c10e32018-07-09 19:00:16 +00008298static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman8938e832016-04-25 05:32:35 +00008299_mm512_permutexvar_ps (__m512i __X, __m512 __Y)
8300{
Craig Topper55b40672018-05-20 23:34:10 +00008301 return (__m512)__builtin_ia32_permvarsf512((__v16sf)__Y, (__v16si)__X);
Michael Zuckerman8938e832016-04-25 05:32:35 +00008302}
8303
Craig Topper74c10e32018-07-09 19:00:16 +00008304static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman8938e832016-04-25 05:32:35 +00008305_mm512_mask_permutexvar_ps (__m512 __W, __mmask16 __U, __m512i __X, __m512 __Y)
8306{
Craig Topper55b40672018-05-20 23:34:10 +00008307 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
8308 (__v16sf)_mm512_permutexvar_ps(__X, __Y),
8309 (__v16sf)__W);
Michael Zuckerman8938e832016-04-25 05:32:35 +00008310}
8311
Craig Topper74c10e32018-07-09 19:00:16 +00008312static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman8938e832016-04-25 05:32:35 +00008313_mm512_maskz_permutexvar_ps (__mmask16 __U, __m512i __X, __m512 __Y)
8314{
Craig Topper55b40672018-05-20 23:34:10 +00008315 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
8316 (__v16sf)_mm512_permutexvar_ps(__X, __Y),
8317 (__v16sf)_mm512_setzero_ps());
Michael Zuckerman8938e832016-04-25 05:32:35 +00008318}
8319
Craig Topper74c10e32018-07-09 19:00:16 +00008320static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman8938e832016-04-25 05:32:35 +00008321_mm512_permutexvar_epi32 (__m512i __X, __m512i __Y)
8322{
Craig Topper55b40672018-05-20 23:34:10 +00008323 return (__m512i)__builtin_ia32_permvarsi512((__v16si)__Y, (__v16si)__X);
Michael Zuckerman8938e832016-04-25 05:32:35 +00008324}
8325
Igor Bregerf050b792017-03-19 08:27:16 +00008326#define _mm512_permutevar_epi32 _mm512_permutexvar_epi32
8327
Craig Topper74c10e32018-07-09 19:00:16 +00008328static __inline__ __m512i __DEFAULT_FN_ATTRS512
Craig Topper55b40672018-05-20 23:34:10 +00008329_mm512_maskz_permutexvar_epi32 (__mmask16 __M, __m512i __X, __m512i __Y)
8330{
8331 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
8332 (__v16si)_mm512_permutexvar_epi32(__X, __Y),
8333 (__v16si)_mm512_setzero_si512());
8334}
8335
Craig Topper74c10e32018-07-09 19:00:16 +00008336static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman8938e832016-04-25 05:32:35 +00008337_mm512_mask_permutexvar_epi32 (__m512i __W, __mmask16 __M, __m512i __X,
8338 __m512i __Y)
8339{
Craig Topper55b40672018-05-20 23:34:10 +00008340 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
8341 (__v16si)_mm512_permutexvar_epi32(__X, __Y),
8342 (__v16si)__W);
Michael Zuckerman8938e832016-04-25 05:32:35 +00008343}
8344
Igor Bregerf050b792017-03-19 08:27:16 +00008345#define _mm512_mask_permutevar_epi32 _mm512_mask_permutexvar_epi32
8346
Craig Topper74c10e32018-07-09 19:00:16 +00008347static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
Michael Zuckermanfa508e82016-04-25 16:42:29 +00008348_mm512_kand (__mmask16 __A, __mmask16 __B)
8349{
8350 return (__mmask16) __builtin_ia32_kandhi ((__mmask16) __A, (__mmask16) __B);
8351}
Michael Zuckerman8938e832016-04-25 05:32:35 +00008352
Craig Topper74c10e32018-07-09 19:00:16 +00008353static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
Michael Zuckermanfa508e82016-04-25 16:42:29 +00008354_mm512_kandn (__mmask16 __A, __mmask16 __B)
8355{
8356 return (__mmask16) __builtin_ia32_kandnhi ((__mmask16) __A, (__mmask16) __B);
8357}
8358
Craig Topper74c10e32018-07-09 19:00:16 +00008359static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
Michael Zuckermanfa508e82016-04-25 16:42:29 +00008360_mm512_kor (__mmask16 __A, __mmask16 __B)
8361{
8362 return (__mmask16) __builtin_ia32_korhi ((__mmask16) __A, (__mmask16) __B);
8363}
8364
Craig Topper74c10e32018-07-09 19:00:16 +00008365static __inline__ int __DEFAULT_FN_ATTRS512
Michael Zuckermanfa508e82016-04-25 16:42:29 +00008366_mm512_kortestc (__mmask16 __A, __mmask16 __B)
8367{
Craig Topper91f23d92016-05-16 01:09:16 +00008368 return __builtin_ia32_kortestchi ((__mmask16) __A, (__mmask16) __B);
Michael Zuckermanfa508e82016-04-25 16:42:29 +00008369}
8370
Craig Topper74c10e32018-07-09 19:00:16 +00008371static __inline__ int __DEFAULT_FN_ATTRS512
Michael Zuckermanfa508e82016-04-25 16:42:29 +00008372_mm512_kortestz (__mmask16 __A, __mmask16 __B)
8373{
Craig Topper91f23d92016-05-16 01:09:16 +00008374 return __builtin_ia32_kortestzhi ((__mmask16) __A, (__mmask16) __B);
Michael Zuckermanfa508e82016-04-25 16:42:29 +00008375}
8376
Craig Topper74c10e32018-07-09 19:00:16 +00008377static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
Michael Zuckermanfa508e82016-04-25 16:42:29 +00008378_mm512_kunpackb (__mmask16 __A, __mmask16 __B)
8379{
Craig Topperf517f1a2018-01-14 19:23:50 +00008380 return (__mmask16) __builtin_ia32_kunpckhi ((__mmask16) __A, (__mmask16) __B);
Michael Zuckermanfa508e82016-04-25 16:42:29 +00008381}
8382
Craig Topper74c10e32018-07-09 19:00:16 +00008383static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
Michael Zuckermanfa508e82016-04-25 16:42:29 +00008384_mm512_kxnor (__mmask16 __A, __mmask16 __B)
8385{
8386 return (__mmask16) __builtin_ia32_kxnorhi ((__mmask16) __A, (__mmask16) __B);
8387}
8388
Craig Topper74c10e32018-07-09 19:00:16 +00008389static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
Michael Zuckermanfa508e82016-04-25 16:42:29 +00008390_mm512_kxor (__mmask16 __A, __mmask16 __B)
8391{
8392 return (__mmask16) __builtin_ia32_kxorhi ((__mmask16) __A, (__mmask16) __B);
8393}
Michael Zuckerman8938e832016-04-25 05:32:35 +00008394
Craig Topper74c10e32018-07-09 19:00:16 +00008395static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckerman7c85a8c2016-04-27 10:44:15 +00008396_mm512_stream_si512 (__m512i * __P, __m512i __A)
8397{
Reid Kleckner89fbd552018-06-04 21:39:20 +00008398 typedef __v8di __v8di_aligned __attribute__((aligned(64)));
8399 __builtin_nontemporal_store((__v8di_aligned)__A, (__v8di_aligned*)__P);
Michael Zuckerman7c85a8c2016-04-27 10:44:15 +00008400}
8401
Craig Topper74c10e32018-07-09 19:00:16 +00008402static __inline__ __m512i __DEFAULT_FN_ATTRS512
Simon Pilgrim1ba2bf22017-09-05 10:06:41 +00008403_mm512_stream_load_si512 (void const *__P)
Michael Zuckerman7c85a8c2016-04-27 10:44:15 +00008404{
Reid Kleckner89fbd552018-06-04 21:39:20 +00008405 typedef __v8di __v8di_aligned __attribute__((aligned(64)));
8406 return (__m512i) __builtin_nontemporal_load((const __v8di_aligned *)__P);
Michael Zuckerman7c85a8c2016-04-27 10:44:15 +00008407}
8408
Craig Topper74c10e32018-07-09 19:00:16 +00008409static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckerman7c85a8c2016-04-27 10:44:15 +00008410_mm512_stream_pd (double *__P, __m512d __A)
8411{
Reid Kleckner89fbd552018-06-04 21:39:20 +00008412 typedef __v8df __v8df_aligned __attribute__((aligned(64)));
8413 __builtin_nontemporal_store((__v8df_aligned)__A, (__v8df_aligned*)__P);
Michael Zuckerman7c85a8c2016-04-27 10:44:15 +00008414}
8415
Craig Topper74c10e32018-07-09 19:00:16 +00008416static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckerman7c85a8c2016-04-27 10:44:15 +00008417_mm512_stream_ps (float *__P, __m512 __A)
8418{
Reid Kleckner89fbd552018-06-04 21:39:20 +00008419 typedef __v16sf __v16sf_aligned __attribute__((aligned(64)));
8420 __builtin_nontemporal_store((__v16sf_aligned)__A, (__v16sf_aligned*)__P);
Michael Zuckerman7c85a8c2016-04-27 10:44:15 +00008421}
8422
Craig Topper74c10e32018-07-09 19:00:16 +00008423static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman41f5a372016-04-29 08:52:02 +00008424_mm512_mask_compress_pd (__m512d __W, __mmask8 __U, __m512d __A)
8425{
8426 return (__m512d) __builtin_ia32_compressdf512_mask ((__v8df) __A,
8427 (__v8df) __W,
8428 (__mmask8) __U);
8429}
8430
Craig Topper74c10e32018-07-09 19:00:16 +00008431static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman41f5a372016-04-29 08:52:02 +00008432_mm512_maskz_compress_pd (__mmask8 __U, __m512d __A)
8433{
8434 return (__m512d) __builtin_ia32_compressdf512_mask ((__v8df) __A,
8435 (__v8df)
8436 _mm512_setzero_pd (),
8437 (__mmask8) __U);
8438}
8439
Craig Topper74c10e32018-07-09 19:00:16 +00008440static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman41f5a372016-04-29 08:52:02 +00008441_mm512_mask_compress_epi64 (__m512i __W, __mmask8 __U, __m512i __A)
8442{
8443 return (__m512i) __builtin_ia32_compressdi512_mask ((__v8di) __A,
8444 (__v8di) __W,
8445 (__mmask8) __U);
8446}
8447
Craig Topper74c10e32018-07-09 19:00:16 +00008448static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman41f5a372016-04-29 08:52:02 +00008449_mm512_maskz_compress_epi64 (__mmask8 __U, __m512i __A)
8450{
8451 return (__m512i) __builtin_ia32_compressdi512_mask ((__v8di) __A,
8452 (__v8di)
8453 _mm512_setzero_si512 (),
8454 (__mmask8) __U);
8455}
8456
Craig Topper74c10e32018-07-09 19:00:16 +00008457static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman41f5a372016-04-29 08:52:02 +00008458_mm512_mask_compress_ps (__m512 __W, __mmask16 __U, __m512 __A)
8459{
8460 return (__m512) __builtin_ia32_compresssf512_mask ((__v16sf) __A,
8461 (__v16sf) __W,
8462 (__mmask16) __U);
8463}
8464
Craig Topper74c10e32018-07-09 19:00:16 +00008465static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman41f5a372016-04-29 08:52:02 +00008466_mm512_maskz_compress_ps (__mmask16 __U, __m512 __A)
8467{
8468 return (__m512) __builtin_ia32_compresssf512_mask ((__v16sf) __A,
8469 (__v16sf)
8470 _mm512_setzero_ps (),
8471 (__mmask16) __U);
8472}
8473
Craig Topper74c10e32018-07-09 19:00:16 +00008474static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman41f5a372016-04-29 08:52:02 +00008475_mm512_mask_compress_epi32 (__m512i __W, __mmask16 __U, __m512i __A)
8476{
8477 return (__m512i) __builtin_ia32_compresssi512_mask ((__v16si) __A,
8478 (__v16si) __W,
8479 (__mmask16) __U);
8480}
8481
Craig Topper74c10e32018-07-09 19:00:16 +00008482static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman41f5a372016-04-29 08:52:02 +00008483_mm512_maskz_compress_epi32 (__mmask16 __U, __m512i __A)
8484{
8485 return (__m512i) __builtin_ia32_compresssi512_mask ((__v16si) __A,
8486 (__v16si)
8487 _mm512_setzero_si512 (),
8488 (__mmask16) __U);
8489}
8490
Craig Topperc6338672018-05-31 00:51:20 +00008491#define _mm_cmp_round_ss_mask(X, Y, P, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00008492 (__mmask8)__builtin_ia32_cmpss_mask((__v4sf)(__m128)(X), \
8493 (__v4sf)(__m128)(Y), (int)(P), \
Craig Topperc6338672018-05-31 00:51:20 +00008494 (__mmask8)-1, (int)(R))
Michael Zuckerman0b9d1052016-04-29 11:01:16 +00008495
Craig Topperc6338672018-05-31 00:51:20 +00008496#define _mm_mask_cmp_round_ss_mask(M, X, Y, P, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00008497 (__mmask8)__builtin_ia32_cmpss_mask((__v4sf)(__m128)(X), \
8498 (__v4sf)(__m128)(Y), (int)(P), \
Craig Topperc6338672018-05-31 00:51:20 +00008499 (__mmask8)(M), (int)(R))
Michael Zuckerman0b9d1052016-04-29 11:01:16 +00008500
Craig Topperc6338672018-05-31 00:51:20 +00008501#define _mm_cmp_ss_mask(X, Y, P) \
Craig Topper8c18e112016-05-17 04:41:50 +00008502 (__mmask8)__builtin_ia32_cmpss_mask((__v4sf)(__m128)(X), \
8503 (__v4sf)(__m128)(Y), (int)(P), \
8504 (__mmask8)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00008505 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman0b9d1052016-04-29 11:01:16 +00008506
Craig Topperc6338672018-05-31 00:51:20 +00008507#define _mm_mask_cmp_ss_mask(M, X, Y, P) \
Craig Topper8c18e112016-05-17 04:41:50 +00008508 (__mmask8)__builtin_ia32_cmpss_mask((__v4sf)(__m128)(X), \
8509 (__v4sf)(__m128)(Y), (int)(P), \
8510 (__mmask8)(M), \
Craig Topperc6338672018-05-31 00:51:20 +00008511 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman0b9d1052016-04-29 11:01:16 +00008512
Craig Topperc6338672018-05-31 00:51:20 +00008513#define _mm_cmp_round_sd_mask(X, Y, P, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00008514 (__mmask8)__builtin_ia32_cmpsd_mask((__v2df)(__m128d)(X), \
8515 (__v2df)(__m128d)(Y), (int)(P), \
Craig Topperc6338672018-05-31 00:51:20 +00008516 (__mmask8)-1, (int)(R))
Michael Zuckerman0b9d1052016-04-29 11:01:16 +00008517
Craig Topperc6338672018-05-31 00:51:20 +00008518#define _mm_mask_cmp_round_sd_mask(M, X, Y, P, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00008519 (__mmask8)__builtin_ia32_cmpsd_mask((__v2df)(__m128d)(X), \
8520 (__v2df)(__m128d)(Y), (int)(P), \
Craig Topperc6338672018-05-31 00:51:20 +00008521 (__mmask8)(M), (int)(R))
Michael Zuckerman0b9d1052016-04-29 11:01:16 +00008522
Craig Topperc6338672018-05-31 00:51:20 +00008523#define _mm_cmp_sd_mask(X, Y, P) \
Craig Topper8c18e112016-05-17 04:41:50 +00008524 (__mmask8)__builtin_ia32_cmpsd_mask((__v2df)(__m128d)(X), \
8525 (__v2df)(__m128d)(Y), (int)(P), \
8526 (__mmask8)-1, \
Craig Topperc6338672018-05-31 00:51:20 +00008527 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman0b9d1052016-04-29 11:01:16 +00008528
Craig Topperc6338672018-05-31 00:51:20 +00008529#define _mm_mask_cmp_sd_mask(M, X, Y, P) \
Craig Topper8c18e112016-05-17 04:41:50 +00008530 (__mmask8)__builtin_ia32_cmpsd_mask((__v2df)(__m128d)(X), \
8531 (__v2df)(__m128d)(Y), (int)(P), \
8532 (__mmask8)(M), \
Craig Topperc6338672018-05-31 00:51:20 +00008533 _MM_FROUND_CUR_DIRECTION)
Michael Zuckerman0b9d1052016-04-29 11:01:16 +00008534
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008535/* Bit Test */
8536
Craig Topper74c10e32018-07-09 19:00:16 +00008537static __inline __mmask16 __DEFAULT_FN_ATTRS512
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008538_mm512_test_epi32_mask (__m512i __A, __m512i __B)
8539{
8540 return _mm512_cmpneq_epi32_mask (_mm512_and_epi32(__A, __B),
Craig Topperdff5b312018-05-30 18:02:11 +00008541 _mm512_setzero_si512());
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008542}
8543
Craig Topper74c10e32018-07-09 19:00:16 +00008544static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008545_mm512_mask_test_epi32_mask (__mmask16 __U, __m512i __A, __m512i __B)
8546{
8547 return _mm512_mask_cmpneq_epi32_mask (__U, _mm512_and_epi32 (__A, __B),
Craig Topperdff5b312018-05-30 18:02:11 +00008548 _mm512_setzero_si512());
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008549}
8550
Craig Topper74c10e32018-07-09 19:00:16 +00008551static __inline __mmask8 __DEFAULT_FN_ATTRS512
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008552_mm512_test_epi64_mask (__m512i __A, __m512i __B)
8553{
8554 return _mm512_cmpneq_epi64_mask (_mm512_and_epi32 (__A, __B),
Craig Topperdff5b312018-05-30 18:02:11 +00008555 _mm512_setzero_si512());
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008556}
8557
Craig Topper74c10e32018-07-09 19:00:16 +00008558static __inline__ __mmask8 __DEFAULT_FN_ATTRS512
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008559_mm512_mask_test_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B)
8560{
8561 return _mm512_mask_cmpneq_epi64_mask (__U, _mm512_and_epi32 (__A, __B),
Craig Topperdff5b312018-05-30 18:02:11 +00008562 _mm512_setzero_si512());
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008563}
8564
Craig Topper74c10e32018-07-09 19:00:16 +00008565static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008566_mm512_testn_epi32_mask (__m512i __A, __m512i __B)
8567{
8568 return _mm512_cmpeq_epi32_mask (_mm512_and_epi32 (__A, __B),
Craig Topperdff5b312018-05-30 18:02:11 +00008569 _mm512_setzero_si512());
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008570}
8571
Craig Topper74c10e32018-07-09 19:00:16 +00008572static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008573_mm512_mask_testn_epi32_mask (__mmask16 __U, __m512i __A, __m512i __B)
8574{
8575 return _mm512_mask_cmpeq_epi32_mask (__U, _mm512_and_epi32 (__A, __B),
Craig Topperdff5b312018-05-30 18:02:11 +00008576 _mm512_setzero_si512());
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008577}
8578
Craig Topper74c10e32018-07-09 19:00:16 +00008579static __inline__ __mmask8 __DEFAULT_FN_ATTRS512
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008580_mm512_testn_epi64_mask (__m512i __A, __m512i __B)
8581{
8582 return _mm512_cmpeq_epi64_mask (_mm512_and_epi32 (__A, __B),
Craig Topperdff5b312018-05-30 18:02:11 +00008583 _mm512_setzero_si512());
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008584}
8585
Craig Topper74c10e32018-07-09 19:00:16 +00008586static __inline__ __mmask8 __DEFAULT_FN_ATTRS512
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008587_mm512_mask_testn_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B)
8588{
8589 return _mm512_mask_cmpeq_epi64_mask (__U, _mm512_and_epi32 (__A, __B),
Craig Topperdff5b312018-05-30 18:02:11 +00008590 _mm512_setzero_si512());
Uriel Korach5b2b71d2017-11-13 12:50:52 +00008591}
8592
Craig Topper74c10e32018-07-09 19:00:16 +00008593static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanac1e5192016-05-01 14:43:43 +00008594_mm512_movehdup_ps (__m512 __A)
8595{
Simon Pilgrim275d7212016-07-02 17:16:25 +00008596 return (__m512)__builtin_shufflevector((__v16sf)__A, (__v16sf)__A,
8597 1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15);
Michael Zuckermanac1e5192016-05-01 14:43:43 +00008598}
8599
Craig Topper74c10e32018-07-09 19:00:16 +00008600static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanac1e5192016-05-01 14:43:43 +00008601_mm512_mask_movehdup_ps (__m512 __W, __mmask16 __U, __m512 __A)
8602{
Simon Pilgrim275d7212016-07-02 17:16:25 +00008603 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
8604 (__v16sf)_mm512_movehdup_ps(__A),
8605 (__v16sf)__W);
Michael Zuckermanac1e5192016-05-01 14:43:43 +00008606}
8607
Craig Topper74c10e32018-07-09 19:00:16 +00008608static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanac1e5192016-05-01 14:43:43 +00008609_mm512_maskz_movehdup_ps (__mmask16 __U, __m512 __A)
8610{
Simon Pilgrim275d7212016-07-02 17:16:25 +00008611 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
8612 (__v16sf)_mm512_movehdup_ps(__A),
8613 (__v16sf)_mm512_setzero_ps());
Michael Zuckermanac1e5192016-05-01 14:43:43 +00008614}
8615
Craig Topper74c10e32018-07-09 19:00:16 +00008616static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanac1e5192016-05-01 14:43:43 +00008617_mm512_moveldup_ps (__m512 __A)
8618{
Simon Pilgrim275d7212016-07-02 17:16:25 +00008619 return (__m512)__builtin_shufflevector((__v16sf)__A, (__v16sf)__A,
8620 0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14);
Michael Zuckermanac1e5192016-05-01 14:43:43 +00008621}
8622
Craig Topper74c10e32018-07-09 19:00:16 +00008623static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanac1e5192016-05-01 14:43:43 +00008624_mm512_mask_moveldup_ps (__m512 __W, __mmask16 __U, __m512 __A)
8625{
Simon Pilgrim275d7212016-07-02 17:16:25 +00008626 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
8627 (__v16sf)_mm512_moveldup_ps(__A),
8628 (__v16sf)__W);
Michael Zuckermanac1e5192016-05-01 14:43:43 +00008629}
8630
Craig Topper74c10e32018-07-09 19:00:16 +00008631static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckermanac1e5192016-05-01 14:43:43 +00008632_mm512_maskz_moveldup_ps (__mmask16 __U, __m512 __A)
8633{
Simon Pilgrim275d7212016-07-02 17:16:25 +00008634 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
8635 (__v16sf)_mm512_moveldup_ps(__A),
8636 (__v16sf)_mm512_setzero_ps());
Michael Zuckermanac1e5192016-05-01 14:43:43 +00008637}
8638
Craig Topper74c10e32018-07-09 19:00:16 +00008639static __inline__ __m128 __DEFAULT_FN_ATTRS128
Michael Zuckerman9e43ccf2016-10-05 12:56:06 +00008640_mm_mask_move_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
8641{
Simon Pilgrim0b37ffb2017-07-28 14:01:51 +00008642 __m128 res = __A;
Ayman Musae60a41c2016-11-08 12:00:30 +00008643 res[0] = (__U & 1) ? __B[0] : __W[0];
Simon Pilgrim0b37ffb2017-07-28 14:01:51 +00008644 return res;
Michael Zuckerman9e43ccf2016-10-05 12:56:06 +00008645}
8646
Craig Topper74c10e32018-07-09 19:00:16 +00008647static __inline__ __m128 __DEFAULT_FN_ATTRS128
Michael Zuckerman9e43ccf2016-10-05 12:56:06 +00008648_mm_maskz_move_ss (__mmask8 __U, __m128 __A, __m128 __B)
8649{
Simon Pilgrim0b37ffb2017-07-28 14:01:51 +00008650 __m128 res = __A;
8651 res[0] = (__U & 1) ? __B[0] : 0;
8652 return res;
Michael Zuckerman9e43ccf2016-10-05 12:56:06 +00008653}
8654
Craig Topper74c10e32018-07-09 19:00:16 +00008655static __inline__ __m128d __DEFAULT_FN_ATTRS128
Michael Zuckerman9e43ccf2016-10-05 12:56:06 +00008656_mm_mask_move_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B)
8657{
Simon Pilgrim0b37ffb2017-07-28 14:01:51 +00008658 __m128d res = __A;
Ayman Musae60a41c2016-11-08 12:00:30 +00008659 res[0] = (__U & 1) ? __B[0] : __W[0];
Simon Pilgrim0b37ffb2017-07-28 14:01:51 +00008660 return res;
Michael Zuckerman9e43ccf2016-10-05 12:56:06 +00008661}
8662
Craig Topper74c10e32018-07-09 19:00:16 +00008663static __inline__ __m128d __DEFAULT_FN_ATTRS128
Michael Zuckerman9e43ccf2016-10-05 12:56:06 +00008664_mm_maskz_move_sd (__mmask8 __U, __m128d __A, __m128d __B)
8665{
Simon Pilgrim0b37ffb2017-07-28 14:01:51 +00008666 __m128d res = __A;
8667 res[0] = (__U & 1) ? __B[0] : 0;
8668 return res;
Ayman Musae60a41c2016-11-08 12:00:30 +00008669}
8670
Craig Topper74c10e32018-07-09 19:00:16 +00008671static __inline__ void __DEFAULT_FN_ATTRS128
Ayman Musae60a41c2016-11-08 12:00:30 +00008672_mm_mask_store_ss (float * __W, __mmask8 __U, __m128 __A)
8673{
Craig Topper74ac0ed2018-05-10 05:43:43 +00008674 __builtin_ia32_storess128_mask ((__v4sf *)__W, __A, __U & 1);
Ayman Musae60a41c2016-11-08 12:00:30 +00008675}
8676
Craig Topper74c10e32018-07-09 19:00:16 +00008677static __inline__ void __DEFAULT_FN_ATTRS128
Ayman Musae60a41c2016-11-08 12:00:30 +00008678_mm_mask_store_sd (double * __W, __mmask8 __U, __m128d __A)
8679{
Craig Topper74ac0ed2018-05-10 05:43:43 +00008680 __builtin_ia32_storesd128_mask ((__v2df *)__W, __A, __U & 1);
Ayman Musae60a41c2016-11-08 12:00:30 +00008681}
8682
Craig Topper74c10e32018-07-09 19:00:16 +00008683static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ayman Musae60a41c2016-11-08 12:00:30 +00008684_mm_mask_load_ss (__m128 __W, __mmask8 __U, const float* __A)
8685{
8686 __m128 src = (__v4sf) __builtin_shufflevector((__v4sf) __W,
Craig Topper63ec0ea2018-05-30 21:08:27 +00008687 (__v4sf)_mm_setzero_ps(),
Ayman Musae60a41c2016-11-08 12:00:30 +00008688 0, 4, 4, 4);
8689
Craig Topper74ac0ed2018-05-10 05:43:43 +00008690 return (__m128) __builtin_ia32_loadss128_mask ((__v4sf *) __A, src, __U & 1);
Ayman Musae60a41c2016-11-08 12:00:30 +00008691}
8692
Craig Topper74c10e32018-07-09 19:00:16 +00008693static __inline__ __m128 __DEFAULT_FN_ATTRS128
Ayman Musae60a41c2016-11-08 12:00:30 +00008694_mm_maskz_load_ss (__mmask8 __U, const float* __A)
8695{
Craig Topper74ac0ed2018-05-10 05:43:43 +00008696 return (__m128)__builtin_ia32_loadss128_mask ((__v4sf *) __A,
8697 (__v4sf) _mm_setzero_ps(),
8698 __U & 1);
Ayman Musae60a41c2016-11-08 12:00:30 +00008699}
8700
Craig Topper74c10e32018-07-09 19:00:16 +00008701static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ayman Musae60a41c2016-11-08 12:00:30 +00008702_mm_mask_load_sd (__m128d __W, __mmask8 __U, const double* __A)
8703{
8704 __m128d src = (__v2df) __builtin_shufflevector((__v2df) __W,
Craig Topper63ec0ea2018-05-30 21:08:27 +00008705 (__v2df)_mm_setzero_pd(),
8706 0, 2);
Ayman Musae60a41c2016-11-08 12:00:30 +00008707
Craig Topper74ac0ed2018-05-10 05:43:43 +00008708 return (__m128d) __builtin_ia32_loadsd128_mask ((__v2df *) __A, src, __U & 1);
Ayman Musae60a41c2016-11-08 12:00:30 +00008709}
8710
Craig Topper74c10e32018-07-09 19:00:16 +00008711static __inline__ __m128d __DEFAULT_FN_ATTRS128
Ayman Musae60a41c2016-11-08 12:00:30 +00008712_mm_maskz_load_sd (__mmask8 __U, const double* __A)
8713{
Craig Topper74ac0ed2018-05-10 05:43:43 +00008714 return (__m128d) __builtin_ia32_loadsd128_mask ((__v2df *) __A,
8715 (__v2df) _mm_setzero_pd(),
8716 __U & 1);
Michael Zuckerman9e43ccf2016-10-05 12:56:06 +00008717}
8718
Craig Topperc6338672018-05-31 00:51:20 +00008719#define _mm512_shuffle_epi32(A, I) \
Craig Topper03de1662018-06-08 06:13:16 +00008720 (__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(A), (int)(I))
Michael Zuckermanc62f27e2016-05-02 07:35:27 +00008721
Craig Topperc6338672018-05-31 00:51:20 +00008722#define _mm512_mask_shuffle_epi32(W, U, A, I) \
Craig Topper7cc92632016-06-11 12:50:19 +00008723 (__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
8724 (__v16si)_mm512_shuffle_epi32((A), (I)), \
Craig Topperc6338672018-05-31 00:51:20 +00008725 (__v16si)(__m512i)(W))
Michael Zuckermanc62f27e2016-05-02 07:35:27 +00008726
Craig Topperc6338672018-05-31 00:51:20 +00008727#define _mm512_maskz_shuffle_epi32(U, A, I) \
Craig Topper7cc92632016-06-11 12:50:19 +00008728 (__m512i)__builtin_ia32_selectd_512((__mmask16)(U), \
8729 (__v16si)_mm512_shuffle_epi32((A), (I)), \
Craig Topperc6338672018-05-31 00:51:20 +00008730 (__v16si)_mm512_setzero_si512())
Michael Zuckermanc62f27e2016-05-02 07:35:27 +00008731
Craig Topper74c10e32018-07-09 19:00:16 +00008732static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008733_mm512_mask_expand_pd (__m512d __W, __mmask8 __U, __m512d __A)
8734{
8735 return (__m512d) __builtin_ia32_expanddf512_mask ((__v8df) __A,
8736 (__v8df) __W,
8737 (__mmask8) __U);
8738}
8739
Craig Topper74c10e32018-07-09 19:00:16 +00008740static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008741_mm512_maskz_expand_pd (__mmask8 __U, __m512d __A)
8742{
8743 return (__m512d) __builtin_ia32_expanddf512_mask ((__v8df) __A,
8744 (__v8df) _mm512_setzero_pd (),
8745 (__mmask8) __U);
8746}
8747
Craig Topper74c10e32018-07-09 19:00:16 +00008748static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008749_mm512_mask_expand_epi64 (__m512i __W, __mmask8 __U, __m512i __A)
8750{
8751 return (__m512i) __builtin_ia32_expanddi512_mask ((__v8di) __A,
8752 (__v8di) __W,
8753 (__mmask8) __U);
8754}
8755
Craig Topper74c10e32018-07-09 19:00:16 +00008756static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008757_mm512_maskz_expand_epi64 ( __mmask8 __U, __m512i __A)
8758{
8759 return (__m512i) __builtin_ia32_expanddi512_mask ((__v8di) __A,
Craig Topper5cbeeed2018-07-07 17:03:32 +00008760 (__v8di) _mm512_setzero_si512 (),
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008761 (__mmask8) __U);
8762}
8763
Craig Topper74c10e32018-07-09 19:00:16 +00008764static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008765_mm512_mask_expandloadu_pd(__m512d __W, __mmask8 __U, void const *__P)
8766{
8767 return (__m512d) __builtin_ia32_expandloaddf512_mask ((const __v8df *)__P,
8768 (__v8df) __W,
8769 (__mmask8) __U);
8770}
8771
Craig Topper74c10e32018-07-09 19:00:16 +00008772static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008773_mm512_maskz_expandloadu_pd(__mmask8 __U, void const *__P)
8774{
8775 return (__m512d) __builtin_ia32_expandloaddf512_mask ((const __v8df *)__P,
8776 (__v8df) _mm512_setzero_pd(),
8777 (__mmask8) __U);
8778}
8779
Craig Topper74c10e32018-07-09 19:00:16 +00008780static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008781_mm512_mask_expandloadu_epi64(__m512i __W, __mmask8 __U, void const *__P)
8782{
8783 return (__m512i) __builtin_ia32_expandloaddi512_mask ((const __v8di *)__P,
8784 (__v8di) __W,
8785 (__mmask8) __U);
8786}
8787
Craig Topper74c10e32018-07-09 19:00:16 +00008788static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008789_mm512_maskz_expandloadu_epi64(__mmask8 __U, void const *__P)
8790{
8791 return (__m512i) __builtin_ia32_expandloaddi512_mask ((const __v8di *)__P,
Craig Topper5cbeeed2018-07-07 17:03:32 +00008792 (__v8di) _mm512_setzero_si512(),
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008793 (__mmask8) __U);
8794}
8795
Craig Topper74c10e32018-07-09 19:00:16 +00008796static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008797_mm512_mask_expandloadu_ps(__m512 __W, __mmask16 __U, void const *__P)
8798{
8799 return (__m512) __builtin_ia32_expandloadsf512_mask ((const __v16sf *)__P,
8800 (__v16sf) __W,
8801 (__mmask16) __U);
8802}
8803
Craig Topper74c10e32018-07-09 19:00:16 +00008804static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008805_mm512_maskz_expandloadu_ps(__mmask16 __U, void const *__P)
8806{
8807 return (__m512) __builtin_ia32_expandloadsf512_mask ((const __v16sf *)__P,
8808 (__v16sf) _mm512_setzero_ps(),
8809 (__mmask16) __U);
8810}
8811
Craig Topper74c10e32018-07-09 19:00:16 +00008812static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008813_mm512_mask_expandloadu_epi32(__m512i __W, __mmask16 __U, void const *__P)
8814{
8815 return (__m512i) __builtin_ia32_expandloadsi512_mask ((const __v16si *)__P,
8816 (__v16si) __W,
8817 (__mmask16) __U);
8818}
8819
Craig Topper74c10e32018-07-09 19:00:16 +00008820static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008821_mm512_maskz_expandloadu_epi32(__mmask16 __U, void const *__P)
8822{
8823 return (__m512i) __builtin_ia32_expandloadsi512_mask ((const __v16si *)__P,
Craig Topper5cbeeed2018-07-07 17:03:32 +00008824 (__v16si) _mm512_setzero_si512(),
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008825 (__mmask16) __U);
8826}
8827
Craig Topper74c10e32018-07-09 19:00:16 +00008828static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008829_mm512_mask_expand_ps (__m512 __W, __mmask16 __U, __m512 __A)
8830{
8831 return (__m512) __builtin_ia32_expandsf512_mask ((__v16sf) __A,
8832 (__v16sf) __W,
8833 (__mmask16) __U);
8834}
8835
Craig Topper74c10e32018-07-09 19:00:16 +00008836static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008837_mm512_maskz_expand_ps (__mmask16 __U, __m512 __A)
8838{
8839 return (__m512) __builtin_ia32_expandsf512_mask ((__v16sf) __A,
8840 (__v16sf) _mm512_setzero_ps(),
8841 (__mmask16) __U);
8842}
8843
Craig Topper74c10e32018-07-09 19:00:16 +00008844static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008845_mm512_mask_expand_epi32 (__m512i __W, __mmask16 __U, __m512i __A)
8846{
8847 return (__m512i) __builtin_ia32_expandsi512_mask ((__v16si) __A,
8848 (__v16si) __W,
8849 (__mmask16) __U);
8850}
8851
Craig Topper74c10e32018-07-09 19:00:16 +00008852static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008853_mm512_maskz_expand_epi32 (__mmask16 __U, __m512i __A)
8854{
8855 return (__m512i) __builtin_ia32_expandsi512_mask ((__v16si) __A,
Craig Topper5cbeeed2018-07-07 17:03:32 +00008856 (__v16si) _mm512_setzero_si512(),
Michael Zuckerman6a0e0872016-05-02 08:36:41 +00008857 (__mmask16) __U);
8858}
8859
Craig Topperc6338672018-05-31 00:51:20 +00008860#define _mm512_cvt_roundps_pd(A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00008861 (__m512d)__builtin_ia32_cvtps2pd512_mask((__v8sf)(__m256)(A), \
8862 (__v8df)_mm512_undefined_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00008863 (__mmask8)-1, (int)(R))
Michael Zuckermand6e68ce2016-05-02 09:42:31 +00008864
Craig Topperc6338672018-05-31 00:51:20 +00008865#define _mm512_mask_cvt_roundps_pd(W, U, A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00008866 (__m512d)__builtin_ia32_cvtps2pd512_mask((__v8sf)(__m256)(A), \
8867 (__v8df)(__m512d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00008868 (__mmask8)(U), (int)(R))
Michael Zuckermand6e68ce2016-05-02 09:42:31 +00008869
Craig Topperc6338672018-05-31 00:51:20 +00008870#define _mm512_maskz_cvt_roundps_pd(U, A, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00008871 (__m512d)__builtin_ia32_cvtps2pd512_mask((__v8sf)(__m256)(A), \
8872 (__v8df)_mm512_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00008873 (__mmask8)(U), (int)(R))
Michael Zuckermand6e68ce2016-05-02 09:42:31 +00008874
Craig Topper74c10e32018-07-09 19:00:16 +00008875static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermand6e68ce2016-05-02 09:42:31 +00008876_mm512_cvtps_pd (__m256 __A)
8877{
Craig Topperdaaf1052018-05-14 04:05:06 +00008878 return (__m512d) __builtin_convertvector((__v8sf)__A, __v8df);
Michael Zuckermand6e68ce2016-05-02 09:42:31 +00008879}
8880
Craig Topper74c10e32018-07-09 19:00:16 +00008881static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermand6e68ce2016-05-02 09:42:31 +00008882_mm512_mask_cvtps_pd (__m512d __W, __mmask8 __U, __m256 __A)
8883{
Craig Topper8cb261e2018-05-14 04:57:46 +00008884 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
8885 (__v8df)_mm512_cvtps_pd(__A),
8886 (__v8df)__W);
Michael Zuckermand6e68ce2016-05-02 09:42:31 +00008887}
8888
Craig Topper74c10e32018-07-09 19:00:16 +00008889static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckermand6e68ce2016-05-02 09:42:31 +00008890_mm512_maskz_cvtps_pd (__mmask8 __U, __m256 __A)
8891{
Craig Topper8cb261e2018-05-14 04:57:46 +00008892 return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
8893 (__v8df)_mm512_cvtps_pd(__A),
8894 (__v8df)_mm512_setzero_pd());
Michael Zuckermand6e68ce2016-05-02 09:42:31 +00008895}
8896
Craig Topper74c10e32018-07-09 19:00:16 +00008897static __inline__ __m512d __DEFAULT_FN_ATTRS512
Ayman Musa17a28192016-09-27 15:37:31 +00008898_mm512_cvtpslo_pd (__m512 __A)
Ayman Musa2e250e82016-09-27 14:06:32 +00008899{
Craig Topper5cbeeed2018-07-07 17:03:32 +00008900 return (__m512d) _mm512_cvtps_pd(_mm512_castps512_ps256(__A));
Ayman Musa2e250e82016-09-27 14:06:32 +00008901}
8902
Craig Topper74c10e32018-07-09 19:00:16 +00008903static __inline__ __m512d __DEFAULT_FN_ATTRS512
Ayman Musa17a28192016-09-27 15:37:31 +00008904_mm512_mask_cvtpslo_pd (__m512d __W, __mmask8 __U, __m512 __A)
Ayman Musa2e250e82016-09-27 14:06:32 +00008905{
Craig Topper5cbeeed2018-07-07 17:03:32 +00008906 return (__m512d) _mm512_mask_cvtps_pd(__W, __U, _mm512_castps512_ps256(__A));
Ayman Musa2e250e82016-09-27 14:06:32 +00008907}
8908
Craig Topper74c10e32018-07-09 19:00:16 +00008909static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman5f0e96e2016-05-02 14:02:01 +00008910_mm512_mask_mov_pd (__m512d __W, __mmask8 __U, __m512d __A)
8911{
Igor Bregeraadb8762016-06-08 13:59:20 +00008912 return (__m512d) __builtin_ia32_selectpd_512 ((__mmask8) __U,
8913 (__v8df) __A,
8914 (__v8df) __W);
Michael Zuckerman5f0e96e2016-05-02 14:02:01 +00008915}
8916
Craig Topper74c10e32018-07-09 19:00:16 +00008917static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman5f0e96e2016-05-02 14:02:01 +00008918_mm512_maskz_mov_pd (__mmask8 __U, __m512d __A)
8919{
Igor Bregeraadb8762016-06-08 13:59:20 +00008920 return (__m512d) __builtin_ia32_selectpd_512 ((__mmask8) __U,
8921 (__v8df) __A,
8922 (__v8df) _mm512_setzero_pd ());
Michael Zuckerman5f0e96e2016-05-02 14:02:01 +00008923}
8924
Craig Topper74c10e32018-07-09 19:00:16 +00008925static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman5f0e96e2016-05-02 14:02:01 +00008926_mm512_mask_mov_ps (__m512 __W, __mmask16 __U, __m512 __A)
8927{
Igor Bregeraadb8762016-06-08 13:59:20 +00008928 return (__m512) __builtin_ia32_selectps_512 ((__mmask16) __U,
8929 (__v16sf) __A,
8930 (__v16sf) __W);
Michael Zuckerman5f0e96e2016-05-02 14:02:01 +00008931}
8932
Craig Topper74c10e32018-07-09 19:00:16 +00008933static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman5f0e96e2016-05-02 14:02:01 +00008934_mm512_maskz_mov_ps (__mmask16 __U, __m512 __A)
8935{
Igor Bregeraadb8762016-06-08 13:59:20 +00008936 return (__m512) __builtin_ia32_selectps_512 ((__mmask16) __U,
8937 (__v16sf) __A,
8938 (__v16sf) _mm512_setzero_ps ());
Michael Zuckerman5f0e96e2016-05-02 14:02:01 +00008939}
8940
Craig Topper74c10e32018-07-09 19:00:16 +00008941static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckerman708e7592016-05-03 10:42:46 +00008942_mm512_mask_compressstoreu_pd (void *__P, __mmask8 __U, __m512d __A)
8943{
8944 __builtin_ia32_compressstoredf512_mask ((__v8df *) __P, (__v8df) __A,
8945 (__mmask8) __U);
8946}
8947
Craig Topper74c10e32018-07-09 19:00:16 +00008948static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckerman708e7592016-05-03 10:42:46 +00008949_mm512_mask_compressstoreu_epi64 (void *__P, __mmask8 __U, __m512i __A)
8950{
8951 __builtin_ia32_compressstoredi512_mask ((__v8di *) __P, (__v8di) __A,
8952 (__mmask8) __U);
8953}
8954
Craig Topper74c10e32018-07-09 19:00:16 +00008955static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckerman708e7592016-05-03 10:42:46 +00008956_mm512_mask_compressstoreu_ps (void *__P, __mmask16 __U, __m512 __A)
8957{
8958 __builtin_ia32_compressstoresf512_mask ((__v16sf *) __P, (__v16sf) __A,
8959 (__mmask16) __U);
8960}
8961
Craig Topper74c10e32018-07-09 19:00:16 +00008962static __inline__ void __DEFAULT_FN_ATTRS512
Michael Zuckerman708e7592016-05-03 10:42:46 +00008963_mm512_mask_compressstoreu_epi32 (void *__P, __mmask16 __U, __m512i __A)
8964{
8965 __builtin_ia32_compressstoresi512_mask ((__v16si *) __P, (__v16si) __A,
8966 (__mmask16) __U);
8967}
Michael Zuckerman5f0e96e2016-05-02 14:02:01 +00008968
Craig Topperc6338672018-05-31 00:51:20 +00008969#define _mm_cvt_roundsd_ss(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00008970 (__m128)__builtin_ia32_cvtsd2ss_round_mask((__v4sf)(__m128)(A), \
8971 (__v2df)(__m128d)(B), \
8972 (__v4sf)_mm_undefined_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00008973 (__mmask8)-1, (int)(R))
Michael Zuckermane6f73892016-05-04 08:55:11 +00008974
Craig Topperc6338672018-05-31 00:51:20 +00008975#define _mm_mask_cvt_roundsd_ss(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00008976 (__m128)__builtin_ia32_cvtsd2ss_round_mask((__v4sf)(__m128)(A), \
8977 (__v2df)(__m128d)(B), \
8978 (__v4sf)(__m128)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00008979 (__mmask8)(U), (int)(R))
Michael Zuckermane6f73892016-05-04 08:55:11 +00008980
Craig Topperc6338672018-05-31 00:51:20 +00008981#define _mm_maskz_cvt_roundsd_ss(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00008982 (__m128)__builtin_ia32_cvtsd2ss_round_mask((__v4sf)(__m128)(A), \
8983 (__v2df)(__m128d)(B), \
8984 (__v4sf)_mm_setzero_ps(), \
Craig Topperc6338672018-05-31 00:51:20 +00008985 (__mmask8)(U), (int)(R))
Michael Zuckermane6f73892016-05-04 08:55:11 +00008986
Craig Topper74c10e32018-07-09 19:00:16 +00008987static __inline__ __m128 __DEFAULT_FN_ATTRS128
Asaf Badouha0b6f8f2016-07-14 08:40:30 +00008988_mm_mask_cvtsd_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128d __B)
Asaf Badouh89f65762016-06-02 08:11:35 +00008989{
Craig Toppercbf39292018-05-31 01:24:40 +00008990 return __builtin_ia32_cvtsd2ss_round_mask ((__v4sf)__A,
8991 (__v2df)__B,
8992 (__v4sf)__W,
8993 (__mmask8)__U, _MM_FROUND_CUR_DIRECTION);
Asaf Badouh89f65762016-06-02 08:11:35 +00008994}
8995
Craig Topper74c10e32018-07-09 19:00:16 +00008996static __inline__ __m128 __DEFAULT_FN_ATTRS128
Asaf Badouha0b6f8f2016-07-14 08:40:30 +00008997_mm_maskz_cvtsd_ss (__mmask8 __U, __m128 __A, __m128d __B)
Asaf Badouh89f65762016-06-02 08:11:35 +00008998{
Craig Toppercbf39292018-05-31 01:24:40 +00008999 return __builtin_ia32_cvtsd2ss_round_mask ((__v4sf)__A,
9000 (__v2df)__B,
Simon Pilgrim0b37ffb2017-07-28 14:01:51 +00009001 (__v4sf)_mm_setzero_ps(),
Craig Toppercbf39292018-05-31 01:24:40 +00009002 (__mmask8)__U, _MM_FROUND_CUR_DIRECTION);
Asaf Badouh89f65762016-06-02 08:11:35 +00009003}
9004
9005#define _mm_cvtss_i32 _mm_cvtss_si32
Asaf Badouh89f65762016-06-02 08:11:35 +00009006#define _mm_cvtsd_i32 _mm_cvtsd_si32
Asaf Badouh89f65762016-06-02 08:11:35 +00009007#define _mm_cvti32_sd _mm_cvtsi32_sd
Asaf Badouh89f65762016-06-02 08:11:35 +00009008#define _mm_cvti32_ss _mm_cvtsi32_ss
Craig Topper45db56c2016-07-21 07:38:39 +00009009#ifdef __x86_64__
9010#define _mm_cvtss_i64 _mm_cvtss_si64
9011#define _mm_cvtsd_i64 _mm_cvtsd_si64
9012#define _mm_cvti64_sd _mm_cvtsi64_sd
Asaf Badouh89f65762016-06-02 08:11:35 +00009013#define _mm_cvti64_ss _mm_cvtsi64_ss
Craig Topper45db56c2016-07-21 07:38:39 +00009014#endif
Asaf Badouh89f65762016-06-02 08:11:35 +00009015
Craig Topper45db56c2016-07-21 07:38:39 +00009016#ifdef __x86_64__
Craig Topperc6338672018-05-31 00:51:20 +00009017#define _mm_cvt_roundi64_sd(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00009018 (__m128d)__builtin_ia32_cvtsi2sd64((__v2df)(__m128d)(A), (long long)(B), \
Craig Topperc6338672018-05-31 00:51:20 +00009019 (int)(R))
Michael Zuckermane6f73892016-05-04 08:55:11 +00009020
Craig Topperc6338672018-05-31 00:51:20 +00009021#define _mm_cvt_roundsi64_sd(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00009022 (__m128d)__builtin_ia32_cvtsi2sd64((__v2df)(__m128d)(A), (long long)(B), \
Craig Topperc6338672018-05-31 00:51:20 +00009023 (int)(R))
Craig Topper45db56c2016-07-21 07:38:39 +00009024#endif
Michael Zuckermane6f73892016-05-04 08:55:11 +00009025
Craig Topperc6338672018-05-31 00:51:20 +00009026#define _mm_cvt_roundsi32_ss(A, B, R) \
9027 (__m128)__builtin_ia32_cvtsi2ss32((__v4sf)(__m128)(A), (int)(B), (int)(R))
Michael Zuckermane6f73892016-05-04 08:55:11 +00009028
Craig Topperc6338672018-05-31 00:51:20 +00009029#define _mm_cvt_roundi32_ss(A, B, R) \
9030 (__m128)__builtin_ia32_cvtsi2ss32((__v4sf)(__m128)(A), (int)(B), (int)(R))
Michael Zuckermane6f73892016-05-04 08:55:11 +00009031
Craig Topper45db56c2016-07-21 07:38:39 +00009032#ifdef __x86_64__
Craig Topperc6338672018-05-31 00:51:20 +00009033#define _mm_cvt_roundsi64_ss(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00009034 (__m128)__builtin_ia32_cvtsi2ss64((__v4sf)(__m128)(A), (long long)(B), \
Craig Topperc6338672018-05-31 00:51:20 +00009035 (int)(R))
Michael Zuckermane6f73892016-05-04 08:55:11 +00009036
Craig Topperc6338672018-05-31 00:51:20 +00009037#define _mm_cvt_roundi64_ss(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00009038 (__m128)__builtin_ia32_cvtsi2ss64((__v4sf)(__m128)(A), (long long)(B), \
Craig Topperc6338672018-05-31 00:51:20 +00009039 (int)(R))
Craig Topper45db56c2016-07-21 07:38:39 +00009040#endif
Michael Zuckermane6f73892016-05-04 08:55:11 +00009041
Craig Topperc6338672018-05-31 00:51:20 +00009042#define _mm_cvt_roundss_sd(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00009043 (__m128d)__builtin_ia32_cvtss2sd_round_mask((__v2df)(__m128d)(A), \
9044 (__v4sf)(__m128)(B), \
9045 (__v2df)_mm_undefined_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00009046 (__mmask8)-1, (int)(R))
Michael Zuckermane6f73892016-05-04 08:55:11 +00009047
Craig Topperc6338672018-05-31 00:51:20 +00009048#define _mm_mask_cvt_roundss_sd(W, U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00009049 (__m128d)__builtin_ia32_cvtss2sd_round_mask((__v2df)(__m128d)(A), \
9050 (__v4sf)(__m128)(B), \
9051 (__v2df)(__m128d)(W), \
Craig Topperc6338672018-05-31 00:51:20 +00009052 (__mmask8)(U), (int)(R))
Michael Zuckermane6f73892016-05-04 08:55:11 +00009053
Craig Topperc6338672018-05-31 00:51:20 +00009054#define _mm_maskz_cvt_roundss_sd(U, A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00009055 (__m128d)__builtin_ia32_cvtss2sd_round_mask((__v2df)(__m128d)(A), \
9056 (__v4sf)(__m128)(B), \
9057 (__v2df)_mm_setzero_pd(), \
Craig Topperc6338672018-05-31 00:51:20 +00009058 (__mmask8)(U), (int)(R))
Michael Zuckermane6f73892016-05-04 08:55:11 +00009059
Craig Topper74c10e32018-07-09 19:00:16 +00009060static __inline__ __m128d __DEFAULT_FN_ATTRS128
Asaf Badouha0b6f8f2016-07-14 08:40:30 +00009061_mm_mask_cvtss_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128 __B)
Asaf Badouh89f65762016-06-02 08:11:35 +00009062{
Craig Toppercbf39292018-05-31 01:24:40 +00009063 return __builtin_ia32_cvtss2sd_round_mask((__v2df)__A,
9064 (__v4sf)__B,
9065 (__v2df)__W,
9066 (__mmask8)__U, _MM_FROUND_CUR_DIRECTION);
Asaf Badouh89f65762016-06-02 08:11:35 +00009067}
9068
Craig Topper74c10e32018-07-09 19:00:16 +00009069static __inline__ __m128d __DEFAULT_FN_ATTRS128
Asaf Badouha0b6f8f2016-07-14 08:40:30 +00009070_mm_maskz_cvtss_sd (__mmask8 __U, __m128d __A, __m128 __B)
Asaf Badouh89f65762016-06-02 08:11:35 +00009071{
Craig Toppercbf39292018-05-31 01:24:40 +00009072 return __builtin_ia32_cvtss2sd_round_mask((__v2df)__A,
9073 (__v4sf)__B,
9074 (__v2df)_mm_setzero_pd(),
9075 (__mmask8)__U, _MM_FROUND_CUR_DIRECTION);
Asaf Badouh89f65762016-06-02 08:11:35 +00009076}
9077
Craig Topper74c10e32018-07-09 19:00:16 +00009078static __inline__ __m128d __DEFAULT_FN_ATTRS128
Michael Zuckermane6f73892016-05-04 08:55:11 +00009079_mm_cvtu32_sd (__m128d __A, unsigned __B)
9080{
Craig Topper6fa91252018-05-13 23:03:30 +00009081 __A[0] = __B;
9082 return __A;
Michael Zuckermane6f73892016-05-04 08:55:11 +00009083}
9084
Craig Topper45db56c2016-07-21 07:38:39 +00009085#ifdef __x86_64__
Craig Topperc6338672018-05-31 00:51:20 +00009086#define _mm_cvt_roundu64_sd(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00009087 (__m128d)__builtin_ia32_cvtusi2sd64((__v2df)(__m128d)(A), \
Craig Topperc6338672018-05-31 00:51:20 +00009088 (unsigned long long)(B), (int)(R))
Michael Zuckermane6f73892016-05-04 08:55:11 +00009089
Craig Topper74c10e32018-07-09 19:00:16 +00009090static __inline__ __m128d __DEFAULT_FN_ATTRS128
Michael Zuckermane6f73892016-05-04 08:55:11 +00009091_mm_cvtu64_sd (__m128d __A, unsigned long long __B)
9092{
Craig Topper6fa91252018-05-13 23:03:30 +00009093 __A[0] = __B;
9094 return __A;
Michael Zuckermane6f73892016-05-04 08:55:11 +00009095}
Craig Topper45db56c2016-07-21 07:38:39 +00009096#endif
Michael Zuckermane6f73892016-05-04 08:55:11 +00009097
Craig Topperc6338672018-05-31 00:51:20 +00009098#define _mm_cvt_roundu32_ss(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00009099 (__m128)__builtin_ia32_cvtusi2ss32((__v4sf)(__m128)(A), (unsigned int)(B), \
Craig Topperc6338672018-05-31 00:51:20 +00009100 (int)(R))
Michael Zuckermane6f73892016-05-04 08:55:11 +00009101
Craig Topper74c10e32018-07-09 19:00:16 +00009102static __inline__ __m128 __DEFAULT_FN_ATTRS128
Michael Zuckermane6f73892016-05-04 08:55:11 +00009103_mm_cvtu32_ss (__m128 __A, unsigned __B)
9104{
Craig Topper6fa91252018-05-13 23:03:30 +00009105 __A[0] = __B;
9106 return __A;
Michael Zuckermane6f73892016-05-04 08:55:11 +00009107}
9108
Craig Topper45db56c2016-07-21 07:38:39 +00009109#ifdef __x86_64__
Craig Topperc6338672018-05-31 00:51:20 +00009110#define _mm_cvt_roundu64_ss(A, B, R) \
Craig Topper8c18e112016-05-17 04:41:50 +00009111 (__m128)__builtin_ia32_cvtusi2ss64((__v4sf)(__m128)(A), \
Craig Topperc6338672018-05-31 00:51:20 +00009112 (unsigned long long)(B), (int)(R))
Michael Zuckermane6f73892016-05-04 08:55:11 +00009113
Craig Topper74c10e32018-07-09 19:00:16 +00009114static __inline__ __m128 __DEFAULT_FN_ATTRS128
Michael Zuckermane6f73892016-05-04 08:55:11 +00009115_mm_cvtu64_ss (__m128 __A, unsigned long long __B)
9116{
Craig Topper6fa91252018-05-13 23:03:30 +00009117 __A[0] = __B;
9118 return __A;
Michael Zuckermane6f73892016-05-04 08:55:11 +00009119}
Craig Topper45db56c2016-07-21 07:38:39 +00009120#endif
Michael Zuckermane6f73892016-05-04 08:55:11 +00009121
Craig Topper74c10e32018-07-09 19:00:16 +00009122static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman13d3c002016-05-11 11:41:29 +00009123_mm512_mask_set1_epi32 (__m512i __O, __mmask16 __M, int __A)
9124{
Jina Nahias3ad702a2017-09-19 11:00:27 +00009125 return (__m512i) __builtin_ia32_selectd_512(__M,
9126 (__v16si) _mm512_set1_epi32(__A),
9127 (__v16si) __O);
Michael Zuckerman13d3c002016-05-11 11:41:29 +00009128}
9129
Craig Topper74c10e32018-07-09 19:00:16 +00009130static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman13d3c002016-05-11 11:41:29 +00009131_mm512_mask_set1_epi64 (__m512i __O, __mmask8 __M, long long __A)
9132{
Jina Nahias3ad702a2017-09-19 11:00:27 +00009133 return (__m512i) __builtin_ia32_selectq_512(__M,
9134 (__v8di) _mm512_set1_epi64(__A),
9135 (__v8di) __O);
Michael Zuckerman13d3c002016-05-11 11:41:29 +00009136}
9137
Craig Topper74c10e32018-07-09 19:00:16 +00009138static __inline __m512i __DEFAULT_FN_ATTRS512
Igor Bregerf050b792017-03-19 08:27:16 +00009139_mm512_set_epi8 (char __e63, char __e62, char __e61, char __e60, char __e59,
9140 char __e58, char __e57, char __e56, char __e55, char __e54, char __e53,
9141 char __e52, char __e51, char __e50, char __e49, char __e48, char __e47,
9142 char __e46, char __e45, char __e44, char __e43, char __e42, char __e41,
9143 char __e40, char __e39, char __e38, char __e37, char __e36, char __e35,
9144 char __e34, char __e33, char __e32, char __e31, char __e30, char __e29,
9145 char __e28, char __e27, char __e26, char __e25, char __e24, char __e23,
9146 char __e22, char __e21, char __e20, char __e19, char __e18, char __e17,
9147 char __e16, char __e15, char __e14, char __e13, char __e12, char __e11,
9148 char __e10, char __e9, char __e8, char __e7, char __e6, char __e5,
9149 char __e4, char __e3, char __e2, char __e1, char __e0) {
9150
9151 return __extension__ (__m512i)(__v64qi)
9152 {__e0, __e1, __e2, __e3, __e4, __e5, __e6, __e7,
9153 __e8, __e9, __e10, __e11, __e12, __e13, __e14, __e15,
9154 __e16, __e17, __e18, __e19, __e20, __e21, __e22, __e23,
9155 __e24, __e25, __e26, __e27, __e28, __e29, __e30, __e31,
9156 __e32, __e33, __e34, __e35, __e36, __e37, __e38, __e39,
9157 __e40, __e41, __e42, __e43, __e44, __e45, __e46, __e47,
9158 __e48, __e49, __e50, __e51, __e52, __e53, __e54, __e55,
9159 __e56, __e57, __e58, __e59, __e60, __e61, __e62, __e63};
9160}
9161
Craig Topper74c10e32018-07-09 19:00:16 +00009162static __inline __m512i __DEFAULT_FN_ATTRS512
Igor Bregerf050b792017-03-19 08:27:16 +00009163_mm512_set_epi16(short __e31, short __e30, short __e29, short __e28,
9164 short __e27, short __e26, short __e25, short __e24, short __e23,
9165 short __e22, short __e21, short __e20, short __e19, short __e18,
9166 short __e17, short __e16, short __e15, short __e14, short __e13,
9167 short __e12, short __e11, short __e10, short __e9, short __e8,
9168 short __e7, short __e6, short __e5, short __e4, short __e3,
9169 short __e2, short __e1, short __e0) {
9170 return __extension__ (__m512i)(__v32hi)
9171 {__e0, __e1, __e2, __e3, __e4, __e5, __e6, __e7,
9172 __e8, __e9, __e10, __e11, __e12, __e13, __e14, __e15,
9173 __e16, __e17, __e18, __e19, __e20, __e21, __e22, __e23,
9174 __e24, __e25, __e26, __e27, __e28, __e29, __e30, __e31 };
9175}
9176
Craig Topper74c10e32018-07-09 19:00:16 +00009177static __inline __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman178113e2016-05-19 12:07:49 +00009178_mm512_set_epi32 (int __A, int __B, int __C, int __D,
9179 int __E, int __F, int __G, int __H,
9180 int __I, int __J, int __K, int __L,
9181 int __M, int __N, int __O, int __P)
9182{
9183 return __extension__ (__m512i)(__v16si)
9184 { __P, __O, __N, __M, __L, __K, __J, __I,
9185 __H, __G, __F, __E, __D, __C, __B, __A };
9186}
9187
9188#define _mm512_setr_epi32(e0,e1,e2,e3,e4,e5,e6,e7, \
9189 e8,e9,e10,e11,e12,e13,e14,e15) \
Craig Topper60589492016-06-08 06:08:04 +00009190 _mm512_set_epi32((e15),(e14),(e13),(e12),(e11),(e10),(e9),(e8),(e7),(e6), \
9191 (e5),(e4),(e3),(e2),(e1),(e0))
Ekaterina Romanova5a7f09c2016-05-28 00:18:59 +00009192
Craig Topper74c10e32018-07-09 19:00:16 +00009193static __inline__ __m512i __DEFAULT_FN_ATTRS512
Michael Zuckerman13d3c002016-05-11 11:41:29 +00009194_mm512_set_epi64 (long long __A, long long __B, long long __C,
9195 long long __D, long long __E, long long __F,
9196 long long __G, long long __H)
9197{
9198 return __extension__ (__m512i) (__v8di)
9199 { __H, __G, __F, __E, __D, __C, __B, __A };
9200}
9201
Michael Zuckerman178113e2016-05-19 12:07:49 +00009202#define _mm512_setr_epi64(e0,e1,e2,e3,e4,e5,e6,e7) \
Craig Topper60589492016-06-08 06:08:04 +00009203 _mm512_set_epi64((e7),(e6),(e5),(e4),(e3),(e2),(e1),(e0))
Michael Zuckerman178113e2016-05-19 12:07:49 +00009204
Craig Topper74c10e32018-07-09 19:00:16 +00009205static __inline__ __m512d __DEFAULT_FN_ATTRS512
Michael Zuckerman13d3c002016-05-11 11:41:29 +00009206_mm512_set_pd (double __A, double __B, double __C, double __D,
9207 double __E, double __F, double __G, double __H)
9208{
9209 return __extension__ (__m512d)
9210 { __H, __G, __F, __E, __D, __C, __B, __A };
9211}
9212
Michael Zuckerman178113e2016-05-19 12:07:49 +00009213#define _mm512_setr_pd(e0,e1,e2,e3,e4,e5,e6,e7) \
Craig Topper60589492016-06-08 06:08:04 +00009214 _mm512_set_pd((e7),(e6),(e5),(e4),(e3),(e2),(e1),(e0))
Michael Zuckerman178113e2016-05-19 12:07:49 +00009215
Craig Topper74c10e32018-07-09 19:00:16 +00009216static __inline__ __m512 __DEFAULT_FN_ATTRS512
Michael Zuckerman13d3c002016-05-11 11:41:29 +00009217_mm512_set_ps (float __A, float __B, float __C, float __D,
9218 float __E, float __F, float __G, float __H,
9219 float __I, float __J, float __K, float __L,
9220 float __M, float __N, float __O, float __P)
9221{
9222 return __extension__ (__m512)
9223 { __P, __O, __N, __M, __L, __K, __J, __I,
9224 __H, __G, __F, __E, __D, __C, __B, __A };
9225}
9226
Michael Zuckerman178113e2016-05-19 12:07:49 +00009227#define _mm512_setr_ps(e0,e1,e2,e3,e4,e5,e6,e7,e8,e9,e10,e11,e12,e13,e14,e15) \
Craig Topper60589492016-06-08 06:08:04 +00009228 _mm512_set_ps((e15),(e14),(e13),(e12),(e11),(e10),(e9),(e8),(e7),(e6),(e5), \
9229 (e4),(e3),(e2),(e1),(e0))
Michael Zuckerman178113e2016-05-19 12:07:49 +00009230
Craig Topper74c10e32018-07-09 19:00:16 +00009231static __inline__ __m512 __DEFAULT_FN_ATTRS512
Asaf Badouh356bb762016-08-21 07:56:47 +00009232_mm512_abs_ps(__m512 __A)
Asaf Badouh13633282016-07-05 12:24:14 +00009233{
Asaf Badouh356bb762016-08-21 07:56:47 +00009234 return (__m512)_mm512_and_epi32(_mm512_set1_epi32(0x7FFFFFFF),(__m512i)__A) ;
Asaf Badouh13633282016-07-05 12:24:14 +00009235}
9236
Craig Topper74c10e32018-07-09 19:00:16 +00009237static __inline__ __m512 __DEFAULT_FN_ATTRS512
Asaf Badouh356bb762016-08-21 07:56:47 +00009238_mm512_mask_abs_ps(__m512 __W, __mmask16 __K, __m512 __A)
Asaf Badouh13633282016-07-05 12:24:14 +00009239{
Asaf Badouh356bb762016-08-21 07:56:47 +00009240 return (__m512)_mm512_mask_and_epi32((__m512i)__W, __K, _mm512_set1_epi32(0x7FFFFFFF),(__m512i)__A) ;
Asaf Badouh13633282016-07-05 12:24:14 +00009241}
9242
Craig Topper74c10e32018-07-09 19:00:16 +00009243static __inline__ __m512d __DEFAULT_FN_ATTRS512
Asaf Badouh356bb762016-08-21 07:56:47 +00009244_mm512_abs_pd(__m512d __A)
Asaf Badouh13633282016-07-05 12:24:14 +00009245{
Asaf Badouh356bb762016-08-21 07:56:47 +00009246 return (__m512d)_mm512_and_epi64(_mm512_set1_epi64(0x7FFFFFFFFFFFFFFF),(__v8di)__A) ;
Asaf Badouh13633282016-07-05 12:24:14 +00009247}
9248
Craig Topper74c10e32018-07-09 19:00:16 +00009249static __inline__ __m512d __DEFAULT_FN_ATTRS512
Asaf Badouh356bb762016-08-21 07:56:47 +00009250_mm512_mask_abs_pd(__m512d __W, __mmask8 __K, __m512d __A)
Asaf Badouh13633282016-07-05 12:24:14 +00009251{
Asaf Badouh356bb762016-08-21 07:56:47 +00009252 return (__m512d)_mm512_mask_and_epi64((__v8di)__W, __K, _mm512_set1_epi64(0x7FFFFFFFFFFFFFFF),(__v8di)__A);
Asaf Badouh13633282016-07-05 12:24:14 +00009253}
9254
Craig Topper73d1d402018-05-30 22:33:21 +00009255/* Vector-reduction arithmetic accepts vectors as inputs and produces scalars as
9256 * outputs. This class of vector operation forms the basis of many scientific
9257 * computations. In vector-reduction arithmetic, the evaluation off is
9258 * independent of the order of the input elements of V.
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009259
Craig Topper73d1d402018-05-30 22:33:21 +00009260 * Used bisection method. At each step, we partition the vector with previous
9261 * step in half, and the operation is performed on its two halves.
9262 * This takes log2(n) steps where n is the number of elements in the vector.
Craig Topper73d1d402018-05-30 22:33:21 +00009263 */
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009264
Craig Topperddfe69c2018-06-21 16:41:28 +00009265#define _mm512_mask_reduce_operator(op) \
9266 __v4du __t1 = (__v4du)_mm512_extracti64x4_epi64(__W, 0); \
9267 __v4du __t2 = (__v4du)_mm512_extracti64x4_epi64(__W, 1); \
9268 __m256i __t3 = (__m256i)(__t1 op __t2); \
9269 __v2du __t4 = (__v2du)_mm256_extracti128_si256(__t3, 0); \
9270 __v2du __t5 = (__v2du)_mm256_extracti128_si256(__t3, 1); \
9271 __v2du __t6 = __t4 op __t5; \
9272 __v2du __t7 = __builtin_shufflevector(__t6, __t6, 1, 0); \
9273 __v2du __t8 = __t6 op __t7; \
9274 return __t8[0];
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009275
Craig Topper74c10e32018-07-09 19:00:16 +00009276static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_add_epi64(__m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009277 _mm512_mask_reduce_operator(+);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009278}
9279
Craig Topper74c10e32018-07-09 19:00:16 +00009280static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_mul_epi64(__m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009281 _mm512_mask_reduce_operator(*);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009282}
9283
Craig Topper74c10e32018-07-09 19:00:16 +00009284static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_and_epi64(__m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009285 _mm512_mask_reduce_operator(&);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009286}
9287
Craig Topper74c10e32018-07-09 19:00:16 +00009288static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_or_epi64(__m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009289 _mm512_mask_reduce_operator(|);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009290}
9291
Craig Topper74c10e32018-07-09 19:00:16 +00009292static __inline__ long long __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009293_mm512_mask_reduce_add_epi64(__mmask8 __M, __m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009294 __W = _mm512_maskz_mov_epi64(__M, __W);
9295 _mm512_mask_reduce_operator(+);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009296}
9297
Craig Topper74c10e32018-07-09 19:00:16 +00009298static __inline__ long long __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009299_mm512_mask_reduce_mul_epi64(__mmask8 __M, __m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009300 __W = _mm512_mask_mov_epi64(_mm512_set1_epi64(1), __M, __W);
9301 _mm512_mask_reduce_operator(*);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009302}
9303
Craig Topper74c10e32018-07-09 19:00:16 +00009304static __inline__ long long __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009305_mm512_mask_reduce_and_epi64(__mmask8 __M, __m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009306 __W = _mm512_mask_mov_epi64(_mm512_set1_epi64(~0ULL), __M, __W);
9307 _mm512_mask_reduce_operator(&);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009308}
9309
Craig Topper74c10e32018-07-09 19:00:16 +00009310static __inline__ long long __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009311_mm512_mask_reduce_or_epi64(__mmask8 __M, __m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009312 __W = _mm512_maskz_mov_epi64(__M, __W);
9313 _mm512_mask_reduce_operator(|);
9314}
9315#undef _mm512_mask_reduce_operator
9316
9317#define _mm512_mask_reduce_operator(op) \
9318 __m256d __t1 = _mm512_extractf64x4_pd(__W, 0); \
9319 __m256d __t2 = _mm512_extractf64x4_pd(__W, 1); \
9320 __m256d __t3 = __t1 op __t2; \
9321 __m128d __t4 = _mm256_extractf128_pd(__t3, 0); \
9322 __m128d __t5 = _mm256_extractf128_pd(__t3, 1); \
9323 __m128d __t6 = __t4 op __t5; \
9324 __m128d __t7 = __builtin_shufflevector(__t6, __t6, 1, 0); \
9325 __m128d __t8 = __t6 op __t7; \
9326 return __t8[0];
9327
Craig Topper74c10e32018-07-09 19:00:16 +00009328static __inline__ double __DEFAULT_FN_ATTRS512 _mm512_reduce_add_pd(__m512d __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009329 _mm512_mask_reduce_operator(+);
9330}
9331
Craig Topper74c10e32018-07-09 19:00:16 +00009332static __inline__ double __DEFAULT_FN_ATTRS512 _mm512_reduce_mul_pd(__m512d __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009333 _mm512_mask_reduce_operator(*);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009334}
9335
Craig Topper74c10e32018-07-09 19:00:16 +00009336static __inline__ double __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009337_mm512_mask_reduce_add_pd(__mmask8 __M, __m512d __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009338 __W = _mm512_maskz_mov_pd(__M, __W);
9339 _mm512_mask_reduce_operator(+);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009340}
9341
Craig Topper74c10e32018-07-09 19:00:16 +00009342static __inline__ double __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009343_mm512_mask_reduce_mul_pd(__mmask8 __M, __m512d __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009344 __W = _mm512_mask_mov_pd(_mm512_set1_pd(1.0), __M, __W);
9345 _mm512_mask_reduce_operator(*);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009346}
Craig Topperddfe69c2018-06-21 16:41:28 +00009347#undef _mm512_mask_reduce_operator
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009348
Craig Topperddfe69c2018-06-21 16:41:28 +00009349#define _mm512_mask_reduce_operator(op) \
9350 __v8su __t1 = (__v8su)_mm512_extracti64x4_epi64(__W, 0); \
9351 __v8su __t2 = (__v8su)_mm512_extracti64x4_epi64(__W, 1); \
9352 __m256i __t3 = (__m256i)(__t1 op __t2); \
9353 __v4su __t4 = (__v4su)_mm256_extracti128_si256(__t3, 0); \
9354 __v4su __t5 = (__v4su)_mm256_extracti128_si256(__t3, 1); \
9355 __v4su __t6 = __t4 op __t5; \
9356 __v4su __t7 = __builtin_shufflevector(__t6, __t6, 2, 3, 0, 1); \
9357 __v4su __t8 = __t6 op __t7; \
9358 __v4su __t9 = __builtin_shufflevector(__t8, __t8, 1, 0, 3, 2); \
9359 __v4su __t10 = __t8 op __t9; \
9360 return __t10[0];
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009361
Craig Topper74c10e32018-07-09 19:00:16 +00009362static __inline__ int __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009363_mm512_reduce_add_epi32(__m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009364 _mm512_mask_reduce_operator(+);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009365}
9366
Craig Topper74c10e32018-07-09 19:00:16 +00009367static __inline__ int __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009368_mm512_reduce_mul_epi32(__m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009369 _mm512_mask_reduce_operator(*);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009370}
9371
Craig Topper74c10e32018-07-09 19:00:16 +00009372static __inline__ int __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009373_mm512_reduce_and_epi32(__m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009374 _mm512_mask_reduce_operator(&);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009375}
9376
Craig Topper74c10e32018-07-09 19:00:16 +00009377static __inline__ int __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009378_mm512_reduce_or_epi32(__m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009379 _mm512_mask_reduce_operator(|);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009380}
9381
Craig Topper74c10e32018-07-09 19:00:16 +00009382static __inline__ int __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009383_mm512_mask_reduce_add_epi32( __mmask16 __M, __m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009384 __W = _mm512_maskz_mov_epi32(__M, __W);
9385 _mm512_mask_reduce_operator(+);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009386}
9387
Craig Topper74c10e32018-07-09 19:00:16 +00009388static __inline__ int __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009389_mm512_mask_reduce_mul_epi32( __mmask16 __M, __m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009390 __W = _mm512_mask_mov_epi32(_mm512_set1_epi32(1), __M, __W);
9391 _mm512_mask_reduce_operator(*);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009392}
9393
Craig Topper74c10e32018-07-09 19:00:16 +00009394static __inline__ int __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009395_mm512_mask_reduce_and_epi32( __mmask16 __M, __m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009396 __W = _mm512_mask_mov_epi32(_mm512_set1_epi32(~0U), __M, __W);
9397 _mm512_mask_reduce_operator(&);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009398}
9399
Craig Topper74c10e32018-07-09 19:00:16 +00009400static __inline__ int __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009401_mm512_mask_reduce_or_epi32(__mmask16 __M, __m512i __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009402 __W = _mm512_maskz_mov_epi32(__M, __W);
9403 _mm512_mask_reduce_operator(|);
9404}
9405#undef _mm512_mask_reduce_operator
9406
9407#define _mm512_mask_reduce_operator(op) \
9408 __m256 __t1 = (__m256)_mm512_extractf64x4_pd((__m512d)__W, 0); \
9409 __m256 __t2 = (__m256)_mm512_extractf64x4_pd((__m512d)__W, 1); \
9410 __m256 __t3 = __t1 op __t2; \
9411 __m128 __t4 = _mm256_extractf128_ps(__t3, 0); \
9412 __m128 __t5 = _mm256_extractf128_ps(__t3, 1); \
9413 __m128 __t6 = __t4 op __t5; \
9414 __m128 __t7 = __builtin_shufflevector(__t6, __t6, 2, 3, 0, 1); \
9415 __m128 __t8 = __t6 op __t7; \
9416 __m128 __t9 = __builtin_shufflevector(__t8, __t8, 1, 0, 3, 2); \
9417 __m128 __t10 = __t8 op __t9; \
9418 return __t10[0];
9419
Craig Topper74c10e32018-07-09 19:00:16 +00009420static __inline__ float __DEFAULT_FN_ATTRS512
Craig Topperddfe69c2018-06-21 16:41:28 +00009421_mm512_reduce_add_ps(__m512 __W) {
9422 _mm512_mask_reduce_operator(+);
9423}
9424
Craig Topper74c10e32018-07-09 19:00:16 +00009425static __inline__ float __DEFAULT_FN_ATTRS512
Craig Topperddfe69c2018-06-21 16:41:28 +00009426_mm512_reduce_mul_ps(__m512 __W) {
9427 _mm512_mask_reduce_operator(*);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009428}
9429
Craig Topper74c10e32018-07-09 19:00:16 +00009430static __inline__ float __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009431_mm512_mask_reduce_add_ps(__mmask16 __M, __m512 __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009432 __W = _mm512_maskz_mov_ps(__M, __W);
9433 _mm512_mask_reduce_operator(+);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009434}
9435
Craig Topper74c10e32018-07-09 19:00:16 +00009436static __inline__ float __DEFAULT_FN_ATTRS512
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009437_mm512_mask_reduce_mul_ps(__mmask16 __M, __m512 __W) {
Craig Topperddfe69c2018-06-21 16:41:28 +00009438 __W = _mm512_mask_mov_ps(_mm512_set1_ps(1.0f), __M, __W);
9439 _mm512_mask_reduce_operator(*);
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009440}
Craig Topperddfe69c2018-06-21 16:41:28 +00009441#undef _mm512_mask_reduce_operator
Michael Zuckermanfacb37c2016-10-25 07:56:04 +00009442
Craig Topper61495b32018-06-19 21:00:30 +00009443#define _mm512_mask_reduce_operator(op) \
9444 __m512i __t1 = (__m512i)__builtin_shufflevector((__v8di)__V, (__v8di)__V, 4, 5, 6, 7, 0, 1, 2, 3); \
9445 __m512i __t2 = _mm512_##op(__V, __t1); \
9446 __m512i __t3 = (__m512i)__builtin_shufflevector((__v8di)__t2, (__v8di)__t2, 2, 3, 0, 1, 6, 7, 4, 5); \
9447 __m512i __t4 = _mm512_##op(__t2, __t3); \
9448 __m512i __t5 = (__m512i)__builtin_shufflevector((__v8di)__t4, (__v8di)__t4, 1, 0, 3, 2, 5, 4, 7, 6); \
9449 __v8di __t6 = (__v8di)_mm512_##op(__t4, __t5); \
9450 return __t6[0];
Michael Zuckerman25eb4202016-10-29 10:29:20 +00009451
Craig Topper74c10e32018-07-09 19:00:16 +00009452static __inline__ long long __DEFAULT_FN_ATTRS512
Michael Zuckerman25eb4202016-10-29 10:29:20 +00009453_mm512_reduce_max_epi64(__m512i __V) {
Craig Topper61495b32018-06-19 21:00:30 +00009454 _mm512_mask_reduce_operator(max_epi64);
Michael Zuckerman25eb4202016-10-29 10:29:20 +00009455}
9456
Craig Topper74c10e32018-07-09 19:00:16 +00009457static __inline__ unsigned long long __DEFAULT_FN_ATTRS512
Michael Zuckerman25eb4202016-10-29 10:29:20 +00009458_mm512_reduce_max_epu64(__m512i __V) {
Craig Topper61495b32018-06-19 21:00:30 +00009459 _mm512_mask_reduce_operator(max_epu64);
Michael Zuckerman25eb4202016-10-29 10:29:20 +00009460}
9461
Craig Topper74c10e32018-07-09 19:00:16 +00009462static __inline__ long long __DEFAULT_FN_ATTRS512
Craig Topper61495b32018-06-19 21:00:30 +00009463_mm512_reduce_min_epi64(__m512i __V) {
9464 _mm512_mask_reduce_operator(min_epi64);
Michael Zuckerman25eb4202016-10-29 10:29:20 +00009465}
9466
Craig Topper74c10e32018-07-09 19:00:16 +00009467static __inline__ unsigned long long __DEFAULT_FN_ATTRS512
Michael Zuckerman25eb4202016-10-29 10:29:20 +00009468_mm512_reduce_min_epu64(__m512i __V) {
Craig Topper61495b32018-06-19 21:00:30 +00009469 _mm512_mask_reduce_operator(min_epu64);
Craig Topper873afd02018-06-19 19:13:54 +00009470}
9471
Craig Topper74c10e32018-07-09 19:00:16 +00009472static __inline__ long long __DEFAULT_FN_ATTRS512
Craig Topper79b13a02018-06-19 19:37:07 +00009473_mm512_mask_reduce_max_epi64(__mmask8 __M, __m512i __V) {
Craig Topper61495b32018-06-19 21:00:30 +00009474 __V = _mm512_mask_mov_epi64(_mm512_set1_epi64(-__LONG_LONG_MAX__ - 1LL), __M, __V);
9475 _mm512_mask_reduce_operator(max_epi64);
Craig Topper79b13a02018-06-19 19:37:07 +00009476}
9477
Craig Topper74c10e32018-07-09 19:00:16 +00009478static __inline__ unsigned long long __DEFAULT_FN_ATTRS512
Craig Topper79b13a02018-06-19 19:37:07 +00009479_mm512_mask_reduce_max_epu64(__mmask8 __M, __m512i __V) {
Craig Topper61495b32018-06-19 21:00:30 +00009480 __V = _mm512_maskz_mov_epi64(__M, __V);
9481 _mm512_mask_reduce_operator(max_epu64);
Craig Topper79b13a02018-06-19 19:37:07 +00009482}
9483
Craig Topper74c10e32018-07-09 19:00:16 +00009484static __inline__ long long __DEFAULT_FN_ATTRS512
Craig Topper79b13a02018-06-19 19:37:07 +00009485_mm512_mask_reduce_min_epi64(__mmask8 __M, __m512i __V) {
Craig Topper61495b32018-06-19 21:00:30 +00009486 __V = _mm512_mask_mov_epi64(_mm512_set1_epi64(__LONG_LONG_MAX__), __M, __V);
9487 _mm512_mask_reduce_operator(min_epi64);
Craig Topper79b13a02018-06-19 19:37:07 +00009488}
9489
Craig Topper74c10e32018-07-09 19:00:16 +00009490static __inline__ unsigned long long __DEFAULT_FN_ATTRS512
Craig Topper79b13a02018-06-19 19:37:07 +00009491_mm512_mask_reduce_min_epu64(__mmask8 __M, __m512i __V) {
Craig Topper61495b32018-06-19 21:00:30 +00009492 __V = _mm512_mask_mov_epi64(_mm512_set1_epi64(~0ULL), __M, __V);
9493 _mm512_mask_reduce_operator(min_epu64);
Craig Topper873afd02018-06-19 19:13:54 +00009494}
Craig Topper61495b32018-06-19 21:00:30 +00009495#undef _mm512_mask_reduce_operator
Craig Topper873afd02018-06-19 19:13:54 +00009496
Craig Topper61495b32018-06-19 21:00:30 +00009497#define _mm512_mask_reduce_operator(op) \
9498 __m256i __t1 = _mm512_extracti64x4_epi64(__V, 0); \
9499 __m256i __t2 = _mm512_extracti64x4_epi64(__V, 1); \
9500 __m256i __t3 = _mm256_##op(__t1, __t2); \
9501 __m128i __t4 = _mm256_extracti128_si256(__t3, 0); \
9502 __m128i __t5 = _mm256_extracti128_si256(__t3, 1); \
9503 __m128i __t6 = _mm_##op(__t4, __t5); \
9504 __m128i __t7 = (__m128i)__builtin_shufflevector((__v4si)__t6, (__v4si)__t6, 2, 3, 0, 1); \
9505 __m128i __t8 = _mm_##op(__t6, __t7); \
9506 __m128i __t9 = (__m128i)__builtin_shufflevector((__v4si)__t8, (__v4si)__t8, 1, 0, 3, 2); \
9507 __v4si __t10 = (__v4si)_mm_##op(__t8, __t9); \
9508 return __t10[0];
Craig Topper873afd02018-06-19 19:13:54 +00009509
Craig Topper74c10e32018-07-09 19:00:16 +00009510static __inline__ int __DEFAULT_FN_ATTRS512
Craig Topper61495b32018-06-19 21:00:30 +00009511_mm512_reduce_max_epi32(__m512i __V) {
9512 _mm512_mask_reduce_operator(max_epi32);
Craig Topper873afd02018-06-19 19:13:54 +00009513}
9514
Craig Topper74c10e32018-07-09 19:00:16 +00009515static __inline__ unsigned int __DEFAULT_FN_ATTRS512
Craig Topper61495b32018-06-19 21:00:30 +00009516_mm512_reduce_max_epu32(__m512i __V) {
9517 _mm512_mask_reduce_operator(max_epu32);
Craig Topper79b13a02018-06-19 19:37:07 +00009518}
9519
Craig Topper74c10e32018-07-09 19:00:16 +00009520static __inline__ int __DEFAULT_FN_ATTRS512
Craig Topper61495b32018-06-19 21:00:30 +00009521_mm512_reduce_min_epi32(__m512i __V) {
9522 _mm512_mask_reduce_operator(min_epi32);
Craig Topper79b13a02018-06-19 19:37:07 +00009523}
9524
Craig Topper74c10e32018-07-09 19:00:16 +00009525static __inline__ unsigned int __DEFAULT_FN_ATTRS512
Craig Topper61495b32018-06-19 21:00:30 +00009526_mm512_reduce_min_epu32(__m512i __V) {
9527 _mm512_mask_reduce_operator(min_epu32);
Craig Topper79b13a02018-06-19 19:37:07 +00009528}
9529
Craig Topper74c10e32018-07-09 19:00:16 +00009530static __inline__ int __DEFAULT_FN_ATTRS512
Craig Topper79b13a02018-06-19 19:37:07 +00009531_mm512_mask_reduce_max_epi32(__mmask16 __M, __m512i __V) {
Craig Topper61495b32018-06-19 21:00:30 +00009532 __V = _mm512_mask_mov_epi32(_mm512_set1_epi32(-__INT_MAX__ - 1), __M, __V);
9533 _mm512_mask_reduce_operator(max_epi32);
Craig Topper79b13a02018-06-19 19:37:07 +00009534}
9535
Craig Topper74c10e32018-07-09 19:00:16 +00009536static __inline__ unsigned int __DEFAULT_FN_ATTRS512
Craig Topper79b13a02018-06-19 19:37:07 +00009537_mm512_mask_reduce_max_epu32(__mmask16 __M, __m512i __V) {
Craig Topper61495b32018-06-19 21:00:30 +00009538 __V = _mm512_maskz_mov_epi32(__M, __V);
9539 _mm512_mask_reduce_operator(max_epu32);
Craig Topper79b13a02018-06-19 19:37:07 +00009540}
9541
Craig Topper74c10e32018-07-09 19:00:16 +00009542static __inline__ int __DEFAULT_FN_ATTRS512
Craig Topper79b13a02018-06-19 19:37:07 +00009543_mm512_mask_reduce_min_epi32(__mmask16 __M, __m512i __V) {
Craig Topper61495b32018-06-19 21:00:30 +00009544 __V = _mm512_mask_mov_epi32(_mm512_set1_epi32(__INT_MAX__), __M, __V);
9545 _mm512_mask_reduce_operator(min_epi32);
Craig Topper79b13a02018-06-19 19:37:07 +00009546}
9547
Craig Topper74c10e32018-07-09 19:00:16 +00009548static __inline__ unsigned int __DEFAULT_FN_ATTRS512
Craig Topper79b13a02018-06-19 19:37:07 +00009549_mm512_mask_reduce_min_epu32(__mmask16 __M, __m512i __V) {
Craig Topper61495b32018-06-19 21:00:30 +00009550 __V = _mm512_mask_mov_epi32(_mm512_set1_epi32(~0U), __M, __V);
9551 _mm512_mask_reduce_operator(min_epu32);
9552}
Craig Topperd3346152018-06-20 00:31:39 +00009553#undef _mm512_mask_reduce_operator
Craig Topper61495b32018-06-19 21:00:30 +00009554
9555#define _mm512_mask_reduce_operator(op) \
9556 __m256d __t1 = _mm512_extractf64x4_pd(__V, 0); \
9557 __m256d __t2 = _mm512_extractf64x4_pd(__V, 1); \
9558 __m256d __t3 = _mm256_##op(__t1, __t2); \
9559 __m128d __t4 = _mm256_extractf128_pd(__t3, 0); \
9560 __m128d __t5 = _mm256_extractf128_pd(__t3, 1); \
9561 __m128d __t6 = _mm_##op(__t4, __t5); \
9562 __m128d __t7 = __builtin_shufflevector(__t6, __t6, 1, 0); \
9563 __m128d __t8 = _mm_##op(__t6, __t7); \
9564 return __t8[0];
9565
Craig Topper74c10e32018-07-09 19:00:16 +00009566static __inline__ double __DEFAULT_FN_ATTRS512
Craig Topper61495b32018-06-19 21:00:30 +00009567_mm512_reduce_max_pd(__m512d __V) {
9568 _mm512_mask_reduce_operator(max_pd);
9569}
9570
Craig Topper74c10e32018-07-09 19:00:16 +00009571static __inline__ double __DEFAULT_FN_ATTRS512
Craig Topper61495b32018-06-19 21:00:30 +00009572_mm512_reduce_min_pd(__m512d __V) {
9573 _mm512_mask_reduce_operator(min_pd);
9574}
9575
Craig Topper74c10e32018-07-09 19:00:16 +00009576static __inline__ double __DEFAULT_FN_ATTRS512
Craig Topper61495b32018-06-19 21:00:30 +00009577_mm512_mask_reduce_max_pd(__mmask8 __M, __m512d __V) {
9578 __V = _mm512_mask_mov_pd(_mm512_set1_pd(-__builtin_inf()), __M, __V);
9579 _mm512_mask_reduce_operator(max_pd);
9580}
9581
Craig Topper74c10e32018-07-09 19:00:16 +00009582static __inline__ double __DEFAULT_FN_ATTRS512
Craig Topper61495b32018-06-19 21:00:30 +00009583_mm512_mask_reduce_min_pd(__mmask8 __M, __m512d __V) {
9584 __V = _mm512_mask_mov_pd(_mm512_set1_pd(__builtin_inf()), __M, __V);
9585 _mm512_mask_reduce_operator(min_pd);
9586}
9587#undef _mm512_mask_reduce_operator
9588
9589#define _mm512_mask_reduce_operator(op) \
9590 __m256 __t1 = (__m256)_mm512_extractf64x4_pd((__m512d)__V, 0); \
9591 __m256 __t2 = (__m256)_mm512_extractf64x4_pd((__m512d)__V, 1); \
9592 __m256 __t3 = _mm256_##op(__t1, __t2); \
9593 __m128 __t4 = _mm256_extractf128_ps(__t3, 0); \
9594 __m128 __t5 = _mm256_extractf128_ps(__t3, 1); \
9595 __m128 __t6 = _mm_##op(__t4, __t5); \
9596 __m128 __t7 = __builtin_shufflevector(__t6, __t6, 2, 3, 0, 1); \
9597 __m128 __t8 = _mm_##op(__t6, __t7); \
9598 __m128 __t9 = __builtin_shufflevector(__t8, __t8, 1, 0, 3, 2); \
9599 __m128 __t10 = _mm_##op(__t8, __t9); \
9600 return __t10[0];
9601
Craig Topper74c10e32018-07-09 19:00:16 +00009602static __inline__ float __DEFAULT_FN_ATTRS512
Craig Topper61495b32018-06-19 21:00:30 +00009603_mm512_reduce_max_ps(__m512 __V) {
9604 _mm512_mask_reduce_operator(max_ps);
9605}
9606
Craig Topper74c10e32018-07-09 19:00:16 +00009607static __inline__ float __DEFAULT_FN_ATTRS512
Craig Topper61495b32018-06-19 21:00:30 +00009608_mm512_reduce_min_ps(__m512 __V) {
9609 _mm512_mask_reduce_operator(min_ps);
9610}
9611
Craig Topper74c10e32018-07-09 19:00:16 +00009612static __inline__ float __DEFAULT_FN_ATTRS512
Craig Topper61495b32018-06-19 21:00:30 +00009613_mm512_mask_reduce_max_ps(__mmask16 __M, __m512 __V) {
9614 __V = _mm512_mask_mov_ps(_mm512_set1_ps(-__builtin_inff()), __M, __V);
9615 _mm512_mask_reduce_operator(max_ps);
Michael Zuckerman25eb4202016-10-29 10:29:20 +00009616}
9617
Craig Topper74c10e32018-07-09 19:00:16 +00009618static __inline__ float __DEFAULT_FN_ATTRS512
Michael Zuckerman25eb4202016-10-29 10:29:20 +00009619_mm512_mask_reduce_min_ps(__mmask16 __M, __m512 __V) {
Craig Topper61495b32018-06-19 21:00:30 +00009620 __V = _mm512_mask_mov_ps(_mm512_set1_ps(__builtin_inff()), __M, __V);
9621 _mm512_mask_reduce_operator(min_ps);
Michael Zuckerman25eb4202016-10-29 10:29:20 +00009622}
Craig Topper61495b32018-06-19 21:00:30 +00009623#undef _mm512_mask_reduce_operator
Michael Zuckerman25eb4202016-10-29 10:29:20 +00009624
Craig Topper74c10e32018-07-09 19:00:16 +00009625#undef __DEFAULT_FN_ATTRS512
9626#undef __DEFAULT_FN_ATTRS128
Eric Christopher4d1851682015-06-17 07:09:20 +00009627
Craig Topper73d1d402018-05-30 22:33:21 +00009628#endif /* __AVX512FINTRIN_H */