Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / prebuilts / clang-tools / d00e613486995ae7e945d04e302246f65de56a0b / . / linux-x86 / lib64 / clang / 14.0.2 / include / avx2intrin.h
blob: 5064c87c2bb19f77276e406a9975fbcee49c91ac [file] [log] [blame]
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1/*===---- avx2intrin.h - AVX2 intrinsics -----------------------------------===
2 *
Logan Chiendf4f7662019-09-04 16:45:23 -0700[diff] [blame]3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error "Never use <avx2intrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifndef __AVX2INTRIN_H
15#define __AVX2INTRIN_H
16
17/* Define the default attributes for the functions in this file. */
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]18#define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("avx2"), __min_vector_width__(256)))
19#define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avx2"), __min_vector_width__(128)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]20
21/* SSE4 Multiple Packed Sums of Absolute Difference. */
22#define _mm256_mpsadbw_epu8(X, Y, M) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]23 ((__m256i)__builtin_ia32_mpsadbw256((__v32qi)(__m256i)(X), \
24 (__v32qi)(__m256i)(Y), (int)(M)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]25
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]26static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]27_mm256_abs_epi8(__m256i __a)
28{
29 return (__m256i)__builtin_ia32_pabsb256((__v32qi)__a);
30}
31
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]32static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]33_mm256_abs_epi16(__m256i __a)
34{
35 return (__m256i)__builtin_ia32_pabsw256((__v16hi)__a);
36}
37
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]38static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]39_mm256_abs_epi32(__m256i __a)
40{
41 return (__m256i)__builtin_ia32_pabsd256((__v8si)__a);
42}
43
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]44static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]45_mm256_packs_epi16(__m256i __a, __m256i __b)
46{
47 return (__m256i)__builtin_ia32_packsswb256((__v16hi)__a, (__v16hi)__b);
48}
49
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]50static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]51_mm256_packs_epi32(__m256i __a, __m256i __b)
52{
53 return (__m256i)__builtin_ia32_packssdw256((__v8si)__a, (__v8si)__b);
54}
55
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]56static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]57_mm256_packus_epi16(__m256i __a, __m256i __b)
58{
59 return (__m256i)__builtin_ia32_packuswb256((__v16hi)__a, (__v16hi)__b);
60}
61
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]62static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]63_mm256_packus_epi32(__m256i __V1, __m256i __V2)
64{
65 return (__m256i) __builtin_ia32_packusdw256((__v8si)__V1, (__v8si)__V2);
66}
67
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]68static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]69_mm256_add_epi8(__m256i __a, __m256i __b)
70{
71 return (__m256i)((__v32qu)__a + (__v32qu)__b);
72}
73
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]74static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]75_mm256_add_epi16(__m256i __a, __m256i __b)
76{
77 return (__m256i)((__v16hu)__a + (__v16hu)__b);
78}
79
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]80static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]81_mm256_add_epi32(__m256i __a, __m256i __b)
82{
83 return (__m256i)((__v8su)__a + (__v8su)__b);
84}
85
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]86static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]87_mm256_add_epi64(__m256i __a, __m256i __b)
88{
89 return (__m256i)((__v4du)__a + (__v4du)__b);
90}
91
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]92static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]93_mm256_adds_epi8(__m256i __a, __m256i __b)
94{
95 return (__m256i)__builtin_ia32_paddsb256((__v32qi)__a, (__v32qi)__b);
96}
97
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]98static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]99_mm256_adds_epi16(__m256i __a, __m256i __b)
100{
101 return (__m256i)__builtin_ia32_paddsw256((__v16hi)__a, (__v16hi)__b);
102}
103
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]104static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]105_mm256_adds_epu8(__m256i __a, __m256i __b)
106{
107 return (__m256i)__builtin_ia32_paddusb256((__v32qi)__a, (__v32qi)__b);
108}
109
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]110static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]111_mm256_adds_epu16(__m256i __a, __m256i __b)
112{
113 return (__m256i)__builtin_ia32_paddusw256((__v16hi)__a, (__v16hi)__b);
114}
115
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]116#define _mm256_alignr_epi8(a, b, n) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]117 ((__m256i)__builtin_ia32_palignr256((__v32qi)(__m256i)(a), \
118 (__v32qi)(__m256i)(b), (n)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]119
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]120static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]121_mm256_and_si256(__m256i __a, __m256i __b)
122{
123 return (__m256i)((__v4du)__a & (__v4du)__b);
124}
125
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]126static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]127_mm256_andnot_si256(__m256i __a, __m256i __b)
128{
129 return (__m256i)(~(__v4du)__a & (__v4du)__b);
130}
131
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]132static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]133_mm256_avg_epu8(__m256i __a, __m256i __b)
134{
Logan Chiendf4f7662019-09-04 16:45:23 -0700[diff] [blame]135 return (__m256i)__builtin_ia32_pavgb256((__v32qi)__a, (__v32qi)__b);
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]136}
137
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]138static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]139_mm256_avg_epu16(__m256i __a, __m256i __b)
140{
Logan Chiendf4f7662019-09-04 16:45:23 -0700[diff] [blame]141 return (__m256i)__builtin_ia32_pavgw256((__v16hi)__a, (__v16hi)__b);
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]142}
143
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]144static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]145_mm256_blendv_epi8(__m256i __V1, __m256i __V2, __m256i __M)
146{
147 return (__m256i)__builtin_ia32_pblendvb256((__v32qi)__V1, (__v32qi)__V2,
148 (__v32qi)__M);
149}
150
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]151#define _mm256_blend_epi16(V1, V2, M) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]152 ((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(V1), \
153 (__v16hi)(__m256i)(V2), (int)(M)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]154
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]155static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]156_mm256_cmpeq_epi8(__m256i __a, __m256i __b)
157{
158 return (__m256i)((__v32qi)__a == (__v32qi)__b);
159}
160
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]161static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]162_mm256_cmpeq_epi16(__m256i __a, __m256i __b)
163{
164 return (__m256i)((__v16hi)__a == (__v16hi)__b);
165}
166
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]167static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]168_mm256_cmpeq_epi32(__m256i __a, __m256i __b)
169{
170 return (__m256i)((__v8si)__a == (__v8si)__b);
171}
172
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]173static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]174_mm256_cmpeq_epi64(__m256i __a, __m256i __b)
175{
176 return (__m256i)((__v4di)__a == (__v4di)__b);
177}
178
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]179static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]180_mm256_cmpgt_epi8(__m256i __a, __m256i __b)
181{
182 /* This function always performs a signed comparison, but __v32qi is a char
183 which may be signed or unsigned, so use __v32qs. */
184 return (__m256i)((__v32qs)__a > (__v32qs)__b);
185}
186
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]187static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]188_mm256_cmpgt_epi16(__m256i __a, __m256i __b)
189{
190 return (__m256i)((__v16hi)__a > (__v16hi)__b);
191}
192
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]193static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]194_mm256_cmpgt_epi32(__m256i __a, __m256i __b)
195{
196 return (__m256i)((__v8si)__a > (__v8si)__b);
197}
198
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]199static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]200_mm256_cmpgt_epi64(__m256i __a, __m256i __b)
201{
202 return (__m256i)((__v4di)__a > (__v4di)__b);
203}
204
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]205static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]206_mm256_hadd_epi16(__m256i __a, __m256i __b)
207{
208 return (__m256i)__builtin_ia32_phaddw256((__v16hi)__a, (__v16hi)__b);
209}
210
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]211static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]212_mm256_hadd_epi32(__m256i __a, __m256i __b)
213{
214 return (__m256i)__builtin_ia32_phaddd256((__v8si)__a, (__v8si)__b);
215}
216
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]217static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]218_mm256_hadds_epi16(__m256i __a, __m256i __b)
219{
220 return (__m256i)__builtin_ia32_phaddsw256((__v16hi)__a, (__v16hi)__b);
221}
222
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]223static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]224_mm256_hsub_epi16(__m256i __a, __m256i __b)
225{
226 return (__m256i)__builtin_ia32_phsubw256((__v16hi)__a, (__v16hi)__b);
227}
228
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]229static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]230_mm256_hsub_epi32(__m256i __a, __m256i __b)
231{
232 return (__m256i)__builtin_ia32_phsubd256((__v8si)__a, (__v8si)__b);
233}
234
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]235static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]236_mm256_hsubs_epi16(__m256i __a, __m256i __b)
237{
238 return (__m256i)__builtin_ia32_phsubsw256((__v16hi)__a, (__v16hi)__b);
239}
240
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]241static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]242_mm256_maddubs_epi16(__m256i __a, __m256i __b)
243{
244 return (__m256i)__builtin_ia32_pmaddubsw256((__v32qi)__a, (__v32qi)__b);
245}
246
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]247static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]248_mm256_madd_epi16(__m256i __a, __m256i __b)
249{
250 return (__m256i)__builtin_ia32_pmaddwd256((__v16hi)__a, (__v16hi)__b);
251}
252
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]253static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]254_mm256_max_epi8(__m256i __a, __m256i __b)
255{
256 return (__m256i)__builtin_ia32_pmaxsb256((__v32qi)__a, (__v32qi)__b);
257}
258
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]259static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]260_mm256_max_epi16(__m256i __a, __m256i __b)
261{
262 return (__m256i)__builtin_ia32_pmaxsw256((__v16hi)__a, (__v16hi)__b);
263}
264
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]265static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]266_mm256_max_epi32(__m256i __a, __m256i __b)
267{
268 return (__m256i)__builtin_ia32_pmaxsd256((__v8si)__a, (__v8si)__b);
269}
270
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]271static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]272_mm256_max_epu8(__m256i __a, __m256i __b)
273{
274 return (__m256i)__builtin_ia32_pmaxub256((__v32qi)__a, (__v32qi)__b);
275}
276
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]277static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]278_mm256_max_epu16(__m256i __a, __m256i __b)
279{
280 return (__m256i)__builtin_ia32_pmaxuw256((__v16hi)__a, (__v16hi)__b);
281}
282
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]283static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]284_mm256_max_epu32(__m256i __a, __m256i __b)
285{
286 return (__m256i)__builtin_ia32_pmaxud256((__v8si)__a, (__v8si)__b);
287}
288
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]289static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]290_mm256_min_epi8(__m256i __a, __m256i __b)
291{
292 return (__m256i)__builtin_ia32_pminsb256((__v32qi)__a, (__v32qi)__b);
293}
294
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]295static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]296_mm256_min_epi16(__m256i __a, __m256i __b)
297{
298 return (__m256i)__builtin_ia32_pminsw256((__v16hi)__a, (__v16hi)__b);
299}
300
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]301static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]302_mm256_min_epi32(__m256i __a, __m256i __b)
303{
304 return (__m256i)__builtin_ia32_pminsd256((__v8si)__a, (__v8si)__b);
305}
306
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]307static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]308_mm256_min_epu8(__m256i __a, __m256i __b)
309{
310 return (__m256i)__builtin_ia32_pminub256((__v32qi)__a, (__v32qi)__b);
311}
312
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]313static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]314_mm256_min_epu16(__m256i __a, __m256i __b)
315{
316 return (__m256i)__builtin_ia32_pminuw256 ((__v16hi)__a, (__v16hi)__b);
317}
318
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]319static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]320_mm256_min_epu32(__m256i __a, __m256i __b)
321{
322 return (__m256i)__builtin_ia32_pminud256((__v8si)__a, (__v8si)__b);
323}
324
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]325static __inline__ int __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]326_mm256_movemask_epi8(__m256i __a)
327{
328 return __builtin_ia32_pmovmskb256((__v32qi)__a);
329}
330
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]331static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]332_mm256_cvtepi8_epi16(__m128i __V)
333{
334 /* This function always performs a signed extension, but __v16qi is a char
335 which may be signed or unsigned, so use __v16qs. */
336 return (__m256i)__builtin_convertvector((__v16qs)__V, __v16hi);
337}
338
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]339static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]340_mm256_cvtepi8_epi32(__m128i __V)
341{
342 /* This function always performs a signed extension, but __v16qi is a char
343 which may be signed or unsigned, so use __v16qs. */
344 return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8si);
345}
346
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]347static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]348_mm256_cvtepi8_epi64(__m128i __V)
349{
350 /* This function always performs a signed extension, but __v16qi is a char
351 which may be signed or unsigned, so use __v16qs. */
352 return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3), __v4di);
353}
354
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]355static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]356_mm256_cvtepi16_epi32(__m128i __V)
357{
358 return (__m256i)__builtin_convertvector((__v8hi)__V, __v8si);
359}
360
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]361static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]362_mm256_cvtepi16_epi64(__m128i __V)
363{
364 return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1, 2, 3), __v4di);
365}
366
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]367static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]368_mm256_cvtepi32_epi64(__m128i __V)
369{
370 return (__m256i)__builtin_convertvector((__v4si)__V, __v4di);
371}
372
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]373static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]374_mm256_cvtepu8_epi16(__m128i __V)
375{
376 return (__m256i)__builtin_convertvector((__v16qu)__V, __v16hi);
377}
378
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]379static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]380_mm256_cvtepu8_epi32(__m128i __V)
381{
382 return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8si);
383}
384
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]385static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]386_mm256_cvtepu8_epi64(__m128i __V)
387{
388 return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3), __v4di);
389}
390
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]391static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]392_mm256_cvtepu16_epi32(__m128i __V)
393{
394 return (__m256i)__builtin_convertvector((__v8hu)__V, __v8si);
395}
396
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]397static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]398_mm256_cvtepu16_epi64(__m128i __V)
399{
400 return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v8hu)__V, (__v8hu)__V, 0, 1, 2, 3), __v4di);
401}
402
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]403static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]404_mm256_cvtepu32_epi64(__m128i __V)
405{
406 return (__m256i)__builtin_convertvector((__v4su)__V, __v4di);
407}
408
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]409static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]410_mm256_mul_epi32(__m256i __a, __m256i __b)
411{
412 return (__m256i)__builtin_ia32_pmuldq256((__v8si)__a, (__v8si)__b);
413}
414
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]415static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]416_mm256_mulhrs_epi16(__m256i __a, __m256i __b)
417{
418 return (__m256i)__builtin_ia32_pmulhrsw256((__v16hi)__a, (__v16hi)__b);
419}
420
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]421static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]422_mm256_mulhi_epu16(__m256i __a, __m256i __b)
423{
424 return (__m256i)__builtin_ia32_pmulhuw256((__v16hi)__a, (__v16hi)__b);
425}
426
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]427static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]428_mm256_mulhi_epi16(__m256i __a, __m256i __b)
429{
430 return (__m256i)__builtin_ia32_pmulhw256((__v16hi)__a, (__v16hi)__b);
431}
432
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]433static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]434_mm256_mullo_epi16(__m256i __a, __m256i __b)
435{
436 return (__m256i)((__v16hu)__a * (__v16hu)__b);
437}
438
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]439static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]440_mm256_mullo_epi32 (__m256i __a, __m256i __b)
441{
442 return (__m256i)((__v8su)__a * (__v8su)__b);
443}
444
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]445static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]446_mm256_mul_epu32(__m256i __a, __m256i __b)
447{
448 return __builtin_ia32_pmuludq256((__v8si)__a, (__v8si)__b);
449}
450
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]451static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]452_mm256_or_si256(__m256i __a, __m256i __b)
453{
454 return (__m256i)((__v4du)__a | (__v4du)__b);
455}
456
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]457static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]458_mm256_sad_epu8(__m256i __a, __m256i __b)
459{
460 return __builtin_ia32_psadbw256((__v32qi)__a, (__v32qi)__b);
461}
462
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]463static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]464_mm256_shuffle_epi8(__m256i __a, __m256i __b)
465{
466 return (__m256i)__builtin_ia32_pshufb256((__v32qi)__a, (__v32qi)__b);
467}
468
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]469#define _mm256_shuffle_epi32(a, imm) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]470 ((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(a), (int)(imm)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]471
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]472#define _mm256_shufflehi_epi16(a, imm) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]473 ((__m256i)__builtin_ia32_pshufhw256((__v16hi)(__m256i)(a), (int)(imm)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]474
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]475#define _mm256_shufflelo_epi16(a, imm) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]476 ((__m256i)__builtin_ia32_pshuflw256((__v16hi)(__m256i)(a), (int)(imm)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]477
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]478static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]479_mm256_sign_epi8(__m256i __a, __m256i __b)
480{
481 return (__m256i)__builtin_ia32_psignb256((__v32qi)__a, (__v32qi)__b);
482}
483
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]484static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]485_mm256_sign_epi16(__m256i __a, __m256i __b)
486{
487 return (__m256i)__builtin_ia32_psignw256((__v16hi)__a, (__v16hi)__b);
488}
489
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]490static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]491_mm256_sign_epi32(__m256i __a, __m256i __b)
492{
493 return (__m256i)__builtin_ia32_psignd256((__v8si)__a, (__v8si)__b);
494}
495
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]496#define _mm256_slli_si256(a, imm) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]497 ((__m256i)__builtin_ia32_pslldqi256_byteshift((__v4di)(__m256i)(a), (int)(imm)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]498
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]499#define _mm256_bslli_epi128(a, imm) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]500 ((__m256i)__builtin_ia32_pslldqi256_byteshift((__v4di)(__m256i)(a), (int)(imm)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]501
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]502static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]503_mm256_slli_epi16(__m256i __a, int __count)
504{
505 return (__m256i)__builtin_ia32_psllwi256((__v16hi)__a, __count);
506}
507
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]508static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]509_mm256_sll_epi16(__m256i __a, __m128i __count)
510{
511 return (__m256i)__builtin_ia32_psllw256((__v16hi)__a, (__v8hi)__count);
512}
513
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]514static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]515_mm256_slli_epi32(__m256i __a, int __count)
516{
517 return (__m256i)__builtin_ia32_pslldi256((__v8si)__a, __count);
518}
519
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]520static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]521_mm256_sll_epi32(__m256i __a, __m128i __count)
522{
523 return (__m256i)__builtin_ia32_pslld256((__v8si)__a, (__v4si)__count);
524}
525
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]526static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]527_mm256_slli_epi64(__m256i __a, int __count)
528{
529 return __builtin_ia32_psllqi256((__v4di)__a, __count);
530}
531
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]532static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]533_mm256_sll_epi64(__m256i __a, __m128i __count)
534{
535 return __builtin_ia32_psllq256((__v4di)__a, __count);
536}
537
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]538static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]539_mm256_srai_epi16(__m256i __a, int __count)
540{
541 return (__m256i)__builtin_ia32_psrawi256((__v16hi)__a, __count);
542}
543
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]544static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]545_mm256_sra_epi16(__m256i __a, __m128i __count)
546{
547 return (__m256i)__builtin_ia32_psraw256((__v16hi)__a, (__v8hi)__count);
548}
549
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]550static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]551_mm256_srai_epi32(__m256i __a, int __count)
552{
553 return (__m256i)__builtin_ia32_psradi256((__v8si)__a, __count);
554}
555
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]556static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]557_mm256_sra_epi32(__m256i __a, __m128i __count)
558{
559 return (__m256i)__builtin_ia32_psrad256((__v8si)__a, (__v4si)__count);
560}
561
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]562#define _mm256_srli_si256(a, imm) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]563 ((__m256i)__builtin_ia32_psrldqi256_byteshift((__m256i)(a), (int)(imm)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]564
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]565#define _mm256_bsrli_epi128(a, imm) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]566 ((__m256i)__builtin_ia32_psrldqi256_byteshift((__m256i)(a), (int)(imm)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]567
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]568static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]569_mm256_srli_epi16(__m256i __a, int __count)
570{
571 return (__m256i)__builtin_ia32_psrlwi256((__v16hi)__a, __count);
572}
573
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]574static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]575_mm256_srl_epi16(__m256i __a, __m128i __count)
576{
577 return (__m256i)__builtin_ia32_psrlw256((__v16hi)__a, (__v8hi)__count);
578}
579
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]580static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]581_mm256_srli_epi32(__m256i __a, int __count)
582{
583 return (__m256i)__builtin_ia32_psrldi256((__v8si)__a, __count);
584}
585
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]586static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]587_mm256_srl_epi32(__m256i __a, __m128i __count)
588{
589 return (__m256i)__builtin_ia32_psrld256((__v8si)__a, (__v4si)__count);
590}
591
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]592static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]593_mm256_srli_epi64(__m256i __a, int __count)
594{
595 return __builtin_ia32_psrlqi256((__v4di)__a, __count);
596}
597
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]598static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]599_mm256_srl_epi64(__m256i __a, __m128i __count)
600{
601 return __builtin_ia32_psrlq256((__v4di)__a, __count);
602}
603
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]604static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]605_mm256_sub_epi8(__m256i __a, __m256i __b)
606{
607 return (__m256i)((__v32qu)__a - (__v32qu)__b);
608}
609
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]610static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]611_mm256_sub_epi16(__m256i __a, __m256i __b)
612{
613 return (__m256i)((__v16hu)__a - (__v16hu)__b);
614}
615
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]616static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]617_mm256_sub_epi32(__m256i __a, __m256i __b)
618{
619 return (__m256i)((__v8su)__a - (__v8su)__b);
620}
621
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]622static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]623_mm256_sub_epi64(__m256i __a, __m256i __b)
624{
625 return (__m256i)((__v4du)__a - (__v4du)__b);
626}
627
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]628static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]629_mm256_subs_epi8(__m256i __a, __m256i __b)
630{
631 return (__m256i)__builtin_ia32_psubsb256((__v32qi)__a, (__v32qi)__b);
632}
633
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]634static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]635_mm256_subs_epi16(__m256i __a, __m256i __b)
636{
637 return (__m256i)__builtin_ia32_psubsw256((__v16hi)__a, (__v16hi)__b);
638}
639
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]640static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]641_mm256_subs_epu8(__m256i __a, __m256i __b)
642{
643 return (__m256i)__builtin_ia32_psubusb256((__v32qi)__a, (__v32qi)__b);
644}
645
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]646static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]647_mm256_subs_epu16(__m256i __a, __m256i __b)
648{
649 return (__m256i)__builtin_ia32_psubusw256((__v16hi)__a, (__v16hi)__b);
650}
651
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]652static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]653_mm256_unpackhi_epi8(__m256i __a, __m256i __b)
654{
655 return (__m256i)__builtin_shufflevector((__v32qi)__a, (__v32qi)__b, 8, 32+8, 9, 32+9, 10, 32+10, 11, 32+11, 12, 32+12, 13, 32+13, 14, 32+14, 15, 32+15, 24, 32+24, 25, 32+25, 26, 32+26, 27, 32+27, 28, 32+28, 29, 32+29, 30, 32+30, 31, 32+31);
656}
657
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]658static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]659_mm256_unpackhi_epi16(__m256i __a, __m256i __b)
660{
661 return (__m256i)__builtin_shufflevector((__v16hi)__a, (__v16hi)__b, 4, 16+4, 5, 16+5, 6, 16+6, 7, 16+7, 12, 16+12, 13, 16+13, 14, 16+14, 15, 16+15);
662}
663
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]664static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]665_mm256_unpackhi_epi32(__m256i __a, __m256i __b)
666{
667 return (__m256i)__builtin_shufflevector((__v8si)__a, (__v8si)__b, 2, 8+2, 3, 8+3, 6, 8+6, 7, 8+7);
668}
669
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]670static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]671_mm256_unpackhi_epi64(__m256i __a, __m256i __b)
672{
673 return (__m256i)__builtin_shufflevector((__v4di)__a, (__v4di)__b, 1, 4+1, 3, 4+3);
674}
675
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]676static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]677_mm256_unpacklo_epi8(__m256i __a, __m256i __b)
678{
679 return (__m256i)__builtin_shufflevector((__v32qi)__a, (__v32qi)__b, 0, 32+0, 1, 32+1, 2, 32+2, 3, 32+3, 4, 32+4, 5, 32+5, 6, 32+6, 7, 32+7, 16, 32+16, 17, 32+17, 18, 32+18, 19, 32+19, 20, 32+20, 21, 32+21, 22, 32+22, 23, 32+23);
680}
681
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]682static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]683_mm256_unpacklo_epi16(__m256i __a, __m256i __b)
684{
685 return (__m256i)__builtin_shufflevector((__v16hi)__a, (__v16hi)__b, 0, 16+0, 1, 16+1, 2, 16+2, 3, 16+3, 8, 16+8, 9, 16+9, 10, 16+10, 11, 16+11);
686}
687
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]688static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]689_mm256_unpacklo_epi32(__m256i __a, __m256i __b)
690{
691 return (__m256i)__builtin_shufflevector((__v8si)__a, (__v8si)__b, 0, 8+0, 1, 8+1, 4, 8+4, 5, 8+5);
692}
693
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]694static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]695_mm256_unpacklo_epi64(__m256i __a, __m256i __b)
696{
697 return (__m256i)__builtin_shufflevector((__v4di)__a, (__v4di)__b, 0, 4+0, 2, 4+2);
698}
699
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]700static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]701_mm256_xor_si256(__m256i __a, __m256i __b)
702{
703 return (__m256i)((__v4du)__a ^ (__v4du)__b);
704}
705
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]706static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]707_mm256_stream_load_si256(__m256i const *__V)
708{
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]709 typedef __v4di __v4di_aligned __attribute__((aligned(32)));
710 return (__m256i)__builtin_nontemporal_load((const __v4di_aligned *)__V);
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]711}
712
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]713static __inline__ __m128 __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]714_mm_broadcastss_ps(__m128 __X)
715{
716 return (__m128)__builtin_shufflevector((__v4sf)__X, (__v4sf)__X, 0, 0, 0, 0);
717}
718
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]719static __inline__ __m128d __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]720_mm_broadcastsd_pd(__m128d __a)
721{
722 return __builtin_shufflevector((__v2df)__a, (__v2df)__a, 0, 0);
723}
724
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]725static __inline__ __m256 __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]726_mm256_broadcastss_ps(__m128 __X)
727{
728 return (__m256)__builtin_shufflevector((__v4sf)__X, (__v4sf)__X, 0, 0, 0, 0, 0, 0, 0, 0);
729}
730
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]731static __inline__ __m256d __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]732_mm256_broadcastsd_pd(__m128d __X)
733{
734 return (__m256d)__builtin_shufflevector((__v2df)__X, (__v2df)__X, 0, 0, 0, 0);
735}
736
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]737static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]738_mm256_broadcastsi128_si256(__m128i __X)
739{
740 return (__m256i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 1, 0, 1);
741}
742
Sasha Smundak0fc590b2020-10-07 08:11:59 -0700[diff] [blame]743#define _mm_broadcastsi128_si256(X) _mm256_broadcastsi128_si256(X)
744
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]745#define _mm_blend_epi32(V1, V2, M) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]746 ((__m128i)__builtin_ia32_pblendd128((__v4si)(__m128i)(V1), \
747 (__v4si)(__m128i)(V2), (int)(M)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]748
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]749#define _mm256_blend_epi32(V1, V2, M) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]750 ((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(V1), \
751 (__v8si)(__m256i)(V2), (int)(M)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]752
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]753static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]754_mm256_broadcastb_epi8(__m128i __X)
755{
756 return (__m256i)__builtin_shufflevector((__v16qi)__X, (__v16qi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
757}
758
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]759static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]760_mm256_broadcastw_epi16(__m128i __X)
761{
762 return (__m256i)__builtin_shufflevector((__v8hi)__X, (__v8hi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
763}
764
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]765static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]766_mm256_broadcastd_epi32(__m128i __X)
767{
768 return (__m256i)__builtin_shufflevector((__v4si)__X, (__v4si)__X, 0, 0, 0, 0, 0, 0, 0, 0);
769}
770
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]771static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]772_mm256_broadcastq_epi64(__m128i __X)
773{
774 return (__m256i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 0, 0, 0);
775}
776
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]777static __inline__ __m128i __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]778_mm_broadcastb_epi8(__m128i __X)
779{
780 return (__m128i)__builtin_shufflevector((__v16qi)__X, (__v16qi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
781}
782
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]783static __inline__ __m128i __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]784_mm_broadcastw_epi16(__m128i __X)
785{
786 return (__m128i)__builtin_shufflevector((__v8hi)__X, (__v8hi)__X, 0, 0, 0, 0, 0, 0, 0, 0);
787}
788
789
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]790static __inline__ __m128i __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]791_mm_broadcastd_epi32(__m128i __X)
792{
793 return (__m128i)__builtin_shufflevector((__v4si)__X, (__v4si)__X, 0, 0, 0, 0);
794}
795
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]796static __inline__ __m128i __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]797_mm_broadcastq_epi64(__m128i __X)
798{
799 return (__m128i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 0);
800}
801
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]802static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]803_mm256_permutevar8x32_epi32(__m256i __a, __m256i __b)
804{
805 return (__m256i)__builtin_ia32_permvarsi256((__v8si)__a, (__v8si)__b);
806}
807
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]808#define _mm256_permute4x64_pd(V, M) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]809 ((__m256d)__builtin_ia32_permdf256((__v4df)(__m256d)(V), (int)(M)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]810
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]811static __inline__ __m256 __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]812_mm256_permutevar8x32_ps(__m256 __a, __m256i __b)
813{
814 return (__m256)__builtin_ia32_permvarsf256((__v8sf)__a, (__v8si)__b);
815}
816
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]817#define _mm256_permute4x64_epi64(V, M) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]818 ((__m256i)__builtin_ia32_permdi256((__v4di)(__m256i)(V), (int)(M)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]819
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]820#define _mm256_permute2x128_si256(V1, V2, M) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]821 ((__m256i)__builtin_ia32_permti256((__m256i)(V1), (__m256i)(V2), (int)(M)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]822
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]823#define _mm256_extracti128_si256(V, M) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]824 ((__m128i)__builtin_ia32_extract128i256((__v4di)(__m256i)(V), (int)(M)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]825
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]826#define _mm256_inserti128_si256(V1, V2, M) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]827 ((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(V1), \
828 (__v2di)(__m128i)(V2), (int)(M)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]829
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]830static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]831_mm256_maskload_epi32(int const *__X, __m256i __M)
832{
833 return (__m256i)__builtin_ia32_maskloadd256((const __v8si *)__X, (__v8si)__M);
834}
835
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]836static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]837_mm256_maskload_epi64(long long const *__X, __m256i __M)
838{
839 return (__m256i)__builtin_ia32_maskloadq256((const __v4di *)__X, (__v4di)__M);
840}
841
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]842static __inline__ __m128i __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]843_mm_maskload_epi32(int const *__X, __m128i __M)
844{
845 return (__m128i)__builtin_ia32_maskloadd((const __v4si *)__X, (__v4si)__M);
846}
847
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]848static __inline__ __m128i __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]849_mm_maskload_epi64(long long const *__X, __m128i __M)
850{
851 return (__m128i)__builtin_ia32_maskloadq((const __v2di *)__X, (__v2di)__M);
852}
853
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]854static __inline__ void __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]855_mm256_maskstore_epi32(int *__X, __m256i __M, __m256i __Y)
856{
857 __builtin_ia32_maskstored256((__v8si *)__X, (__v8si)__M, (__v8si)__Y);
858}
859
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]860static __inline__ void __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]861_mm256_maskstore_epi64(long long *__X, __m256i __M, __m256i __Y)
862{
863 __builtin_ia32_maskstoreq256((__v4di *)__X, (__v4di)__M, (__v4di)__Y);
864}
865
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]866static __inline__ void __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]867_mm_maskstore_epi32(int *__X, __m128i __M, __m128i __Y)
868{
869 __builtin_ia32_maskstored((__v4si *)__X, (__v4si)__M, (__v4si)__Y);
870}
871
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]872static __inline__ void __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]873_mm_maskstore_epi64(long long *__X, __m128i __M, __m128i __Y)
874{
875 __builtin_ia32_maskstoreq(( __v2di *)__X, (__v2di)__M, (__v2di)__Y);
876}
877
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]878static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]879_mm256_sllv_epi32(__m256i __X, __m256i __Y)
880{
881 return (__m256i)__builtin_ia32_psllv8si((__v8si)__X, (__v8si)__Y);
882}
883
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]884static __inline__ __m128i __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]885_mm_sllv_epi32(__m128i __X, __m128i __Y)
886{
887 return (__m128i)__builtin_ia32_psllv4si((__v4si)__X, (__v4si)__Y);
888}
889
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]890static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]891_mm256_sllv_epi64(__m256i __X, __m256i __Y)
892{
893 return (__m256i)__builtin_ia32_psllv4di((__v4di)__X, (__v4di)__Y);
894}
895
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]896static __inline__ __m128i __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]897_mm_sllv_epi64(__m128i __X, __m128i __Y)
898{
899 return (__m128i)__builtin_ia32_psllv2di((__v2di)__X, (__v2di)__Y);
900}
901
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]902static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]903_mm256_srav_epi32(__m256i __X, __m256i __Y)
904{
905 return (__m256i)__builtin_ia32_psrav8si((__v8si)__X, (__v8si)__Y);
906}
907
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]908static __inline__ __m128i __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]909_mm_srav_epi32(__m128i __X, __m128i __Y)
910{
911 return (__m128i)__builtin_ia32_psrav4si((__v4si)__X, (__v4si)__Y);
912}
913
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]914static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]915_mm256_srlv_epi32(__m256i __X, __m256i __Y)
916{
917 return (__m256i)__builtin_ia32_psrlv8si((__v8si)__X, (__v8si)__Y);
918}
919
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]920static __inline__ __m128i __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]921_mm_srlv_epi32(__m128i __X, __m128i __Y)
922{
923 return (__m128i)__builtin_ia32_psrlv4si((__v4si)__X, (__v4si)__Y);
924}
925
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]926static __inline__ __m256i __DEFAULT_FN_ATTRS256
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]927_mm256_srlv_epi64(__m256i __X, __m256i __Y)
928{
929 return (__m256i)__builtin_ia32_psrlv4di((__v4di)__X, (__v4di)__Y);
930}
931
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]932static __inline__ __m128i __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]933_mm_srlv_epi64(__m128i __X, __m128i __Y)
934{
935 return (__m128i)__builtin_ia32_psrlv2di((__v2di)__X, (__v2di)__Y);
936}
937
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]938#define _mm_mask_i32gather_pd(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]939 ((__m128d)__builtin_ia32_gatherd_pd((__v2df)(__m128i)(a), \
940 (double const *)(m), \
941 (__v4si)(__m128i)(i), \
942 (__v2df)(__m128d)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]943
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]944#define _mm256_mask_i32gather_pd(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]945 ((__m256d)__builtin_ia32_gatherd_pd256((__v4df)(__m256d)(a), \
946 (double const *)(m), \
947 (__v4si)(__m128i)(i), \
948 (__v4df)(__m256d)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]949
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]950#define _mm_mask_i64gather_pd(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]951 ((__m128d)__builtin_ia32_gatherq_pd((__v2df)(__m128d)(a), \
952 (double const *)(m), \
953 (__v2di)(__m128i)(i), \
954 (__v2df)(__m128d)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]955
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]956#define _mm256_mask_i64gather_pd(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]957 ((__m256d)__builtin_ia32_gatherq_pd256((__v4df)(__m256d)(a), \
958 (double const *)(m), \
959 (__v4di)(__m256i)(i), \
960 (__v4df)(__m256d)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]961
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]962#define _mm_mask_i32gather_ps(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]963 ((__m128)__builtin_ia32_gatherd_ps((__v4sf)(__m128)(a), \
964 (float const *)(m), \
965 (__v4si)(__m128i)(i), \
966 (__v4sf)(__m128)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]967
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]968#define _mm256_mask_i32gather_ps(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]969 ((__m256)__builtin_ia32_gatherd_ps256((__v8sf)(__m256)(a), \
970 (float const *)(m), \
971 (__v8si)(__m256i)(i), \
972 (__v8sf)(__m256)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]973
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]974#define _mm_mask_i64gather_ps(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]975 ((__m128)__builtin_ia32_gatherq_ps((__v4sf)(__m128)(a), \
976 (float const *)(m), \
977 (__v2di)(__m128i)(i), \
978 (__v4sf)(__m128)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]979
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]980#define _mm256_mask_i64gather_ps(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]981 ((__m128)__builtin_ia32_gatherq_ps256((__v4sf)(__m128)(a), \
982 (float const *)(m), \
983 (__v4di)(__m256i)(i), \
984 (__v4sf)(__m128)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]985
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]986#define _mm_mask_i32gather_epi32(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]987 ((__m128i)__builtin_ia32_gatherd_d((__v4si)(__m128i)(a), \
988 (int const *)(m), \
989 (__v4si)(__m128i)(i), \
990 (__v4si)(__m128i)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]991
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]992#define _mm256_mask_i32gather_epi32(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]993 ((__m256i)__builtin_ia32_gatherd_d256((__v8si)(__m256i)(a), \
994 (int const *)(m), \
995 (__v8si)(__m256i)(i), \
996 (__v8si)(__m256i)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]997
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]998#define _mm_mask_i64gather_epi32(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]999 ((__m128i)__builtin_ia32_gatherq_d((__v4si)(__m128i)(a), \
1000 (int const *)(m), \
1001 (__v2di)(__m128i)(i), \
1002 (__v4si)(__m128i)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1003
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1004#define _mm256_mask_i64gather_epi32(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1005 ((__m128i)__builtin_ia32_gatherq_d256((__v4si)(__m128i)(a), \
1006 (int const *)(m), \
1007 (__v4di)(__m256i)(i), \
1008 (__v4si)(__m128i)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1009
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1010#define _mm_mask_i32gather_epi64(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1011 ((__m128i)__builtin_ia32_gatherd_q((__v2di)(__m128i)(a), \
1012 (long long const *)(m), \
1013 (__v4si)(__m128i)(i), \
1014 (__v2di)(__m128i)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1015
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1016#define _mm256_mask_i32gather_epi64(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1017 ((__m256i)__builtin_ia32_gatherd_q256((__v4di)(__m256i)(a), \
1018 (long long const *)(m), \
1019 (__v4si)(__m128i)(i), \
1020 (__v4di)(__m256i)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1021
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1022#define _mm_mask_i64gather_epi64(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1023 ((__m128i)__builtin_ia32_gatherq_q((__v2di)(__m128i)(a), \
1024 (long long const *)(m), \
1025 (__v2di)(__m128i)(i), \
1026 (__v2di)(__m128i)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1027
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1028#define _mm256_mask_i64gather_epi64(a, m, i, mask, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1029 ((__m256i)__builtin_ia32_gatherq_q256((__v4di)(__m256i)(a), \
1030 (long long const *)(m), \
1031 (__v4di)(__m256i)(i), \
1032 (__v4di)(__m256i)(mask), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1033
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1034#define _mm_i32gather_pd(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1035 ((__m128d)__builtin_ia32_gatherd_pd((__v2df)_mm_undefined_pd(), \
1036 (double const *)(m), \
1037 (__v4si)(__m128i)(i), \
1038 (__v2df)_mm_cmpeq_pd(_mm_setzero_pd(), \
1039 _mm_setzero_pd()), \
1040 (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1041
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1042#define _mm256_i32gather_pd(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1043 ((__m256d)__builtin_ia32_gatherd_pd256((__v4df)_mm256_undefined_pd(), \
1044 (double const *)(m), \
1045 (__v4si)(__m128i)(i), \
1046 (__v4df)_mm256_cmp_pd(_mm256_setzero_pd(), \
1047 _mm256_setzero_pd(), \
1048 _CMP_EQ_OQ), \
1049 (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1050
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1051#define _mm_i64gather_pd(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1052 ((__m128d)__builtin_ia32_gatherq_pd((__v2df)_mm_undefined_pd(), \
1053 (double const *)(m), \
1054 (__v2di)(__m128i)(i), \
1055 (__v2df)_mm_cmpeq_pd(_mm_setzero_pd(), \
1056 _mm_setzero_pd()), \
1057 (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1058
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1059#define _mm256_i64gather_pd(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1060 ((__m256d)__builtin_ia32_gatherq_pd256((__v4df)_mm256_undefined_pd(), \
1061 (double const *)(m), \
1062 (__v4di)(__m256i)(i), \
1063 (__v4df)_mm256_cmp_pd(_mm256_setzero_pd(), \
1064 _mm256_setzero_pd(), \
1065 _CMP_EQ_OQ), \
1066 (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1067
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1068#define _mm_i32gather_ps(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1069 ((__m128)__builtin_ia32_gatherd_ps((__v4sf)_mm_undefined_ps(), \
1070 (float const *)(m), \
1071 (__v4si)(__m128i)(i), \
1072 (__v4sf)_mm_cmpeq_ps(_mm_setzero_ps(), \
1073 _mm_setzero_ps()), \
1074 (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1075
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1076#define _mm256_i32gather_ps(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1077 ((__m256)__builtin_ia32_gatherd_ps256((__v8sf)_mm256_undefined_ps(), \
1078 (float const *)(m), \
1079 (__v8si)(__m256i)(i), \
1080 (__v8sf)_mm256_cmp_ps(_mm256_setzero_ps(), \
1081 _mm256_setzero_ps(), \
1082 _CMP_EQ_OQ), \
1083 (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1084
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1085#define _mm_i64gather_ps(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1086 ((__m128)__builtin_ia32_gatherq_ps((__v4sf)_mm_undefined_ps(), \
1087 (float const *)(m), \
1088 (__v2di)(__m128i)(i), \
1089 (__v4sf)_mm_cmpeq_ps(_mm_setzero_ps(), \
1090 _mm_setzero_ps()), \
1091 (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1092
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1093#define _mm256_i64gather_ps(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1094 ((__m128)__builtin_ia32_gatherq_ps256((__v4sf)_mm_undefined_ps(), \
1095 (float const *)(m), \
1096 (__v4di)(__m256i)(i), \
1097 (__v4sf)_mm_cmpeq_ps(_mm_setzero_ps(), \
1098 _mm_setzero_ps()), \
1099 (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1100
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1101#define _mm_i32gather_epi32(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1102 ((__m128i)__builtin_ia32_gatherd_d((__v4si)_mm_undefined_si128(), \
1103 (int const *)(m), (__v4si)(__m128i)(i), \
1104 (__v4si)_mm_set1_epi32(-1), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1105
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1106#define _mm256_i32gather_epi32(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1107 ((__m256i)__builtin_ia32_gatherd_d256((__v8si)_mm256_undefined_si256(), \
1108 (int const *)(m), (__v8si)(__m256i)(i), \
1109 (__v8si)_mm256_set1_epi32(-1), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1110
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1111#define _mm_i64gather_epi32(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1112 ((__m128i)__builtin_ia32_gatherq_d((__v4si)_mm_undefined_si128(), \
1113 (int const *)(m), (__v2di)(__m128i)(i), \
1114 (__v4si)_mm_set1_epi32(-1), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1115
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1116#define _mm256_i64gather_epi32(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1117 ((__m128i)__builtin_ia32_gatherq_d256((__v4si)_mm_undefined_si128(), \
1118 (int const *)(m), (__v4di)(__m256i)(i), \
1119 (__v4si)_mm_set1_epi32(-1), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1120
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1121#define _mm_i32gather_epi64(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1122 ((__m128i)__builtin_ia32_gatherd_q((__v2di)_mm_undefined_si128(), \
1123 (long long const *)(m), \
1124 (__v4si)(__m128i)(i), \
1125 (__v2di)_mm_set1_epi64x(-1), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1126
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1127#define _mm256_i32gather_epi64(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1128 ((__m256i)__builtin_ia32_gatherd_q256((__v4di)_mm256_undefined_si256(), \
1129 (long long const *)(m), \
1130 (__v4si)(__m128i)(i), \
1131 (__v4di)_mm256_set1_epi64x(-1), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1132
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1133#define _mm_i64gather_epi64(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1134 ((__m128i)__builtin_ia32_gatherq_q((__v2di)_mm_undefined_si128(), \
1135 (long long const *)(m), \
1136 (__v2di)(__m128i)(i), \
1137 (__v2di)_mm_set1_epi64x(-1), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1138
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1139#define _mm256_i64gather_epi64(m, i, s) \
Pirama Arumuga Nainar494f6452021-12-02 10:42:14 -0800[diff] [blame]1140 ((__m256i)__builtin_ia32_gatherq_q256((__v4di)_mm256_undefined_si256(), \
1141 (long long const *)(m), \
1142 (__v4di)(__m256i)(i), \
1143 (__v4di)_mm256_set1_epi64x(-1), (s)))
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1144
Logan Chien55afb0a2018-10-15 10:42:14 +0800[diff] [blame]1145#undef __DEFAULT_FN_ATTRS256
1146#undef __DEFAULT_FN_ATTRS128
Logan Chien2833ffb2018-10-09 10:03:24 +0800[diff] [blame]1147
1148#endif /* __AVX2INTRIN_H */