Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 1 | /*===---- avx512vlcdintrin.h - AVX512VL and AVX512CD intrinsics ------------=== |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 2 | * |
Logan Chien | df4f766 | 2019-09-04 16:45:23 -0700 | [diff] [blame] | 3 | * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | * See https://llvm.org/LICENSE.txt for license information. |
| 5 | * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 6 | * |
| 7 | *===-----------------------------------------------------------------------=== |
| 8 | */ |
| 9 | #ifndef __IMMINTRIN_H |
| 10 | #error "Never use <avx512vlcdintrin.h> directly; include <immintrin.h> instead." |
| 11 | #endif |
| 12 | |
| 13 | #ifndef __AVX512VLCDINTRIN_H |
| 14 | #define __AVX512VLCDINTRIN_H |
| 15 | |
| 16 | /* Define the default attributes for the functions in this file. */ |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 17 | #define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avx512vl,avx512cd"), __min_vector_width__(128))) |
| 18 | #define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("avx512vl,avx512cd"), __min_vector_width__(256))) |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 19 | |
| 20 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 21 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 22 | _mm_broadcastmb_epi64 (__mmask8 __A) |
| 23 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 24 | return (__m128i) _mm_set1_epi64x((long long) __A); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 25 | } |
| 26 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 27 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 28 | _mm256_broadcastmb_epi64 (__mmask8 __A) |
| 29 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 30 | return (__m256i) _mm256_set1_epi64x((long long)__A); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 31 | } |
| 32 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 33 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 34 | _mm_broadcastmw_epi32 (__mmask16 __A) |
| 35 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 36 | return (__m128i) _mm_set1_epi32((int)__A); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 37 | } |
| 38 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 39 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 40 | _mm256_broadcastmw_epi32 (__mmask16 __A) |
| 41 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 42 | return (__m256i) _mm256_set1_epi32((int)__A); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 43 | } |
| 44 | |
| 45 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 46 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 47 | _mm_conflict_epi64 (__m128i __A) |
| 48 | { |
Logan Chien | dbcf412 | 2019-03-21 10:50:25 +0800 | [diff] [blame] | 49 | return (__m128i) __builtin_ia32_vpconflictdi_128 ((__v2di) __A); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 50 | } |
| 51 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 52 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 53 | _mm_mask_conflict_epi64 (__m128i __W, __mmask8 __U, __m128i __A) |
| 54 | { |
Logan Chien | dbcf412 | 2019-03-21 10:50:25 +0800 | [diff] [blame] | 55 | return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, |
| 56 | (__v2di)_mm_conflict_epi64(__A), |
| 57 | (__v2di)__W); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 58 | } |
| 59 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 60 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 61 | _mm_maskz_conflict_epi64 (__mmask8 __U, __m128i __A) |
| 62 | { |
Logan Chien | dbcf412 | 2019-03-21 10:50:25 +0800 | [diff] [blame] | 63 | return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, |
| 64 | (__v2di)_mm_conflict_epi64(__A), |
| 65 | (__v2di)_mm_setzero_si128()); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 66 | } |
| 67 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 68 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 69 | _mm256_conflict_epi64 (__m256i __A) |
| 70 | { |
Logan Chien | dbcf412 | 2019-03-21 10:50:25 +0800 | [diff] [blame] | 71 | return (__m256i) __builtin_ia32_vpconflictdi_256 ((__v4di) __A); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 72 | } |
| 73 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 74 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 75 | _mm256_mask_conflict_epi64 (__m256i __W, __mmask8 __U, __m256i __A) |
| 76 | { |
Logan Chien | dbcf412 | 2019-03-21 10:50:25 +0800 | [diff] [blame] | 77 | return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, |
| 78 | (__v4di)_mm256_conflict_epi64(__A), |
| 79 | (__v4di)__W); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 80 | } |
| 81 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 82 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 83 | _mm256_maskz_conflict_epi64 (__mmask8 __U, __m256i __A) |
| 84 | { |
Logan Chien | dbcf412 | 2019-03-21 10:50:25 +0800 | [diff] [blame] | 85 | return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, |
| 86 | (__v4di)_mm256_conflict_epi64(__A), |
| 87 | (__v4di)_mm256_setzero_si256()); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 88 | } |
| 89 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 90 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 91 | _mm_conflict_epi32 (__m128i __A) |
| 92 | { |
Logan Chien | dbcf412 | 2019-03-21 10:50:25 +0800 | [diff] [blame] | 93 | return (__m128i) __builtin_ia32_vpconflictsi_128 ((__v4si) __A); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 94 | } |
| 95 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 96 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 97 | _mm_mask_conflict_epi32 (__m128i __W, __mmask8 __U, __m128i __A) |
| 98 | { |
Logan Chien | dbcf412 | 2019-03-21 10:50:25 +0800 | [diff] [blame] | 99 | return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U, |
| 100 | (__v4si)_mm_conflict_epi32(__A), |
| 101 | (__v4si)__W); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 102 | } |
| 103 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 104 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 105 | _mm_maskz_conflict_epi32 (__mmask8 __U, __m128i __A) |
| 106 | { |
Logan Chien | dbcf412 | 2019-03-21 10:50:25 +0800 | [diff] [blame] | 107 | return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U, |
| 108 | (__v4si)_mm_conflict_epi32(__A), |
| 109 | (__v4si)_mm_setzero_si128()); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 110 | } |
| 111 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 112 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 113 | _mm256_conflict_epi32 (__m256i __A) |
| 114 | { |
Logan Chien | dbcf412 | 2019-03-21 10:50:25 +0800 | [diff] [blame] | 115 | return (__m256i) __builtin_ia32_vpconflictsi_256 ((__v8si) __A); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 116 | } |
| 117 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 118 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 119 | _mm256_mask_conflict_epi32 (__m256i __W, __mmask8 __U, __m256i __A) |
| 120 | { |
Logan Chien | dbcf412 | 2019-03-21 10:50:25 +0800 | [diff] [blame] | 121 | return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U, |
| 122 | (__v8si)_mm256_conflict_epi32(__A), |
| 123 | (__v8si)__W); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 124 | } |
| 125 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 126 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 127 | _mm256_maskz_conflict_epi32 (__mmask8 __U, __m256i __A) |
| 128 | { |
Logan Chien | dbcf412 | 2019-03-21 10:50:25 +0800 | [diff] [blame] | 129 | return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U, |
| 130 | (__v8si)_mm256_conflict_epi32(__A), |
| 131 | (__v8si)_mm256_setzero_si256()); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 132 | } |
| 133 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 134 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 135 | _mm_lzcnt_epi32 (__m128i __A) |
| 136 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 137 | return (__m128i) __builtin_ia32_vplzcntd_128 ((__v4si) __A); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 138 | } |
| 139 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 140 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 141 | _mm_mask_lzcnt_epi32 (__m128i __W, __mmask8 __U, __m128i __A) |
| 142 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 143 | return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U, |
| 144 | (__v4si)_mm_lzcnt_epi32(__A), |
| 145 | (__v4si)__W); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 146 | } |
| 147 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 148 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 149 | _mm_maskz_lzcnt_epi32 (__mmask8 __U, __m128i __A) |
| 150 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 151 | return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U, |
| 152 | (__v4si)_mm_lzcnt_epi32(__A), |
| 153 | (__v4si)_mm_setzero_si128()); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 154 | } |
| 155 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 156 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 157 | _mm256_lzcnt_epi32 (__m256i __A) |
| 158 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 159 | return (__m256i) __builtin_ia32_vplzcntd_256 ((__v8si) __A); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 160 | } |
| 161 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 162 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 163 | _mm256_mask_lzcnt_epi32 (__m256i __W, __mmask8 __U, __m256i __A) |
| 164 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 165 | return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U, |
| 166 | (__v8si)_mm256_lzcnt_epi32(__A), |
| 167 | (__v8si)__W); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 168 | } |
| 169 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 170 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 171 | _mm256_maskz_lzcnt_epi32 (__mmask8 __U, __m256i __A) |
| 172 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 173 | return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U, |
| 174 | (__v8si)_mm256_lzcnt_epi32(__A), |
| 175 | (__v8si)_mm256_setzero_si256()); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 176 | } |
| 177 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 178 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 179 | _mm_lzcnt_epi64 (__m128i __A) |
| 180 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 181 | return (__m128i) __builtin_ia32_vplzcntq_128 ((__v2di) __A); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 182 | } |
| 183 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 184 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 185 | _mm_mask_lzcnt_epi64 (__m128i __W, __mmask8 __U, __m128i __A) |
| 186 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 187 | return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, |
| 188 | (__v2di)_mm_lzcnt_epi64(__A), |
| 189 | (__v2di)__W); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 190 | } |
| 191 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 192 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 193 | _mm_maskz_lzcnt_epi64 (__mmask8 __U, __m128i __A) |
| 194 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 195 | return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, |
| 196 | (__v2di)_mm_lzcnt_epi64(__A), |
| 197 | (__v2di)_mm_setzero_si128()); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 198 | } |
| 199 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 200 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 201 | _mm256_lzcnt_epi64 (__m256i __A) |
| 202 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 203 | return (__m256i) __builtin_ia32_vplzcntq_256 ((__v4di) __A); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 204 | } |
| 205 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 206 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 207 | _mm256_mask_lzcnt_epi64 (__m256i __W, __mmask8 __U, __m256i __A) |
| 208 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 209 | return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, |
| 210 | (__v4di)_mm256_lzcnt_epi64(__A), |
| 211 | (__v4di)__W); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 212 | } |
| 213 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 214 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 215 | _mm256_maskz_lzcnt_epi64 (__mmask8 __U, __m256i __A) |
| 216 | { |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 217 | return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, |
| 218 | (__v4di)_mm256_lzcnt_epi64(__A), |
| 219 | (__v4di)_mm256_setzero_si256()); |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 220 | } |
| 221 | |
Logan Chien | 55afb0a | 2018-10-15 10:42:14 +0800 | [diff] [blame] | 222 | #undef __DEFAULT_FN_ATTRS128 |
| 223 | #undef __DEFAULT_FN_ATTRS256 |
Logan Chien | 2833ffb | 2018-10-09 10:03:24 +0800 | [diff] [blame] | 224 | |
| 225 | #endif /* __AVX512VLCDINTRIN_H */ |