| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 1 | /*===---- tbmintrin.h - TBM intrinsics -------------------------------------=== | 
|  | 2 | * | 
|  | 3 | * Permission is hereby granted, free of charge, to any person obtaining a copy | 
|  | 4 | * of this software and associated documentation files (the "Software"), to deal | 
|  | 5 | * in the Software without restriction, including without limitation the rights | 
|  | 6 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | 
|  | 7 | * copies of the Software, and to permit persons to whom the Software is | 
|  | 8 | * furnished to do so, subject to the following conditions: | 
|  | 9 | * | 
|  | 10 | * The above copyright notice and this permission notice shall be included in | 
|  | 11 | * all copies or substantial portions of the Software. | 
|  | 12 | * | 
|  | 13 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | 
|  | 14 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | 
|  | 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | 
|  | 16 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | 
|  | 17 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | 
|  | 18 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN | 
|  | 19 | * THE SOFTWARE. | 
|  | 20 | * | 
|  | 21 | *===-----------------------------------------------------------------------=== | 
|  | 22 | */ | 
|  | 23 |  | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 24 | #ifndef __X86INTRIN_H | 
|  | 25 | #error "Never use <tbmintrin.h> directly; include <x86intrin.h> instead." | 
|  | 26 | #endif | 
|  | 27 |  | 
|  | 28 | #ifndef __TBMINTRIN_H | 
|  | 29 | #define __TBMINTRIN_H | 
|  | 30 |  | 
| Eric Christopher | 4d185168 | 2015-06-17 07:09:20 +0000 | [diff] [blame] | 31 | /* Define the default attributes for the functions in this file. */ | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 32 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("tbm"))) | 
| Eric Christopher | 4d185168 | 2015-06-17 07:09:20 +0000 | [diff] [blame] | 33 |  | 
| Craig Topper | d619eaaa | 2015-11-11 03:47:10 +0000 | [diff] [blame] | 34 | #define __bextri_u32(a, b) \ | 
|  | 35 | ((unsigned int)__builtin_ia32_bextri_u32((unsigned int)(a), \ | 
|  | 36 | (unsigned int)(b))) | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 37 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 38 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 39 | __blcfill_u32(unsigned int __a) | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 40 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 41 | return __a & (__a + 1); | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 42 | } | 
|  | 43 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 44 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 45 | __blci_u32(unsigned int __a) | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 46 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 47 | return __a | ~(__a + 1); | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 48 | } | 
|  | 49 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 50 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 51 | __blcic_u32(unsigned int __a) | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 52 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 53 | return ~__a & (__a + 1); | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 54 | } | 
|  | 55 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 56 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 57 | __blcmsk_u32(unsigned int __a) | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 58 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 59 | return __a ^ (__a + 1); | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 60 | } | 
|  | 61 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 62 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 63 | __blcs_u32(unsigned int __a) | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 64 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 65 | return __a | (__a + 1); | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 66 | } | 
|  | 67 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 68 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 69 | __blsfill_u32(unsigned int __a) | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 70 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 71 | return __a | (__a - 1); | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 72 | } | 
|  | 73 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 74 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 75 | __blsic_u32(unsigned int __a) | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 76 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 77 | return ~__a | (__a - 1); | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 78 | } | 
|  | 79 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 80 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 81 | __t1mskc_u32(unsigned int __a) | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 82 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 83 | return ~__a | (__a + 1); | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 84 | } | 
|  | 85 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 86 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 87 | __tzmsk_u32(unsigned int __a) | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 88 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 89 | return ~__a & (__a - 1); | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 90 | } | 
|  | 91 |  | 
|  | 92 | #ifdef __x86_64__ | 
| Craig Topper | d619eaaa | 2015-11-11 03:47:10 +0000 | [diff] [blame] | 93 | #define __bextri_u64(a, b) \ | 
|  | 94 | ((unsigned long long)__builtin_ia32_bextri_u64((unsigned long long)(a), \ | 
|  | 95 | (unsigned long long)(b))) | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 96 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 97 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 98 | __blcfill_u64(unsigned long long __a) | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 99 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 100 | return __a & (__a + 1); | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 101 | } | 
|  | 102 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 103 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 104 | __blci_u64(unsigned long long __a) | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 105 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 106 | return __a | ~(__a + 1); | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 107 | } | 
|  | 108 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 109 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 110 | __blcic_u64(unsigned long long __a) | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 111 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 112 | return ~__a & (__a + 1); | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 113 | } | 
|  | 114 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 115 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 116 | __blcmsk_u64(unsigned long long __a) | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 117 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 118 | return __a ^ (__a + 1); | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 119 | } | 
|  | 120 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 121 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 122 | __blcs_u64(unsigned long long __a) | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 123 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 124 | return __a | (__a + 1); | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 125 | } | 
|  | 126 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 127 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 128 | __blsfill_u64(unsigned long long __a) | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 129 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 130 | return __a | (__a - 1); | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 131 | } | 
|  | 132 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 133 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 134 | __blsic_u64(unsigned long long __a) | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 135 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 136 | return ~__a | (__a - 1); | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 137 | } | 
|  | 138 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 139 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 140 | __t1mskc_u64(unsigned long long __a) | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 141 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 142 | return ~__a | (__a + 1); | 
| Craig Topper | d335c9d | 2013-10-05 17:08:42 +0000 | [diff] [blame] | 143 | } | 
|  | 144 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 145 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 146 | __tzmsk_u64(unsigned long long __a) | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 147 | { | 
| Eric Christopher | 39a84d0 | 2016-02-12 02:22:53 +0000 | [diff] [blame] | 148 | return ~__a & (__a - 1); | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 149 | } | 
|  | 150 | #endif | 
|  | 151 |  | 
| Michael Kuperstein | e45af54 | 2015-06-30 13:36:19 +0000 | [diff] [blame] | 152 | #undef __DEFAULT_FN_ATTRS | 
| Eric Christopher | 4d185168 | 2015-06-17 07:09:20 +0000 | [diff] [blame] | 153 |  | 
| Yunzhong Gao | f4e0b10 | 2013-09-30 17:25:14 +0000 | [diff] [blame] | 154 | #endif /* __TBMINTRIN_H */ |