blob: 9ab0a73aad6c50923e17b2c85914abf0fd945a7c [file] [log] [blame]
Benjamin Kramer0930b6e2010-08-20 16:47:17 +00001/*===---- emmintrin.h - SSE2 intrinsics ------------------------------------===
Anders Carlsson37f2f002008-12-24 01:45:22 +00002 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy
4 * of this software and associated documentation files (the "Software"), to deal
5 * in the Software without restriction, including without limitation the rights
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *
21 *===-----------------------------------------------------------------------===
22 */
Benjamin Kramer0930b6e2010-08-20 16:47:17 +000023
Anders Carlsson37f2f002008-12-24 01:45:22 +000024#ifndef __EMMINTRIN_H
25#define __EMMINTRIN_H
26
27#ifndef __SSE2__
28#error "SSE2 instruction set not enabled"
29#else
30
31#include <xmmintrin.h>
32
33typedef double __m128d __attribute__((__vector_size__(16)));
34typedef long long __m128i __attribute__((__vector_size__(16)));
35
Eric Christopherfaf94042010-08-26 02:09:25 +000036/* Type defines. */
37typedef double __v2df __attribute__ ((__vector_size__ (16)));
38typedef long long __v2di __attribute__ ((__vector_size__ (16)));
Anders Carlsson07603aa2008-12-24 02:41:00 +000039typedef short __v8hi __attribute__((__vector_size__(16)));
Anders Carlsson4bf4e302009-09-18 19:18:19 +000040typedef char __v16qi __attribute__((__vector_size__(16)));
Anders Carlsson37f2f002008-12-24 01:45:22 +000041
Chris Lattner1bddbcb2010-03-22 18:14:12 +000042static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +000043_mm_add_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +000044{
David Blaikie4f918ae2013-01-16 23:08:36 +000045 __a[0] += __b[0];
46 return __a;
Anders Carlsson37f2f002008-12-24 01:45:22 +000047}
48
Chris Lattner1bddbcb2010-03-22 18:14:12 +000049static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +000050_mm_add_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +000051{
David Blaikie4f918ae2013-01-16 23:08:36 +000052 return __a + __b;
Anders Carlsson37f2f002008-12-24 01:45:22 +000053}
54
Chris Lattner1bddbcb2010-03-22 18:14:12 +000055static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +000056_mm_sub_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +000057{
David Blaikie4f918ae2013-01-16 23:08:36 +000058 __a[0] -= __b[0];
59 return __a;
Anders Carlsson37f2f002008-12-24 01:45:22 +000060}
61
Chris Lattner1bddbcb2010-03-22 18:14:12 +000062static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +000063_mm_sub_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +000064{
David Blaikie4f918ae2013-01-16 23:08:36 +000065 return __a - __b;
Anders Carlsson37f2f002008-12-24 01:45:22 +000066}
67
Chris Lattner1bddbcb2010-03-22 18:14:12 +000068static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +000069_mm_mul_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +000070{
David Blaikie4f918ae2013-01-16 23:08:36 +000071 __a[0] *= __b[0];
72 return __a;
Anders Carlsson37f2f002008-12-24 01:45:22 +000073}
74
Chris Lattner1bddbcb2010-03-22 18:14:12 +000075static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +000076_mm_mul_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +000077{
David Blaikie4f918ae2013-01-16 23:08:36 +000078 return __a * __b;
Anders Carlsson37f2f002008-12-24 01:45:22 +000079}
80
Chris Lattner1bddbcb2010-03-22 18:14:12 +000081static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +000082_mm_div_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +000083{
David Blaikie4f918ae2013-01-16 23:08:36 +000084 __a[0] /= __b[0];
85 return __a;
Anders Carlsson37f2f002008-12-24 01:45:22 +000086}
87
Chris Lattner1bddbcb2010-03-22 18:14:12 +000088static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +000089_mm_div_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +000090{
David Blaikie4f918ae2013-01-16 23:08:36 +000091 return __a / __b;
Anders Carlsson37f2f002008-12-24 01:45:22 +000092}
93
Chris Lattner1bddbcb2010-03-22 18:14:12 +000094static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +000095_mm_sqrt_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +000096{
David Blaikie4f918ae2013-01-16 23:08:36 +000097 __m128d __c = __builtin_ia32_sqrtsd(__b);
98 return (__m128d) { __c[0], __a[1] };
Anders Carlsson37f2f002008-12-24 01:45:22 +000099}
100
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000101static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000102_mm_sqrt_pd(__m128d __a)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000103{
David Blaikie4f918ae2013-01-16 23:08:36 +0000104 return __builtin_ia32_sqrtpd(__a);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000105}
106
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000107static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000108_mm_min_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000109{
David Blaikie4f918ae2013-01-16 23:08:36 +0000110 return __builtin_ia32_minsd(__a, __b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000111}
112
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000113static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000114_mm_min_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000115{
David Blaikie4f918ae2013-01-16 23:08:36 +0000116 return __builtin_ia32_minpd(__a, __b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000117}
118
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000119static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000120_mm_max_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000121{
David Blaikie4f918ae2013-01-16 23:08:36 +0000122 return __builtin_ia32_maxsd(__a, __b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000123}
124
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000125static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000126_mm_max_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000127{
David Blaikie4f918ae2013-01-16 23:08:36 +0000128 return __builtin_ia32_maxpd(__a, __b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000129}
130
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000131static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000132_mm_and_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000133{
David Blaikie4f918ae2013-01-16 23:08:36 +0000134 return (__m128d)((__v4si)__a & (__v4si)__b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000135}
136
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000137static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000138_mm_andnot_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000139{
David Blaikie4f918ae2013-01-16 23:08:36 +0000140 return (__m128d)(~(__v4si)__a & (__v4si)__b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000141}
142
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000143static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000144_mm_or_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000145{
David Blaikie4f918ae2013-01-16 23:08:36 +0000146 return (__m128d)((__v4si)__a | (__v4si)__b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000147}
148
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000149static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000150_mm_xor_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000151{
David Blaikie4f918ae2013-01-16 23:08:36 +0000152 return (__m128d)((__v4si)__a ^ (__v4si)__b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000153}
154
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000155static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000156_mm_cmpeq_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000157{
David Blaikie4f918ae2013-01-16 23:08:36 +0000158 return (__m128d)__builtin_ia32_cmppd(__a, __b, 0);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000159}
160
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000161static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000162_mm_cmplt_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000163{
David Blaikie4f918ae2013-01-16 23:08:36 +0000164 return (__m128d)__builtin_ia32_cmppd(__a, __b, 1);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000165}
166
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000167static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000168_mm_cmple_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000169{
David Blaikie4f918ae2013-01-16 23:08:36 +0000170 return (__m128d)__builtin_ia32_cmppd(__a, __b, 2);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000171}
172
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000173static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000174_mm_cmpgt_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000175{
David Blaikie4f918ae2013-01-16 23:08:36 +0000176 return (__m128d)__builtin_ia32_cmppd(__b, __a, 1);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000177}
178
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000179static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000180_mm_cmpge_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000181{
David Blaikie4f918ae2013-01-16 23:08:36 +0000182 return (__m128d)__builtin_ia32_cmppd(__b, __a, 2);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000183}
184
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000185static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000186_mm_cmpord_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000187{
David Blaikie4f918ae2013-01-16 23:08:36 +0000188 return (__m128d)__builtin_ia32_cmppd(__a, __b, 7);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000189}
190
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000191static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000192_mm_cmpunord_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000193{
David Blaikie4f918ae2013-01-16 23:08:36 +0000194 return (__m128d)__builtin_ia32_cmppd(__a, __b, 3);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000195}
196
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000197static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000198_mm_cmpneq_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000199{
David Blaikie4f918ae2013-01-16 23:08:36 +0000200 return (__m128d)__builtin_ia32_cmppd(__a, __b, 4);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000201}
202
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000203static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000204_mm_cmpnlt_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000205{
David Blaikie4f918ae2013-01-16 23:08:36 +0000206 return (__m128d)__builtin_ia32_cmppd(__a, __b, 5);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000207}
208
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000209static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000210_mm_cmpnle_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000211{
David Blaikie4f918ae2013-01-16 23:08:36 +0000212 return (__m128d)__builtin_ia32_cmppd(__a, __b, 6);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000213}
214
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000215static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000216_mm_cmpngt_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000217{
David Blaikie4f918ae2013-01-16 23:08:36 +0000218 return (__m128d)__builtin_ia32_cmppd(__b, __a, 5);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000219}
220
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000221static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000222_mm_cmpnge_pd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000223{
David Blaikie4f918ae2013-01-16 23:08:36 +0000224 return (__m128d)__builtin_ia32_cmppd(__b, __a, 6);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000225}
226
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000227static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000228_mm_cmpeq_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000229{
David Blaikie4f918ae2013-01-16 23:08:36 +0000230 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 0);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000231}
232
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000233static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000234_mm_cmplt_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000235{
David Blaikie4f918ae2013-01-16 23:08:36 +0000236 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 1);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000237}
238
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000239static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000240_mm_cmple_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000241{
David Blaikie4f918ae2013-01-16 23:08:36 +0000242 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 2);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000243}
244
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000245static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000246_mm_cmpgt_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000247{
Manman Ren492d84c02013-06-17 19:42:49 +0000248 __m128d __c = __builtin_ia32_cmpsd(__b, __a, 1);
249 return (__m128d) { __c[0], __a[1] };
Anders Carlsson37f2f002008-12-24 01:45:22 +0000250}
251
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000252static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000253_mm_cmpge_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000254{
Manman Ren492d84c02013-06-17 19:42:49 +0000255 __m128d __c = __builtin_ia32_cmpsd(__b, __a, 2);
256 return (__m128d) { __c[0], __a[1] };
Anders Carlsson37f2f002008-12-24 01:45:22 +0000257}
258
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000259static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000260_mm_cmpord_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000261{
David Blaikie4f918ae2013-01-16 23:08:36 +0000262 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 7);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000263}
264
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000265static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000266_mm_cmpunord_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000267{
David Blaikie4f918ae2013-01-16 23:08:36 +0000268 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 3);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000269}
270
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000271static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000272_mm_cmpneq_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000273{
David Blaikie4f918ae2013-01-16 23:08:36 +0000274 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 4);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000275}
276
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000277static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000278_mm_cmpnlt_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000279{
David Blaikie4f918ae2013-01-16 23:08:36 +0000280 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 5);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000281}
282
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000283static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000284_mm_cmpnle_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000285{
David Blaikie4f918ae2013-01-16 23:08:36 +0000286 return (__m128d)__builtin_ia32_cmpsd(__a, __b, 6);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000287}
288
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000289static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000290_mm_cmpngt_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000291{
Manman Ren492d84c02013-06-17 19:42:49 +0000292 __m128d __c = __builtin_ia32_cmpsd(__b, __a, 5);
293 return (__m128d) { __c[0], __a[1] };
Anders Carlsson37f2f002008-12-24 01:45:22 +0000294}
295
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000296static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000297_mm_cmpnge_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000298{
Manman Ren492d84c02013-06-17 19:42:49 +0000299 __m128d __c = __builtin_ia32_cmpsd(__b, __a, 6);
300 return (__m128d) { __c[0], __a[1] };
Anders Carlsson37f2f002008-12-24 01:45:22 +0000301}
302
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000303static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000304_mm_comieq_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000305{
David Blaikie4f918ae2013-01-16 23:08:36 +0000306 return __builtin_ia32_comisdeq(__a, __b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000307}
308
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000309static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000310_mm_comilt_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000311{
David Blaikie4f918ae2013-01-16 23:08:36 +0000312 return __builtin_ia32_comisdlt(__a, __b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000313}
314
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000315static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000316_mm_comile_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000317{
David Blaikie4f918ae2013-01-16 23:08:36 +0000318 return __builtin_ia32_comisdle(__a, __b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000319}
320
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000321static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000322_mm_comigt_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000323{
David Blaikie4f918ae2013-01-16 23:08:36 +0000324 return __builtin_ia32_comisdgt(__a, __b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000325}
326
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000327static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000328_mm_comige_sd(__m128d __a, __m128d __b)
Eli Friedman8052a4f2011-10-06 20:31:50 +0000329{
David Blaikie4f918ae2013-01-16 23:08:36 +0000330 return __builtin_ia32_comisdge(__a, __b);
Eli Friedman8052a4f2011-10-06 20:31:50 +0000331}
332
333static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000334_mm_comineq_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000335{
David Blaikie4f918ae2013-01-16 23:08:36 +0000336 return __builtin_ia32_comisdneq(__a, __b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000337}
338
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000339static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000340_mm_ucomieq_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000341{
David Blaikie4f918ae2013-01-16 23:08:36 +0000342 return __builtin_ia32_ucomisdeq(__a, __b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000343}
344
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000345static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000346_mm_ucomilt_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000347{
David Blaikie4f918ae2013-01-16 23:08:36 +0000348 return __builtin_ia32_ucomisdlt(__a, __b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000349}
350
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000351static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000352_mm_ucomile_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000353{
David Blaikie4f918ae2013-01-16 23:08:36 +0000354 return __builtin_ia32_ucomisdle(__a, __b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000355}
356
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000357static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000358_mm_ucomigt_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000359{
David Blaikie4f918ae2013-01-16 23:08:36 +0000360 return __builtin_ia32_ucomisdgt(__a, __b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000361}
362
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000363static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000364_mm_ucomige_sd(__m128d __a, __m128d __b)
Eli Friedmanef534ce2011-08-29 21:26:24 +0000365{
David Blaikie4f918ae2013-01-16 23:08:36 +0000366 return __builtin_ia32_ucomisdge(__a, __b);
Eli Friedmanef534ce2011-08-29 21:26:24 +0000367}
368
369static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000370_mm_ucomineq_sd(__m128d __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000371{
David Blaikie4f918ae2013-01-16 23:08:36 +0000372 return __builtin_ia32_ucomisdneq(__a, __b);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000373}
374
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000375static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000376_mm_cvtpd_ps(__m128d __a)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000377{
David Blaikie4f918ae2013-01-16 23:08:36 +0000378 return __builtin_ia32_cvtpd2ps(__a);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000379}
380
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000381static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000382_mm_cvtps_pd(__m128 __a)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000383{
David Blaikie4f918ae2013-01-16 23:08:36 +0000384 return __builtin_ia32_cvtps2pd(__a);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000385}
386
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000387static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000388_mm_cvtepi32_pd(__m128i __a)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000389{
David Blaikie4f918ae2013-01-16 23:08:36 +0000390 return __builtin_ia32_cvtdq2pd((__v4si)__a);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000391}
392
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000393static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000394_mm_cvtpd_epi32(__m128d __a)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000395{
David Blaikie4f918ae2013-01-16 23:08:36 +0000396 return __builtin_ia32_cvtpd2dq(__a);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000397}
398
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000399static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000400_mm_cvtsd_si32(__m128d __a)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000401{
David Blaikie4f918ae2013-01-16 23:08:36 +0000402 return __builtin_ia32_cvtsd2si(__a);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000403}
404
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000405static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000406_mm_cvtsd_ss(__m128 __a, __m128d __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000407{
David Blaikie4f918ae2013-01-16 23:08:36 +0000408 __a[0] = __b[0];
409 return __a;
Anders Carlsson37f2f002008-12-24 01:45:22 +0000410}
411
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000412static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000413_mm_cvtsi32_sd(__m128d __a, int __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000414{
David Blaikie4f918ae2013-01-16 23:08:36 +0000415 __a[0] = __b;
416 return __a;
Anders Carlsson37f2f002008-12-24 01:45:22 +0000417}
418
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000419static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000420_mm_cvtss_sd(__m128d __a, __m128 __b)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000421{
David Blaikie4f918ae2013-01-16 23:08:36 +0000422 __a[0] = __b[0];
423 return __a;
Anders Carlsson37f2f002008-12-24 01:45:22 +0000424}
425
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000426static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000427_mm_cvttpd_epi32(__m128d __a)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000428{
David Blaikie4f918ae2013-01-16 23:08:36 +0000429 return (__m128i)__builtin_ia32_cvttpd2dq(__a);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000430}
431
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000432static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000433_mm_cvttsd_si32(__m128d __a)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000434{
David Blaikie4f918ae2013-01-16 23:08:36 +0000435 return __a[0];
Anders Carlsson37f2f002008-12-24 01:45:22 +0000436}
437
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000438static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000439_mm_cvtpd_pi32(__m128d __a)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000440{
David Blaikie4f918ae2013-01-16 23:08:36 +0000441 return (__m64)__builtin_ia32_cvtpd2pi(__a);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000442}
443
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000444static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000445_mm_cvttpd_pi32(__m128d __a)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000446{
David Blaikie4f918ae2013-01-16 23:08:36 +0000447 return (__m64)__builtin_ia32_cvttpd2pi(__a);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000448}
449
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000450static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000451_mm_cvtpi32_pd(__m64 __a)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000452{
David Blaikie4f918ae2013-01-16 23:08:36 +0000453 return __builtin_ia32_cvtpi2pd((__v2si)__a);
Anders Carlsson37f2f002008-12-24 01:45:22 +0000454}
455
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000456static __inline__ double __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000457_mm_cvtsd_f64(__m128d __a)
Anders Carlsson37f2f002008-12-24 01:45:22 +0000458{
David Blaikie4f918ae2013-01-16 23:08:36 +0000459 return __a[0];
Anders Carlsson37f2f002008-12-24 01:45:22 +0000460}
461
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000462static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000463_mm_load_pd(double const *__dp)
Anders Carlsson445afa02008-12-24 02:11:54 +0000464{
David Blaikie4f918ae2013-01-16 23:08:36 +0000465 return *(__m128d*)__dp;
Anders Carlsson445afa02008-12-24 02:11:54 +0000466}
467
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000468static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000469_mm_load1_pd(double const *__dp)
Anders Carlsson445afa02008-12-24 02:11:54 +0000470{
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000471 struct __mm_load1_pd_struct {
David Blaikie4f918ae2013-01-16 23:08:36 +0000472 double __u;
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000473 } __attribute__((__packed__, __may_alias__));
David Blaikie4f918ae2013-01-16 23:08:36 +0000474 double __u = ((struct __mm_load1_pd_struct*)__dp)->__u;
475 return (__m128d){ __u, __u };
Anders Carlsson445afa02008-12-24 02:11:54 +0000476}
477
Eli Friedmandb7351a2009-06-02 05:55:48 +0000478#define _mm_load_pd1(dp) _mm_load1_pd(dp)
479
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000480static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000481_mm_loadr_pd(double const *__dp)
Anders Carlsson445afa02008-12-24 02:11:54 +0000482{
David Blaikie4f918ae2013-01-16 23:08:36 +0000483 __m128d __u = *(__m128d*)__dp;
484 return __builtin_shufflevector(__u, __u, 1, 0);
Anders Carlsson445afa02008-12-24 02:11:54 +0000485}
486
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000487static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000488_mm_loadu_pd(double const *__dp)
Anders Carlsson445afa02008-12-24 02:11:54 +0000489{
Bill Wendlingeed92a12011-05-13 00:11:39 +0000490 struct __loadu_pd {
David Blaikie4f918ae2013-01-16 23:08:36 +0000491 __m128d __v;
Bill Wendling8e3ec9c2011-05-13 01:24:00 +0000492 } __attribute__((packed, may_alias));
David Blaikie4f918ae2013-01-16 23:08:36 +0000493 return ((struct __loadu_pd*)__dp)->__v;
Anders Carlsson445afa02008-12-24 02:11:54 +0000494}
495
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000496static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000497_mm_load_sd(double const *__dp)
Anders Carlsson445afa02008-12-24 02:11:54 +0000498{
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000499 struct __mm_load_sd_struct {
David Blaikie4f918ae2013-01-16 23:08:36 +0000500 double __u;
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000501 } __attribute__((__packed__, __may_alias__));
David Blaikie4f918ae2013-01-16 23:08:36 +0000502 double __u = ((struct __mm_load_sd_struct*)__dp)->__u;
503 return (__m128d){ __u, 0 };
Anders Carlsson445afa02008-12-24 02:11:54 +0000504}
505
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000506static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000507_mm_loadh_pd(__m128d __a, double const *__dp)
Anders Carlsson445afa02008-12-24 02:11:54 +0000508{
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000509 struct __mm_loadh_pd_struct {
David Blaikie4f918ae2013-01-16 23:08:36 +0000510 double __u;
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000511 } __attribute__((__packed__, __may_alias__));
David Blaikie4f918ae2013-01-16 23:08:36 +0000512 double __u = ((struct __mm_loadh_pd_struct*)__dp)->__u;
513 return (__m128d){ __a[0], __u };
Anders Carlsson445afa02008-12-24 02:11:54 +0000514}
515
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000516static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000517_mm_loadl_pd(__m128d __a, double const *__dp)
Anders Carlsson445afa02008-12-24 02:11:54 +0000518{
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000519 struct __mm_loadl_pd_struct {
David Blaikie4f918ae2013-01-16 23:08:36 +0000520 double __u;
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000521 } __attribute__((__packed__, __may_alias__));
David Blaikie4f918ae2013-01-16 23:08:36 +0000522 double __u = ((struct __mm_loadl_pd_struct*)__dp)->__u;
523 return (__m128d){ __u, __a[1] };
Anders Carlsson445afa02008-12-24 02:11:54 +0000524}
525
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000526static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000527_mm_set_sd(double __w)
Anders Carlsson445afa02008-12-24 02:11:54 +0000528{
David Blaikie4f918ae2013-01-16 23:08:36 +0000529 return (__m128d){ __w, 0 };
Anders Carlsson445afa02008-12-24 02:11:54 +0000530}
531
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000532static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000533_mm_set1_pd(double __w)
Anders Carlsson445afa02008-12-24 02:11:54 +0000534{
David Blaikie4f918ae2013-01-16 23:08:36 +0000535 return (__m128d){ __w, __w };
Anders Carlsson445afa02008-12-24 02:11:54 +0000536}
537
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000538static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000539_mm_set_pd(double __w, double __x)
Anders Carlsson445afa02008-12-24 02:11:54 +0000540{
David Blaikie4f918ae2013-01-16 23:08:36 +0000541 return (__m128d){ __x, __w };
Anders Carlsson445afa02008-12-24 02:11:54 +0000542}
543
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000544static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000545_mm_setr_pd(double __w, double __x)
Anders Carlsson445afa02008-12-24 02:11:54 +0000546{
David Blaikie4f918ae2013-01-16 23:08:36 +0000547 return (__m128d){ __w, __x };
Anders Carlsson445afa02008-12-24 02:11:54 +0000548}
549
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000550static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
Mike Stumpdae44132009-02-13 14:24:50 +0000551_mm_setzero_pd(void)
Anders Carlsson445afa02008-12-24 02:11:54 +0000552{
553 return (__m128d){ 0, 0 };
554}
555
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000556static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000557_mm_move_sd(__m128d __a, __m128d __b)
Anders Carlsson445afa02008-12-24 02:11:54 +0000558{
David Blaikie4f918ae2013-01-16 23:08:36 +0000559 return (__m128d){ __b[0], __a[1] };
Anders Carlsson445afa02008-12-24 02:11:54 +0000560}
561
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000562static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000563_mm_store_sd(double *__dp, __m128d __a)
Anders Carlsson445afa02008-12-24 02:11:54 +0000564{
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000565 struct __mm_store_sd_struct {
David Blaikie4f918ae2013-01-16 23:08:36 +0000566 double __u;
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000567 } __attribute__((__packed__, __may_alias__));
David Blaikie4f918ae2013-01-16 23:08:36 +0000568 ((struct __mm_store_sd_struct*)__dp)->__u = __a[0];
Anders Carlsson445afa02008-12-24 02:11:54 +0000569}
570
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000571static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000572_mm_store1_pd(double *__dp, __m128d __a)
Anders Carlsson445afa02008-12-24 02:11:54 +0000573{
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000574 struct __mm_store1_pd_struct {
David Blaikie4f918ae2013-01-16 23:08:36 +0000575 double __u[2];
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000576 } __attribute__((__packed__, __may_alias__));
David Blaikie4f918ae2013-01-16 23:08:36 +0000577 ((struct __mm_store1_pd_struct*)__dp)->__u[0] = __a[0];
578 ((struct __mm_store1_pd_struct*)__dp)->__u[1] = __a[0];
Anders Carlsson445afa02008-12-24 02:11:54 +0000579}
580
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000581static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000582_mm_store_pd(double *__dp, __m128d __a)
Anders Carlsson445afa02008-12-24 02:11:54 +0000583{
David Blaikie4f918ae2013-01-16 23:08:36 +0000584 *(__m128d *)__dp = __a;
Anders Carlsson445afa02008-12-24 02:11:54 +0000585}
586
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000587static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000588_mm_storeu_pd(double *__dp, __m128d __a)
Anders Carlsson445afa02008-12-24 02:11:54 +0000589{
David Blaikie4f918ae2013-01-16 23:08:36 +0000590 __builtin_ia32_storeupd(__dp, __a);
Anders Carlsson445afa02008-12-24 02:11:54 +0000591}
592
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000593static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000594_mm_storer_pd(double *__dp, __m128d __a)
Anders Carlsson445afa02008-12-24 02:11:54 +0000595{
David Blaikie4f918ae2013-01-16 23:08:36 +0000596 __a = __builtin_shufflevector(__a, __a, 1, 0);
597 *(__m128d *)__dp = __a;
Anders Carlsson445afa02008-12-24 02:11:54 +0000598}
599
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000600static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000601_mm_storeh_pd(double *__dp, __m128d __a)
Anders Carlsson445afa02008-12-24 02:11:54 +0000602{
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000603 struct __mm_storeh_pd_struct {
David Blaikie4f918ae2013-01-16 23:08:36 +0000604 double __u;
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000605 } __attribute__((__packed__, __may_alias__));
David Blaikie4f918ae2013-01-16 23:08:36 +0000606 ((struct __mm_storeh_pd_struct*)__dp)->__u = __a[1];
Anders Carlsson445afa02008-12-24 02:11:54 +0000607}
608
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000609static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000610_mm_storel_pd(double *__dp, __m128d __a)
Anders Carlsson445afa02008-12-24 02:11:54 +0000611{
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000612 struct __mm_storeh_pd_struct {
David Blaikie4f918ae2013-01-16 23:08:36 +0000613 double __u;
Eli Friedman7c06f6b2011-09-15 23:15:27 +0000614 } __attribute__((__packed__, __may_alias__));
David Blaikie4f918ae2013-01-16 23:08:36 +0000615 ((struct __mm_storeh_pd_struct*)__dp)->__u = __a[0];
Anders Carlsson445afa02008-12-24 02:11:54 +0000616}
617
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000618static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000619_mm_add_epi8(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000620{
David Blaikie4f918ae2013-01-16 23:08:36 +0000621 return (__m128i)((__v16qi)__a + (__v16qi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000622}
623
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000624static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000625_mm_add_epi16(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000626{
David Blaikie4f918ae2013-01-16 23:08:36 +0000627 return (__m128i)((__v8hi)__a + (__v8hi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000628}
629
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000630static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000631_mm_add_epi32(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000632{
David Blaikie4f918ae2013-01-16 23:08:36 +0000633 return (__m128i)((__v4si)__a + (__v4si)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000634}
635
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000636static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000637_mm_add_si64(__m64 __a, __m64 __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000638{
David Blaikie4f918ae2013-01-16 23:08:36 +0000639 return __a + __b;
Anders Carlsson07603aa2008-12-24 02:41:00 +0000640}
641
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000642static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000643_mm_add_epi64(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000644{
David Blaikie4f918ae2013-01-16 23:08:36 +0000645 return __a + __b;
Anders Carlsson07603aa2008-12-24 02:41:00 +0000646}
647
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000648static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000649_mm_adds_epi8(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000650{
David Blaikie4f918ae2013-01-16 23:08:36 +0000651 return (__m128i)__builtin_ia32_paddsb128((__v16qi)__a, (__v16qi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000652}
653
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000654static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000655_mm_adds_epi16(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000656{
David Blaikie4f918ae2013-01-16 23:08:36 +0000657 return (__m128i)__builtin_ia32_paddsw128((__v8hi)__a, (__v8hi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000658}
659
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000660static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000661_mm_adds_epu8(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000662{
David Blaikie4f918ae2013-01-16 23:08:36 +0000663 return (__m128i)__builtin_ia32_paddusb128((__v16qi)__a, (__v16qi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000664}
665
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000666static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000667_mm_adds_epu16(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000668{
David Blaikie4f918ae2013-01-16 23:08:36 +0000669 return (__m128i)__builtin_ia32_paddusw128((__v8hi)__a, (__v8hi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000670}
671
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000672static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000673_mm_avg_epu8(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000674{
David Blaikie4f918ae2013-01-16 23:08:36 +0000675 return (__m128i)__builtin_ia32_pavgb128((__v16qi)__a, (__v16qi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000676}
677
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000678static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000679_mm_avg_epu16(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000680{
David Blaikie4f918ae2013-01-16 23:08:36 +0000681 return (__m128i)__builtin_ia32_pavgw128((__v8hi)__a, (__v8hi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000682}
683
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000684static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000685_mm_madd_epi16(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000686{
David Blaikie4f918ae2013-01-16 23:08:36 +0000687 return (__m128i)__builtin_ia32_pmaddwd128((__v8hi)__a, (__v8hi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000688}
689
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000690static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000691_mm_max_epi16(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000692{
David Blaikie4f918ae2013-01-16 23:08:36 +0000693 return (__m128i)__builtin_ia32_pmaxsw128((__v8hi)__a, (__v8hi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000694}
695
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000696static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000697_mm_max_epu8(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000698{
David Blaikie4f918ae2013-01-16 23:08:36 +0000699 return (__m128i)__builtin_ia32_pmaxub128((__v16qi)__a, (__v16qi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000700}
701
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000702static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000703_mm_min_epi16(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000704{
David Blaikie4f918ae2013-01-16 23:08:36 +0000705 return (__m128i)__builtin_ia32_pminsw128((__v8hi)__a, (__v8hi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000706}
707
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000708static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000709_mm_min_epu8(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000710{
David Blaikie4f918ae2013-01-16 23:08:36 +0000711 return (__m128i)__builtin_ia32_pminub128((__v16qi)__a, (__v16qi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000712}
713
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000714static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000715_mm_mulhi_epi16(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000716{
David Blaikie4f918ae2013-01-16 23:08:36 +0000717 return (__m128i)__builtin_ia32_pmulhw128((__v8hi)__a, (__v8hi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000718}
719
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000720static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000721_mm_mulhi_epu16(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000722{
David Blaikie4f918ae2013-01-16 23:08:36 +0000723 return (__m128i)__builtin_ia32_pmulhuw128((__v8hi)__a, (__v8hi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000724}
725
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000726static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000727_mm_mullo_epi16(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000728{
David Blaikie4f918ae2013-01-16 23:08:36 +0000729 return (__m128i)((__v8hi)__a * (__v8hi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000730}
731
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000732static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000733_mm_mul_su32(__m64 __a, __m64 __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000734{
David Blaikie4f918ae2013-01-16 23:08:36 +0000735 return __builtin_ia32_pmuludq((__v2si)__a, (__v2si)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000736}
737
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000738static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000739_mm_mul_epu32(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000740{
David Blaikie4f918ae2013-01-16 23:08:36 +0000741 return __builtin_ia32_pmuludq128((__v4si)__a, (__v4si)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000742}
743
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000744static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000745_mm_sad_epu8(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000746{
David Blaikie4f918ae2013-01-16 23:08:36 +0000747 return __builtin_ia32_psadbw128((__v16qi)__a, (__v16qi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000748}
749
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000750static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000751_mm_sub_epi8(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000752{
David Blaikie4f918ae2013-01-16 23:08:36 +0000753 return (__m128i)((__v16qi)__a - (__v16qi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000754}
755
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000756static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000757_mm_sub_epi16(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000758{
David Blaikie4f918ae2013-01-16 23:08:36 +0000759 return (__m128i)((__v8hi)__a - (__v8hi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000760}
761
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000762static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000763_mm_sub_epi32(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000764{
David Blaikie4f918ae2013-01-16 23:08:36 +0000765 return (__m128i)((__v4si)__a - (__v4si)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000766}
767
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000768static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000769_mm_sub_si64(__m64 __a, __m64 __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000770{
David Blaikie4f918ae2013-01-16 23:08:36 +0000771 return __a - __b;
Anders Carlsson07603aa2008-12-24 02:41:00 +0000772}
773
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000774static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000775_mm_sub_epi64(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000776{
David Blaikie4f918ae2013-01-16 23:08:36 +0000777 return __a - __b;
Anders Carlsson07603aa2008-12-24 02:41:00 +0000778}
779
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000780static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000781_mm_subs_epi8(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000782{
David Blaikie4f918ae2013-01-16 23:08:36 +0000783 return (__m128i)__builtin_ia32_psubsb128((__v16qi)__a, (__v16qi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000784}
785
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000786static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000787_mm_subs_epi16(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000788{
David Blaikie4f918ae2013-01-16 23:08:36 +0000789 return (__m128i)__builtin_ia32_psubsw128((__v8hi)__a, (__v8hi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000790}
791
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000792static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000793_mm_subs_epu8(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000794{
David Blaikie4f918ae2013-01-16 23:08:36 +0000795 return (__m128i)__builtin_ia32_psubusb128((__v16qi)__a, (__v16qi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000796}
797
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000798static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000799_mm_subs_epu16(__m128i __a, __m128i __b)
Anders Carlsson07603aa2008-12-24 02:41:00 +0000800{
David Blaikie4f918ae2013-01-16 23:08:36 +0000801 return (__m128i)__builtin_ia32_psubusw128((__v8hi)__a, (__v8hi)__b);
Anders Carlsson07603aa2008-12-24 02:41:00 +0000802}
803
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000804static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000805_mm_and_si128(__m128i __a, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +0000806{
David Blaikie4f918ae2013-01-16 23:08:36 +0000807 return __a & __b;
Anders Carlsson0727df02008-12-25 23:48:58 +0000808}
809
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000810static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000811_mm_andnot_si128(__m128i __a, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +0000812{
David Blaikie4f918ae2013-01-16 23:08:36 +0000813 return ~__a & __b;
Anders Carlsson0727df02008-12-25 23:48:58 +0000814}
815
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000816static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000817_mm_or_si128(__m128i __a, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +0000818{
David Blaikie4f918ae2013-01-16 23:08:36 +0000819 return __a | __b;
Anders Carlsson0727df02008-12-25 23:48:58 +0000820}
821
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000822static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000823_mm_xor_si128(__m128i __a, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +0000824{
David Blaikie4f918ae2013-01-16 23:08:36 +0000825 return __a ^ __b;
Anders Carlsson0727df02008-12-25 23:48:58 +0000826}
827
Bob Wilson32bae372011-11-05 06:08:06 +0000828#define _mm_slli_si128(a, count) __extension__ ({ \
Ted Kremenek185f1562013-10-07 23:51:11 +0000829 _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wshadow\""); \
Bob Wilson32bae372011-11-05 06:08:06 +0000830 __m128i __a = (a); \
Ted Kremenek185f1562013-10-07 23:51:11 +0000831 _Pragma("clang diagnostic pop"); \
Bob Wilson32bae372011-11-05 06:08:06 +0000832 (__m128i)__builtin_ia32_pslldqi128(__a, (count)*8); })
Anders Carlsson0727df02008-12-25 23:48:58 +0000833
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000834static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000835_mm_slli_epi16(__m128i __a, int __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000836{
David Blaikie4f918ae2013-01-16 23:08:36 +0000837 return (__m128i)__builtin_ia32_psllwi128((__v8hi)__a, __count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000838}
839
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000840static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000841_mm_sll_epi16(__m128i __a, __m128i __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000842{
David Blaikie4f918ae2013-01-16 23:08:36 +0000843 return (__m128i)__builtin_ia32_psllw128((__v8hi)__a, (__v8hi)__count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000844}
845
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000846static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000847_mm_slli_epi32(__m128i __a, int __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000848{
David Blaikie4f918ae2013-01-16 23:08:36 +0000849 return (__m128i)__builtin_ia32_pslldi128((__v4si)__a, __count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000850}
851
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000852static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000853_mm_sll_epi32(__m128i __a, __m128i __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000854{
David Blaikie4f918ae2013-01-16 23:08:36 +0000855 return (__m128i)__builtin_ia32_pslld128((__v4si)__a, (__v4si)__count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000856}
857
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000858static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000859_mm_slli_epi64(__m128i __a, int __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000860{
David Blaikie4f918ae2013-01-16 23:08:36 +0000861 return __builtin_ia32_psllqi128(__a, __count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000862}
863
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000864static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000865_mm_sll_epi64(__m128i __a, __m128i __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000866{
David Blaikie4f918ae2013-01-16 23:08:36 +0000867 return __builtin_ia32_psllq128(__a, __count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000868}
869
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000870static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000871_mm_srai_epi16(__m128i __a, int __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000872{
David Blaikie4f918ae2013-01-16 23:08:36 +0000873 return (__m128i)__builtin_ia32_psrawi128((__v8hi)__a, __count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000874}
875
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000876static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000877_mm_sra_epi16(__m128i __a, __m128i __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000878{
David Blaikie4f918ae2013-01-16 23:08:36 +0000879 return (__m128i)__builtin_ia32_psraw128((__v8hi)__a, (__v8hi)__count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000880}
881
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000882static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000883_mm_srai_epi32(__m128i __a, int __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000884{
David Blaikie4f918ae2013-01-16 23:08:36 +0000885 return (__m128i)__builtin_ia32_psradi128((__v4si)__a, __count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000886}
887
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000888static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000889_mm_sra_epi32(__m128i __a, __m128i __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000890{
David Blaikie4f918ae2013-01-16 23:08:36 +0000891 return (__m128i)__builtin_ia32_psrad128((__v4si)__a, (__v4si)__count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000892}
893
Chris Lattnerfcb0b252010-10-01 06:58:49 +0000894
Bob Wilson32bae372011-11-05 06:08:06 +0000895#define _mm_srli_si128(a, count) __extension__ ({ \
Ted Kremenek185f1562013-10-07 23:51:11 +0000896 _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wshadow\""); \
Bob Wilson32bae372011-11-05 06:08:06 +0000897 __m128i __a = (a); \
Ted Kremenek185f1562013-10-07 23:51:11 +0000898 _Pragma("clang diagnostic pop"); \
Bob Wilson32bae372011-11-05 06:08:06 +0000899 (__m128i)__builtin_ia32_psrldqi128(__a, (count)*8); })
Anders Carlsson0727df02008-12-25 23:48:58 +0000900
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000901static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000902_mm_srli_epi16(__m128i __a, int __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000903{
David Blaikie4f918ae2013-01-16 23:08:36 +0000904 return (__m128i)__builtin_ia32_psrlwi128((__v8hi)__a, __count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000905}
906
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000907static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000908_mm_srl_epi16(__m128i __a, __m128i __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000909{
David Blaikie4f918ae2013-01-16 23:08:36 +0000910 return (__m128i)__builtin_ia32_psrlw128((__v8hi)__a, (__v8hi)__count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000911}
912
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000913static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000914_mm_srli_epi32(__m128i __a, int __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000915{
David Blaikie4f918ae2013-01-16 23:08:36 +0000916 return (__m128i)__builtin_ia32_psrldi128((__v4si)__a, __count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000917}
918
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000919static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000920_mm_srl_epi32(__m128i __a, __m128i __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000921{
David Blaikie4f918ae2013-01-16 23:08:36 +0000922 return (__m128i)__builtin_ia32_psrld128((__v4si)__a, (__v4si)__count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000923}
924
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000925static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000926_mm_srli_epi64(__m128i __a, int __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000927{
David Blaikie4f918ae2013-01-16 23:08:36 +0000928 return __builtin_ia32_psrlqi128(__a, __count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000929}
930
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000931static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000932_mm_srl_epi64(__m128i __a, __m128i __count)
Anders Carlsson0727df02008-12-25 23:48:58 +0000933{
David Blaikie4f918ae2013-01-16 23:08:36 +0000934 return __builtin_ia32_psrlq128(__a, __count);
Anders Carlsson0727df02008-12-25 23:48:58 +0000935}
936
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000937static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000938_mm_cmpeq_epi8(__m128i __a, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +0000939{
David Blaikie4f918ae2013-01-16 23:08:36 +0000940 return (__m128i)((__v16qi)__a == (__v16qi)__b);
Anders Carlsson0727df02008-12-25 23:48:58 +0000941}
942
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000943static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000944_mm_cmpeq_epi16(__m128i __a, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +0000945{
David Blaikie4f918ae2013-01-16 23:08:36 +0000946 return (__m128i)((__v8hi)__a == (__v8hi)__b);
Anders Carlsson0727df02008-12-25 23:48:58 +0000947}
948
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000949static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000950_mm_cmpeq_epi32(__m128i __a, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +0000951{
David Blaikie4f918ae2013-01-16 23:08:36 +0000952 return (__m128i)((__v4si)__a == (__v4si)__b);
Anders Carlsson0727df02008-12-25 23:48:58 +0000953}
954
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000955static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000956_mm_cmpgt_epi8(__m128i __a, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +0000957{
Nick Lewycky279114c2012-02-04 02:16:48 +0000958 /* This function always performs a signed comparison, but __v16qi is a char
959 which may be signed or unsigned. */
Nick Lewyckyf42f85c2012-02-03 23:57:48 +0000960 typedef signed char __v16qs __attribute__((__vector_size__(16)));
David Blaikie4f918ae2013-01-16 23:08:36 +0000961 return (__m128i)((__v16qs)__a > (__v16qs)__b);
Anders Carlsson0727df02008-12-25 23:48:58 +0000962}
963
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000964static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000965_mm_cmpgt_epi16(__m128i __a, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +0000966{
David Blaikie4f918ae2013-01-16 23:08:36 +0000967 return (__m128i)((__v8hi)__a > (__v8hi)__b);
Anders Carlsson0727df02008-12-25 23:48:58 +0000968}
969
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000970static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000971_mm_cmpgt_epi32(__m128i __a, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +0000972{
David Blaikie4f918ae2013-01-16 23:08:36 +0000973 return (__m128i)((__v4si)__a > (__v4si)__b);
Anders Carlsson0727df02008-12-25 23:48:58 +0000974}
975
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000976static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000977_mm_cmplt_epi8(__m128i __a, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +0000978{
David Blaikie4f918ae2013-01-16 23:08:36 +0000979 return _mm_cmpgt_epi8(__b, __a);
Anders Carlsson0727df02008-12-25 23:48:58 +0000980}
981
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000982static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000983_mm_cmplt_epi16(__m128i __a, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +0000984{
David Blaikie4f918ae2013-01-16 23:08:36 +0000985 return _mm_cmpgt_epi16(__b, __a);
Anders Carlsson0727df02008-12-25 23:48:58 +0000986}
987
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000988static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000989_mm_cmplt_epi32(__m128i __a, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +0000990{
David Blaikie4f918ae2013-01-16 23:08:36 +0000991 return _mm_cmpgt_epi32(__b, __a);
Anders Carlsson0727df02008-12-25 23:48:58 +0000992}
993
994#ifdef __x86_64__
Chris Lattner1bddbcb2010-03-22 18:14:12 +0000995static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +0000996_mm_cvtsi64_sd(__m128d __a, long long __b)
Anders Carlsson0727df02008-12-25 23:48:58 +0000997{
David Blaikie4f918ae2013-01-16 23:08:36 +0000998 __a[0] = __b;
999 return __a;
Anders Carlsson0727df02008-12-25 23:48:58 +00001000}
1001
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001002static __inline__ long long __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001003_mm_cvtsd_si64(__m128d __a)
Anders Carlsson0727df02008-12-25 23:48:58 +00001004{
David Blaikie4f918ae2013-01-16 23:08:36 +00001005 return __builtin_ia32_cvtsd2si64(__a);
Anders Carlsson0727df02008-12-25 23:48:58 +00001006}
1007
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001008static __inline__ long long __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001009_mm_cvttsd_si64(__m128d __a)
Anders Carlsson0727df02008-12-25 23:48:58 +00001010{
David Blaikie4f918ae2013-01-16 23:08:36 +00001011 return __a[0];
Anders Carlsson0727df02008-12-25 23:48:58 +00001012}
1013#endif
1014
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001015static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001016_mm_cvtepi32_ps(__m128i __a)
Anders Carlsson0727df02008-12-25 23:48:58 +00001017{
David Blaikie4f918ae2013-01-16 23:08:36 +00001018 return __builtin_ia32_cvtdq2ps((__v4si)__a);
Anders Carlsson0727df02008-12-25 23:48:58 +00001019}
1020
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001021static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001022_mm_cvtps_epi32(__m128 __a)
Anders Carlsson0727df02008-12-25 23:48:58 +00001023{
David Blaikie4f918ae2013-01-16 23:08:36 +00001024 return (__m128i)__builtin_ia32_cvtps2dq(__a);
Anders Carlsson0727df02008-12-25 23:48:58 +00001025}
1026
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001027static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001028_mm_cvttps_epi32(__m128 __a)
Anders Carlsson0727df02008-12-25 23:48:58 +00001029{
David Blaikie4f918ae2013-01-16 23:08:36 +00001030 return (__m128i)__builtin_ia32_cvttps2dq(__a);
Anders Carlsson0727df02008-12-25 23:48:58 +00001031}
1032
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001033static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001034_mm_cvtsi32_si128(int __a)
Anders Carlsson0727df02008-12-25 23:48:58 +00001035{
David Blaikie4f918ae2013-01-16 23:08:36 +00001036 return (__m128i)(__v4si){ __a, 0, 0, 0 };
Anders Carlsson0727df02008-12-25 23:48:58 +00001037}
1038
1039#ifdef __x86_64__
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001040static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001041_mm_cvtsi64_si128(long long __a)
Anders Carlsson0727df02008-12-25 23:48:58 +00001042{
David Blaikie4f918ae2013-01-16 23:08:36 +00001043 return (__m128i){ __a, 0 };
Anders Carlsson0727df02008-12-25 23:48:58 +00001044}
1045#endif
1046
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001047static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001048_mm_cvtsi128_si32(__m128i __a)
Anders Carlsson0727df02008-12-25 23:48:58 +00001049{
David Blaikie4f918ae2013-01-16 23:08:36 +00001050 __v4si __b = (__v4si)__a;
1051 return __b[0];
Anders Carlsson0727df02008-12-25 23:48:58 +00001052}
1053
1054#ifdef __x86_64__
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001055static __inline__ long long __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001056_mm_cvtsi128_si64(__m128i __a)
Anders Carlsson0727df02008-12-25 23:48:58 +00001057{
David Blaikie4f918ae2013-01-16 23:08:36 +00001058 return __a[0];
Anders Carlsson0727df02008-12-25 23:48:58 +00001059}
1060#endif
1061
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001062static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001063_mm_load_si128(__m128i const *__p)
Anders Carlsson0727df02008-12-25 23:48:58 +00001064{
David Blaikie4f918ae2013-01-16 23:08:36 +00001065 return *__p;
Anders Carlsson0727df02008-12-25 23:48:58 +00001066}
1067
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001068static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001069_mm_loadu_si128(__m128i const *__p)
Anders Carlsson0727df02008-12-25 23:48:58 +00001070{
Bill Wendlingeed92a12011-05-13 00:11:39 +00001071 struct __loadu_si128 {
David Blaikie4f918ae2013-01-16 23:08:36 +00001072 __m128i __v;
Bill Wendling8e3ec9c2011-05-13 01:24:00 +00001073 } __attribute__((packed, may_alias));
David Blaikie4f918ae2013-01-16 23:08:36 +00001074 return ((struct __loadu_si128*)__p)->__v;
Anders Carlsson0727df02008-12-25 23:48:58 +00001075}
1076
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001077static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001078_mm_loadl_epi64(__m128i const *__p)
Anders Carlsson0727df02008-12-25 23:48:58 +00001079{
Eli Friedman7c06f6b2011-09-15 23:15:27 +00001080 struct __mm_loadl_epi64_struct {
David Blaikie4f918ae2013-01-16 23:08:36 +00001081 long long __u;
Eli Friedman7c06f6b2011-09-15 23:15:27 +00001082 } __attribute__((__packed__, __may_alias__));
David Blaikie4f918ae2013-01-16 23:08:36 +00001083 return (__m128i) { ((struct __mm_loadl_epi64_struct*)__p)->__u, 0};
Anders Carlsson0727df02008-12-25 23:48:58 +00001084}
1085
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001086static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
Anders Carlsson9436ed52009-09-18 17:03:55 +00001087_mm_set_epi64x(long long q1, long long q0)
1088{
1089 return (__m128i){ q0, q1 };
1090}
1091
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001092static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
Mike Stumpdae44132009-02-13 14:24:50 +00001093_mm_set_epi64(__m64 q1, __m64 q0)
Anders Carlsson0727df02008-12-25 23:48:58 +00001094{
1095 return (__m128i){ (long long)q0, (long long)q1 };
1096}
1097
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001098static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
Mike Stumpdae44132009-02-13 14:24:50 +00001099_mm_set_epi32(int i3, int i2, int i1, int i0)
Anders Carlsson0727df02008-12-25 23:48:58 +00001100{
1101 return (__m128i)(__v4si){ i0, i1, i2, i3};
1102}
1103
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001104static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
Mike Stumpdae44132009-02-13 14:24:50 +00001105_mm_set_epi16(short w7, short w6, short w5, short w4, short w3, short w2, short w1, short w0)
Anders Carlsson0727df02008-12-25 23:48:58 +00001106{
1107 return (__m128i)(__v8hi){ w0, w1, w2, w3, w4, w5, w6, w7 };
1108}
1109
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001110static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
Mike Stumpdae44132009-02-13 14:24:50 +00001111_mm_set_epi8(char b15, char b14, char b13, char b12, char b11, char b10, char b9, char b8, char b7, char b6, char b5, char b4, char b3, char b2, char b1, char b0)
Anders Carlsson0727df02008-12-25 23:48:58 +00001112{
1113 return (__m128i)(__v16qi){ b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15 };
1114}
1115
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001116static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001117_mm_set1_epi64x(long long __q)
Anders Carlsson9436ed52009-09-18 17:03:55 +00001118{
David Blaikie4f918ae2013-01-16 23:08:36 +00001119 return (__m128i){ __q, __q };
Anders Carlsson9436ed52009-09-18 17:03:55 +00001120}
1121
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001122static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001123_mm_set1_epi64(__m64 __q)
Anders Carlsson0727df02008-12-25 23:48:58 +00001124{
David Blaikie4f918ae2013-01-16 23:08:36 +00001125 return (__m128i){ (long long)__q, (long long)__q };
Anders Carlsson0727df02008-12-25 23:48:58 +00001126}
1127
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001128static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001129_mm_set1_epi32(int __i)
Anders Carlsson0727df02008-12-25 23:48:58 +00001130{
David Blaikie4f918ae2013-01-16 23:08:36 +00001131 return (__m128i)(__v4si){ __i, __i, __i, __i };
Anders Carlsson0727df02008-12-25 23:48:58 +00001132}
1133
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001134static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001135_mm_set1_epi16(short __w)
Anders Carlsson0727df02008-12-25 23:48:58 +00001136{
David Blaikie4f918ae2013-01-16 23:08:36 +00001137 return (__m128i)(__v8hi){ __w, __w, __w, __w, __w, __w, __w, __w };
Anders Carlsson0727df02008-12-25 23:48:58 +00001138}
1139
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001140static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001141_mm_set1_epi8(char __b)
Anders Carlsson0727df02008-12-25 23:48:58 +00001142{
David Blaikie4f918ae2013-01-16 23:08:36 +00001143 return (__m128i)(__v16qi){ __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b };
Anders Carlsson0727df02008-12-25 23:48:58 +00001144}
1145
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001146static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
Mike Stumpdae44132009-02-13 14:24:50 +00001147_mm_setr_epi64(__m64 q0, __m64 q1)
Anders Carlsson0727df02008-12-25 23:48:58 +00001148{
1149 return (__m128i){ (long long)q0, (long long)q1 };
1150}
1151
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001152static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
Mike Stumpdae44132009-02-13 14:24:50 +00001153_mm_setr_epi32(int i0, int i1, int i2, int i3)
Anders Carlsson0727df02008-12-25 23:48:58 +00001154{
1155 return (__m128i)(__v4si){ i0, i1, i2, i3};
1156}
1157
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001158static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
Mike Stumpdae44132009-02-13 14:24:50 +00001159_mm_setr_epi16(short w0, short w1, short w2, short w3, short w4, short w5, short w6, short w7)
Anders Carlsson0727df02008-12-25 23:48:58 +00001160{
1161 return (__m128i)(__v8hi){ w0, w1, w2, w3, w4, w5, w6, w7 };
1162}
1163
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001164static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
Mike Stumpdae44132009-02-13 14:24:50 +00001165_mm_setr_epi8(char b0, char b1, char b2, char b3, char b4, char b5, char b6, char b7, char b8, char b9, char b10, char b11, char b12, char b13, char b14, char b15)
Anders Carlsson0727df02008-12-25 23:48:58 +00001166{
1167 return (__m128i)(__v16qi){ b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15 };
1168}
1169
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001170static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
Mike Stumpdae44132009-02-13 14:24:50 +00001171_mm_setzero_si128(void)
Anders Carlsson0727df02008-12-25 23:48:58 +00001172{
1173 return (__m128i){ 0LL, 0LL };
1174}
1175
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001176static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001177_mm_store_si128(__m128i *__p, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +00001178{
David Blaikie4f918ae2013-01-16 23:08:36 +00001179 *__p = __b;
Anders Carlsson0727df02008-12-25 23:48:58 +00001180}
1181
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001182static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001183_mm_storeu_si128(__m128i *__p, __m128i __b)
Anders Carlsson0727df02008-12-25 23:48:58 +00001184{
David Blaikie4f918ae2013-01-16 23:08:36 +00001185 __builtin_ia32_storedqu((char *)__p, (__v16qi)__b);
Anders Carlsson0727df02008-12-25 23:48:58 +00001186}
1187
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001188static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001189_mm_maskmoveu_si128(__m128i __d, __m128i __n, char *__p)
Anders Carlsson0727df02008-12-25 23:48:58 +00001190{
David Blaikie4f918ae2013-01-16 23:08:36 +00001191 __builtin_ia32_maskmovdqu((__v16qi)__d, (__v16qi)__n, __p);
Anders Carlsson0727df02008-12-25 23:48:58 +00001192}
1193
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001194static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001195_mm_storel_epi64(__m128i *__p, __m128i __a)
Anders Carlsson0727df02008-12-25 23:48:58 +00001196{
Chad Rosierc5713cf2012-05-01 18:11:51 +00001197 struct __mm_storel_epi64_struct {
David Blaikie4f918ae2013-01-16 23:08:36 +00001198 long long __u;
Chad Rosierc5713cf2012-05-01 18:11:51 +00001199 } __attribute__((__packed__, __may_alias__));
David Blaikie4f918ae2013-01-16 23:08:36 +00001200 ((struct __mm_storel_epi64_struct*)__p)->__u = __a[0];
Anders Carlsson0727df02008-12-25 23:48:58 +00001201}
1202
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001203static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001204_mm_stream_pd(double *__p, __m128d __a)
Anders Carlsson0727df02008-12-25 23:48:58 +00001205{
David Blaikie4f918ae2013-01-16 23:08:36 +00001206 __builtin_ia32_movntpd(__p, __a);
Anders Carlsson0727df02008-12-25 23:48:58 +00001207}
1208
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001209static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001210_mm_stream_si128(__m128i *__p, __m128i __a)
Anders Carlsson0727df02008-12-25 23:48:58 +00001211{
David Blaikie4f918ae2013-01-16 23:08:36 +00001212 __builtin_ia32_movntdq(__p, __a);
Anders Carlsson0727df02008-12-25 23:48:58 +00001213}
1214
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001215static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001216_mm_stream_si32(int *__p, int __a)
Anders Carlsson0727df02008-12-25 23:48:58 +00001217{
David Blaikie4f918ae2013-01-16 23:08:36 +00001218 __builtin_ia32_movnti(__p, __a);
Anders Carlsson0727df02008-12-25 23:48:58 +00001219}
1220
Eli Friedman440a5f42013-09-23 23:38:39 +00001221#ifdef __x86_64__
1222static __inline__ void __attribute__((__always_inline__, __nodebug__))
1223_mm_stream_si64(long long *__p, long long __a)
1224{
1225 __builtin_ia32_movnti64(__p, __a);
1226}
1227#endif
1228
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001229static __inline__ void __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001230_mm_clflush(void const *__p)
Anders Carlsson0727df02008-12-25 23:48:58 +00001231{
David Blaikie4f918ae2013-01-16 23:08:36 +00001232 __builtin_ia32_clflush(__p);
Anders Carlsson0727df02008-12-25 23:48:58 +00001233}
1234
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001235static __inline__ void __attribute__((__always_inline__, __nodebug__))
Mike Stumpdae44132009-02-13 14:24:50 +00001236_mm_lfence(void)
Anders Carlsson0727df02008-12-25 23:48:58 +00001237{
1238 __builtin_ia32_lfence();
1239}
1240
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001241static __inline__ void __attribute__((__always_inline__, __nodebug__))
Mike Stumpdae44132009-02-13 14:24:50 +00001242_mm_mfence(void)
Anders Carlsson0727df02008-12-25 23:48:58 +00001243{
1244 __builtin_ia32_mfence();
1245}
1246
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001247static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001248_mm_packs_epi16(__m128i __a, __m128i __b)
Anders Carlsson45470752008-12-26 00:45:50 +00001249{
David Blaikie4f918ae2013-01-16 23:08:36 +00001250 return (__m128i)__builtin_ia32_packsswb128((__v8hi)__a, (__v8hi)__b);
Anders Carlsson45470752008-12-26 00:45:50 +00001251}
1252
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001253static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001254_mm_packs_epi32(__m128i __a, __m128i __b)
Anders Carlsson45470752008-12-26 00:45:50 +00001255{
David Blaikie4f918ae2013-01-16 23:08:36 +00001256 return (__m128i)__builtin_ia32_packssdw128((__v4si)__a, (__v4si)__b);
Anders Carlsson45470752008-12-26 00:45:50 +00001257}
1258
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001259static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001260_mm_packus_epi16(__m128i __a, __m128i __b)
Anders Carlsson45470752008-12-26 00:45:50 +00001261{
David Blaikie4f918ae2013-01-16 23:08:36 +00001262 return (__m128i)__builtin_ia32_packuswb128((__v8hi)__a, (__v8hi)__b);
Anders Carlsson45470752008-12-26 00:45:50 +00001263}
1264
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001265static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001266_mm_extract_epi16(__m128i __a, int __imm)
Anders Carlsson45470752008-12-26 00:45:50 +00001267{
David Blaikie4f918ae2013-01-16 23:08:36 +00001268 __v8hi __b = (__v8hi)__a;
1269 return (unsigned short)__b[__imm];
Anders Carlsson45470752008-12-26 00:45:50 +00001270}
1271
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001272static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001273_mm_insert_epi16(__m128i __a, int __b, int __imm)
Anders Carlsson45470752008-12-26 00:45:50 +00001274{
David Blaikie4f918ae2013-01-16 23:08:36 +00001275 __v8hi __c = (__v8hi)__a;
1276 __c[__imm & 7] = __b;
1277 return (__m128i)__c;
Anders Carlsson45470752008-12-26 00:45:50 +00001278}
1279
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001280static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001281_mm_movemask_epi8(__m128i __a)
Anders Carlsson45470752008-12-26 00:45:50 +00001282{
David Blaikie4f918ae2013-01-16 23:08:36 +00001283 return __builtin_ia32_pmovmskb128((__v16qi)__a);
Anders Carlsson45470752008-12-26 00:45:50 +00001284}
1285
Bob Wilson32bae372011-11-05 06:08:06 +00001286#define _mm_shuffle_epi32(a, imm) __extension__ ({ \
Ted Kremenek185f1562013-10-07 23:51:11 +00001287 _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wshadow\""); \
Bob Wilson32bae372011-11-05 06:08:06 +00001288 __m128i __a = (a); \
Ted Kremenek185f1562013-10-07 23:51:11 +00001289 _Pragma("clang diagnostic pop"); \
Bob Wilson32bae372011-11-05 06:08:06 +00001290 (__m128i)__builtin_shufflevector((__v4si)__a, (__v4si) _mm_set1_epi32(0), \
1291 (imm) & 0x3, ((imm) & 0xc) >> 2, \
1292 ((imm) & 0x30) >> 4, ((imm) & 0xc0) >> 6); })
Chris Lattner85e59d72011-04-25 20:42:40 +00001293
Bob Wilson32bae372011-11-05 06:08:06 +00001294#define _mm_shufflelo_epi16(a, imm) __extension__ ({ \
Ted Kremenek185f1562013-10-07 23:51:11 +00001295 _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wshadow\""); \
Bob Wilson32bae372011-11-05 06:08:06 +00001296 __m128i __a = (a); \
Ted Kremenek185f1562013-10-07 23:51:11 +00001297 _Pragma("clang diagnostic pop"); \
Bob Wilson32bae372011-11-05 06:08:06 +00001298 (__m128i)__builtin_shufflevector((__v8hi)__a, (__v8hi) _mm_set1_epi16(0), \
1299 (imm) & 0x3, ((imm) & 0xc) >> 2, \
1300 ((imm) & 0x30) >> 4, ((imm) & 0xc0) >> 6, \
1301 4, 5, 6, 7); })
Chris Lattner85e59d72011-04-25 20:42:40 +00001302
Bob Wilson32bae372011-11-05 06:08:06 +00001303#define _mm_shufflehi_epi16(a, imm) __extension__ ({ \
Ted Kremenek185f1562013-10-07 23:51:11 +00001304 _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wshadow\""); \
Bob Wilson32bae372011-11-05 06:08:06 +00001305 __m128i __a = (a); \
Ted Kremenek185f1562013-10-07 23:51:11 +00001306 _Pragma("clang diagnostic pop"); \
Bob Wilson32bae372011-11-05 06:08:06 +00001307 (__m128i)__builtin_shufflevector((__v8hi)__a, (__v8hi) _mm_set1_epi16(0), \
1308 0, 1, 2, 3, \
1309 4 + (((imm) & 0x03) >> 0), \
1310 4 + (((imm) & 0x0c) >> 2), \
1311 4 + (((imm) & 0x30) >> 4), \
1312 4 + (((imm) & 0xc0) >> 6)); })
Anders Carlsson45470752008-12-26 00:45:50 +00001313
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001314static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001315_mm_unpackhi_epi8(__m128i __a, __m128i __b)
Anders Carlsson45470752008-12-26 00:45:50 +00001316{
David Blaikie4f918ae2013-01-16 23:08:36 +00001317 return (__m128i)__builtin_shufflevector((__v16qi)__a, (__v16qi)__b, 8, 16+8, 9, 16+9, 10, 16+10, 11, 16+11, 12, 16+12, 13, 16+13, 14, 16+14, 15, 16+15);
Anders Carlsson45470752008-12-26 00:45:50 +00001318}
1319
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001320static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001321_mm_unpackhi_epi16(__m128i __a, __m128i __b)
Anders Carlsson45470752008-12-26 00:45:50 +00001322{
David Blaikie4f918ae2013-01-16 23:08:36 +00001323 return (__m128i)__builtin_shufflevector((__v8hi)__a, (__v8hi)__b, 4, 8+4, 5, 8+5, 6, 8+6, 7, 8+7);
Anders Carlsson45470752008-12-26 00:45:50 +00001324}
1325
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001326static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001327_mm_unpackhi_epi32(__m128i __a, __m128i __b)
Anders Carlsson45470752008-12-26 00:45:50 +00001328{
David Blaikie4f918ae2013-01-16 23:08:36 +00001329 return (__m128i)__builtin_shufflevector((__v4si)__a, (__v4si)__b, 2, 4+2, 3, 4+3);
Anders Carlsson45470752008-12-26 00:45:50 +00001330}
1331
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001332static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001333_mm_unpackhi_epi64(__m128i __a, __m128i __b)
Anders Carlsson45470752008-12-26 00:45:50 +00001334{
David Blaikie4f918ae2013-01-16 23:08:36 +00001335 return (__m128i)__builtin_shufflevector(__a, __b, 1, 2+1);
Anders Carlsson45470752008-12-26 00:45:50 +00001336}
1337
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001338static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001339_mm_unpacklo_epi8(__m128i __a, __m128i __b)
Anders Carlsson45470752008-12-26 00:45:50 +00001340{
David Blaikie4f918ae2013-01-16 23:08:36 +00001341 return (__m128i)__builtin_shufflevector((__v16qi)__a, (__v16qi)__b, 0, 16+0, 1, 16+1, 2, 16+2, 3, 16+3, 4, 16+4, 5, 16+5, 6, 16+6, 7, 16+7);
Anders Carlsson45470752008-12-26 00:45:50 +00001342}
1343
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001344static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001345_mm_unpacklo_epi16(__m128i __a, __m128i __b)
Anders Carlsson45470752008-12-26 00:45:50 +00001346{
David Blaikie4f918ae2013-01-16 23:08:36 +00001347 return (__m128i)__builtin_shufflevector((__v8hi)__a, (__v8hi)__b, 0, 8+0, 1, 8+1, 2, 8+2, 3, 8+3);
Anders Carlsson45470752008-12-26 00:45:50 +00001348}
1349
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001350static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001351_mm_unpacklo_epi32(__m128i __a, __m128i __b)
Anders Carlsson45470752008-12-26 00:45:50 +00001352{
David Blaikie4f918ae2013-01-16 23:08:36 +00001353 return (__m128i)__builtin_shufflevector((__v4si)__a, (__v4si)__b, 0, 4+0, 1, 4+1);
Anders Carlsson45470752008-12-26 00:45:50 +00001354}
1355
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001356static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001357_mm_unpacklo_epi64(__m128i __a, __m128i __b)
Anders Carlsson45470752008-12-26 00:45:50 +00001358{
David Blaikie4f918ae2013-01-16 23:08:36 +00001359 return (__m128i)__builtin_shufflevector(__a, __b, 0, 2+0);
Anders Carlsson45470752008-12-26 00:45:50 +00001360}
1361
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001362static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001363_mm_movepi64_pi64(__m128i __a)
Anders Carlsson45470752008-12-26 00:45:50 +00001364{
David Blaikie4f918ae2013-01-16 23:08:36 +00001365 return (__m64)__a[0];
Anders Carlsson45470752008-12-26 00:45:50 +00001366}
1367
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001368static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001369_mm_movpi64_pi64(__m64 __a)
Anders Carlsson45470752008-12-26 00:45:50 +00001370{
David Blaikie4f918ae2013-01-16 23:08:36 +00001371 return (__m128i){ (long long)__a, 0 };
Anders Carlsson45470752008-12-26 00:45:50 +00001372}
1373
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001374static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001375_mm_move_epi64(__m128i __a)
Anders Carlsson45470752008-12-26 00:45:50 +00001376{
David Blaikie4f918ae2013-01-16 23:08:36 +00001377 return __builtin_shufflevector(__a, (__m128i){ 0 }, 0, 2);
Anders Carlsson45470752008-12-26 00:45:50 +00001378}
1379
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001380static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001381_mm_unpackhi_pd(__m128d __a, __m128d __b)
Anders Carlsson45470752008-12-26 00:45:50 +00001382{
David Blaikie4f918ae2013-01-16 23:08:36 +00001383 return __builtin_shufflevector(__a, __b, 1, 2+1);
Anders Carlsson45470752008-12-26 00:45:50 +00001384}
1385
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001386static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001387_mm_unpacklo_pd(__m128d __a, __m128d __b)
Anders Carlsson45470752008-12-26 00:45:50 +00001388{
David Blaikie4f918ae2013-01-16 23:08:36 +00001389 return __builtin_shufflevector(__a, __b, 0, 2+0);
Anders Carlsson45470752008-12-26 00:45:50 +00001390}
1391
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001392static __inline__ int __attribute__((__always_inline__, __nodebug__))
David Blaikie4f918ae2013-01-16 23:08:36 +00001393_mm_movemask_pd(__m128d __a)
Anders Carlsson45470752008-12-26 00:45:50 +00001394{
David Blaikie4f918ae2013-01-16 23:08:36 +00001395 return __builtin_ia32_movmskpd(__a);
Anders Carlsson45470752008-12-26 00:45:50 +00001396}
1397
Bob Wilson32bae372011-11-05 06:08:06 +00001398#define _mm_shuffle_pd(a, b, i) __extension__ ({ \
Ted Kremenek185f1562013-10-07 23:51:11 +00001399 _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wshadow\""); \
Bob Wilson32bae372011-11-05 06:08:06 +00001400 __m128d __a = (a); \
1401 __m128d __b = (b); \
Ted Kremenek185f1562013-10-07 23:51:11 +00001402 _Pragma("clang diagnostic pop"); \
Bob Wilson32bae372011-11-05 06:08:06 +00001403 __builtin_shufflevector(__a, __b, (i) & 1, (((i) & 2) >> 1) + 2); })
Anders Carlsson45470752008-12-26 00:45:50 +00001404
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001405static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
Reid Klecknerf0cdc842013-04-19 17:00:14 +00001406_mm_castpd_ps(__m128d __a)
Anders Carlsson45470752008-12-26 00:45:50 +00001407{
Reid Klecknerf0cdc842013-04-19 17:00:14 +00001408 return (__m128)__a;
Anders Carlsson45470752008-12-26 00:45:50 +00001409}
1410
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001411static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
Reid Klecknerf0cdc842013-04-19 17:00:14 +00001412_mm_castpd_si128(__m128d __a)
Anders Carlsson45470752008-12-26 00:45:50 +00001413{
Reid Klecknerf0cdc842013-04-19 17:00:14 +00001414 return (__m128i)__a;
Anders Carlsson45470752008-12-26 00:45:50 +00001415}
1416
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001417static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
Reid Klecknerf0cdc842013-04-19 17:00:14 +00001418_mm_castps_pd(__m128 __a)
Anders Carlsson45470752008-12-26 00:45:50 +00001419{
Reid Klecknerf0cdc842013-04-19 17:00:14 +00001420 return (__m128d)__a;
Anders Carlsson45470752008-12-26 00:45:50 +00001421}
1422
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001423static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
Reid Klecknerf0cdc842013-04-19 17:00:14 +00001424_mm_castps_si128(__m128 __a)
Anders Carlsson45470752008-12-26 00:45:50 +00001425{
Reid Klecknerf0cdc842013-04-19 17:00:14 +00001426 return (__m128i)__a;
Anders Carlsson45470752008-12-26 00:45:50 +00001427}
1428
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001429static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
Reid Klecknerf0cdc842013-04-19 17:00:14 +00001430_mm_castsi128_ps(__m128i __a)
Anders Carlsson45470752008-12-26 00:45:50 +00001431{
Reid Klecknerf0cdc842013-04-19 17:00:14 +00001432 return (__m128)__a;
Anders Carlsson45470752008-12-26 00:45:50 +00001433}
1434
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001435static __inline__ __m128d __attribute__((__always_inline__, __nodebug__))
Reid Klecknerf0cdc842013-04-19 17:00:14 +00001436_mm_castsi128_pd(__m128i __a)
Anders Carlsson45470752008-12-26 00:45:50 +00001437{
Reid Klecknerf0cdc842013-04-19 17:00:14 +00001438 return (__m128d)__a;
Anders Carlsson45470752008-12-26 00:45:50 +00001439}
1440
Chris Lattner1bddbcb2010-03-22 18:14:12 +00001441static __inline__ void __attribute__((__always_inline__, __nodebug__))
Mike Stumpdae44132009-02-13 14:24:50 +00001442_mm_pause(void)
Anders Carlssonf1bc6602008-12-26 00:49:43 +00001443{
Anders Carlsson4bcd44d2008-12-26 02:22:10 +00001444 __asm__ volatile ("pause");
Anders Carlssonf1bc6602008-12-26 00:49:43 +00001445}
1446
Anders Carlssonbbd1fa22009-01-21 01:49:39 +00001447#define _MM_SHUFFLE2(x, y) (((x) << 1) | (y))
Anders Carlssonf1bc6602008-12-26 00:49:43 +00001448
Anders Carlsson37f2f002008-12-24 01:45:22 +00001449#endif /* __SSE2__ */
1450
1451#endif /* __EMMINTRIN_H */