authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-07-19 16:50:45-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-07-19 16:50:45-04:00
log2117fbdae35dddf368c4ce5bb39cc73fa0f78d4c
tree54b38682f94cc183a7b8c87a4693c09902db23aa
parent70da0762f7aa2d800da4a238499fc3f31dc4d31f
signaturelock-open Commit is signed but in an unrecognized format.

update C headers to llvm9

upstream commit 1931d3cb20a00da732c5210b123656632982fde0

130 files changed, 9519 insertions(+), 3542 deletions(-)

lib/include/__clang_cuda_builtin_vars.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- cuda_builtin_vars.h - CUDA built-in variables ---------------------===1/*===---- cuda_builtin_vars.h - CUDA built-in variables ---------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/__clang_cuda_cmath.h+31-18
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- __clang_cuda_cmath.h - Device-side CUDA cmath support ------------===1/*===---- __clang_cuda_cmath.h - Device-side CUDA cmath support ------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -44,12 +30,32 @@...@@ -44,12 +30,32 @@
44// implementation. Declaring in the global namespace and pulling into namespace30// implementation. Declaring in the global namespace and pulling into namespace
45// std covers all of the known knowns.31// std covers all of the known knowns.
4632
33#ifdef _OPENMP
34#define __DEVICE__ static __attribute__((always_inline))
35#else
47#define __DEVICE__ static __device__ __inline__ __attribute__((always_inline))36#define __DEVICE__ static __device__ __inline__ __attribute__((always_inline))
37#endif
4838
39// For C++ 17 we need to include noexcept attribute to be compatible
40// with the header-defined version. This may be removed once
41// variant is supported.
42#if defined(_OPENMP) && defined(__cplusplus) && __cplusplus >= 201703L
43#define __NOEXCEPT noexcept
44#else
45#define __NOEXCEPT
46#endif
47
48#if !(defined(_OPENMP) && defined(__cplusplus))
49__DEVICE__ long long abs(long long __n) { return ::llabs(__n); }49__DEVICE__ long long abs(long long __n) { return ::llabs(__n); }
50__DEVICE__ long abs(long __n) { return ::labs(__n); }50__DEVICE__ long abs(long __n) { return ::labs(__n); }
51__DEVICE__ float abs(float __x) { return ::fabsf(__x); }51__DEVICE__ float abs(float __x) { return ::fabsf(__x); }
52__DEVICE__ double abs(double __x) { return ::fabs(__x); }52__DEVICE__ double abs(double __x) { return ::fabs(__x); }
53#endif
54// TODO: remove once variat is supported.
55#if defined(_OPENMP) && defined(__cplusplus)
56__DEVICE__ const float abs(const float __x) { return ::fabsf((float)__x); }
57__DEVICE__ const double abs(const double __x) { return ::fabs((double)__x); }
58#endif
53__DEVICE__ float acos(float __x) { return ::acosf(__x); }59__DEVICE__ float acos(float __x) { return ::acosf(__x); }
54__DEVICE__ float asin(float __x) { return ::asinf(__x); }60__DEVICE__ float asin(float __x) { return ::asinf(__x); }
55__DEVICE__ float atan(float __x) { return ::atanf(__x); }61__DEVICE__ float atan(float __x) { return ::atanf(__x); }
...@@ -58,9 +64,11 @@ __DEVICE__ float ceil(float __x) { return ::ceilf(__x); }...@@ -58,9 +64,11 @@ __DEVICE__ float ceil(float __x) { return ::ceilf(__x); }
58__DEVICE__ float cos(float __x) { return ::cosf(__x); }64__DEVICE__ float cos(float __x) { return ::cosf(__x); }
59__DEVICE__ float cosh(float __x) { return ::coshf(__x); }65__DEVICE__ float cosh(float __x) { return ::coshf(__x); }
60__DEVICE__ float exp(float __x) { return ::expf(__x); }66__DEVICE__ float exp(float __x) { return ::expf(__x); }
61__DEVICE__ float fabs(float __x) { return ::fabsf(__x); }67__DEVICE__ float fabs(float __x) __NOEXCEPT { return ::fabsf(__x); }
62__DEVICE__ float floor(float __x) { return ::floorf(__x); }68__DEVICE__ float floor(float __x) { return ::floorf(__x); }
63__DEVICE__ float fmod(float __x, float __y) { return ::fmodf(__x, __y); }69__DEVICE__ float fmod(float __x, float __y) { return ::fmodf(__x, __y); }
70// TODO: remove when variant is supported
71#ifndef _OPENMP
64__DEVICE__ int fpclassify(float __x) {72__DEVICE__ int fpclassify(float __x) {
65 return __builtin_fpclassify(FP_NAN, FP_INFINITE, FP_NORMAL, FP_SUBNORMAL,73 return __builtin_fpclassify(FP_NAN, FP_INFINITE, FP_NORMAL, FP_SUBNORMAL,
66 FP_ZERO, __x);74 FP_ZERO, __x);
...@@ -69,6 +77,7 @@ __DEVICE__ int fpclassify(double __x) {...@@ -69,6 +77,7 @@ __DEVICE__ int fpclassify(double __x) {
69 return __builtin_fpclassify(FP_NAN, FP_INFINITE, FP_NORMAL, FP_SUBNORMAL,77 return __builtin_fpclassify(FP_NAN, FP_INFINITE, FP_NORMAL, FP_SUBNORMAL,
70 FP_ZERO, __x);78 FP_ZERO, __x);
71}79}
80#endif
72__DEVICE__ float frexp(float __arg, int *__exp) {81__DEVICE__ float frexp(float __arg, int *__exp) {
73 return ::frexpf(__arg, __exp);82 return ::frexpf(__arg, __exp);
74}83}
...@@ -448,7 +457,10 @@ using ::remainderf;...@@ -448,7 +457,10 @@ using ::remainderf;
448using ::remquof;457using ::remquof;
449using ::rintf;458using ::rintf;
450using ::roundf;459using ::roundf;
460// TODO: remove once variant is supported
461#ifndef _OPENMP
451using ::scalblnf;462using ::scalblnf;
463#endif
452using ::scalbnf;464using ::scalbnf;
453using ::sinf;465using ::sinf;
454using ::sinhf;466using ::sinhf;
...@@ -467,6 +479,7 @@ _GLIBCXX_END_NAMESPACE_VERSION...@@ -467,6 +479,7 @@ _GLIBCXX_END_NAMESPACE_VERSION
467} // namespace std479} // namespace std
468#endif480#endif
469481
482#undef __NOEXCEPT
470#undef __DEVICE__483#undef __DEVICE__
471484
472#endif485#endif
lib/include/__clang_cuda_complex_builtins.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===-- __clang_cuda_complex_builtins - CUDA impls of runtime complex fns ---===1/*===-- __clang_cuda_complex_builtins - CUDA impls of runtime complex fns ---===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/__clang_cuda_device_functions.h+59-34
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- __clang_cuda_device_functions.h - CUDA runtime support -----------===1/*===---- __clang_cuda_device_functions.h - CUDA runtime support -----------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -24,15 +10,21 @@...@@ -24,15 +10,21 @@
24#ifndef __CLANG_CUDA_DEVICE_FUNCTIONS_H__10#ifndef __CLANG_CUDA_DEVICE_FUNCTIONS_H__
25#define __CLANG_CUDA_DEVICE_FUNCTIONS_H__11#define __CLANG_CUDA_DEVICE_FUNCTIONS_H__
2612
13#ifndef _OPENMP
27#if CUDA_VERSION < 900014#if CUDA_VERSION < 9000
28#error This file is intended to be used with CUDA-9+ only.15#error This file is intended to be used with CUDA-9+ only.
29#endif16#endif
17#endif
3018
31// __DEVICE__ is a helper macro with common set of attributes for the wrappers19// __DEVICE__ is a helper macro with common set of attributes for the wrappers
32// we implement in this file. We need static in order to avoid emitting unused20// we implement in this file. We need static in order to avoid emitting unused
33// functions and __forceinline__ helps inlining these wrappers at -O1.21// functions and __forceinline__ helps inlining these wrappers at -O1.
34#pragma push_macro("__DEVICE__")22#pragma push_macro("__DEVICE__")
23#ifdef _OPENMP
24#define __DEVICE__ static __attribute__((always_inline))
25#else
35#define __DEVICE__ static __device__ __forceinline__26#define __DEVICE__ static __device__ __forceinline__
27#endif
3628
37// libdevice provides fast low precision and slow full-recision implementations29// libdevice provides fast low precision and slow full-recision implementations
38// for some functions. Which one gets selected depends on30// for some functions. Which one gets selected depends on
...@@ -45,6 +37,15 @@...@@ -45,6 +37,15 @@
45#define __FAST_OR_SLOW(fast, slow) slow37#define __FAST_OR_SLOW(fast, slow) slow
46#endif38#endif
4739
40// For C++ 17 we need to include noexcept attribute to be compatible
41// with the header-defined version. This may be removed once
42// variant is supported.
43#if defined(_OPENMP) && defined(__cplusplus) && __cplusplus >= 201703L
44#define __NOEXCEPT noexcept
45#else
46#define __NOEXCEPT
47#endif
48
48__DEVICE__ int __all(int __a) { return __nvvm_vote_all(__a); }49__DEVICE__ int __all(int __a) { return __nvvm_vote_all(__a); }
49__DEVICE__ int __any(int __a) { return __nvvm_vote_any(__a); }50__DEVICE__ int __any(int __a) { return __nvvm_vote_any(__a); }
50__DEVICE__ unsigned int __ballot(int __a) { return __nvvm_vote_ballot(__a); }51__DEVICE__ unsigned int __ballot(int __a) { return __nvvm_vote_ballot(__a); }
...@@ -52,8 +53,13 @@ __DEVICE__ unsigned int __brev(unsigned int __a) { return __nv_brev(__a); }...@@ -52,8 +53,13 @@ __DEVICE__ unsigned int __brev(unsigned int __a) { return __nv_brev(__a); }
52__DEVICE__ unsigned long long __brevll(unsigned long long __a) {53__DEVICE__ unsigned long long __brevll(unsigned long long __a) {
53 return __nv_brevll(__a);54 return __nv_brevll(__a);
54}55}
56#if defined(__cplusplus)
55__DEVICE__ void __brkpt() { asm volatile("brkpt;"); }57__DEVICE__ void __brkpt() { asm volatile("brkpt;"); }
56__DEVICE__ void __brkpt(int __a) { __brkpt(); }58__DEVICE__ void __brkpt(int __a) { __brkpt(); }
59#else
60__DEVICE__ void __attribute__((overloadable)) __brkpt(void) { asm volatile("brkpt;"); }
61__DEVICE__ void __attribute__((overloadable)) __brkpt(int __a) { __brkpt(); }
62#endif
57__DEVICE__ unsigned int __byte_perm(unsigned int __a, unsigned int __b,63__DEVICE__ unsigned int __byte_perm(unsigned int __a, unsigned int __b,
58 unsigned int __c) {64 unsigned int __c) {
59 return __nv_byte_perm(__a, __b, __c);65 return __nv_byte_perm(__a, __b, __c);
...@@ -237,6 +243,9 @@ __DEVICE__ int __ffs(int __a) { return __nv_ffs(__a); }...@@ -237,6 +243,9 @@ __DEVICE__ int __ffs(int __a) { return __nv_ffs(__a); }
237__DEVICE__ int __ffsll(long long __a) { return __nv_ffsll(__a); }243__DEVICE__ int __ffsll(long long __a) { return __nv_ffsll(__a); }
238__DEVICE__ int __finite(double __a) { return __nv_isfinited(__a); }244__DEVICE__ int __finite(double __a) { return __nv_isfinited(__a); }
239__DEVICE__ int __finitef(float __a) { return __nv_finitef(__a); }245__DEVICE__ int __finitef(float __a) { return __nv_finitef(__a); }
246#ifdef _MSC_VER
247__DEVICE__ int __finitel(long double __a);
248#endif
240__DEVICE__ int __float2int_rd(float __a) { return __nv_float2int_rd(__a); }249__DEVICE__ int __float2int_rd(float __a) { return __nv_float2int_rd(__a); }
241__DEVICE__ int __float2int_rn(float __a) { return __nv_float2int_rn(__a); }250__DEVICE__ int __float2int_rn(float __a) { return __nv_float2int_rn(__a); }
242__DEVICE__ int __float2int_ru(float __a) { return __nv_float2int_ru(__a); }251__DEVICE__ int __float2int_ru(float __a) { return __nv_float2int_ru(__a); }
...@@ -445,8 +454,14 @@ __DEVICE__ float __int_as_float(int __a) { return __nv_int_as_float(__a); }...@@ -445,8 +454,14 @@ __DEVICE__ float __int_as_float(int __a) { return __nv_int_as_float(__a); }
445__DEVICE__ int __isfinited(double __a) { return __nv_isfinited(__a); }454__DEVICE__ int __isfinited(double __a) { return __nv_isfinited(__a); }
446__DEVICE__ int __isinf(double __a) { return __nv_isinfd(__a); }455__DEVICE__ int __isinf(double __a) { return __nv_isinfd(__a); }
447__DEVICE__ int __isinff(float __a) { return __nv_isinff(__a); }456__DEVICE__ int __isinff(float __a) { return __nv_isinff(__a); }
457#ifdef _MSC_VER
458__DEVICE__ int __isinfl(long double __a);
459#endif
448__DEVICE__ int __isnan(double __a) { return __nv_isnand(__a); }460__DEVICE__ int __isnan(double __a) { return __nv_isnand(__a); }
449__DEVICE__ int __isnanf(float __a) { return __nv_isnanf(__a); }461__DEVICE__ int __isnanf(float __a) { return __nv_isnanf(__a); }
462#ifdef _MSC_VER
463__DEVICE__ int __isnanl(long double __a);
464#endif
450__DEVICE__ double __ll2double_rd(long long __a) {465__DEVICE__ double __ll2double_rd(long long __a) {
451 return __nv_ll2double_rd(__a);466 return __nv_ll2double_rd(__a);
452}467}
...@@ -520,8 +535,8 @@ __DEVICE__ unsigned int __sad(int __a, int __b, unsigned int __c) {...@@ -520,8 +535,8 @@ __DEVICE__ unsigned int __sad(int __a, int __b, unsigned int __c) {
520__DEVICE__ float __saturatef(float __a) { return __nv_saturatef(__a); }535__DEVICE__ float __saturatef(float __a) { return __nv_saturatef(__a); }
521__DEVICE__ int __signbitd(double __a) { return __nv_signbitd(__a); }536__DEVICE__ int __signbitd(double __a) { return __nv_signbitd(__a); }
522__DEVICE__ int __signbitf(float __a) { return __nv_signbitf(__a); }537__DEVICE__ int __signbitf(float __a) { return __nv_signbitf(__a); }
523__DEVICE__ void __sincosf(float __a, float *__sptr, float *__cptr) {538__DEVICE__ void __sincosf(float __a, float *__s, float *__c) {
524 return __nv_fast_sincosf(__a, __sptr, __cptr);539 return __nv_fast_sincosf(__a, __s, __c);
525}540}
526__DEVICE__ float __sinf(float __a) { return __nv_fast_sinf(__a); }541__DEVICE__ float __sinf(float __a) { return __nv_fast_sinf(__a); }
527__DEVICE__ int __syncthreads_and(int __a) { return __nvvm_bar0_and(__a); }542__DEVICE__ int __syncthreads_and(int __a) { return __nvvm_bar0_and(__a); }
...@@ -1468,7 +1483,8 @@ __DEVICE__ unsigned int __vsubus4(unsigned int __a, unsigned int __b) {...@@ -1468,7 +1483,8 @@ __DEVICE__ unsigned int __vsubus4(unsigned int __a, unsigned int __b) {
1468 return r;1483 return r;
1469}1484}
1470#endif // CUDA_VERSION >= 90201485#endif // CUDA_VERSION >= 9020
1471__DEVICE__ int abs(int __a) { return __nv_abs(__a); }1486__DEVICE__ int abs(int __a) __NOEXCEPT { return __nv_abs(__a); }
1487__DEVICE__ double fabs(double __a) __NOEXCEPT { return __nv_fabs(__a); }
1472__DEVICE__ double acos(double __a) { return __nv_acos(__a); }1488__DEVICE__ double acos(double __a) { return __nv_acos(__a); }
1473__DEVICE__ float acosf(float __a) { return __nv_acosf(__a); }1489__DEVICE__ float acosf(float __a) { return __nv_acosf(__a); }
1474__DEVICE__ double acosh(double __a) { return __nv_acosh(__a); }1490__DEVICE__ double acosh(double __a) { return __nv_acosh(__a); }
...@@ -1487,8 +1503,10 @@ __DEVICE__ double cbrt(double __a) { return __nv_cbrt(__a); }...@@ -1487,8 +1503,10 @@ __DEVICE__ double cbrt(double __a) { return __nv_cbrt(__a); }
1487__DEVICE__ float cbrtf(float __a) { return __nv_cbrtf(__a); }1503__DEVICE__ float cbrtf(float __a) { return __nv_cbrtf(__a); }
1488__DEVICE__ double ceil(double __a) { return __nv_ceil(__a); }1504__DEVICE__ double ceil(double __a) { return __nv_ceil(__a); }
1489__DEVICE__ float ceilf(float __a) { return __nv_ceilf(__a); }1505__DEVICE__ float ceilf(float __a) { return __nv_ceilf(__a); }
1506#ifndef _OPENMP
1490__DEVICE__ int clock() { return __nvvm_read_ptx_sreg_clock(); }1507__DEVICE__ int clock() { return __nvvm_read_ptx_sreg_clock(); }
1491__DEVICE__ long long clock64() { return __nvvm_read_ptx_sreg_clock64(); }1508__DEVICE__ long long clock64() { return __nvvm_read_ptx_sreg_clock64(); }
1509#endif
1492__DEVICE__ double copysign(double __a, double __b) {1510__DEVICE__ double copysign(double __a, double __b) {
1493 return __nv_copysign(__a, __b);1511 return __nv_copysign(__a, __b);
1494}1512}
...@@ -1525,7 +1543,6 @@ __DEVICE__ float exp2f(float __a) { return __nv_exp2f(__a); }...@@ -1525,7 +1543,6 @@ __DEVICE__ float exp2f(float __a) { return __nv_exp2f(__a); }
1525__DEVICE__ float expf(float __a) { return __nv_expf(__a); }1543__DEVICE__ float expf(float __a) { return __nv_expf(__a); }
1526__DEVICE__ double expm1(double __a) { return __nv_expm1(__a); }1544__DEVICE__ double expm1(double __a) { return __nv_expm1(__a); }
1527__DEVICE__ float expm1f(float __a) { return __nv_expm1f(__a); }1545__DEVICE__ float expm1f(float __a) { return __nv_expm1f(__a); }
1528__DEVICE__ double fabs(double __a) { return __nv_fabs(__a); }
1529__DEVICE__ float fabsf(float __a) { return __nv_fabsf(__a); }1546__DEVICE__ float fabsf(float __a) { return __nv_fabsf(__a); }
1530__DEVICE__ double fdim(double __a, double __b) { return __nv_fdim(__a, __b); }1547__DEVICE__ double fdim(double __a, double __b) { return __nv_fdim(__a, __b); }
1531__DEVICE__ float fdimf(float __a, float __b) { return __nv_fdimf(__a, __b); }1548__DEVICE__ float fdimf(float __a, float __b) { return __nv_fdimf(__a, __b); }
...@@ -1563,16 +1580,16 @@ __DEVICE__ double j1(double __a) { return __nv_j1(__a); }...@@ -1563,16 +1580,16 @@ __DEVICE__ double j1(double __a) { return __nv_j1(__a); }
1563__DEVICE__ float j1f(float __a) { return __nv_j1f(__a); }1580__DEVICE__ float j1f(float __a) { return __nv_j1f(__a); }
1564__DEVICE__ double jn(int __n, double __a) { return __nv_jn(__n, __a); }1581__DEVICE__ double jn(int __n, double __a) { return __nv_jn(__n, __a); }
1565__DEVICE__ float jnf(int __n, float __a) { return __nv_jnf(__n, __a); }1582__DEVICE__ float jnf(int __n, float __a) { return __nv_jnf(__n, __a); }
1566#if defined(__LP64__)1583#if defined(__LP64__) || defined(_WIN64)
1567__DEVICE__ long labs(long __a) { return llabs(__a); };1584__DEVICE__ long labs(long __a) __NOEXCEPT { return __nv_llabs(__a); };
1568#else1585#else
1569__DEVICE__ long labs(long __a) { return __nv_abs(__a); };1586__DEVICE__ long labs(long __a) __NOEXCEPT { return __nv_abs(__a); };
1570#endif1587#endif
1571__DEVICE__ double ldexp(double __a, int __b) { return __nv_ldexp(__a, __b); }1588__DEVICE__ double ldexp(double __a, int __b) { return __nv_ldexp(__a, __b); }
1572__DEVICE__ float ldexpf(float __a, int __b) { return __nv_ldexpf(__a, __b); }1589__DEVICE__ float ldexpf(float __a, int __b) { return __nv_ldexpf(__a, __b); }
1573__DEVICE__ double lgamma(double __a) { return __nv_lgamma(__a); }1590__DEVICE__ double lgamma(double __a) { return __nv_lgamma(__a); }
1574__DEVICE__ float lgammaf(float __a) { return __nv_lgammaf(__a); }1591__DEVICE__ float lgammaf(float __a) { return __nv_lgammaf(__a); }
1575__DEVICE__ long long llabs(long long __a) { return __nv_llabs(__a); }1592__DEVICE__ long long llabs(long long __a) __NOEXCEPT { return __nv_llabs(__a); }
1576__DEVICE__ long long llmax(long long __a, long long __b) {1593__DEVICE__ long long llmax(long long __a, long long __b) {
1577 return __nv_llmax(__a, __b);1594 return __nv_llmax(__a, __b);
1578}1595}
...@@ -1597,7 +1614,7 @@ __DEVICE__ float logbf(float __a) { return __nv_logbf(__a); }...@@ -1597,7 +1614,7 @@ __DEVICE__ float logbf(float __a) { return __nv_logbf(__a); }
1597__DEVICE__ float logf(float __a) {1614__DEVICE__ float logf(float __a) {
1598 return __FAST_OR_SLOW(__nv_fast_logf, __nv_logf)(__a);1615 return __FAST_OR_SLOW(__nv_fast_logf, __nv_logf)(__a);
1599}1616}
1600#if defined(__LP64__)1617#if defined(__LP64__) || defined(_WIN64)
1601__DEVICE__ long lrint(double __a) { return llrint(__a); }1618__DEVICE__ long lrint(double __a) { return llrint(__a); }
1602__DEVICE__ long lrintf(float __a) { return __float2ll_rn(__a); }1619__DEVICE__ long lrintf(float __a) { return __float2ll_rn(__a); }
1603__DEVICE__ long lround(double __a) { return llround(__a); }1620__DEVICE__ long lround(double __a) { return llround(__a); }
...@@ -1609,12 +1626,16 @@ __DEVICE__ long lround(double __a) { return round(__a); }...@@ -1609,12 +1626,16 @@ __DEVICE__ long lround(double __a) { return round(__a); }
1609__DEVICE__ long lroundf(float __a) { return roundf(__a); }1626__DEVICE__ long lroundf(float __a) { return roundf(__a); }
1610#endif1627#endif
1611__DEVICE__ int max(int __a, int __b) { return __nv_max(__a, __b); }1628__DEVICE__ int max(int __a, int __b) { return __nv_max(__a, __b); }
1629// These functions shouldn't be declared when including this header
1630// for math function resolution purposes.
1631#ifndef _OPENMP
1612__DEVICE__ void *memcpy(void *__a, const void *__b, size_t __c) {1632__DEVICE__ void *memcpy(void *__a, const void *__b, size_t __c) {
1613 return __builtin_memcpy(__a, __b, __c);1633 return __builtin_memcpy(__a, __b, __c);
1614}1634}
1615__DEVICE__ void *memset(void *__a, int __b, size_t __c) {1635__DEVICE__ void *memset(void *__a, int __b, size_t __c) {
1616 return __builtin_memset(__a, __b, __c);1636 return __builtin_memset(__a, __b, __c);
1617}1637}
1638#endif
1618__DEVICE__ int min(int __a, int __b) { return __nv_min(__a, __b); }1639__DEVICE__ int min(int __a, int __b) { return __nv_min(__a, __b); }
1619__DEVICE__ double modf(double __a, double *__b) { return __nv_modf(__a, __b); }1640__DEVICE__ double modf(double __a, double *__b) { return __nv_modf(__a, __b); }
1620__DEVICE__ float modff(float __a, float *__b) { return __nv_modff(__a, __b); }1641__DEVICE__ float modff(float __a, float *__b) { return __nv_modff(__a, __b); }
...@@ -1698,6 +1719,8 @@ __DEVICE__ double rsqrt(double __a) { return __nv_rsqrt(__a); }...@@ -1698,6 +1719,8 @@ __DEVICE__ double rsqrt(double __a) { return __nv_rsqrt(__a); }
1698__DEVICE__ float rsqrtf(float __a) { return __nv_rsqrtf(__a); }1719__DEVICE__ float rsqrtf(float __a) { return __nv_rsqrtf(__a); }
1699__DEVICE__ double scalbn(double __a, int __b) { return __nv_scalbn(__a, __b); }1720__DEVICE__ double scalbn(double __a, int __b) { return __nv_scalbn(__a, __b); }
1700__DEVICE__ float scalbnf(float __a, int __b) { return __nv_scalbnf(__a, __b); }1721__DEVICE__ float scalbnf(float __a, int __b) { return __nv_scalbnf(__a, __b); }
1722// TODO: remove once variant is supported
1723#ifndef _OPENMP
1701__DEVICE__ double scalbln(double __a, long __b) {1724__DEVICE__ double scalbln(double __a, long __b) {
1702 if (__b > INT_MAX)1725 if (__b > INT_MAX)
1703 return __a > 0 ? HUGE_VAL : -HUGE_VAL;1726 return __a > 0 ? HUGE_VAL : -HUGE_VAL;
...@@ -1712,18 +1735,19 @@ __DEVICE__ float scalblnf(float __a, long __b) {...@@ -1712,18 +1735,19 @@ __DEVICE__ float scalblnf(float __a, long __b) {
1712 return __a > 0 ? 0.f : -0.f;1735 return __a > 0 ? 0.f : -0.f;
1713 return scalbnf(__a, (int)__b);1736 return scalbnf(__a, (int)__b);
1714}1737}
1738#endif
1715__DEVICE__ double sin(double __a) { return __nv_sin(__a); }1739__DEVICE__ double sin(double __a) { return __nv_sin(__a); }
1716__DEVICE__ void sincos(double __a, double *__sptr, double *__cptr) {1740__DEVICE__ void sincos(double __a, double *__s, double *__c) {
1717 return __nv_sincos(__a, __sptr, __cptr);1741 return __nv_sincos(__a, __s, __c);
1718}1742}
1719__DEVICE__ void sincosf(float __a, float *__sptr, float *__cptr) {1743__DEVICE__ void sincosf(float __a, float *__s, float *__c) {
1720 return __FAST_OR_SLOW(__nv_fast_sincosf, __nv_sincosf)(__a, __sptr, __cptr);1744 return __FAST_OR_SLOW(__nv_fast_sincosf, __nv_sincosf)(__a, __s, __c);
1721}1745}
1722__DEVICE__ void sincospi(double __a, double *__sptr, double *__cptr) {1746__DEVICE__ void sincospi(double __a, double *__s, double *__c) {
1723 return __nv_sincospi(__a, __sptr, __cptr);1747 return __nv_sincospi(__a, __s, __c);
1724}1748}
1725__DEVICE__ void sincospif(float __a, float *__sptr, float *__cptr) {1749__DEVICE__ void sincospif(float __a, float *__s, float *__c) {
1726 return __nv_sincospif(__a, __sptr, __cptr);1750 return __nv_sincospif(__a, __s, __c);
1727}1751}
1728__DEVICE__ float sinf(float __a) {1752__DEVICE__ float sinf(float __a) {
1729 return __FAST_OR_SLOW(__nv_fast_sinf, __nv_sinf)(__a);1753 return __FAST_OR_SLOW(__nv_fast_sinf, __nv_sinf)(__a);
...@@ -1763,6 +1787,7 @@ __DEVICE__ float y1f(float __a) { return __nv_y1f(__a); }...@@ -1763,6 +1787,7 @@ __DEVICE__ float y1f(float __a) { return __nv_y1f(__a); }
1763__DEVICE__ double yn(int __a, double __b) { return __nv_yn(__a, __b); }1787__DEVICE__ double yn(int __a, double __b) { return __nv_yn(__a, __b); }
1764__DEVICE__ float ynf(int __a, float __b) { return __nv_ynf(__a, __b); }1788__DEVICE__ float ynf(int __a, float __b) { return __nv_ynf(__a, __b); }
17651789
1790#undef __NOEXCEPT
1766#pragma pop_macro("__DEVICE__")1791#pragma pop_macro("__DEVICE__")
1767#pragma pop_macro("__FAST_OR_SLOW")1792#pragma pop_macro("__FAST_OR_SLOW")
1768#endif // __CLANG_CUDA_DEVICE_FUNCTIONS_H__1793#endif // __CLANG_CUDA_DEVICE_FUNCTIONS_H__
lib/include/__clang_cuda_intrinsics.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===--- __clang_cuda_intrinsics.h - Device-side CUDA intrinsic wrappers ---===1/*===--- __clang_cuda_intrinsics.h - Device-side CUDA intrinsic wrappers ---===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/__clang_cuda_libdevice_declares.h+443-447
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===-- __clang_cuda_libdevice_declares.h - decls for libdevice functions --===1/*===-- __clang_cuda_libdevice_declares.h - decls for libdevice functions --===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -24,443 +10,453 @@...@@ -24,443 +10,453 @@
24#ifndef __CLANG_CUDA_LIBDEVICE_DECLARES_H__10#ifndef __CLANG_CUDA_LIBDEVICE_DECLARES_H__
25#define __CLANG_CUDA_LIBDEVICE_DECLARES_H__11#define __CLANG_CUDA_LIBDEVICE_DECLARES_H__
2612
13#if defined(__cplusplus)
27extern "C" {14extern "C" {
15#endif
16
17#if defined(_OPENMP)
18#define __DEVICE__
19#elif defined(__CUDA__)
20#define __DEVICE__ __device__
21#endif
2822
29__device__ int __nv_abs(int __a);23__DEVICE__ int __nv_abs(int __a);
30__device__ double __nv_acos(double __a);24__DEVICE__ double __nv_acos(double __a);
31__device__ float __nv_acosf(float __a);25__DEVICE__ float __nv_acosf(float __a);
32__device__ double __nv_acosh(double __a);26__DEVICE__ double __nv_acosh(double __a);
33__device__ float __nv_acoshf(float __a);27__DEVICE__ float __nv_acoshf(float __a);
34__device__ double __nv_asin(double __a);28__DEVICE__ double __nv_asin(double __a);
35__device__ float __nv_asinf(float __a);29__DEVICE__ float __nv_asinf(float __a);
36__device__ double __nv_asinh(double __a);30__DEVICE__ double __nv_asinh(double __a);
37__device__ float __nv_asinhf(float __a);31__DEVICE__ float __nv_asinhf(float __a);
38__device__ double __nv_atan2(double __a, double __b);32__DEVICE__ double __nv_atan2(double __a, double __b);
39__device__ float __nv_atan2f(float __a, float __b);33__DEVICE__ float __nv_atan2f(float __a, float __b);
40__device__ double __nv_atan(double __a);34__DEVICE__ double __nv_atan(double __a);
41__device__ float __nv_atanf(float __a);35__DEVICE__ float __nv_atanf(float __a);
42__device__ double __nv_atanh(double __a);36__DEVICE__ double __nv_atanh(double __a);
43__device__ float __nv_atanhf(float __a);37__DEVICE__ float __nv_atanhf(float __a);
44__device__ int __nv_brev(int __a);38__DEVICE__ int __nv_brev(int __a);
45__device__ long long __nv_brevll(long long __a);39__DEVICE__ long long __nv_brevll(long long __a);
46__device__ int __nv_byte_perm(int __a, int __b, int __c);40__DEVICE__ int __nv_byte_perm(int __a, int __b, int __c);
47__device__ double __nv_cbrt(double __a);41__DEVICE__ double __nv_cbrt(double __a);
48__device__ float __nv_cbrtf(float __a);42__DEVICE__ float __nv_cbrtf(float __a);
49__device__ double __nv_ceil(double __a);43__DEVICE__ double __nv_ceil(double __a);
50__device__ float __nv_ceilf(float __a);44__DEVICE__ float __nv_ceilf(float __a);
51__device__ int __nv_clz(int __a);45__DEVICE__ int __nv_clz(int __a);
52__device__ int __nv_clzll(long long __a);46__DEVICE__ int __nv_clzll(long long __a);
53__device__ double __nv_copysign(double __a, double __b);47__DEVICE__ double __nv_copysign(double __a, double __b);
54__device__ float __nv_copysignf(float __a, float __b);48__DEVICE__ float __nv_copysignf(float __a, float __b);
55__device__ double __nv_cos(double __a);49__DEVICE__ double __nv_cos(double __a);
56__device__ float __nv_cosf(float __a);50__DEVICE__ float __nv_cosf(float __a);
57__device__ double __nv_cosh(double __a);51__DEVICE__ double __nv_cosh(double __a);
58__device__ float __nv_coshf(float __a);52__DEVICE__ float __nv_coshf(float __a);
59__device__ double __nv_cospi(double __a);53__DEVICE__ double __nv_cospi(double __a);
60__device__ float __nv_cospif(float __a);54__DEVICE__ float __nv_cospif(float __a);
61__device__ double __nv_cyl_bessel_i0(double __a);55__DEVICE__ double __nv_cyl_bessel_i0(double __a);
62__device__ float __nv_cyl_bessel_i0f(float __a);56__DEVICE__ float __nv_cyl_bessel_i0f(float __a);
63__device__ double __nv_cyl_bessel_i1(double __a);57__DEVICE__ double __nv_cyl_bessel_i1(double __a);
64__device__ float __nv_cyl_bessel_i1f(float __a);58__DEVICE__ float __nv_cyl_bessel_i1f(float __a);
65__device__ double __nv_dadd_rd(double __a, double __b);59__DEVICE__ double __nv_dadd_rd(double __a, double __b);
66__device__ double __nv_dadd_rn(double __a, double __b);60__DEVICE__ double __nv_dadd_rn(double __a, double __b);
67__device__ double __nv_dadd_ru(double __a, double __b);61__DEVICE__ double __nv_dadd_ru(double __a, double __b);
68__device__ double __nv_dadd_rz(double __a, double __b);62__DEVICE__ double __nv_dadd_rz(double __a, double __b);
69__device__ double __nv_ddiv_rd(double __a, double __b);63__DEVICE__ double __nv_ddiv_rd(double __a, double __b);
70__device__ double __nv_ddiv_rn(double __a, double __b);64__DEVICE__ double __nv_ddiv_rn(double __a, double __b);
71__device__ double __nv_ddiv_ru(double __a, double __b);65__DEVICE__ double __nv_ddiv_ru(double __a, double __b);
72__device__ double __nv_ddiv_rz(double __a, double __b);66__DEVICE__ double __nv_ddiv_rz(double __a, double __b);
73__device__ double __nv_dmul_rd(double __a, double __b);67__DEVICE__ double __nv_dmul_rd(double __a, double __b);
74__device__ double __nv_dmul_rn(double __a, double __b);68__DEVICE__ double __nv_dmul_rn(double __a, double __b);
75__device__ double __nv_dmul_ru(double __a, double __b);69__DEVICE__ double __nv_dmul_ru(double __a, double __b);
76__device__ double __nv_dmul_rz(double __a, double __b);70__DEVICE__ double __nv_dmul_rz(double __a, double __b);
77__device__ float __nv_double2float_rd(double __a);71__DEVICE__ float __nv_double2float_rd(double __a);
78__device__ float __nv_double2float_rn(double __a);72__DEVICE__ float __nv_double2float_rn(double __a);
79__device__ float __nv_double2float_ru(double __a);73__DEVICE__ float __nv_double2float_ru(double __a);
80__device__ float __nv_double2float_rz(double __a);74__DEVICE__ float __nv_double2float_rz(double __a);
81__device__ int __nv_double2hiint(double __a);75__DEVICE__ int __nv_double2hiint(double __a);
82__device__ int __nv_double2int_rd(double __a);76__DEVICE__ int __nv_double2int_rd(double __a);
83__device__ int __nv_double2int_rn(double __a);77__DEVICE__ int __nv_double2int_rn(double __a);
84__device__ int __nv_double2int_ru(double __a);78__DEVICE__ int __nv_double2int_ru(double __a);
85__device__ int __nv_double2int_rz(double __a);79__DEVICE__ int __nv_double2int_rz(double __a);
86__device__ long long __nv_double2ll_rd(double __a);80__DEVICE__ long long __nv_double2ll_rd(double __a);
87__device__ long long __nv_double2ll_rn(double __a);81__DEVICE__ long long __nv_double2ll_rn(double __a);
88__device__ long long __nv_double2ll_ru(double __a);82__DEVICE__ long long __nv_double2ll_ru(double __a);
89__device__ long long __nv_double2ll_rz(double __a);83__DEVICE__ long long __nv_double2ll_rz(double __a);
90__device__ int __nv_double2loint(double __a);84__DEVICE__ int __nv_double2loint(double __a);
91__device__ unsigned int __nv_double2uint_rd(double __a);85__DEVICE__ unsigned int __nv_double2uint_rd(double __a);
92__device__ unsigned int __nv_double2uint_rn(double __a);86__DEVICE__ unsigned int __nv_double2uint_rn(double __a);
93__device__ unsigned int __nv_double2uint_ru(double __a);87__DEVICE__ unsigned int __nv_double2uint_ru(double __a);
94__device__ unsigned int __nv_double2uint_rz(double __a);88__DEVICE__ unsigned int __nv_double2uint_rz(double __a);
95__device__ unsigned long long __nv_double2ull_rd(double __a);89__DEVICE__ unsigned long long __nv_double2ull_rd(double __a);
96__device__ unsigned long long __nv_double2ull_rn(double __a);90__DEVICE__ unsigned long long __nv_double2ull_rn(double __a);
97__device__ unsigned long long __nv_double2ull_ru(double __a);91__DEVICE__ unsigned long long __nv_double2ull_ru(double __a);
98__device__ unsigned long long __nv_double2ull_rz(double __a);92__DEVICE__ unsigned long long __nv_double2ull_rz(double __a);
99__device__ unsigned long long __nv_double_as_longlong(double __a);93__DEVICE__ unsigned long long __nv_double_as_longlong(double __a);
100__device__ double __nv_drcp_rd(double __a);94__DEVICE__ double __nv_drcp_rd(double __a);
101__device__ double __nv_drcp_rn(double __a);95__DEVICE__ double __nv_drcp_rn(double __a);
102__device__ double __nv_drcp_ru(double __a);96__DEVICE__ double __nv_drcp_ru(double __a);
103__device__ double __nv_drcp_rz(double __a);97__DEVICE__ double __nv_drcp_rz(double __a);
104__device__ double __nv_dsqrt_rd(double __a);98__DEVICE__ double __nv_dsqrt_rd(double __a);
105__device__ double __nv_dsqrt_rn(double __a);99__DEVICE__ double __nv_dsqrt_rn(double __a);
106__device__ double __nv_dsqrt_ru(double __a);100__DEVICE__ double __nv_dsqrt_ru(double __a);
107__device__ double __nv_dsqrt_rz(double __a);101__DEVICE__ double __nv_dsqrt_rz(double __a);
108__device__ double __nv_dsub_rd(double __a, double __b);102__DEVICE__ double __nv_dsub_rd(double __a, double __b);
109__device__ double __nv_dsub_rn(double __a, double __b);103__DEVICE__ double __nv_dsub_rn(double __a, double __b);
110__device__ double __nv_dsub_ru(double __a, double __b);104__DEVICE__ double __nv_dsub_ru(double __a, double __b);
111__device__ double __nv_dsub_rz(double __a, double __b);105__DEVICE__ double __nv_dsub_rz(double __a, double __b);
112__device__ double __nv_erfc(double __a);106__DEVICE__ double __nv_erfc(double __a);
113__device__ float __nv_erfcf(float __a);107__DEVICE__ float __nv_erfcf(float __a);
114__device__ double __nv_erfcinv(double __a);108__DEVICE__ double __nv_erfcinv(double __a);
115__device__ float __nv_erfcinvf(float __a);109__DEVICE__ float __nv_erfcinvf(float __a);
116__device__ double __nv_erfcx(double __a);110__DEVICE__ double __nv_erfcx(double __a);
117__device__ float __nv_erfcxf(float __a);111__DEVICE__ float __nv_erfcxf(float __a);
118__device__ double __nv_erf(double __a);112__DEVICE__ double __nv_erf(double __a);
119__device__ float __nv_erff(float __a);113__DEVICE__ float __nv_erff(float __a);
120__device__ double __nv_erfinv(double __a);114__DEVICE__ double __nv_erfinv(double __a);
121__device__ float __nv_erfinvf(float __a);115__DEVICE__ float __nv_erfinvf(float __a);
122__device__ double __nv_exp10(double __a);116__DEVICE__ double __nv_exp10(double __a);
123__device__ float __nv_exp10f(float __a);117__DEVICE__ float __nv_exp10f(float __a);
124__device__ double __nv_exp2(double __a);118__DEVICE__ double __nv_exp2(double __a);
125__device__ float __nv_exp2f(float __a);119__DEVICE__ float __nv_exp2f(float __a);
126__device__ double __nv_exp(double __a);120__DEVICE__ double __nv_exp(double __a);
127__device__ float __nv_expf(float __a);121__DEVICE__ float __nv_expf(float __a);
128__device__ double __nv_expm1(double __a);122__DEVICE__ double __nv_expm1(double __a);
129__device__ float __nv_expm1f(float __a);123__DEVICE__ float __nv_expm1f(float __a);
130__device__ double __nv_fabs(double __a);124__DEVICE__ double __nv_fabs(double __a);
131__device__ float __nv_fabsf(float __a);125__DEVICE__ float __nv_fabsf(float __a);
132__device__ float __nv_fadd_rd(float __a, float __b);126__DEVICE__ float __nv_fadd_rd(float __a, float __b);
133__device__ float __nv_fadd_rn(float __a, float __b);127__DEVICE__ float __nv_fadd_rn(float __a, float __b);
134__device__ float __nv_fadd_ru(float __a, float __b);128__DEVICE__ float __nv_fadd_ru(float __a, float __b);
135__device__ float __nv_fadd_rz(float __a, float __b);129__DEVICE__ float __nv_fadd_rz(float __a, float __b);
136__device__ float __nv_fast_cosf(float __a);130__DEVICE__ float __nv_fast_cosf(float __a);
137__device__ float __nv_fast_exp10f(float __a);131__DEVICE__ float __nv_fast_exp10f(float __a);
138__device__ float __nv_fast_expf(float __a);132__DEVICE__ float __nv_fast_expf(float __a);
139__device__ float __nv_fast_fdividef(float __a, float __b);133__DEVICE__ float __nv_fast_fdividef(float __a, float __b);
140__device__ float __nv_fast_log10f(float __a);134__DEVICE__ float __nv_fast_log10f(float __a);
141__device__ float __nv_fast_log2f(float __a);135__DEVICE__ float __nv_fast_log2f(float __a);
142__device__ float __nv_fast_logf(float __a);136__DEVICE__ float __nv_fast_logf(float __a);
143__device__ float __nv_fast_powf(float __a, float __b);137__DEVICE__ float __nv_fast_powf(float __a, float __b);
144__device__ void __nv_fast_sincosf(float __a, float *__sptr, float *__cptr);138__DEVICE__ void __nv_fast_sincosf(float __a, float *__s, float *__c);
145__device__ float __nv_fast_sinf(float __a);139__DEVICE__ float __nv_fast_sinf(float __a);
146__device__ float __nv_fast_tanf(float __a);140__DEVICE__ float __nv_fast_tanf(float __a);
147__device__ double __nv_fdim(double __a, double __b);141__DEVICE__ double __nv_fdim(double __a, double __b);
148__device__ float __nv_fdimf(float __a, float __b);142__DEVICE__ float __nv_fdimf(float __a, float __b);
149__device__ float __nv_fdiv_rd(float __a, float __b);143__DEVICE__ float __nv_fdiv_rd(float __a, float __b);
150__device__ float __nv_fdiv_rn(float __a, float __b);144__DEVICE__ float __nv_fdiv_rn(float __a, float __b);
151__device__ float __nv_fdiv_ru(float __a, float __b);145__DEVICE__ float __nv_fdiv_ru(float __a, float __b);
152__device__ float __nv_fdiv_rz(float __a, float __b);146__DEVICE__ float __nv_fdiv_rz(float __a, float __b);
153__device__ int __nv_ffs(int __a);147__DEVICE__ int __nv_ffs(int __a);
154__device__ int __nv_ffsll(long long __a);148__DEVICE__ int __nv_ffsll(long long __a);
155__device__ int __nv_finitef(float __a);149__DEVICE__ int __nv_finitef(float __a);
156__device__ unsigned short __nv_float2half_rn(float __a);150__DEVICE__ unsigned short __nv_float2half_rn(float __a);
157__device__ int __nv_float2int_rd(float __a);151__DEVICE__ int __nv_float2int_rd(float __a);
158__device__ int __nv_float2int_rn(float __a);152__DEVICE__ int __nv_float2int_rn(float __a);
159__device__ int __nv_float2int_ru(float __a);153__DEVICE__ int __nv_float2int_ru(float __a);
160__device__ int __nv_float2int_rz(float __a);154__DEVICE__ int __nv_float2int_rz(float __a);
161__device__ long long __nv_float2ll_rd(float __a);155__DEVICE__ long long __nv_float2ll_rd(float __a);
162__device__ long long __nv_float2ll_rn(float __a);156__DEVICE__ long long __nv_float2ll_rn(float __a);
163__device__ long long __nv_float2ll_ru(float __a);157__DEVICE__ long long __nv_float2ll_ru(float __a);
164__device__ long long __nv_float2ll_rz(float __a);158__DEVICE__ long long __nv_float2ll_rz(float __a);
165__device__ unsigned int __nv_float2uint_rd(float __a);159__DEVICE__ unsigned int __nv_float2uint_rd(float __a);
166__device__ unsigned int __nv_float2uint_rn(float __a);160__DEVICE__ unsigned int __nv_float2uint_rn(float __a);
167__device__ unsigned int __nv_float2uint_ru(float __a);161__DEVICE__ unsigned int __nv_float2uint_ru(float __a);
168__device__ unsigned int __nv_float2uint_rz(float __a);162__DEVICE__ unsigned int __nv_float2uint_rz(float __a);
169__device__ unsigned long long __nv_float2ull_rd(float __a);163__DEVICE__ unsigned long long __nv_float2ull_rd(float __a);
170__device__ unsigned long long __nv_float2ull_rn(float __a);164__DEVICE__ unsigned long long __nv_float2ull_rn(float __a);
171__device__ unsigned long long __nv_float2ull_ru(float __a);165__DEVICE__ unsigned long long __nv_float2ull_ru(float __a);
172__device__ unsigned long long __nv_float2ull_rz(float __a);166__DEVICE__ unsigned long long __nv_float2ull_rz(float __a);
173__device__ int __nv_float_as_int(float __a);167__DEVICE__ int __nv_float_as_int(float __a);
174__device__ unsigned int __nv_float_as_uint(float __a);168__DEVICE__ unsigned int __nv_float_as_uint(float __a);
175__device__ double __nv_floor(double __a);169__DEVICE__ double __nv_floor(double __a);
176__device__ float __nv_floorf(float __a);170__DEVICE__ float __nv_floorf(float __a);
177__device__ double __nv_fma(double __a, double __b, double __c);171__DEVICE__ double __nv_fma(double __a, double __b, double __c);
178__device__ float __nv_fmaf(float __a, float __b, float __c);172__DEVICE__ float __nv_fmaf(float __a, float __b, float __c);
179__device__ float __nv_fmaf_ieee_rd(float __a, float __b, float __c);173__DEVICE__ float __nv_fmaf_ieee_rd(float __a, float __b, float __c);
180__device__ float __nv_fmaf_ieee_rn(float __a, float __b, float __c);174__DEVICE__ float __nv_fmaf_ieee_rn(float __a, float __b, float __c);
181__device__ float __nv_fmaf_ieee_ru(float __a, float __b, float __c);175__DEVICE__ float __nv_fmaf_ieee_ru(float __a, float __b, float __c);
182__device__ float __nv_fmaf_ieee_rz(float __a, float __b, float __c);176__DEVICE__ float __nv_fmaf_ieee_rz(float __a, float __b, float __c);
183__device__ float __nv_fmaf_rd(float __a, float __b, float __c);177__DEVICE__ float __nv_fmaf_rd(float __a, float __b, float __c);
184__device__ float __nv_fmaf_rn(float __a, float __b, float __c);178__DEVICE__ float __nv_fmaf_rn(float __a, float __b, float __c);
185__device__ float __nv_fmaf_ru(float __a, float __b, float __c);179__DEVICE__ float __nv_fmaf_ru(float __a, float __b, float __c);
186__device__ float __nv_fmaf_rz(float __a, float __b, float __c);180__DEVICE__ float __nv_fmaf_rz(float __a, float __b, float __c);
187__device__ double __nv_fma_rd(double __a, double __b, double __c);181__DEVICE__ double __nv_fma_rd(double __a, double __b, double __c);
188__device__ double __nv_fma_rn(double __a, double __b, double __c);182__DEVICE__ double __nv_fma_rn(double __a, double __b, double __c);
189__device__ double __nv_fma_ru(double __a, double __b, double __c);183__DEVICE__ double __nv_fma_ru(double __a, double __b, double __c);
190__device__ double __nv_fma_rz(double __a, double __b, double __c);184__DEVICE__ double __nv_fma_rz(double __a, double __b, double __c);
191__device__ double __nv_fmax(double __a, double __b);185__DEVICE__ double __nv_fmax(double __a, double __b);
192__device__ float __nv_fmaxf(float __a, float __b);186__DEVICE__ float __nv_fmaxf(float __a, float __b);
193__device__ double __nv_fmin(double __a, double __b);187__DEVICE__ double __nv_fmin(double __a, double __b);
194__device__ float __nv_fminf(float __a, float __b);188__DEVICE__ float __nv_fminf(float __a, float __b);
195__device__ double __nv_fmod(double __a, double __b);189__DEVICE__ double __nv_fmod(double __a, double __b);
196__device__ float __nv_fmodf(float __a, float __b);190__DEVICE__ float __nv_fmodf(float __a, float __b);
197__device__ float __nv_fmul_rd(float __a, float __b);191__DEVICE__ float __nv_fmul_rd(float __a, float __b);
198__device__ float __nv_fmul_rn(float __a, float __b);192__DEVICE__ float __nv_fmul_rn(float __a, float __b);
199__device__ float __nv_fmul_ru(float __a, float __b);193__DEVICE__ float __nv_fmul_ru(float __a, float __b);
200__device__ float __nv_fmul_rz(float __a, float __b);194__DEVICE__ float __nv_fmul_rz(float __a, float __b);
201__device__ float __nv_frcp_rd(float __a);195__DEVICE__ float __nv_frcp_rd(float __a);
202__device__ float __nv_frcp_rn(float __a);196__DEVICE__ float __nv_frcp_rn(float __a);
203__device__ float __nv_frcp_ru(float __a);197__DEVICE__ float __nv_frcp_ru(float __a);
204__device__ float __nv_frcp_rz(float __a);198__DEVICE__ float __nv_frcp_rz(float __a);
205__device__ double __nv_frexp(double __a, int *__b);199__DEVICE__ double __nv_frexp(double __a, int *__b);
206__device__ float __nv_frexpf(float __a, int *__b);200__DEVICE__ float __nv_frexpf(float __a, int *__b);
207__device__ float __nv_frsqrt_rn(float __a);201__DEVICE__ float __nv_frsqrt_rn(float __a);
208__device__ float __nv_fsqrt_rd(float __a);202__DEVICE__ float __nv_fsqrt_rd(float __a);
209__device__ float __nv_fsqrt_rn(float __a);203__DEVICE__ float __nv_fsqrt_rn(float __a);
210__device__ float __nv_fsqrt_ru(float __a);204__DEVICE__ float __nv_fsqrt_ru(float __a);
211__device__ float __nv_fsqrt_rz(float __a);205__DEVICE__ float __nv_fsqrt_rz(float __a);
212__device__ float __nv_fsub_rd(float __a, float __b);206__DEVICE__ float __nv_fsub_rd(float __a, float __b);
213__device__ float __nv_fsub_rn(float __a, float __b);207__DEVICE__ float __nv_fsub_rn(float __a, float __b);
214__device__ float __nv_fsub_ru(float __a, float __b);208__DEVICE__ float __nv_fsub_ru(float __a, float __b);
215__device__ float __nv_fsub_rz(float __a, float __b);209__DEVICE__ float __nv_fsub_rz(float __a, float __b);
216__device__ int __nv_hadd(int __a, int __b);210__DEVICE__ int __nv_hadd(int __a, int __b);
217__device__ float __nv_half2float(unsigned short __h);211__DEVICE__ float __nv_half2float(unsigned short __h);
218__device__ double __nv_hiloint2double(int __a, int __b);212__DEVICE__ double __nv_hiloint2double(int __a, int __b);
219__device__ double __nv_hypot(double __a, double __b);213__DEVICE__ double __nv_hypot(double __a, double __b);
220__device__ float __nv_hypotf(float __a, float __b);214__DEVICE__ float __nv_hypotf(float __a, float __b);
221__device__ int __nv_ilogb(double __a);215__DEVICE__ int __nv_ilogb(double __a);
222__device__ int __nv_ilogbf(float __a);216__DEVICE__ int __nv_ilogbf(float __a);
223__device__ double __nv_int2double_rn(int __a);217__DEVICE__ double __nv_int2double_rn(int __a);
224__device__ float __nv_int2float_rd(int __a);218__DEVICE__ float __nv_int2float_rd(int __a);
225__device__ float __nv_int2float_rn(int __a);219__DEVICE__ float __nv_int2float_rn(int __a);
226__device__ float __nv_int2float_ru(int __a);220__DEVICE__ float __nv_int2float_ru(int __a);
227__device__ float __nv_int2float_rz(int __a);221__DEVICE__ float __nv_int2float_rz(int __a);
228__device__ float __nv_int_as_float(int __a);222__DEVICE__ float __nv_int_as_float(int __a);
229__device__ int __nv_isfinited(double __a);223__DEVICE__ int __nv_isfinited(double __a);
230__device__ int __nv_isinfd(double __a);224__DEVICE__ int __nv_isinfd(double __a);
231__device__ int __nv_isinff(float __a);225__DEVICE__ int __nv_isinff(float __a);
232__device__ int __nv_isnand(double __a);226__DEVICE__ int __nv_isnand(double __a);
233__device__ int __nv_isnanf(float __a);227__DEVICE__ int __nv_isnanf(float __a);
234__device__ double __nv_j0(double __a);228__DEVICE__ double __nv_j0(double __a);
235__device__ float __nv_j0f(float __a);229__DEVICE__ float __nv_j0f(float __a);
236__device__ double __nv_j1(double __a);230__DEVICE__ double __nv_j1(double __a);
237__device__ float __nv_j1f(float __a);231__DEVICE__ float __nv_j1f(float __a);
238__device__ float __nv_jnf(int __a, float __b);232__DEVICE__ float __nv_jnf(int __a, float __b);
239__device__ double __nv_jn(int __a, double __b);233__DEVICE__ double __nv_jn(int __a, double __b);
240__device__ double __nv_ldexp(double __a, int __b);234__DEVICE__ double __nv_ldexp(double __a, int __b);
241__device__ float __nv_ldexpf(float __a, int __b);235__DEVICE__ float __nv_ldexpf(float __a, int __b);
242__device__ double __nv_lgamma(double __a);236__DEVICE__ double __nv_lgamma(double __a);
243__device__ float __nv_lgammaf(float __a);237__DEVICE__ float __nv_lgammaf(float __a);
244__device__ double __nv_ll2double_rd(long long __a);238__DEVICE__ double __nv_ll2double_rd(long long __a);
245__device__ double __nv_ll2double_rn(long long __a);239__DEVICE__ double __nv_ll2double_rn(long long __a);
246__device__ double __nv_ll2double_ru(long long __a);240__DEVICE__ double __nv_ll2double_ru(long long __a);
247__device__ double __nv_ll2double_rz(long long __a);241__DEVICE__ double __nv_ll2double_rz(long long __a);
248__device__ float __nv_ll2float_rd(long long __a);242__DEVICE__ float __nv_ll2float_rd(long long __a);
249__device__ float __nv_ll2float_rn(long long __a);243__DEVICE__ float __nv_ll2float_rn(long long __a);
250__device__ float __nv_ll2float_ru(long long __a);244__DEVICE__ float __nv_ll2float_ru(long long __a);
251__device__ float __nv_ll2float_rz(long long __a);245__DEVICE__ float __nv_ll2float_rz(long long __a);
252__device__ long long __nv_llabs(long long __a);246__DEVICE__ long long __nv_llabs(long long __a);
253__device__ long long __nv_llmax(long long __a, long long __b);247__DEVICE__ long long __nv_llmax(long long __a, long long __b);
254__device__ long long __nv_llmin(long long __a, long long __b);248__DEVICE__ long long __nv_llmin(long long __a, long long __b);
255__device__ long long __nv_llrint(double __a);249__DEVICE__ long long __nv_llrint(double __a);
256__device__ long long __nv_llrintf(float __a);250__DEVICE__ long long __nv_llrintf(float __a);
257__device__ long long __nv_llround(double __a);251__DEVICE__ long long __nv_llround(double __a);
258__device__ long long __nv_llroundf(float __a);252__DEVICE__ long long __nv_llroundf(float __a);
259__device__ double __nv_log10(double __a);253__DEVICE__ double __nv_log10(double __a);
260__device__ float __nv_log10f(float __a);254__DEVICE__ float __nv_log10f(float __a);
261__device__ double __nv_log1p(double __a);255__DEVICE__ double __nv_log1p(double __a);
262__device__ float __nv_log1pf(float __a);256__DEVICE__ float __nv_log1pf(float __a);
263__device__ double __nv_log2(double __a);257__DEVICE__ double __nv_log2(double __a);
264__device__ float __nv_log2f(float __a);258__DEVICE__ float __nv_log2f(float __a);
265__device__ double __nv_logb(double __a);259__DEVICE__ double __nv_logb(double __a);
266__device__ float __nv_logbf(float __a);260__DEVICE__ float __nv_logbf(float __a);
267__device__ double __nv_log(double __a);261__DEVICE__ double __nv_log(double __a);
268__device__ float __nv_logf(float __a);262__DEVICE__ float __nv_logf(float __a);
269__device__ double __nv_longlong_as_double(long long __a);263__DEVICE__ double __nv_longlong_as_double(long long __a);
270__device__ int __nv_max(int __a, int __b);264__DEVICE__ int __nv_max(int __a, int __b);
271__device__ int __nv_min(int __a, int __b);265__DEVICE__ int __nv_min(int __a, int __b);
272__device__ double __nv_modf(double __a, double *__b);266__DEVICE__ double __nv_modf(double __a, double *__b);
273__device__ float __nv_modff(float __a, float *__b);267__DEVICE__ float __nv_modff(float __a, float *__b);
274__device__ int __nv_mul24(int __a, int __b);268__DEVICE__ int __nv_mul24(int __a, int __b);
275__device__ long long __nv_mul64hi(long long __a, long long __b);269__DEVICE__ long long __nv_mul64hi(long long __a, long long __b);
276__device__ int __nv_mulhi(int __a, int __b);270__DEVICE__ int __nv_mulhi(int __a, int __b);
277__device__ double __nv_nan(const signed char *__a);271__DEVICE__ double __nv_nan(const signed char *__a);
278__device__ float __nv_nanf(const signed char *__a);272__DEVICE__ float __nv_nanf(const signed char *__a);
279__device__ double __nv_nearbyint(double __a);273__DEVICE__ double __nv_nearbyint(double __a);
280__device__ float __nv_nearbyintf(float __a);274__DEVICE__ float __nv_nearbyintf(float __a);
281__device__ double __nv_nextafter(double __a, double __b);275__DEVICE__ double __nv_nextafter(double __a, double __b);
282__device__ float __nv_nextafterf(float __a, float __b);276__DEVICE__ float __nv_nextafterf(float __a, float __b);
283__device__ double __nv_norm3d(double __a, double __b, double __c);277__DEVICE__ double __nv_norm3d(double __a, double __b, double __c);
284__device__ float __nv_norm3df(float __a, float __b, float __c);278__DEVICE__ float __nv_norm3df(float __a, float __b, float __c);
285__device__ double __nv_norm4d(double __a, double __b, double __c, double __d);279__DEVICE__ double __nv_norm4d(double __a, double __b, double __c, double __d);
286__device__ float __nv_norm4df(float __a, float __b, float __c, float __d);280__DEVICE__ float __nv_norm4df(float __a, float __b, float __c, float __d);
287__device__ double __nv_normcdf(double __a);281__DEVICE__ double __nv_normcdf(double __a);
288__device__ float __nv_normcdff(float __a);282__DEVICE__ float __nv_normcdff(float __a);
289__device__ double __nv_normcdfinv(double __a);283__DEVICE__ double __nv_normcdfinv(double __a);
290__device__ float __nv_normcdfinvf(float __a);284__DEVICE__ float __nv_normcdfinvf(float __a);
291__device__ float __nv_normf(int __a, const float *__b);285__DEVICE__ float __nv_normf(int __a, const float *__b);
292__device__ double __nv_norm(int __a, const double *__b);286__DEVICE__ double __nv_norm(int __a, const double *__b);
293__device__ int __nv_popc(int __a);287__DEVICE__ int __nv_popc(int __a);
294__device__ int __nv_popcll(long long __a);288__DEVICE__ int __nv_popcll(long long __a);
295__device__ double __nv_pow(double __a, double __b);289__DEVICE__ double __nv_pow(double __a, double __b);
296__device__ float __nv_powf(float __a, float __b);290__DEVICE__ float __nv_powf(float __a, float __b);
297__device__ double __nv_powi(double __a, int __b);291__DEVICE__ double __nv_powi(double __a, int __b);
298__device__ float __nv_powif(float __a, int __b);292__DEVICE__ float __nv_powif(float __a, int __b);
299__device__ double __nv_rcbrt(double __a);293__DEVICE__ double __nv_rcbrt(double __a);
300__device__ float __nv_rcbrtf(float __a);294__DEVICE__ float __nv_rcbrtf(float __a);
301__device__ double __nv_rcp64h(double __a);295__DEVICE__ double __nv_rcp64h(double __a);
302__device__ double __nv_remainder(double __a, double __b);296__DEVICE__ double __nv_remainder(double __a, double __b);
303__device__ float __nv_remainderf(float __a, float __b);297__DEVICE__ float __nv_remainderf(float __a, float __b);
304__device__ double __nv_remquo(double __a, double __b, int *__c);298__DEVICE__ double __nv_remquo(double __a, double __b, int *__c);
305__device__ float __nv_remquof(float __a, float __b, int *__c);299__DEVICE__ float __nv_remquof(float __a, float __b, int *__c);
306__device__ int __nv_rhadd(int __a, int __b);300__DEVICE__ int __nv_rhadd(int __a, int __b);
307__device__ double __nv_rhypot(double __a, double __b);301__DEVICE__ double __nv_rhypot(double __a, double __b);
308__device__ float __nv_rhypotf(float __a, float __b);302__DEVICE__ float __nv_rhypotf(float __a, float __b);
309__device__ double __nv_rint(double __a);303__DEVICE__ double __nv_rint(double __a);
310__device__ float __nv_rintf(float __a);304__DEVICE__ float __nv_rintf(float __a);
311__device__ double __nv_rnorm3d(double __a, double __b, double __c);305__DEVICE__ double __nv_rnorm3d(double __a, double __b, double __c);
312__device__ float __nv_rnorm3df(float __a, float __b, float __c);306__DEVICE__ float __nv_rnorm3df(float __a, float __b, float __c);
313__device__ double __nv_rnorm4d(double __a, double __b, double __c, double __d);307__DEVICE__ double __nv_rnorm4d(double __a, double __b, double __c, double __d);
314__device__ float __nv_rnorm4df(float __a, float __b, float __c, float __d);308__DEVICE__ float __nv_rnorm4df(float __a, float __b, float __c, float __d);
315__device__ float __nv_rnormf(int __a, const float *__b);309__DEVICE__ float __nv_rnormf(int __a, const float *__b);
316__device__ double __nv_rnorm(int __a, const double *__b);310__DEVICE__ double __nv_rnorm(int __a, const double *__b);
317__device__ double __nv_round(double __a);311__DEVICE__ double __nv_round(double __a);
318__device__ float __nv_roundf(float __a);312__DEVICE__ float __nv_roundf(float __a);
319__device__ double __nv_rsqrt(double __a);313__DEVICE__ double __nv_rsqrt(double __a);
320__device__ float __nv_rsqrtf(float __a);314__DEVICE__ float __nv_rsqrtf(float __a);
321__device__ int __nv_sad(int __a, int __b, int __c);315__DEVICE__ int __nv_sad(int __a, int __b, int __c);
322__device__ float __nv_saturatef(float __a);316__DEVICE__ float __nv_saturatef(float __a);
323__device__ double __nv_scalbn(double __a, int __b);317__DEVICE__ double __nv_scalbn(double __a, int __b);
324__device__ float __nv_scalbnf(float __a, int __b);318__DEVICE__ float __nv_scalbnf(float __a, int __b);
325__device__ int __nv_signbitd(double __a);319__DEVICE__ int __nv_signbitd(double __a);
326__device__ int __nv_signbitf(float __a);320__DEVICE__ int __nv_signbitf(float __a);
327__device__ void __nv_sincos(double __a, double *__b, double *__c);321__DEVICE__ void __nv_sincos(double __a, double *__b, double *__c);
328__device__ void __nv_sincosf(float __a, float *__b, float *__c);322__DEVICE__ void __nv_sincosf(float __a, float *__b, float *__c);
329__device__ void __nv_sincospi(double __a, double *__b, double *__c);323__DEVICE__ void __nv_sincospi(double __a, double *__b, double *__c);
330__device__ void __nv_sincospif(float __a, float *__b, float *__c);324__DEVICE__ void __nv_sincospif(float __a, float *__b, float *__c);
331__device__ double __nv_sin(double __a);325__DEVICE__ double __nv_sin(double __a);
332__device__ float __nv_sinf(float __a);326__DEVICE__ float __nv_sinf(float __a);
333__device__ double __nv_sinh(double __a);327__DEVICE__ double __nv_sinh(double __a);
334__device__ float __nv_sinhf(float __a);328__DEVICE__ float __nv_sinhf(float __a);
335__device__ double __nv_sinpi(double __a);329__DEVICE__ double __nv_sinpi(double __a);
336__device__ float __nv_sinpif(float __a);330__DEVICE__ float __nv_sinpif(float __a);
337__device__ double __nv_sqrt(double __a);331__DEVICE__ double __nv_sqrt(double __a);
338__device__ float __nv_sqrtf(float __a);332__DEVICE__ float __nv_sqrtf(float __a);
339__device__ double __nv_tan(double __a);333__DEVICE__ double __nv_tan(double __a);
340__device__ float __nv_tanf(float __a);334__DEVICE__ float __nv_tanf(float __a);
341__device__ double __nv_tanh(double __a);335__DEVICE__ double __nv_tanh(double __a);
342__device__ float __nv_tanhf(float __a);336__DEVICE__ float __nv_tanhf(float __a);
343__device__ double __nv_tgamma(double __a);337__DEVICE__ double __nv_tgamma(double __a);
344__device__ float __nv_tgammaf(float __a);338__DEVICE__ float __nv_tgammaf(float __a);
345__device__ double __nv_trunc(double __a);339__DEVICE__ double __nv_trunc(double __a);
346__device__ float __nv_truncf(float __a);340__DEVICE__ float __nv_truncf(float __a);
347__device__ int __nv_uhadd(unsigned int __a, unsigned int __b);341__DEVICE__ int __nv_uhadd(unsigned int __a, unsigned int __b);
348__device__ double __nv_uint2double_rn(unsigned int __i);342__DEVICE__ double __nv_uint2double_rn(unsigned int __i);
349__device__ float __nv_uint2float_rd(unsigned int __a);343__DEVICE__ float __nv_uint2float_rd(unsigned int __a);
350__device__ float __nv_uint2float_rn(unsigned int __a);344__DEVICE__ float __nv_uint2float_rn(unsigned int __a);
351__device__ float __nv_uint2float_ru(unsigned int __a);345__DEVICE__ float __nv_uint2float_ru(unsigned int __a);
352__device__ float __nv_uint2float_rz(unsigned int __a);346__DEVICE__ float __nv_uint2float_rz(unsigned int __a);
353__device__ float __nv_uint_as_float(unsigned int __a);347__DEVICE__ float __nv_uint_as_float(unsigned int __a);
354__device__ double __nv_ull2double_rd(unsigned long long __a);348__DEVICE__ double __nv_ull2double_rd(unsigned long long __a);
355__device__ double __nv_ull2double_rn(unsigned long long __a);349__DEVICE__ double __nv_ull2double_rn(unsigned long long __a);
356__device__ double __nv_ull2double_ru(unsigned long long __a);350__DEVICE__ double __nv_ull2double_ru(unsigned long long __a);
357__device__ double __nv_ull2double_rz(unsigned long long __a);351__DEVICE__ double __nv_ull2double_rz(unsigned long long __a);
358__device__ float __nv_ull2float_rd(unsigned long long __a);352__DEVICE__ float __nv_ull2float_rd(unsigned long long __a);
359__device__ float __nv_ull2float_rn(unsigned long long __a);353__DEVICE__ float __nv_ull2float_rn(unsigned long long __a);
360__device__ float __nv_ull2float_ru(unsigned long long __a);354__DEVICE__ float __nv_ull2float_ru(unsigned long long __a);
361__device__ float __nv_ull2float_rz(unsigned long long __a);355__DEVICE__ float __nv_ull2float_rz(unsigned long long __a);
362__device__ unsigned long long __nv_ullmax(unsigned long long __a,356__DEVICE__ unsigned long long __nv_ullmax(unsigned long long __a,
363 unsigned long long __b);357 unsigned long long __b);
364__device__ unsigned long long __nv_ullmin(unsigned long long __a,358__DEVICE__ unsigned long long __nv_ullmin(unsigned long long __a,
365 unsigned long long __b);359 unsigned long long __b);
366__device__ unsigned int __nv_umax(unsigned int __a, unsigned int __b);360__DEVICE__ unsigned int __nv_umax(unsigned int __a, unsigned int __b);
367__device__ unsigned int __nv_umin(unsigned int __a, unsigned int __b);361__DEVICE__ unsigned int __nv_umin(unsigned int __a, unsigned int __b);
368__device__ unsigned int __nv_umul24(unsigned int __a, unsigned int __b);362__DEVICE__ unsigned int __nv_umul24(unsigned int __a, unsigned int __b);
369__device__ unsigned long long __nv_umul64hi(unsigned long long __a,363__DEVICE__ unsigned long long __nv_umul64hi(unsigned long long __a,
370 unsigned long long __b);364 unsigned long long __b);
371__device__ unsigned int __nv_umulhi(unsigned int __a, unsigned int __b);365__DEVICE__ unsigned int __nv_umulhi(unsigned int __a, unsigned int __b);
372__device__ unsigned int __nv_urhadd(unsigned int __a, unsigned int __b);366__DEVICE__ unsigned int __nv_urhadd(unsigned int __a, unsigned int __b);
373__device__ unsigned int __nv_usad(unsigned int __a, unsigned int __b,367__DEVICE__ unsigned int __nv_usad(unsigned int __a, unsigned int __b,
374 unsigned int __c);368 unsigned int __c);
375#if CUDA_VERSION >= 9000 && CUDA_VERSION < 9020369#if CUDA_VERSION >= 9000 && CUDA_VERSION < 9020
376__device__ int __nv_vabs2(int __a);370__DEVICE__ int __nv_vabs2(int __a);
377__device__ int __nv_vabs4(int __a);371__DEVICE__ int __nv_vabs4(int __a);
378__device__ int __nv_vabsdiffs2(int __a, int __b);372__DEVICE__ int __nv_vabsdiffs2(int __a, int __b);
379__device__ int __nv_vabsdiffs4(int __a, int __b);373__DEVICE__ int __nv_vabsdiffs4(int __a, int __b);
380__device__ int __nv_vabsdiffu2(int __a, int __b);374__DEVICE__ int __nv_vabsdiffu2(int __a, int __b);
381__device__ int __nv_vabsdiffu4(int __a, int __b);375__DEVICE__ int __nv_vabsdiffu4(int __a, int __b);
382__device__ int __nv_vabsss2(int __a);376__DEVICE__ int __nv_vabsss2(int __a);
383__device__ int __nv_vabsss4(int __a);377__DEVICE__ int __nv_vabsss4(int __a);
384__device__ int __nv_vadd2(int __a, int __b);378__DEVICE__ int __nv_vadd2(int __a, int __b);
385__device__ int __nv_vadd4(int __a, int __b);379__DEVICE__ int __nv_vadd4(int __a, int __b);
386__device__ int __nv_vaddss2(int __a, int __b);380__DEVICE__ int __nv_vaddss2(int __a, int __b);
387__device__ int __nv_vaddss4(int __a, int __b);381__DEVICE__ int __nv_vaddss4(int __a, int __b);
388__device__ int __nv_vaddus2(int __a, int __b);382__DEVICE__ int __nv_vaddus2(int __a, int __b);
389__device__ int __nv_vaddus4(int __a, int __b);383__DEVICE__ int __nv_vaddus4(int __a, int __b);
390__device__ int __nv_vavgs2(int __a, int __b);384__DEVICE__ int __nv_vavgs2(int __a, int __b);
391__device__ int __nv_vavgs4(int __a, int __b);385__DEVICE__ int __nv_vavgs4(int __a, int __b);
392__device__ int __nv_vavgu2(int __a, int __b);386__DEVICE__ int __nv_vavgu2(int __a, int __b);
393__device__ int __nv_vavgu4(int __a, int __b);387__DEVICE__ int __nv_vavgu4(int __a, int __b);
394__device__ int __nv_vcmpeq2(int __a, int __b);388__DEVICE__ int __nv_vcmpeq2(int __a, int __b);
395__device__ int __nv_vcmpeq4(int __a, int __b);389__DEVICE__ int __nv_vcmpeq4(int __a, int __b);
396__device__ int __nv_vcmpges2(int __a, int __b);390__DEVICE__ int __nv_vcmpges2(int __a, int __b);
397__device__ int __nv_vcmpges4(int __a, int __b);391__DEVICE__ int __nv_vcmpges4(int __a, int __b);
398__device__ int __nv_vcmpgeu2(int __a, int __b);392__DEVICE__ int __nv_vcmpgeu2(int __a, int __b);
399__device__ int __nv_vcmpgeu4(int __a, int __b);393__DEVICE__ int __nv_vcmpgeu4(int __a, int __b);
400__device__ int __nv_vcmpgts2(int __a, int __b);394__DEVICE__ int __nv_vcmpgts2(int __a, int __b);
401__device__ int __nv_vcmpgts4(int __a, int __b);395__DEVICE__ int __nv_vcmpgts4(int __a, int __b);
402__device__ int __nv_vcmpgtu2(int __a, int __b);396__DEVICE__ int __nv_vcmpgtu2(int __a, int __b);
403__device__ int __nv_vcmpgtu4(int __a, int __b);397__DEVICE__ int __nv_vcmpgtu4(int __a, int __b);
404__device__ int __nv_vcmples2(int __a, int __b);398__DEVICE__ int __nv_vcmples2(int __a, int __b);
405__device__ int __nv_vcmples4(int __a, int __b);399__DEVICE__ int __nv_vcmples4(int __a, int __b);
406__device__ int __nv_vcmpleu2(int __a, int __b);400__DEVICE__ int __nv_vcmpleu2(int __a, int __b);
407__device__ int __nv_vcmpleu4(int __a, int __b);401__DEVICE__ int __nv_vcmpleu4(int __a, int __b);
408__device__ int __nv_vcmplts2(int __a, int __b);402__DEVICE__ int __nv_vcmplts2(int __a, int __b);
409__device__ int __nv_vcmplts4(int __a, int __b);403__DEVICE__ int __nv_vcmplts4(int __a, int __b);
410__device__ int __nv_vcmpltu2(int __a, int __b);404__DEVICE__ int __nv_vcmpltu2(int __a, int __b);
411__device__ int __nv_vcmpltu4(int __a, int __b);405__DEVICE__ int __nv_vcmpltu4(int __a, int __b);
412__device__ int __nv_vcmpne2(int __a, int __b);406__DEVICE__ int __nv_vcmpne2(int __a, int __b);
413__device__ int __nv_vcmpne4(int __a, int __b);407__DEVICE__ int __nv_vcmpne4(int __a, int __b);
414__device__ int __nv_vhaddu2(int __a, int __b);408__DEVICE__ int __nv_vhaddu2(int __a, int __b);
415__device__ int __nv_vhaddu4(int __a, int __b);409__DEVICE__ int __nv_vhaddu4(int __a, int __b);
416__device__ int __nv_vmaxs2(int __a, int __b);410__DEVICE__ int __nv_vmaxs2(int __a, int __b);
417__device__ int __nv_vmaxs4(int __a, int __b);411__DEVICE__ int __nv_vmaxs4(int __a, int __b);
418__device__ int __nv_vmaxu2(int __a, int __b);412__DEVICE__ int __nv_vmaxu2(int __a, int __b);
419__device__ int __nv_vmaxu4(int __a, int __b);413__DEVICE__ int __nv_vmaxu4(int __a, int __b);
420__device__ int __nv_vmins2(int __a, int __b);414__DEVICE__ int __nv_vmins2(int __a, int __b);
421__device__ int __nv_vmins4(int __a, int __b);415__DEVICE__ int __nv_vmins4(int __a, int __b);
422__device__ int __nv_vminu2(int __a, int __b);416__DEVICE__ int __nv_vminu2(int __a, int __b);
423__device__ int __nv_vminu4(int __a, int __b);417__DEVICE__ int __nv_vminu4(int __a, int __b);
424__device__ int __nv_vneg2(int __a);418__DEVICE__ int __nv_vneg2(int __a);
425__device__ int __nv_vneg4(int __a);419__DEVICE__ int __nv_vneg4(int __a);
426__device__ int __nv_vnegss2(int __a);420__DEVICE__ int __nv_vnegss2(int __a);
427__device__ int __nv_vnegss4(int __a);421__DEVICE__ int __nv_vnegss4(int __a);
428__device__ int __nv_vsads2(int __a, int __b);422__DEVICE__ int __nv_vsads2(int __a, int __b);
429__device__ int __nv_vsads4(int __a, int __b);423__DEVICE__ int __nv_vsads4(int __a, int __b);
430__device__ int __nv_vsadu2(int __a, int __b);424__DEVICE__ int __nv_vsadu2(int __a, int __b);
431__device__ int __nv_vsadu4(int __a, int __b);425__DEVICE__ int __nv_vsadu4(int __a, int __b);
432__device__ int __nv_vseteq2(int __a, int __b);426__DEVICE__ int __nv_vseteq2(int __a, int __b);
433__device__ int __nv_vseteq4(int __a, int __b);427__DEVICE__ int __nv_vseteq4(int __a, int __b);
434__device__ int __nv_vsetges2(int __a, int __b);428__DEVICE__ int __nv_vsetges2(int __a, int __b);
435__device__ int __nv_vsetges4(int __a, int __b);429__DEVICE__ int __nv_vsetges4(int __a, int __b);
436__device__ int __nv_vsetgeu2(int __a, int __b);430__DEVICE__ int __nv_vsetgeu2(int __a, int __b);
437__device__ int __nv_vsetgeu4(int __a, int __b);431__DEVICE__ int __nv_vsetgeu4(int __a, int __b);
438__device__ int __nv_vsetgts2(int __a, int __b);432__DEVICE__ int __nv_vsetgts2(int __a, int __b);
439__device__ int __nv_vsetgts4(int __a, int __b);433__DEVICE__ int __nv_vsetgts4(int __a, int __b);
440__device__ int __nv_vsetgtu2(int __a, int __b);434__DEVICE__ int __nv_vsetgtu2(int __a, int __b);
441__device__ int __nv_vsetgtu4(int __a, int __b);435__DEVICE__ int __nv_vsetgtu4(int __a, int __b);
442__device__ int __nv_vsetles2(int __a, int __b);436__DEVICE__ int __nv_vsetles2(int __a, int __b);
443__device__ int __nv_vsetles4(int __a, int __b);437__DEVICE__ int __nv_vsetles4(int __a, int __b);
444__device__ int __nv_vsetleu2(int __a, int __b);438__DEVICE__ int __nv_vsetleu2(int __a, int __b);
445__device__ int __nv_vsetleu4(int __a, int __b);439__DEVICE__ int __nv_vsetleu4(int __a, int __b);
446__device__ int __nv_vsetlts2(int __a, int __b);440__DEVICE__ int __nv_vsetlts2(int __a, int __b);
447__device__ int __nv_vsetlts4(int __a, int __b);441__DEVICE__ int __nv_vsetlts4(int __a, int __b);
448__device__ int __nv_vsetltu2(int __a, int __b);442__DEVICE__ int __nv_vsetltu2(int __a, int __b);
449__device__ int __nv_vsetltu4(int __a, int __b);443__DEVICE__ int __nv_vsetltu4(int __a, int __b);
450__device__ int __nv_vsetne2(int __a, int __b);444__DEVICE__ int __nv_vsetne2(int __a, int __b);
451__device__ int __nv_vsetne4(int __a, int __b);445__DEVICE__ int __nv_vsetne4(int __a, int __b);
452__device__ int __nv_vsub2(int __a, int __b);446__DEVICE__ int __nv_vsub2(int __a, int __b);
453__device__ int __nv_vsub4(int __a, int __b);447__DEVICE__ int __nv_vsub4(int __a, int __b);
454__device__ int __nv_vsubss2(int __a, int __b);448__DEVICE__ int __nv_vsubss2(int __a, int __b);
455__device__ int __nv_vsubss4(int __a, int __b);449__DEVICE__ int __nv_vsubss4(int __a, int __b);
456__device__ int __nv_vsubus2(int __a, int __b);450__DEVICE__ int __nv_vsubus2(int __a, int __b);
457__device__ int __nv_vsubus4(int __a, int __b);451__DEVICE__ int __nv_vsubus4(int __a, int __b);
458#endif // CUDA_VERSION452#endif // CUDA_VERSION
459__device__ double __nv_y0(double __a);453__DEVICE__ double __nv_y0(double __a);
460__device__ float __nv_y0f(float __a);454__DEVICE__ float __nv_y0f(float __a);
461__device__ double __nv_y1(double __a);455__DEVICE__ double __nv_y1(double __a);
462__device__ float __nv_y1f(float __a);456__DEVICE__ float __nv_y1f(float __a);
463__device__ float __nv_ynf(int __a, float __b);457__DEVICE__ float __nv_ynf(int __a, float __b);
464__device__ double __nv_yn(int __a, double __b);458__DEVICE__ double __nv_yn(int __a, double __b);
459#if defined(__cplusplus)
465} // extern "C"460} // extern "C"
461#endif
466#endif // __CLANG_CUDA_LIBDEVICE_DECLARES_H__462#endif // __CLANG_CUDA_LIBDEVICE_DECLARES_H__
lib/include/__clang_cuda_math_forward_declares.h+46-24
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===- __clang_math_forward_declares.h - Prototypes of __device__ math fns --===1/*===- __clang_math_forward_declares.h - Prototypes of __device__ math fns --===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -34,14 +20,37 @@...@@ -34,14 +20,37 @@
34// would preclude the use of our own __device__ overloads for these functions.20// would preclude the use of our own __device__ overloads for these functions.
3521
36#pragma push_macro("__DEVICE__")22#pragma push_macro("__DEVICE__")
23#ifdef _OPENMP
24#define __DEVICE__ static __inline__ __attribute__((always_inline))
25#else
37#define __DEVICE__ \26#define __DEVICE__ \
38 static __inline__ __attribute__((always_inline)) __attribute__((device))27 static __inline__ __attribute__((always_inline)) __attribute__((device))
28#endif
3929
40__DEVICE__ double abs(double);30// For C++ 17 we need to include noexcept attribute to be compatible
41__DEVICE__ float abs(float);31// with the header-defined version. This may be removed once
42__DEVICE__ int abs(int);32// variant is supported.
33#if defined(_OPENMP) && defined(__cplusplus) && __cplusplus >= 201703L
34#define __NOEXCEPT noexcept
35#else
36#define __NOEXCEPT
37#endif
38
39#if !(defined(_OPENMP) && defined(__cplusplus))
43__DEVICE__ long abs(long);40__DEVICE__ long abs(long);
44__DEVICE__ long long abs(long long);41__DEVICE__ long long abs(long long);
42__DEVICE__ double abs(double);
43__DEVICE__ float abs(float);
44#endif
45// While providing the CUDA declarations and definitions for math functions,
46// we may manually define additional functions.
47// TODO: Once variant is supported the additional functions will have
48// to be removed.
49#if defined(_OPENMP) && defined(__cplusplus)
50__DEVICE__ const double abs(const double);
51__DEVICE__ const float abs(const float);
52#endif
53__DEVICE__ int abs(int) __NOEXCEPT;
45__DEVICE__ double acos(double);54__DEVICE__ double acos(double);
46__DEVICE__ float acos(float);55__DEVICE__ float acos(float);
47__DEVICE__ double acosh(double);56__DEVICE__ double acosh(double);
...@@ -76,8 +85,8 @@ __DEVICE__ double exp(double);...@@ -76,8 +85,8 @@ __DEVICE__ double exp(double);
76__DEVICE__ float exp(float);85__DEVICE__ float exp(float);
77__DEVICE__ double expm1(double);86__DEVICE__ double expm1(double);
78__DEVICE__ float expm1(float);87__DEVICE__ float expm1(float);
79__DEVICE__ double fabs(double);88__DEVICE__ double fabs(double) __NOEXCEPT;
80__DEVICE__ float fabs(float);89__DEVICE__ float fabs(float) __NOEXCEPT;
81__DEVICE__ double fdim(double, double);90__DEVICE__ double fdim(double, double);
82__DEVICE__ float fdim(float, float);91__DEVICE__ float fdim(float, float);
83__DEVICE__ double floor(double);92__DEVICE__ double floor(double);
...@@ -98,12 +107,18 @@ __DEVICE__ double hypot(double, double);...@@ -98,12 +107,18 @@ __DEVICE__ double hypot(double, double);
98__DEVICE__ float hypot(float, float);107__DEVICE__ float hypot(float, float);
99__DEVICE__ int ilogb(double);108__DEVICE__ int ilogb(double);
100__DEVICE__ int ilogb(float);109__DEVICE__ int ilogb(float);
110#ifdef _MSC_VER
111__DEVICE__ bool isfinite(long double);
112#endif
101__DEVICE__ bool isfinite(double);113__DEVICE__ bool isfinite(double);
102__DEVICE__ bool isfinite(float);114__DEVICE__ bool isfinite(float);
103__DEVICE__ bool isgreater(double, double);115__DEVICE__ bool isgreater(double, double);
104__DEVICE__ bool isgreaterequal(double, double);116__DEVICE__ bool isgreaterequal(double, double);
105__DEVICE__ bool isgreaterequal(float, float);117__DEVICE__ bool isgreaterequal(float, float);
106__DEVICE__ bool isgreater(float, float);118__DEVICE__ bool isgreater(float, float);
119#ifdef _MSC_VER
120__DEVICE__ bool isinf(long double);
121#endif
107__DEVICE__ bool isinf(double);122__DEVICE__ bool isinf(double);
108__DEVICE__ bool isinf(float);123__DEVICE__ bool isinf(float);
109__DEVICE__ bool isless(double, double);124__DEVICE__ bool isless(double, double);
...@@ -112,18 +127,21 @@ __DEVICE__ bool islessequal(float, float);...@@ -112,18 +127,21 @@ __DEVICE__ bool islessequal(float, float);
112__DEVICE__ bool isless(float, float);127__DEVICE__ bool isless(float, float);
113__DEVICE__ bool islessgreater(double, double);128__DEVICE__ bool islessgreater(double, double);
114__DEVICE__ bool islessgreater(float, float);129__DEVICE__ bool islessgreater(float, float);
130#ifdef _MSC_VER
131__DEVICE__ bool isnan(long double);
132#endif
115__DEVICE__ bool isnan(double);133__DEVICE__ bool isnan(double);
116__DEVICE__ bool isnan(float);134__DEVICE__ bool isnan(float);
117__DEVICE__ bool isnormal(double);135__DEVICE__ bool isnormal(double);
118__DEVICE__ bool isnormal(float);136__DEVICE__ bool isnormal(float);
119__DEVICE__ bool isunordered(double, double);137__DEVICE__ bool isunordered(double, double);
120__DEVICE__ bool isunordered(float, float);138__DEVICE__ bool isunordered(float, float);
121__DEVICE__ long labs(long);139__DEVICE__ long labs(long) __NOEXCEPT;
122__DEVICE__ double ldexp(double, int);140__DEVICE__ double ldexp(double, int);
123__DEVICE__ float ldexp(float, int);141__DEVICE__ float ldexp(float, int);
124__DEVICE__ double lgamma(double);142__DEVICE__ double lgamma(double);
125__DEVICE__ float lgamma(float);143__DEVICE__ float lgamma(float);
126__DEVICE__ long long llabs(long long);144__DEVICE__ long long llabs(long long) __NOEXCEPT;
127__DEVICE__ long long llrint(double);145__DEVICE__ long long llrint(double);
128__DEVICE__ long long llrint(float);146__DEVICE__ long long llrint(float);
129__DEVICE__ double log10(double);147__DEVICE__ double log10(double);
...@@ -134,6 +152,9 @@ __DEVICE__ double log2(double);...@@ -134,6 +152,9 @@ __DEVICE__ double log2(double);
134__DEVICE__ float log2(float);152__DEVICE__ float log2(float);
135__DEVICE__ double logb(double);153__DEVICE__ double logb(double);
136__DEVICE__ float logb(float);154__DEVICE__ float logb(float);
155#if defined(_OPENMP) && defined(__cplusplus)
156__DEVICE__ long double log(long double);
157#endif
137__DEVICE__ double log(double);158__DEVICE__ double log(double);
138__DEVICE__ float log(float);159__DEVICE__ float log(float);
139__DEVICE__ long lrint(double);160__DEVICE__ long lrint(double);
...@@ -281,6 +302,7 @@ _GLIBCXX_END_NAMESPACE_VERSION...@@ -281,6 +302,7 @@ _GLIBCXX_END_NAMESPACE_VERSION
281} // namespace std302} // namespace std
282#endif303#endif
283304
305#undef __NOEXCEPT
284#pragma pop_macro("__DEVICE__")306#pragma pop_macro("__DEVICE__")
285307
286#endif308#endif
lib/include/__clang_cuda_runtime_wrapper.h+14-18
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- __clang_cuda_runtime_wrapper.h - CUDA runtime support -------------===1/*===---- __clang_cuda_runtime_wrapper.h - CUDA runtime support -------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -62,7 +48,7 @@...@@ -62,7 +48,7 @@
62#include "cuda.h"48#include "cuda.h"
63#if !defined(CUDA_VERSION)49#if !defined(CUDA_VERSION)
64#error "cuda.h did not define CUDA_VERSION"50#error "cuda.h did not define CUDA_VERSION"
65#elif CUDA_VERSION < 7000 || CUDA_VERSION > 1000051#elif CUDA_VERSION < 7000 || CUDA_VERSION > 10010
66#error "Unsupported CUDA version!"52#error "Unsupported CUDA version!"
67#endif53#endif
6854
...@@ -426,5 +412,15 @@ __device__ inline __cuda_builtin_gridDim_t::operator dim3() const {...@@ -426,5 +412,15 @@ __device__ inline __cuda_builtin_gridDim_t::operator dim3() const {
426#pragma pop_macro("__USE_FAST_MATH__")412#pragma pop_macro("__USE_FAST_MATH__")
427#pragma pop_macro("__CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__")413#pragma pop_macro("__CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__")
428414
415// CUDA runtime uses this undocumented function to access kernel launch
416// configuration. The declaration is in crt/device_functions.h but that file
417// includes a lot of other stuff we don't want. Instead, we'll provide our own
418// declaration for it here.
419#if CUDA_VERSION >= 9020
420extern "C" unsigned __cudaPushCallConfiguration(dim3 gridDim, dim3 blockDim,
421 size_t sharedMem = 0,
422 void *stream = 0);
423#endif
424
429#endif // __CUDA__425#endif // __CUDA__
430#endif // __CLANG_CUDA_RUNTIME_WRAPPER_H__426#endif // __CLANG_CUDA_RUNTIME_WRAPPER_H__
lib/include/__stddef_max_align_t.h+3-19
...@@ -1,24 +1,8 @@...@@ -1,24 +1,8 @@
1/*===---- __stddef_max_align_t.h - Definition of max_align_t for modules ---===1/*===---- __stddef_max_align_t.h - Definition of max_align_t for modules ---===
2 *2 *
3 * Copyright (c) 2014 Chandler Carruth3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 *4 * See https://llvm.org/LICENSE.txt for license information.
5 * Permission is hereby granted, free of charge, to any person obtaining a copy5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * of this software and associated documentation files (the "Software"), to deal
7 * in the Software without restriction, including without limitation the rights
8 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 * copies of the Software, and to permit persons to whom the Software is
10 * furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 * THE SOFTWARE.
22 *6 *
23 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
24 */8 */
lib/include/__wmmintrin_aes.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- __wmmintrin_aes.h - AES intrinsics -------------------------------===1/*===---- __wmmintrin_aes.h - AES intrinsics -------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/__wmmintrin_pclmul.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- __wmmintrin_pclmul.h - PCMUL intrinsics ---------------------------===1/*===---- __wmmintrin_pclmul.h - PCMUL intrinsics ---------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/adxintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- adxintrin.h - ADX intrinsics -------------------------------------===1/*===---- adxintrin.h - ADX intrinsics -------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/altivec.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- altivec.h - Standard header for type generic math ---------------===*\1/*===---- altivec.h - Standard header for type generic math ---------------===*\
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21\*===----------------------------------------------------------------------===*/7\*===----------------------------------------------------------------------===*/
228
lib/include/ammintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- ammintrin.h - SSE4a intrinsics -----------------------------------===1/*===---- ammintrin.h - SSE4a intrinsics -----------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/arm64intr.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- arm64intr.h - ARM64 Windows intrinsics -------------------------------===1/*===---- arm64intr.h - ARM64 Windows intrinsics -------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/arm_acle.h+21-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- arm_acle.h - ARM Non-Neon intrinsics -----------------------------===1/*===---- arm_acle.h - ARM Non-Neon intrinsics -----------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -611,6 +597,14 @@ __crc32cd(uint32_t __a, uint64_t __b) {...@@ -611,6 +597,14 @@ __crc32cd(uint32_t __a, uint64_t __b) {
611}597}
612#endif598#endif
613599
600/* Armv8.3-A Javascript conversion intrinsic */
601#if __ARM_64BIT_STATE && defined(__ARM_FEATURE_JCVT)
602static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
603__jcvt(double __a) {
604 return __builtin_arm_jcvt(__a);
605}
606#endif
607
614/* 10.1 Special register intrinsics */608/* 10.1 Special register intrinsics */
615#define __arm_rsr(sysreg) __builtin_arm_rsr(sysreg)609#define __arm_rsr(sysreg) __builtin_arm_rsr(sysreg)
616#define __arm_rsr64(sysreg) __builtin_arm_rsr64(sysreg)610#define __arm_rsr64(sysreg) __builtin_arm_rsr64(sysreg)
...@@ -619,6 +613,16 @@ __crc32cd(uint32_t __a, uint64_t __b) {...@@ -619,6 +613,16 @@ __crc32cd(uint32_t __a, uint64_t __b) {
619#define __arm_wsr64(sysreg, v) __builtin_arm_wsr64(sysreg, v)613#define __arm_wsr64(sysreg, v) __builtin_arm_wsr64(sysreg, v)
620#define __arm_wsrp(sysreg, v) __builtin_arm_wsrp(sysreg, v)614#define __arm_wsrp(sysreg, v) __builtin_arm_wsrp(sysreg, v)
621615
616// Memory Tagging Extensions (MTE) Intrinsics
617#if __ARM_FEATURE_MEMORY_TAGGING
618#define __arm_mte_create_random_tag(__ptr, __mask) __builtin_arm_irg(__ptr, __mask)
619#define __arm_mte_increment_tag(__ptr, __tag_offset) __builtin_arm_addg(__ptr, __tag_offset)
620#define __arm_mte_exclude_tag(__ptr, __excluded) __builtin_arm_gmi(__ptr, __excluded)
621#define __arm_mte_get_tag(__ptr) __builtin_arm_ldg(__ptr)
622#define __arm_mte_set_tag(__ptr) __builtin_arm_stg(__ptr)
623#define __arm_mte_ptrdiff(__ptra, __ptrb) __builtin_arm_subp(__ptra, __ptrb)
624#endif
625
622#if defined(__cplusplus)626#if defined(__cplusplus)
623}627}
624#endif628#endif
lib/include/arm_neon.h+198-198
...@@ -44247,13 +44247,13 @@ __ai float32x2_t vfms_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2)...@@ -44247,13 +44247,13 @@ __ai float32x2_t vfms_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2)
44247#endif44247#endif
44248#if defined(__ARM_FEATURE_FP16FML) && defined(__aarch64__)44248#if defined(__ARM_FEATURE_FP16FML) && defined(__aarch64__)
44249#ifdef __LITTLE_ENDIAN__44249#ifdef __LITTLE_ENDIAN__
44250__ai float32x4_t vfmlalq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {44250__ai float32x4_t vfmlalq_high_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {
44251 float32x4_t __ret;44251 float32x4_t __ret;
44252 __ret = (float32x4_t) __builtin_neon_vfmlalq_high_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);44252 __ret = (float32x4_t) __builtin_neon_vfmlalq_high_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);
44253 return __ret;44253 return __ret;
44254}44254}
44255#else44255#else
44256__ai float32x4_t vfmlalq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {44256__ai float32x4_t vfmlalq_high_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {
44257 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);44257 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
44258 float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);44258 float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
44259 float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);44259 float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -44262,7 +44262,7 @@ __ai float32x4_t vfmlalq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_...@@ -44262,7 +44262,7 @@ __ai float32x4_t vfmlalq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_
44262 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);44262 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
44263 return __ret;44263 return __ret;
44264}44264}
44265__ai float32x4_t __noswap_vfmlalq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {44265__ai float32x4_t __noswap_vfmlalq_high_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {
44266 float32x4_t __ret;44266 float32x4_t __ret;
44267 __ret = (float32x4_t) __builtin_neon_vfmlalq_high_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);44267 __ret = (float32x4_t) __builtin_neon_vfmlalq_high_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);
44268 return __ret;44268 return __ret;
...@@ -44270,13 +44270,13 @@ __ai float32x4_t __noswap_vfmlalq_high_u32(float32x4_t __p0, float16x8_t __p1, f...@@ -44270,13 +44270,13 @@ __ai float32x4_t __noswap_vfmlalq_high_u32(float32x4_t __p0, float16x8_t __p1, f
44270#endif44270#endif
4427144271
44272#ifdef __LITTLE_ENDIAN__44272#ifdef __LITTLE_ENDIAN__
44273__ai float32x2_t vfmlal_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {44273__ai float32x2_t vfmlal_high_f16(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {
44274 float32x2_t __ret;44274 float32x2_t __ret;
44275 __ret = (float32x2_t) __builtin_neon_vfmlal_high_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);44275 __ret = (float32x2_t) __builtin_neon_vfmlal_high_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);
44276 return __ret;44276 return __ret;
44277}44277}
44278#else44278#else
44279__ai float32x2_t vfmlal_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {44279__ai float32x2_t vfmlal_high_f16(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {
44280 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);44280 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
44281 float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);44281 float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
44282 float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);44282 float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);
...@@ -44285,7 +44285,7 @@ __ai float32x2_t vfmlal_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t...@@ -44285,7 +44285,7 @@ __ai float32x2_t vfmlal_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t
44285 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);44285 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
44286 return __ret;44286 return __ret;
44287}44287}
44288__ai float32x2_t __noswap_vfmlal_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {44288__ai float32x2_t __noswap_vfmlal_high_f16(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {
44289 float32x2_t __ret;44289 float32x2_t __ret;
44290 __ret = (float32x2_t) __builtin_neon_vfmlal_high_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);44290 __ret = (float32x2_t) __builtin_neon_vfmlal_high_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);
44291 return __ret;44291 return __ret;
...@@ -44293,13 +44293,13 @@ __ai float32x2_t __noswap_vfmlal_high_u32(float32x2_t __p0, float16x4_t __p1, fl...@@ -44293,13 +44293,13 @@ __ai float32x2_t __noswap_vfmlal_high_u32(float32x2_t __p0, float16x4_t __p1, fl
44293#endif44293#endif
4429444294
44295#ifdef __LITTLE_ENDIAN__44295#ifdef __LITTLE_ENDIAN__
44296__ai float32x4_t vfmlalq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {44296__ai float32x4_t vfmlalq_low_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {
44297 float32x4_t __ret;44297 float32x4_t __ret;
44298 __ret = (float32x4_t) __builtin_neon_vfmlalq_low_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);44298 __ret = (float32x4_t) __builtin_neon_vfmlalq_low_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);
44299 return __ret;44299 return __ret;
44300}44300}
44301#else44301#else
44302__ai float32x4_t vfmlalq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {44302__ai float32x4_t vfmlalq_low_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {
44303 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);44303 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
44304 float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);44304 float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
44305 float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);44305 float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -44308,7 +44308,7 @@ __ai float32x4_t vfmlalq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t...@@ -44308,7 +44308,7 @@ __ai float32x4_t vfmlalq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t
44308 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);44308 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
44309 return __ret;44309 return __ret;
44310}44310}
44311__ai float32x4_t __noswap_vfmlalq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {44311__ai float32x4_t __noswap_vfmlalq_low_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {
44312 float32x4_t __ret;44312 float32x4_t __ret;
44313 __ret = (float32x4_t) __builtin_neon_vfmlalq_low_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);44313 __ret = (float32x4_t) __builtin_neon_vfmlalq_low_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);
44314 return __ret;44314 return __ret;
...@@ -44316,13 +44316,13 @@ __ai float32x4_t __noswap_vfmlalq_low_u32(float32x4_t __p0, float16x8_t __p1, fl...@@ -44316,13 +44316,13 @@ __ai float32x4_t __noswap_vfmlalq_low_u32(float32x4_t __p0, float16x8_t __p1, fl
44316#endif44316#endif
4431744317
44318#ifdef __LITTLE_ENDIAN__44318#ifdef __LITTLE_ENDIAN__
44319__ai float32x2_t vfmlal_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {44319__ai float32x2_t vfmlal_low_f16(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {
44320 float32x2_t __ret;44320 float32x2_t __ret;
44321 __ret = (float32x2_t) __builtin_neon_vfmlal_low_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);44321 __ret = (float32x2_t) __builtin_neon_vfmlal_low_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);
44322 return __ret;44322 return __ret;
44323}44323}
44324#else44324#else
44325__ai float32x2_t vfmlal_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {44325__ai float32x2_t vfmlal_low_f16(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {
44326 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);44326 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
44327 float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);44327 float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
44328 float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);44328 float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);
...@@ -44331,7 +44331,7 @@ __ai float32x2_t vfmlal_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t...@@ -44331,7 +44331,7 @@ __ai float32x2_t vfmlal_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t
44331 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);44331 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
44332 return __ret;44332 return __ret;
44333}44333}
44334__ai float32x2_t __noswap_vfmlal_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {44334__ai float32x2_t __noswap_vfmlal_low_f16(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {
44335 float32x2_t __ret;44335 float32x2_t __ret;
44336 __ret = (float32x2_t) __builtin_neon_vfmlal_low_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);44336 __ret = (float32x2_t) __builtin_neon_vfmlal_low_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);
44337 return __ret;44337 return __ret;
...@@ -44339,13 +44339,13 @@ __ai float32x2_t __noswap_vfmlal_low_u32(float32x2_t __p0, float16x4_t __p1, flo...@@ -44339,13 +44339,13 @@ __ai float32x2_t __noswap_vfmlal_low_u32(float32x2_t __p0, float16x4_t __p1, flo
44339#endif44339#endif
4434044340
44341#ifdef __LITTLE_ENDIAN__44341#ifdef __LITTLE_ENDIAN__
44342__ai float32x4_t vfmlslq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {44342__ai float32x4_t vfmlslq_high_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {
44343 float32x4_t __ret;44343 float32x4_t __ret;
44344 __ret = (float32x4_t) __builtin_neon_vfmlslq_high_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);44344 __ret = (float32x4_t) __builtin_neon_vfmlslq_high_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);
44345 return __ret;44345 return __ret;
44346}44346}
44347#else44347#else
44348__ai float32x4_t vfmlslq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {44348__ai float32x4_t vfmlslq_high_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {
44349 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);44349 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
44350 float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);44350 float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
44351 float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);44351 float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -44354,7 +44354,7 @@ __ai float32x4_t vfmlslq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_...@@ -44354,7 +44354,7 @@ __ai float32x4_t vfmlslq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_
44354 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);44354 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
44355 return __ret;44355 return __ret;
44356}44356}
44357__ai float32x4_t __noswap_vfmlslq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {44357__ai float32x4_t __noswap_vfmlslq_high_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {
44358 float32x4_t __ret;44358 float32x4_t __ret;
44359 __ret = (float32x4_t) __builtin_neon_vfmlslq_high_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);44359 __ret = (float32x4_t) __builtin_neon_vfmlslq_high_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);
44360 return __ret;44360 return __ret;
...@@ -44362,13 +44362,13 @@ __ai float32x4_t __noswap_vfmlslq_high_u32(float32x4_t __p0, float16x8_t __p1, f...@@ -44362,13 +44362,13 @@ __ai float32x4_t __noswap_vfmlslq_high_u32(float32x4_t __p0, float16x8_t __p1, f
44362#endif44362#endif
4436344363
44364#ifdef __LITTLE_ENDIAN__44364#ifdef __LITTLE_ENDIAN__
44365__ai float32x2_t vfmlsl_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {44365__ai float32x2_t vfmlsl_high_f16(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {
44366 float32x2_t __ret;44366 float32x2_t __ret;
44367 __ret = (float32x2_t) __builtin_neon_vfmlsl_high_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);44367 __ret = (float32x2_t) __builtin_neon_vfmlsl_high_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);
44368 return __ret;44368 return __ret;
44369}44369}
44370#else44370#else
44371__ai float32x2_t vfmlsl_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {44371__ai float32x2_t vfmlsl_high_f16(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {
44372 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);44372 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
44373 float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);44373 float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
44374 float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);44374 float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);
...@@ -44377,7 +44377,7 @@ __ai float32x2_t vfmlsl_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t...@@ -44377,7 +44377,7 @@ __ai float32x2_t vfmlsl_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t
44377 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);44377 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
44378 return __ret;44378 return __ret;
44379}44379}
44380__ai float32x2_t __noswap_vfmlsl_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {44380__ai float32x2_t __noswap_vfmlsl_high_f16(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {
44381 float32x2_t __ret;44381 float32x2_t __ret;
44382 __ret = (float32x2_t) __builtin_neon_vfmlsl_high_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);44382 __ret = (float32x2_t) __builtin_neon_vfmlsl_high_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);
44383 return __ret;44383 return __ret;
...@@ -44385,13 +44385,13 @@ __ai float32x2_t __noswap_vfmlsl_high_u32(float32x2_t __p0, float16x4_t __p1, fl...@@ -44385,13 +44385,13 @@ __ai float32x2_t __noswap_vfmlsl_high_u32(float32x2_t __p0, float16x4_t __p1, fl
44385#endif44385#endif
4438644386
44387#ifdef __LITTLE_ENDIAN__44387#ifdef __LITTLE_ENDIAN__
44388__ai float32x4_t vfmlslq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {44388__ai float32x4_t vfmlslq_low_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {
44389 float32x4_t __ret;44389 float32x4_t __ret;
44390 __ret = (float32x4_t) __builtin_neon_vfmlslq_low_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);44390 __ret = (float32x4_t) __builtin_neon_vfmlslq_low_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);
44391 return __ret;44391 return __ret;
44392}44392}
44393#else44393#else
44394__ai float32x4_t vfmlslq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {44394__ai float32x4_t vfmlslq_low_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {
44395 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);44395 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
44396 float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);44396 float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
44397 float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);44397 float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -44400,7 +44400,7 @@ __ai float32x4_t vfmlslq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t...@@ -44400,7 +44400,7 @@ __ai float32x4_t vfmlslq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t
44400 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);44400 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
44401 return __ret;44401 return __ret;
44402}44402}
44403__ai float32x4_t __noswap_vfmlslq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {44403__ai float32x4_t __noswap_vfmlslq_low_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) {
44404 float32x4_t __ret;44404 float32x4_t __ret;
44405 __ret = (float32x4_t) __builtin_neon_vfmlslq_low_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);44405 __ret = (float32x4_t) __builtin_neon_vfmlslq_low_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41);
44406 return __ret;44406 return __ret;
...@@ -44408,13 +44408,13 @@ __ai float32x4_t __noswap_vfmlslq_low_u32(float32x4_t __p0, float16x8_t __p1, fl...@@ -44408,13 +44408,13 @@ __ai float32x4_t __noswap_vfmlslq_low_u32(float32x4_t __p0, float16x8_t __p1, fl
44408#endif44408#endif
4440944409
44410#ifdef __LITTLE_ENDIAN__44410#ifdef __LITTLE_ENDIAN__
44411__ai float32x2_t vfmlsl_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {44411__ai float32x2_t vfmlsl_low_f16(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {
44412 float32x2_t __ret;44412 float32x2_t __ret;
44413 __ret = (float32x2_t) __builtin_neon_vfmlsl_low_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);44413 __ret = (float32x2_t) __builtin_neon_vfmlsl_low_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);
44414 return __ret;44414 return __ret;
44415}44415}
44416#else44416#else
44417__ai float32x2_t vfmlsl_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {44417__ai float32x2_t vfmlsl_low_f16(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {
44418 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);44418 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
44419 float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);44419 float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
44420 float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);44420 float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);
...@@ -44423,7 +44423,7 @@ __ai float32x2_t vfmlsl_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t...@@ -44423,7 +44423,7 @@ __ai float32x2_t vfmlsl_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t
44423 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);44423 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
44424 return __ret;44424 return __ret;
44425}44425}
44426__ai float32x2_t __noswap_vfmlsl_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {44426__ai float32x2_t __noswap_vfmlsl_low_f16(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) {
44427 float32x2_t __ret;44427 float32x2_t __ret;
44428 __ret = (float32x2_t) __builtin_neon_vfmlsl_low_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);44428 __ret = (float32x2_t) __builtin_neon_vfmlsl_low_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9);
44429 return __ret;44429 return __ret;
...@@ -64095,15 +64095,15 @@ __ai uint8x16_t vqtbl1q_u8(uint8x16_t __p0, uint8x16_t __p1) {...@@ -64095,15 +64095,15 @@ __ai uint8x16_t vqtbl1q_u8(uint8x16_t __p0, uint8x16_t __p1) {
64095#endif64095#endif
6409664096
64097#ifdef __LITTLE_ENDIAN__64097#ifdef __LITTLE_ENDIAN__
64098__ai int8x16_t vqtbl1q_s8(int8x16_t __p0, int8x16_t __p1) {64098__ai int8x16_t vqtbl1q_s8(int8x16_t __p0, uint8x16_t __p1) {
64099 int8x16_t __ret;64099 int8x16_t __ret;
64100 __ret = (int8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__p0, (int8x16_t)__p1, 32);64100 __ret = (int8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__p0, (int8x16_t)__p1, 32);
64101 return __ret;64101 return __ret;
64102}64102}
64103#else64103#else
64104__ai int8x16_t vqtbl1q_s8(int8x16_t __p0, int8x16_t __p1) {64104__ai int8x16_t vqtbl1q_s8(int8x16_t __p0, uint8x16_t __p1) {
64105 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64105 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64106 int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64106 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64107 int8x16_t __ret;64107 int8x16_t __ret;
64108 __ret = (int8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32);64108 __ret = (int8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32);
64109 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64109 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64129,15 +64129,15 @@ __ai uint8x8_t vqtbl1_u8(uint8x16_t __p0, uint8x8_t __p1) {...@@ -64129,15 +64129,15 @@ __ai uint8x8_t vqtbl1_u8(uint8x16_t __p0, uint8x8_t __p1) {
64129#endif64129#endif
6413064130
64131#ifdef __LITTLE_ENDIAN__64131#ifdef __LITTLE_ENDIAN__
64132__ai int8x8_t vqtbl1_s8(int8x16_t __p0, int8x8_t __p1) {64132__ai int8x8_t vqtbl1_s8(int8x16_t __p0, uint8x8_t __p1) {
64133 int8x8_t __ret;64133 int8x8_t __ret;
64134 __ret = (int8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__p0, (int8x8_t)__p1, 0);64134 __ret = (int8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__p0, (int8x8_t)__p1, 0);
64135 return __ret;64135 return __ret;
64136}64136}
64137#else64137#else
64138__ai int8x8_t vqtbl1_s8(int8x16_t __p0, int8x8_t __p1) {64138__ai int8x8_t vqtbl1_s8(int8x16_t __p0, uint8x8_t __p1) {
64139 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64139 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64140 int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);64140 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
64141 int8x8_t __ret;64141 int8x8_t __ret;
64142 __ret = (int8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__rev0, (int8x8_t)__rev1, 0);64142 __ret = (int8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__rev0, (int8x8_t)__rev1, 0);
64143 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);64143 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64203,17 +64203,17 @@ __ai uint8x16_t vqtbl2q_u8(uint8x16x2_t __p0, uint8x16_t __p1) {...@@ -64203,17 +64203,17 @@ __ai uint8x16_t vqtbl2q_u8(uint8x16x2_t __p0, uint8x16_t __p1) {
64203#endif64203#endif
6420464204
64205#ifdef __LITTLE_ENDIAN__64205#ifdef __LITTLE_ENDIAN__
64206__ai int8x16_t vqtbl2q_s8(int8x16x2_t __p0, int8x16_t __p1) {64206__ai int8x16_t vqtbl2q_s8(int8x16x2_t __p0, uint8x16_t __p1) {
64207 int8x16_t __ret;64207 int8x16_t __ret;
64208 __ret = (int8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p1, 32);64208 __ret = (int8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p1, 32);
64209 return __ret;64209 return __ret;
64210}64210}
64211#else64211#else
64212__ai int8x16_t vqtbl2q_s8(int8x16x2_t __p0, int8x16_t __p1) {64212__ai int8x16_t vqtbl2q_s8(int8x16x2_t __p0, uint8x16_t __p1) {
64213 int8x16x2_t __rev0;64213 int8x16x2_t __rev0;
64214 __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64214 __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64215 __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64215 __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64216 int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64216 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64217 int8x16_t __ret;64217 int8x16_t __ret;
64218 __ret = (int8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, 32);64218 __ret = (int8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, 32);
64219 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64219 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64241,17 +64241,17 @@ __ai uint8x8_t vqtbl2_u8(uint8x16x2_t __p0, uint8x8_t __p1) {...@@ -64241,17 +64241,17 @@ __ai uint8x8_t vqtbl2_u8(uint8x16x2_t __p0, uint8x8_t __p1) {
64241#endif64241#endif
6424264242
64243#ifdef __LITTLE_ENDIAN__64243#ifdef __LITTLE_ENDIAN__
64244__ai int8x8_t vqtbl2_s8(int8x16x2_t __p0, int8x8_t __p1) {64244__ai int8x8_t vqtbl2_s8(int8x16x2_t __p0, uint8x8_t __p1) {
64245 int8x8_t __ret;64245 int8x8_t __ret;
64246 __ret = (int8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x8_t)__p1, 0);64246 __ret = (int8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x8_t)__p1, 0);
64247 return __ret;64247 return __ret;
64248}64248}
64249#else64249#else
64250__ai int8x8_t vqtbl2_s8(int8x16x2_t __p0, int8x8_t __p1) {64250__ai int8x8_t vqtbl2_s8(int8x16x2_t __p0, uint8x8_t __p1) {
64251 int8x16x2_t __rev0;64251 int8x16x2_t __rev0;
64252 __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64252 __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64253 __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64253 __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64254 int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);64254 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
64255 int8x8_t __ret;64255 int8x8_t __ret;
64256 __ret = (int8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, 0);64256 __ret = (int8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, 0);
64257 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);64257 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64320,18 +64320,18 @@ __ai uint8x16_t vqtbl3q_u8(uint8x16x3_t __p0, uint8x16_t __p1) {...@@ -64320,18 +64320,18 @@ __ai uint8x16_t vqtbl3q_u8(uint8x16x3_t __p0, uint8x16_t __p1) {
64320#endif64320#endif
6432164321
64322#ifdef __LITTLE_ENDIAN__64322#ifdef __LITTLE_ENDIAN__
64323__ai int8x16_t vqtbl3q_s8(int8x16x3_t __p0, int8x16_t __p1) {64323__ai int8x16_t vqtbl3q_s8(int8x16x3_t __p0, uint8x16_t __p1) {
64324 int8x16_t __ret;64324 int8x16_t __ret;
64325 __ret = (int8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p1, 32);64325 __ret = (int8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p1, 32);
64326 return __ret;64326 return __ret;
64327}64327}
64328#else64328#else
64329__ai int8x16_t vqtbl3q_s8(int8x16x3_t __p0, int8x16_t __p1) {64329__ai int8x16_t vqtbl3q_s8(int8x16x3_t __p0, uint8x16_t __p1) {
64330 int8x16x3_t __rev0;64330 int8x16x3_t __rev0;
64331 __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64331 __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64332 __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64332 __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64333 __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64333 __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64334 int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64334 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64335 int8x16_t __ret;64335 int8x16_t __ret;
64336 __ret = (int8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev1, 32);64336 __ret = (int8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev1, 32);
64337 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64337 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64360,18 +64360,18 @@ __ai uint8x8_t vqtbl3_u8(uint8x16x3_t __p0, uint8x8_t __p1) {...@@ -64360,18 +64360,18 @@ __ai uint8x8_t vqtbl3_u8(uint8x16x3_t __p0, uint8x8_t __p1) {
64360#endif64360#endif
6436164361
64362#ifdef __LITTLE_ENDIAN__64362#ifdef __LITTLE_ENDIAN__
64363__ai int8x8_t vqtbl3_s8(int8x16x3_t __p0, int8x8_t __p1) {64363__ai int8x8_t vqtbl3_s8(int8x16x3_t __p0, uint8x8_t __p1) {
64364 int8x8_t __ret;64364 int8x8_t __ret;
64365 __ret = (int8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x8_t)__p1, 0);64365 __ret = (int8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x8_t)__p1, 0);
64366 return __ret;64366 return __ret;
64367}64367}
64368#else64368#else
64369__ai int8x8_t vqtbl3_s8(int8x16x3_t __p0, int8x8_t __p1) {64369__ai int8x8_t vqtbl3_s8(int8x16x3_t __p0, uint8x8_t __p1) {
64370 int8x16x3_t __rev0;64370 int8x16x3_t __rev0;
64371 __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64371 __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64372 __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64372 __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64373 __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64373 __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64374 int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);64374 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
64375 int8x8_t __ret;64375 int8x8_t __ret;
64376 __ret = (int8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x8_t)__rev1, 0);64376 __ret = (int8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x8_t)__rev1, 0);
64377 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);64377 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64443,19 +64443,19 @@ __ai uint8x16_t vqtbl4q_u8(uint8x16x4_t __p0, uint8x16_t __p1) {...@@ -64443,19 +64443,19 @@ __ai uint8x16_t vqtbl4q_u8(uint8x16x4_t __p0, uint8x16_t __p1) {
64443#endif64443#endif
6444464444
64445#ifdef __LITTLE_ENDIAN__64445#ifdef __LITTLE_ENDIAN__
64446__ai int8x16_t vqtbl4q_s8(int8x16x4_t __p0, int8x16_t __p1) {64446__ai int8x16_t vqtbl4q_s8(int8x16x4_t __p0, uint8x16_t __p1) {
64447 int8x16_t __ret;64447 int8x16_t __ret;
64448 __ret = (int8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x16_t)__p1, 32);64448 __ret = (int8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x16_t)__p1, 32);
64449 return __ret;64449 return __ret;
64450}64450}
64451#else64451#else
64452__ai int8x16_t vqtbl4q_s8(int8x16x4_t __p0, int8x16_t __p1) {64452__ai int8x16_t vqtbl4q_s8(int8x16x4_t __p0, uint8x16_t __p1) {
64453 int8x16x4_t __rev0;64453 int8x16x4_t __rev0;
64454 __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64454 __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64455 __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64455 __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64456 __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64456 __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64457 __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64457 __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64458 int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64458 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64459 int8x16_t __ret;64459 int8x16_t __ret;
64460 __ret = (int8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x16_t)__rev1, 32);64460 __ret = (int8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x16_t)__rev1, 32);
64461 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64461 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64485,19 +64485,19 @@ __ai uint8x8_t vqtbl4_u8(uint8x16x4_t __p0, uint8x8_t __p1) {...@@ -64485,19 +64485,19 @@ __ai uint8x8_t vqtbl4_u8(uint8x16x4_t __p0, uint8x8_t __p1) {
64485#endif64485#endif
6448664486
64487#ifdef __LITTLE_ENDIAN__64487#ifdef __LITTLE_ENDIAN__
64488__ai int8x8_t vqtbl4_s8(int8x16x4_t __p0, int8x8_t __p1) {64488__ai int8x8_t vqtbl4_s8(int8x16x4_t __p0, uint8x8_t __p1) {
64489 int8x8_t __ret;64489 int8x8_t __ret;
64490 __ret = (int8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x8_t)__p1, 0);64490 __ret = (int8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x8_t)__p1, 0);
64491 return __ret;64491 return __ret;
64492}64492}
64493#else64493#else
64494__ai int8x8_t vqtbl4_s8(int8x16x4_t __p0, int8x8_t __p1) {64494__ai int8x8_t vqtbl4_s8(int8x16x4_t __p0, uint8x8_t __p1) {
64495 int8x16x4_t __rev0;64495 int8x16x4_t __rev0;
64496 __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64496 __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64497 __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64497 __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64498 __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64498 __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64499 __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64499 __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64500 int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);64500 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
64501 int8x8_t __ret;64501 int8x8_t __ret;
64502 __ret = (int8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x8_t)__rev1, 0);64502 __ret = (int8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x8_t)__rev1, 0);
64503 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);64503 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64560,16 +64560,16 @@ __ai uint8x16_t vqtbx1q_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) {...@@ -64560,16 +64560,16 @@ __ai uint8x16_t vqtbx1q_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) {
64560#endif64560#endif
6456164561
64562#ifdef __LITTLE_ENDIAN__64562#ifdef __LITTLE_ENDIAN__
64563__ai int8x16_t vqtbx1q_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) {64563__ai int8x16_t vqtbx1q_s8(int8x16_t __p0, int8x16_t __p1, uint8x16_t __p2) {
64564 int8x16_t __ret;64564 int8x16_t __ret;
64565 __ret = (int8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 32);64565 __ret = (int8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 32);
64566 return __ret;64566 return __ret;
64567}64567}
64568#else64568#else
64569__ai int8x16_t vqtbx1q_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) {64569__ai int8x16_t vqtbx1q_s8(int8x16_t __p0, int8x16_t __p1, uint8x16_t __p2) {
64570 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64570 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64571 int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64571 int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64572 int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64572 uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64573 int8x16_t __ret;64573 int8x16_t __ret;
64574 __ret = (int8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 32);64574 __ret = (int8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 32);
64575 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64575 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64596,16 +64596,16 @@ __ai uint8x8_t vqtbx1_u8(uint8x8_t __p0, uint8x16_t __p1, uint8x8_t __p2) {...@@ -64596,16 +64596,16 @@ __ai uint8x8_t vqtbx1_u8(uint8x8_t __p0, uint8x16_t __p1, uint8x8_t __p2) {
64596#endif64596#endif
6459764597
64598#ifdef __LITTLE_ENDIAN__64598#ifdef __LITTLE_ENDIAN__
64599__ai int8x8_t vqtbx1_s8(int8x8_t __p0, int8x16_t __p1, int8x8_t __p2) {64599__ai int8x8_t vqtbx1_s8(int8x8_t __p0, int8x16_t __p1, uint8x8_t __p2) {
64600 int8x8_t __ret;64600 int8x8_t __ret;
64601 __ret = (int8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__p0, (int8x16_t)__p1, (int8x8_t)__p2, 0);64601 __ret = (int8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__p0, (int8x16_t)__p1, (int8x8_t)__p2, 0);
64602 return __ret;64602 return __ret;
64603}64603}
64604#else64604#else
64605__ai int8x8_t vqtbx1_s8(int8x8_t __p0, int8x16_t __p1, int8x8_t __p2) {64605__ai int8x8_t vqtbx1_s8(int8x8_t __p0, int8x16_t __p1, uint8x8_t __p2) {
64606 int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);64606 int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
64607 int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64607 int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64608 int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);64608 uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
64609 int8x8_t __ret;64609 int8x8_t __ret;
64610 __ret = (int8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__rev0, (int8x16_t)__rev1, (int8x8_t)__rev2, 0);64610 __ret = (int8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__rev0, (int8x16_t)__rev1, (int8x8_t)__rev2, 0);
64611 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);64611 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64674,18 +64674,18 @@ __ai uint8x16_t vqtbx2q_u8(uint8x16_t __p0, uint8x16x2_t __p1, uint8x16_t __p2)...@@ -64674,18 +64674,18 @@ __ai uint8x16_t vqtbx2q_u8(uint8x16_t __p0, uint8x16x2_t __p1, uint8x16_t __p2)
64674#endif64674#endif
6467564675
64676#ifdef __LITTLE_ENDIAN__64676#ifdef __LITTLE_ENDIAN__
64677__ai int8x16_t vqtbx2q_s8(int8x16_t __p0, int8x16x2_t __p1, int8x16_t __p2) {64677__ai int8x16_t vqtbx2q_s8(int8x16_t __p0, int8x16x2_t __p1, uint8x16_t __p2) {
64678 int8x16_t __ret;64678 int8x16_t __ret;
64679 __ret = (int8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p2, 32);64679 __ret = (int8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p2, 32);
64680 return __ret;64680 return __ret;
64681}64681}
64682#else64682#else
64683__ai int8x16_t vqtbx2q_s8(int8x16_t __p0, int8x16x2_t __p1, int8x16_t __p2) {64683__ai int8x16_t vqtbx2q_s8(int8x16_t __p0, int8x16x2_t __p1, uint8x16_t __p2) {
64684 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64684 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64685 int8x16x2_t __rev1;64685 int8x16x2_t __rev1;
64686 __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64686 __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64687 __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64687 __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64688 int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64688 uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64689 int8x16_t __ret;64689 int8x16_t __ret;
64690 __ret = (int8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev2, 32);64690 __ret = (int8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev2, 32);
64691 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64691 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64714,18 +64714,18 @@ __ai uint8x8_t vqtbx2_u8(uint8x8_t __p0, uint8x16x2_t __p1, uint8x8_t __p2) {...@@ -64714,18 +64714,18 @@ __ai uint8x8_t vqtbx2_u8(uint8x8_t __p0, uint8x16x2_t __p1, uint8x8_t __p2) {
64714#endif64714#endif
6471564715
64716#ifdef __LITTLE_ENDIAN__64716#ifdef __LITTLE_ENDIAN__
64717__ai int8x8_t vqtbx2_s8(int8x8_t __p0, int8x16x2_t __p1, int8x8_t __p2) {64717__ai int8x8_t vqtbx2_s8(int8x8_t __p0, int8x16x2_t __p1, uint8x8_t __p2) {
64718 int8x8_t __ret;64718 int8x8_t __ret;
64719 __ret = (int8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x8_t)__p2, 0);64719 __ret = (int8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x8_t)__p2, 0);
64720 return __ret;64720 return __ret;
64721}64721}
64722#else64722#else
64723__ai int8x8_t vqtbx2_s8(int8x8_t __p0, int8x16x2_t __p1, int8x8_t __p2) {64723__ai int8x8_t vqtbx2_s8(int8x8_t __p0, int8x16x2_t __p1, uint8x8_t __p2) {
64724 int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);64724 int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
64725 int8x16x2_t __rev1;64725 int8x16x2_t __rev1;
64726 __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64726 __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64727 __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64727 __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64728 int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);64728 uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
64729 int8x8_t __ret;64729 int8x8_t __ret;
64730 __ret = (int8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x8_t)__rev2, 0);64730 __ret = (int8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x8_t)__rev2, 0);
64731 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);64731 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64797,19 +64797,19 @@ __ai uint8x16_t vqtbx3q_u8(uint8x16_t __p0, uint8x16x3_t __p1, uint8x16_t __p2)...@@ -64797,19 +64797,19 @@ __ai uint8x16_t vqtbx3q_u8(uint8x16_t __p0, uint8x16x3_t __p1, uint8x16_t __p2)
64797#endif64797#endif
6479864798
64799#ifdef __LITTLE_ENDIAN__64799#ifdef __LITTLE_ENDIAN__
64800__ai int8x16_t vqtbx3q_s8(int8x16_t __p0, int8x16x3_t __p1, int8x16_t __p2) {64800__ai int8x16_t vqtbx3q_s8(int8x16_t __p0, int8x16x3_t __p1, uint8x16_t __p2) {
64801 int8x16_t __ret;64801 int8x16_t __ret;
64802 __ret = (int8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p2, 32);64802 __ret = (int8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p2, 32);
64803 return __ret;64803 return __ret;
64804}64804}
64805#else64805#else
64806__ai int8x16_t vqtbx3q_s8(int8x16_t __p0, int8x16x3_t __p1, int8x16_t __p2) {64806__ai int8x16_t vqtbx3q_s8(int8x16_t __p0, int8x16x3_t __p1, uint8x16_t __p2) {
64807 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64807 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64808 int8x16x3_t __rev1;64808 int8x16x3_t __rev1;
64809 __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64809 __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64810 __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64810 __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64811 __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64811 __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64812 int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64812 uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64813 int8x16_t __ret;64813 int8x16_t __ret;
64814 __ret = (int8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev2, 32);64814 __ret = (int8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev2, 32);
64815 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64815 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64839,19 +64839,19 @@ __ai uint8x8_t vqtbx3_u8(uint8x8_t __p0, uint8x16x3_t __p1, uint8x8_t __p2) {...@@ -64839,19 +64839,19 @@ __ai uint8x8_t vqtbx3_u8(uint8x8_t __p0, uint8x16x3_t __p1, uint8x8_t __p2) {
64839#endif64839#endif
6484064840
64841#ifdef __LITTLE_ENDIAN__64841#ifdef __LITTLE_ENDIAN__
64842__ai int8x8_t vqtbx3_s8(int8x8_t __p0, int8x16x3_t __p1, int8x8_t __p2) {64842__ai int8x8_t vqtbx3_s8(int8x8_t __p0, int8x16x3_t __p1, uint8x8_t __p2) {
64843 int8x8_t __ret;64843 int8x8_t __ret;
64844 __ret = (int8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x8_t)__p2, 0);64844 __ret = (int8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x8_t)__p2, 0);
64845 return __ret;64845 return __ret;
64846}64846}
64847#else64847#else
64848__ai int8x8_t vqtbx3_s8(int8x8_t __p0, int8x16x3_t __p1, int8x8_t __p2) {64848__ai int8x8_t vqtbx3_s8(int8x8_t __p0, int8x16x3_t __p1, uint8x8_t __p2) {
64849 int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);64849 int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
64850 int8x16x3_t __rev1;64850 int8x16x3_t __rev1;
64851 __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64851 __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64852 __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64852 __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64853 __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64853 __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64854 int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);64854 uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
64855 int8x8_t __ret;64855 int8x8_t __ret;
64856 __ret = (int8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x8_t)__rev2, 0);64856 __ret = (int8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x8_t)__rev2, 0);
64857 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);64857 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64926,20 +64926,20 @@ __ai uint8x16_t vqtbx4q_u8(uint8x16_t __p0, uint8x16x4_t __p1, uint8x16_t __p2)...@@ -64926,20 +64926,20 @@ __ai uint8x16_t vqtbx4q_u8(uint8x16_t __p0, uint8x16x4_t __p1, uint8x16_t __p2)
64926#endif64926#endif
6492764927
64928#ifdef __LITTLE_ENDIAN__64928#ifdef __LITTLE_ENDIAN__
64929__ai int8x16_t vqtbx4q_s8(int8x16_t __p0, int8x16x4_t __p1, int8x16_t __p2) {64929__ai int8x16_t vqtbx4q_s8(int8x16_t __p0, int8x16x4_t __p1, uint8x16_t __p2) {
64930 int8x16_t __ret;64930 int8x16_t __ret;
64931 __ret = (int8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x16_t)__p2, 32);64931 __ret = (int8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x16_t)__p2, 32);
64932 return __ret;64932 return __ret;
64933}64933}
64934#else64934#else
64935__ai int8x16_t vqtbx4q_s8(int8x16_t __p0, int8x16x4_t __p1, int8x16_t __p2) {64935__ai int8x16_t vqtbx4q_s8(int8x16_t __p0, int8x16x4_t __p1, uint8x16_t __p2) {
64936 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64936 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64937 int8x16x4_t __rev1;64937 int8x16x4_t __rev1;
64938 __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64938 __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64939 __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64939 __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64940 __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64940 __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64941 __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64941 __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64942 int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64942 uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64943 int8x16_t __ret;64943 int8x16_t __ret;
64944 __ret = (int8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x16_t)__rev2, 32);64944 __ret = (int8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x16_t)__rev2, 32);
64945 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64945 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -64970,20 +64970,20 @@ __ai uint8x8_t vqtbx4_u8(uint8x8_t __p0, uint8x16x4_t __p1, uint8x8_t __p2) {...@@ -64970,20 +64970,20 @@ __ai uint8x8_t vqtbx4_u8(uint8x8_t __p0, uint8x16x4_t __p1, uint8x8_t __p2) {
64970#endif64970#endif
6497164971
64972#ifdef __LITTLE_ENDIAN__64972#ifdef __LITTLE_ENDIAN__
64973__ai int8x8_t vqtbx4_s8(int8x8_t __p0, int8x16x4_t __p1, int8x8_t __p2) {64973__ai int8x8_t vqtbx4_s8(int8x8_t __p0, int8x16x4_t __p1, uint8x8_t __p2) {
64974 int8x8_t __ret;64974 int8x8_t __ret;
64975 __ret = (int8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x8_t)__p2, 0);64975 __ret = (int8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x8_t)__p2, 0);
64976 return __ret;64976 return __ret;
64977}64977}
64978#else64978#else
64979__ai int8x8_t vqtbx4_s8(int8x8_t __p0, int8x16x4_t __p1, int8x8_t __p2) {64979__ai int8x8_t vqtbx4_s8(int8x8_t __p0, int8x16x4_t __p1, uint8x8_t __p2) {
64980 int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);64980 int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
64981 int8x16x4_t __rev1;64981 int8x16x4_t __rev1;
64982 __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64982 __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64983 __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64983 __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64984 __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64984 __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64985 __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);64985 __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
64986 int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);64986 uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
64987 int8x8_t __ret;64987 int8x8_t __ret;
64988 __ret = (int8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x8_t)__rev2, 0);64988 __ret = (int8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x8_t)__rev2, 0);
64989 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);64989 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -66293,13 +66293,13 @@ __ai int64_t vshld_s64(int64_t __p0, int64_t __p1) {...@@ -66293,13 +66293,13 @@ __ai int64_t vshld_s64(int64_t __p0, int64_t __p1) {
66293#endif66293#endif
6629466294
66295#ifdef __LITTLE_ENDIAN__66295#ifdef __LITTLE_ENDIAN__
66296__ai uint8_t vsqaddb_u8(uint8_t __p0, uint8_t __p1) {66296__ai uint8_t vsqaddb_u8(uint8_t __p0, int8_t __p1) {
66297 uint8_t __ret;66297 uint8_t __ret;
66298 __ret = (uint8_t) __builtin_neon_vsqaddb_u8(__p0, __p1);66298 __ret = (uint8_t) __builtin_neon_vsqaddb_u8(__p0, __p1);
66299 return __ret;66299 return __ret;
66300}66300}
66301#else66301#else
66302__ai uint8_t vsqaddb_u8(uint8_t __p0, uint8_t __p1) {66302__ai uint8_t vsqaddb_u8(uint8_t __p0, int8_t __p1) {
66303 uint8_t __ret;66303 uint8_t __ret;
66304 __ret = (uint8_t) __builtin_neon_vsqaddb_u8(__p0, __p1);66304 __ret = (uint8_t) __builtin_neon_vsqaddb_u8(__p0, __p1);
66305 return __ret;66305 return __ret;
...@@ -66307,13 +66307,13 @@ __ai uint8_t vsqaddb_u8(uint8_t __p0, uint8_t __p1) {...@@ -66307,13 +66307,13 @@ __ai uint8_t vsqaddb_u8(uint8_t __p0, uint8_t __p1) {
66307#endif66307#endif
6630866308
66309#ifdef __LITTLE_ENDIAN__66309#ifdef __LITTLE_ENDIAN__
66310__ai uint32_t vsqadds_u32(uint32_t __p0, uint32_t __p1) {66310__ai uint32_t vsqadds_u32(uint32_t __p0, int32_t __p1) {
66311 uint32_t __ret;66311 uint32_t __ret;
66312 __ret = (uint32_t) __builtin_neon_vsqadds_u32(__p0, __p1);66312 __ret = (uint32_t) __builtin_neon_vsqadds_u32(__p0, __p1);
66313 return __ret;66313 return __ret;
66314}66314}
66315#else66315#else
66316__ai uint32_t vsqadds_u32(uint32_t __p0, uint32_t __p1) {66316__ai uint32_t vsqadds_u32(uint32_t __p0, int32_t __p1) {
66317 uint32_t __ret;66317 uint32_t __ret;
66318 __ret = (uint32_t) __builtin_neon_vsqadds_u32(__p0, __p1);66318 __ret = (uint32_t) __builtin_neon_vsqadds_u32(__p0, __p1);
66319 return __ret;66319 return __ret;
...@@ -66321,13 +66321,13 @@ __ai uint32_t vsqadds_u32(uint32_t __p0, uint32_t __p1) {...@@ -66321,13 +66321,13 @@ __ai uint32_t vsqadds_u32(uint32_t __p0, uint32_t __p1) {
66321#endif66321#endif
6632266322
66323#ifdef __LITTLE_ENDIAN__66323#ifdef __LITTLE_ENDIAN__
66324__ai uint64_t vsqaddd_u64(uint64_t __p0, uint64_t __p1) {66324__ai uint64_t vsqaddd_u64(uint64_t __p0, int64_t __p1) {
66325 uint64_t __ret;66325 uint64_t __ret;
66326 __ret = (uint64_t) __builtin_neon_vsqaddd_u64(__p0, __p1);66326 __ret = (uint64_t) __builtin_neon_vsqaddd_u64(__p0, __p1);
66327 return __ret;66327 return __ret;
66328}66328}
66329#else66329#else
66330__ai uint64_t vsqaddd_u64(uint64_t __p0, uint64_t __p1) {66330__ai uint64_t vsqaddd_u64(uint64_t __p0, int64_t __p1) {
66331 uint64_t __ret;66331 uint64_t __ret;
66332 __ret = (uint64_t) __builtin_neon_vsqaddd_u64(__p0, __p1);66332 __ret = (uint64_t) __builtin_neon_vsqaddd_u64(__p0, __p1);
66333 return __ret;66333 return __ret;
...@@ -66335,13 +66335,13 @@ __ai uint64_t vsqaddd_u64(uint64_t __p0, uint64_t __p1) {...@@ -66335,13 +66335,13 @@ __ai uint64_t vsqaddd_u64(uint64_t __p0, uint64_t __p1) {
66335#endif66335#endif
6633666336
66337#ifdef __LITTLE_ENDIAN__66337#ifdef __LITTLE_ENDIAN__
66338__ai uint16_t vsqaddh_u16(uint16_t __p0, uint16_t __p1) {66338__ai uint16_t vsqaddh_u16(uint16_t __p0, int16_t __p1) {
66339 uint16_t __ret;66339 uint16_t __ret;
66340 __ret = (uint16_t) __builtin_neon_vsqaddh_u16(__p0, __p1);66340 __ret = (uint16_t) __builtin_neon_vsqaddh_u16(__p0, __p1);
66341 return __ret;66341 return __ret;
66342}66342}
66343#else66343#else
66344__ai uint16_t vsqaddh_u16(uint16_t __p0, uint16_t __p1) {66344__ai uint16_t vsqaddh_u16(uint16_t __p0, int16_t __p1) {
66345 uint16_t __ret;66345 uint16_t __ret;
66346 __ret = (uint16_t) __builtin_neon_vsqaddh_u16(__p0, __p1);66346 __ret = (uint16_t) __builtin_neon_vsqaddh_u16(__p0, __p1);
66347 return __ret;66347 return __ret;
...@@ -66349,15 +66349,15 @@ __ai uint16_t vsqaddh_u16(uint16_t __p0, uint16_t __p1) {...@@ -66349,15 +66349,15 @@ __ai uint16_t vsqaddh_u16(uint16_t __p0, uint16_t __p1) {
66349#endif66349#endif
6635066350
66351#ifdef __LITTLE_ENDIAN__66351#ifdef __LITTLE_ENDIAN__
66352__ai uint8x16_t vsqaddq_u8(uint8x16_t __p0, uint8x16_t __p1) {66352__ai uint8x16_t vsqaddq_u8(uint8x16_t __p0, int8x16_t __p1) {
66353 uint8x16_t __ret;66353 uint8x16_t __ret;
66354 __ret = (uint8x16_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 48);66354 __ret = (uint8x16_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 48);
66355 return __ret;66355 return __ret;
66356}66356}
66357#else66357#else
66358__ai uint8x16_t vsqaddq_u8(uint8x16_t __p0, uint8x16_t __p1) {66358__ai uint8x16_t vsqaddq_u8(uint8x16_t __p0, int8x16_t __p1) {
66359 uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);66359 uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
66360 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);66360 int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
66361 uint8x16_t __ret;66361 uint8x16_t __ret;
66362 __ret = (uint8x16_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48);66362 __ret = (uint8x16_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48);
66363 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);66363 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -66366,15 +66366,15 @@ __ai uint8x16_t vsqaddq_u8(uint8x16_t __p0, uint8x16_t __p1) {...@@ -66366,15 +66366,15 @@ __ai uint8x16_t vsqaddq_u8(uint8x16_t __p0, uint8x16_t __p1) {
66366#endif66366#endif
6636766367
66368#ifdef __LITTLE_ENDIAN__66368#ifdef __LITTLE_ENDIAN__
66369__ai uint32x4_t vsqaddq_u32(uint32x4_t __p0, uint32x4_t __p1) {66369__ai uint32x4_t vsqaddq_u32(uint32x4_t __p0, int32x4_t __p1) {
66370 uint32x4_t __ret;66370 uint32x4_t __ret;
66371 __ret = (uint32x4_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 50);66371 __ret = (uint32x4_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 50);
66372 return __ret;66372 return __ret;
66373}66373}
66374#else66374#else
66375__ai uint32x4_t vsqaddq_u32(uint32x4_t __p0, uint32x4_t __p1) {66375__ai uint32x4_t vsqaddq_u32(uint32x4_t __p0, int32x4_t __p1) {
66376 uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);66376 uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
66377 uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);66377 int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
66378 uint32x4_t __ret;66378 uint32x4_t __ret;
66379 __ret = (uint32x4_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50);66379 __ret = (uint32x4_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50);
66380 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);66380 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
...@@ -66383,15 +66383,15 @@ __ai uint32x4_t vsqaddq_u32(uint32x4_t __p0, uint32x4_t __p1) {...@@ -66383,15 +66383,15 @@ __ai uint32x4_t vsqaddq_u32(uint32x4_t __p0, uint32x4_t __p1) {
66383#endif66383#endif
6638466384
66385#ifdef __LITTLE_ENDIAN__66385#ifdef __LITTLE_ENDIAN__
66386__ai uint64x2_t vsqaddq_u64(uint64x2_t __p0, uint64x2_t __p1) {66386__ai uint64x2_t vsqaddq_u64(uint64x2_t __p0, int64x2_t __p1) {
66387 uint64x2_t __ret;66387 uint64x2_t __ret;
66388 __ret = (uint64x2_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 51);66388 __ret = (uint64x2_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 51);
66389 return __ret;66389 return __ret;
66390}66390}
66391#else66391#else
66392__ai uint64x2_t vsqaddq_u64(uint64x2_t __p0, uint64x2_t __p1) {66392__ai uint64x2_t vsqaddq_u64(uint64x2_t __p0, int64x2_t __p1) {
66393 uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);66393 uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
66394 uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);66394 int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
66395 uint64x2_t __ret;66395 uint64x2_t __ret;
66396 __ret = (uint64x2_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51);66396 __ret = (uint64x2_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51);
66397 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);66397 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
...@@ -66400,15 +66400,15 @@ __ai uint64x2_t vsqaddq_u64(uint64x2_t __p0, uint64x2_t __p1) {...@@ -66400,15 +66400,15 @@ __ai uint64x2_t vsqaddq_u64(uint64x2_t __p0, uint64x2_t __p1) {
66400#endif66400#endif
6640166401
66402#ifdef __LITTLE_ENDIAN__66402#ifdef __LITTLE_ENDIAN__
66403__ai uint16x8_t vsqaddq_u16(uint16x8_t __p0, uint16x8_t __p1) {66403__ai uint16x8_t vsqaddq_u16(uint16x8_t __p0, int16x8_t __p1) {
66404 uint16x8_t __ret;66404 uint16x8_t __ret;
66405 __ret = (uint16x8_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 49);66405 __ret = (uint16x8_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 49);
66406 return __ret;66406 return __ret;
66407}66407}
66408#else66408#else
66409__ai uint16x8_t vsqaddq_u16(uint16x8_t __p0, uint16x8_t __p1) {66409__ai uint16x8_t vsqaddq_u16(uint16x8_t __p0, int16x8_t __p1) {
66410 uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);66410 uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
66411 uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);66411 int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
66412 uint16x8_t __ret;66412 uint16x8_t __ret;
66413 __ret = (uint16x8_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49);66413 __ret = (uint16x8_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49);
66414 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);66414 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -66417,15 +66417,15 @@ __ai uint16x8_t vsqaddq_u16(uint16x8_t __p0, uint16x8_t __p1) {...@@ -66417,15 +66417,15 @@ __ai uint16x8_t vsqaddq_u16(uint16x8_t __p0, uint16x8_t __p1) {
66417#endif66417#endif
6641866418
66419#ifdef __LITTLE_ENDIAN__66419#ifdef __LITTLE_ENDIAN__
66420__ai uint8x8_t vsqadd_u8(uint8x8_t __p0, uint8x8_t __p1) {66420__ai uint8x8_t vsqadd_u8(uint8x8_t __p0, int8x8_t __p1) {
66421 uint8x8_t __ret;66421 uint8x8_t __ret;
66422 __ret = (uint8x8_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 16);66422 __ret = (uint8x8_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 16);
66423 return __ret;66423 return __ret;
66424}66424}
66425#else66425#else
66426__ai uint8x8_t vsqadd_u8(uint8x8_t __p0, uint8x8_t __p1) {66426__ai uint8x8_t vsqadd_u8(uint8x8_t __p0, int8x8_t __p1) {
66427 uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);66427 uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
66428 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);66428 int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
66429 uint8x8_t __ret;66429 uint8x8_t __ret;
66430 __ret = (uint8x8_t) __builtin_neon_vsqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16);66430 __ret = (uint8x8_t) __builtin_neon_vsqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16);
66431 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);66431 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -66434,15 +66434,15 @@ __ai uint8x8_t vsqadd_u8(uint8x8_t __p0, uint8x8_t __p1) {...@@ -66434,15 +66434,15 @@ __ai uint8x8_t vsqadd_u8(uint8x8_t __p0, uint8x8_t __p1) {
66434#endif66434#endif
6643566435
66436#ifdef __LITTLE_ENDIAN__66436#ifdef __LITTLE_ENDIAN__
66437__ai uint32x2_t vsqadd_u32(uint32x2_t __p0, uint32x2_t __p1) {66437__ai uint32x2_t vsqadd_u32(uint32x2_t __p0, int32x2_t __p1) {
66438 uint32x2_t __ret;66438 uint32x2_t __ret;
66439 __ret = (uint32x2_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 18);66439 __ret = (uint32x2_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 18);
66440 return __ret;66440 return __ret;
66441}66441}
66442#else66442#else
66443__ai uint32x2_t vsqadd_u32(uint32x2_t __p0, uint32x2_t __p1) {66443__ai uint32x2_t vsqadd_u32(uint32x2_t __p0, int32x2_t __p1) {
66444 uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);66444 uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
66445 uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);66445 int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
66446 uint32x2_t __ret;66446 uint32x2_t __ret;
66447 __ret = (uint32x2_t) __builtin_neon_vsqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18);66447 __ret = (uint32x2_t) __builtin_neon_vsqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18);
66448 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);66448 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
...@@ -66451,13 +66451,13 @@ __ai uint32x2_t vsqadd_u32(uint32x2_t __p0, uint32x2_t __p1) {...@@ -66451,13 +66451,13 @@ __ai uint32x2_t vsqadd_u32(uint32x2_t __p0, uint32x2_t __p1) {
66451#endif66451#endif
6645266452
66453#ifdef __LITTLE_ENDIAN__66453#ifdef __LITTLE_ENDIAN__
66454__ai uint64x1_t vsqadd_u64(uint64x1_t __p0, uint64x1_t __p1) {66454__ai uint64x1_t vsqadd_u64(uint64x1_t __p0, int64x1_t __p1) {
66455 uint64x1_t __ret;66455 uint64x1_t __ret;
66456 __ret = (uint64x1_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 19);66456 __ret = (uint64x1_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 19);
66457 return __ret;66457 return __ret;
66458}66458}
66459#else66459#else
66460__ai uint64x1_t vsqadd_u64(uint64x1_t __p0, uint64x1_t __p1) {66460__ai uint64x1_t vsqadd_u64(uint64x1_t __p0, int64x1_t __p1) {
66461 uint64x1_t __ret;66461 uint64x1_t __ret;
66462 __ret = (uint64x1_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 19);66462 __ret = (uint64x1_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 19);
66463 return __ret;66463 return __ret;
...@@ -66465,15 +66465,15 @@ __ai uint64x1_t vsqadd_u64(uint64x1_t __p0, uint64x1_t __p1) {...@@ -66465,15 +66465,15 @@ __ai uint64x1_t vsqadd_u64(uint64x1_t __p0, uint64x1_t __p1) {
66465#endif66465#endif
6646666466
66467#ifdef __LITTLE_ENDIAN__66467#ifdef __LITTLE_ENDIAN__
66468__ai uint16x4_t vsqadd_u16(uint16x4_t __p0, uint16x4_t __p1) {66468__ai uint16x4_t vsqadd_u16(uint16x4_t __p0, int16x4_t __p1) {
66469 uint16x4_t __ret;66469 uint16x4_t __ret;
66470 __ret = (uint16x4_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 17);66470 __ret = (uint16x4_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 17);
66471 return __ret;66471 return __ret;
66472}66472}
66473#else66473#else
66474__ai uint16x4_t vsqadd_u16(uint16x4_t __p0, uint16x4_t __p1) {66474__ai uint16x4_t vsqadd_u16(uint16x4_t __p0, int16x4_t __p1) {
66475 uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);66475 uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
66476 uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);66476 int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
66477 uint16x4_t __ret;66477 uint16x4_t __ret;
66478 __ret = (uint16x4_t) __builtin_neon_vsqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17);66478 __ret = (uint16x4_t) __builtin_neon_vsqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17);
66479 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);66479 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
...@@ -68919,13 +68919,13 @@ __ai int64_t vtstd_s64(int64_t __p0, int64_t __p1) {...@@ -68919,13 +68919,13 @@ __ai int64_t vtstd_s64(int64_t __p0, int64_t __p1) {
68919#endif68919#endif
6892068920
68921#ifdef __LITTLE_ENDIAN__68921#ifdef __LITTLE_ENDIAN__
68922__ai int8_t vuqaddb_s8(int8_t __p0, int8_t __p1) {68922__ai int8_t vuqaddb_s8(int8_t __p0, uint8_t __p1) {
68923 int8_t __ret;68923 int8_t __ret;
68924 __ret = (int8_t) __builtin_neon_vuqaddb_s8(__p0, __p1);68924 __ret = (int8_t) __builtin_neon_vuqaddb_s8(__p0, __p1);
68925 return __ret;68925 return __ret;
68926}68926}
68927#else68927#else
68928__ai int8_t vuqaddb_s8(int8_t __p0, int8_t __p1) {68928__ai int8_t vuqaddb_s8(int8_t __p0, uint8_t __p1) {
68929 int8_t __ret;68929 int8_t __ret;
68930 __ret = (int8_t) __builtin_neon_vuqaddb_s8(__p0, __p1);68930 __ret = (int8_t) __builtin_neon_vuqaddb_s8(__p0, __p1);
68931 return __ret;68931 return __ret;
...@@ -68933,13 +68933,13 @@ __ai int8_t vuqaddb_s8(int8_t __p0, int8_t __p1) {...@@ -68933,13 +68933,13 @@ __ai int8_t vuqaddb_s8(int8_t __p0, int8_t __p1) {
68933#endif68933#endif
6893468934
68935#ifdef __LITTLE_ENDIAN__68935#ifdef __LITTLE_ENDIAN__
68936__ai int32_t vuqadds_s32(int32_t __p0, int32_t __p1) {68936__ai int32_t vuqadds_s32(int32_t __p0, uint32_t __p1) {
68937 int32_t __ret;68937 int32_t __ret;
68938 __ret = (int32_t) __builtin_neon_vuqadds_s32(__p0, __p1);68938 __ret = (int32_t) __builtin_neon_vuqadds_s32(__p0, __p1);
68939 return __ret;68939 return __ret;
68940}68940}
68941#else68941#else
68942__ai int32_t vuqadds_s32(int32_t __p0, int32_t __p1) {68942__ai int32_t vuqadds_s32(int32_t __p0, uint32_t __p1) {
68943 int32_t __ret;68943 int32_t __ret;
68944 __ret = (int32_t) __builtin_neon_vuqadds_s32(__p0, __p1);68944 __ret = (int32_t) __builtin_neon_vuqadds_s32(__p0, __p1);
68945 return __ret;68945 return __ret;
...@@ -68947,13 +68947,13 @@ __ai int32_t vuqadds_s32(int32_t __p0, int32_t __p1) {...@@ -68947,13 +68947,13 @@ __ai int32_t vuqadds_s32(int32_t __p0, int32_t __p1) {
68947#endif68947#endif
6894868948
68949#ifdef __LITTLE_ENDIAN__68949#ifdef __LITTLE_ENDIAN__
68950__ai int64_t vuqaddd_s64(int64_t __p0, int64_t __p1) {68950__ai int64_t vuqaddd_s64(int64_t __p0, uint64_t __p1) {
68951 int64_t __ret;68951 int64_t __ret;
68952 __ret = (int64_t) __builtin_neon_vuqaddd_s64(__p0, __p1);68952 __ret = (int64_t) __builtin_neon_vuqaddd_s64(__p0, __p1);
68953 return __ret;68953 return __ret;
68954}68954}
68955#else68955#else
68956__ai int64_t vuqaddd_s64(int64_t __p0, int64_t __p1) {68956__ai int64_t vuqaddd_s64(int64_t __p0, uint64_t __p1) {
68957 int64_t __ret;68957 int64_t __ret;
68958 __ret = (int64_t) __builtin_neon_vuqaddd_s64(__p0, __p1);68958 __ret = (int64_t) __builtin_neon_vuqaddd_s64(__p0, __p1);
68959 return __ret;68959 return __ret;
...@@ -68961,13 +68961,13 @@ __ai int64_t vuqaddd_s64(int64_t __p0, int64_t __p1) {...@@ -68961,13 +68961,13 @@ __ai int64_t vuqaddd_s64(int64_t __p0, int64_t __p1) {
68961#endif68961#endif
6896268962
68963#ifdef __LITTLE_ENDIAN__68963#ifdef __LITTLE_ENDIAN__
68964__ai int16_t vuqaddh_s16(int16_t __p0, int16_t __p1) {68964__ai int16_t vuqaddh_s16(int16_t __p0, uint16_t __p1) {
68965 int16_t __ret;68965 int16_t __ret;
68966 __ret = (int16_t) __builtin_neon_vuqaddh_s16(__p0, __p1);68966 __ret = (int16_t) __builtin_neon_vuqaddh_s16(__p0, __p1);
68967 return __ret;68967 return __ret;
68968}68968}
68969#else68969#else
68970__ai int16_t vuqaddh_s16(int16_t __p0, int16_t __p1) {68970__ai int16_t vuqaddh_s16(int16_t __p0, uint16_t __p1) {
68971 int16_t __ret;68971 int16_t __ret;
68972 __ret = (int16_t) __builtin_neon_vuqaddh_s16(__p0, __p1);68972 __ret = (int16_t) __builtin_neon_vuqaddh_s16(__p0, __p1);
68973 return __ret;68973 return __ret;
...@@ -68975,15 +68975,15 @@ __ai int16_t vuqaddh_s16(int16_t __p0, int16_t __p1) {...@@ -68975,15 +68975,15 @@ __ai int16_t vuqaddh_s16(int16_t __p0, int16_t __p1) {
68975#endif68975#endif
6897668976
68977#ifdef __LITTLE_ENDIAN__68977#ifdef __LITTLE_ENDIAN__
68978__ai int8x16_t vuqaddq_s8(int8x16_t __p0, int8x16_t __p1) {68978__ai int8x16_t vuqaddq_s8(int8x16_t __p0, uint8x16_t __p1) {
68979 int8x16_t __ret;68979 int8x16_t __ret;
68980 __ret = (int8x16_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 32);68980 __ret = (int8x16_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 32);
68981 return __ret;68981 return __ret;
68982}68982}
68983#else68983#else
68984__ai int8x16_t vuqaddq_s8(int8x16_t __p0, int8x16_t __p1) {68984__ai int8x16_t vuqaddq_s8(int8x16_t __p0, uint8x16_t __p1) {
68985 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);68985 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
68986 int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);68986 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
68987 int8x16_t __ret;68987 int8x16_t __ret;
68988 __ret = (int8x16_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32);68988 __ret = (int8x16_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32);
68989 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);68989 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -68992,15 +68992,15 @@ __ai int8x16_t vuqaddq_s8(int8x16_t __p0, int8x16_t __p1) {...@@ -68992,15 +68992,15 @@ __ai int8x16_t vuqaddq_s8(int8x16_t __p0, int8x16_t __p1) {
68992#endif68992#endif
6899368993
68994#ifdef __LITTLE_ENDIAN__68994#ifdef __LITTLE_ENDIAN__
68995__ai int32x4_t vuqaddq_s32(int32x4_t __p0, int32x4_t __p1) {68995__ai int32x4_t vuqaddq_s32(int32x4_t __p0, uint32x4_t __p1) {
68996 int32x4_t __ret;68996 int32x4_t __ret;
68997 __ret = (int32x4_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 34);68997 __ret = (int32x4_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 34);
68998 return __ret;68998 return __ret;
68999}68999}
69000#else69000#else
69001__ai int32x4_t vuqaddq_s32(int32x4_t __p0, int32x4_t __p1) {69001__ai int32x4_t vuqaddq_s32(int32x4_t __p0, uint32x4_t __p1) {
69002 int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);69002 int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
69003 int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);69003 uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
69004 int32x4_t __ret;69004 int32x4_t __ret;
69005 __ret = (int32x4_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34);69005 __ret = (int32x4_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34);
69006 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);69006 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
...@@ -69009,15 +69009,15 @@ __ai int32x4_t vuqaddq_s32(int32x4_t __p0, int32x4_t __p1) {...@@ -69009,15 +69009,15 @@ __ai int32x4_t vuqaddq_s32(int32x4_t __p0, int32x4_t __p1) {
69009#endif69009#endif
6901069010
69011#ifdef __LITTLE_ENDIAN__69011#ifdef __LITTLE_ENDIAN__
69012__ai int64x2_t vuqaddq_s64(int64x2_t __p0, int64x2_t __p1) {69012__ai int64x2_t vuqaddq_s64(int64x2_t __p0, uint64x2_t __p1) {
69013 int64x2_t __ret;69013 int64x2_t __ret;
69014 __ret = (int64x2_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 35);69014 __ret = (int64x2_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 35);
69015 return __ret;69015 return __ret;
69016}69016}
69017#else69017#else
69018__ai int64x2_t vuqaddq_s64(int64x2_t __p0, int64x2_t __p1) {69018__ai int64x2_t vuqaddq_s64(int64x2_t __p0, uint64x2_t __p1) {
69019 int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);69019 int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
69020 int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);69020 uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
69021 int64x2_t __ret;69021 int64x2_t __ret;
69022 __ret = (int64x2_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35);69022 __ret = (int64x2_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35);
69023 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);69023 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
...@@ -69026,15 +69026,15 @@ __ai int64x2_t vuqaddq_s64(int64x2_t __p0, int64x2_t __p1) {...@@ -69026,15 +69026,15 @@ __ai int64x2_t vuqaddq_s64(int64x2_t __p0, int64x2_t __p1) {
69026#endif69026#endif
6902769027
69028#ifdef __LITTLE_ENDIAN__69028#ifdef __LITTLE_ENDIAN__
69029__ai int16x8_t vuqaddq_s16(int16x8_t __p0, int16x8_t __p1) {69029__ai int16x8_t vuqaddq_s16(int16x8_t __p0, uint16x8_t __p1) {
69030 int16x8_t __ret;69030 int16x8_t __ret;
69031 __ret = (int16x8_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 33);69031 __ret = (int16x8_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 33);
69032 return __ret;69032 return __ret;
69033}69033}
69034#else69034#else
69035__ai int16x8_t vuqaddq_s16(int16x8_t __p0, int16x8_t __p1) {69035__ai int16x8_t vuqaddq_s16(int16x8_t __p0, uint16x8_t __p1) {
69036 int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);69036 int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
69037 int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);69037 uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
69038 int16x8_t __ret;69038 int16x8_t __ret;
69039 __ret = (int16x8_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33);69039 __ret = (int16x8_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33);
69040 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);69040 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -69043,15 +69043,15 @@ __ai int16x8_t vuqaddq_s16(int16x8_t __p0, int16x8_t __p1) {...@@ -69043,15 +69043,15 @@ __ai int16x8_t vuqaddq_s16(int16x8_t __p0, int16x8_t __p1) {
69043#endif69043#endif
6904469044
69045#ifdef __LITTLE_ENDIAN__69045#ifdef __LITTLE_ENDIAN__
69046__ai int8x8_t vuqadd_s8(int8x8_t __p0, int8x8_t __p1) {69046__ai int8x8_t vuqadd_s8(int8x8_t __p0, uint8x8_t __p1) {
69047 int8x8_t __ret;69047 int8x8_t __ret;
69048 __ret = (int8x8_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 0);69048 __ret = (int8x8_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 0);
69049 return __ret;69049 return __ret;
69050}69050}
69051#else69051#else
69052__ai int8x8_t vuqadd_s8(int8x8_t __p0, int8x8_t __p1) {69052__ai int8x8_t vuqadd_s8(int8x8_t __p0, uint8x8_t __p1) {
69053 int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);69053 int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
69054 int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);69054 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
69055 int8x8_t __ret;69055 int8x8_t __ret;
69056 __ret = (int8x8_t) __builtin_neon_vuqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0);69056 __ret = (int8x8_t) __builtin_neon_vuqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0);
69057 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);69057 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
...@@ -69060,15 +69060,15 @@ __ai int8x8_t vuqadd_s8(int8x8_t __p0, int8x8_t __p1) {...@@ -69060,15 +69060,15 @@ __ai int8x8_t vuqadd_s8(int8x8_t __p0, int8x8_t __p1) {
69060#endif69060#endif
6906169061
69062#ifdef __LITTLE_ENDIAN__69062#ifdef __LITTLE_ENDIAN__
69063__ai int32x2_t vuqadd_s32(int32x2_t __p0, int32x2_t __p1) {69063__ai int32x2_t vuqadd_s32(int32x2_t __p0, uint32x2_t __p1) {
69064 int32x2_t __ret;69064 int32x2_t __ret;
69065 __ret = (int32x2_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 2);69065 __ret = (int32x2_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 2);
69066 return __ret;69066 return __ret;
69067}69067}
69068#else69068#else
69069__ai int32x2_t vuqadd_s32(int32x2_t __p0, int32x2_t __p1) {69069__ai int32x2_t vuqadd_s32(int32x2_t __p0, uint32x2_t __p1) {
69070 int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);69070 int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
69071 int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);69071 uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
69072 int32x2_t __ret;69072 int32x2_t __ret;
69073 __ret = (int32x2_t) __builtin_neon_vuqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2);69073 __ret = (int32x2_t) __builtin_neon_vuqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2);
69074 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);69074 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
...@@ -69077,13 +69077,13 @@ __ai int32x2_t vuqadd_s32(int32x2_t __p0, int32x2_t __p1) {...@@ -69077,13 +69077,13 @@ __ai int32x2_t vuqadd_s32(int32x2_t __p0, int32x2_t __p1) {
69077#endif69077#endif
6907869078
69079#ifdef __LITTLE_ENDIAN__69079#ifdef __LITTLE_ENDIAN__
69080__ai int64x1_t vuqadd_s64(int64x1_t __p0, int64x1_t __p1) {69080__ai int64x1_t vuqadd_s64(int64x1_t __p0, uint64x1_t __p1) {
69081 int64x1_t __ret;69081 int64x1_t __ret;
69082 __ret = (int64x1_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 3);69082 __ret = (int64x1_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 3);
69083 return __ret;69083 return __ret;
69084}69084}
69085#else69085#else
69086__ai int64x1_t vuqadd_s64(int64x1_t __p0, int64x1_t __p1) {69086__ai int64x1_t vuqadd_s64(int64x1_t __p0, uint64x1_t __p1) {
69087 int64x1_t __ret;69087 int64x1_t __ret;
69088 __ret = (int64x1_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 3);69088 __ret = (int64x1_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 3);
69089 return __ret;69089 return __ret;
...@@ -69091,15 +69091,15 @@ __ai int64x1_t vuqadd_s64(int64x1_t __p0, int64x1_t __p1) {...@@ -69091,15 +69091,15 @@ __ai int64x1_t vuqadd_s64(int64x1_t __p0, int64x1_t __p1) {
69091#endif69091#endif
6909269092
69093#ifdef __LITTLE_ENDIAN__69093#ifdef __LITTLE_ENDIAN__
69094__ai int16x4_t vuqadd_s16(int16x4_t __p0, int16x4_t __p1) {69094__ai int16x4_t vuqadd_s16(int16x4_t __p0, uint16x4_t __p1) {
69095 int16x4_t __ret;69095 int16x4_t __ret;
69096 __ret = (int16x4_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 1);69096 __ret = (int16x4_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 1);
69097 return __ret;69097 return __ret;
69098}69098}
69099#else69099#else
69100__ai int16x4_t vuqadd_s16(int16x4_t __p0, int16x4_t __p1) {69100__ai int16x4_t vuqadd_s16(int16x4_t __p0, uint16x4_t __p1) {
69101 int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);69101 int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
69102 int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);69102 uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
69103 int16x4_t __ret;69103 int16x4_t __ret;
69104 __ret = (int16x4_t) __builtin_neon_vuqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1);69104 __ret = (int16x4_t) __builtin_neon_vuqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1);
69105 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);69105 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
...@@ -71912,16 +71912,16 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -71912,16 +71912,16 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
7191271912
71913#if defined(__ARM_FEATURE_FP16FML) && defined(__aarch64__)71913#if defined(__ARM_FEATURE_FP16FML) && defined(__aarch64__)
71914#ifdef __LITTLE_ENDIAN__71914#ifdef __LITTLE_ENDIAN__
71915#define vfmlalq_lane_high_u32(__p0_264, __p1_264, __p2_264, __p3_264) __extension__ ({ \71915#define vfmlalq_lane_high_f16(__p0_264, __p1_264, __p2_264, __p3_264) __extension__ ({ \
71916 float32x4_t __s0_264 = __p0_264; \71916 float32x4_t __s0_264 = __p0_264; \
71917 float16x8_t __s1_264 = __p1_264; \71917 float16x8_t __s1_264 = __p1_264; \
71918 float16x4_t __s2_264 = __p2_264; \71918 float16x4_t __s2_264 = __p2_264; \
71919 float32x4_t __ret_264; \71919 float32x4_t __ret_264; \
71920 __ret_264 = vfmlalq_high_u32(__s0_264, __s1_264, (float16x8_t) {vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264)}); \71920 __ret_264 = vfmlalq_high_f16(__s0_264, __s1_264, (float16x8_t) {vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264)}); \
71921 __ret_264; \71921 __ret_264; \
71922})71922})
71923#else71923#else
71924#define vfmlalq_lane_high_u32(__p0_265, __p1_265, __p2_265, __p3_265) __extension__ ({ \71924#define vfmlalq_lane_high_f16(__p0_265, __p1_265, __p2_265, __p3_265) __extension__ ({ \
71925 float32x4_t __s0_265 = __p0_265; \71925 float32x4_t __s0_265 = __p0_265; \
71926 float16x8_t __s1_265 = __p1_265; \71926 float16x8_t __s1_265 = __p1_265; \
71927 float16x4_t __s2_265 = __p2_265; \71927 float16x4_t __s2_265 = __p2_265; \
...@@ -71929,23 +71929,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -71929,23 +71929,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
71929 float16x8_t __rev1_265; __rev1_265 = __builtin_shufflevector(__s1_265, __s1_265, 7, 6, 5, 4, 3, 2, 1, 0); \71929 float16x8_t __rev1_265; __rev1_265 = __builtin_shufflevector(__s1_265, __s1_265, 7, 6, 5, 4, 3, 2, 1, 0); \
71930 float16x4_t __rev2_265; __rev2_265 = __builtin_shufflevector(__s2_265, __s2_265, 3, 2, 1, 0); \71930 float16x4_t __rev2_265; __rev2_265 = __builtin_shufflevector(__s2_265, __s2_265, 3, 2, 1, 0); \
71931 float32x4_t __ret_265; \71931 float32x4_t __ret_265; \
71932 __ret_265 = __noswap_vfmlalq_high_u32(__rev0_265, __rev1_265, (float16x8_t) {__noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265)}); \71932 __ret_265 = __noswap_vfmlalq_high_f16(__rev0_265, __rev1_265, (float16x8_t) {__noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265)}); \
71933 __ret_265 = __builtin_shufflevector(__ret_265, __ret_265, 3, 2, 1, 0); \71933 __ret_265 = __builtin_shufflevector(__ret_265, __ret_265, 3, 2, 1, 0); \
71934 __ret_265; \71934 __ret_265; \
71935})71935})
71936#endif71936#endif
7193771937
71938#ifdef __LITTLE_ENDIAN__71938#ifdef __LITTLE_ENDIAN__
71939#define vfmlal_lane_high_u32(__p0_266, __p1_266, __p2_266, __p3_266) __extension__ ({ \71939#define vfmlal_lane_high_f16(__p0_266, __p1_266, __p2_266, __p3_266) __extension__ ({ \
71940 float32x2_t __s0_266 = __p0_266; \71940 float32x2_t __s0_266 = __p0_266; \
71941 float16x4_t __s1_266 = __p1_266; \71941 float16x4_t __s1_266 = __p1_266; \
71942 float16x4_t __s2_266 = __p2_266; \71942 float16x4_t __s2_266 = __p2_266; \
71943 float32x2_t __ret_266; \71943 float32x2_t __ret_266; \
71944 __ret_266 = vfmlal_high_u32(__s0_266, __s1_266, (float16x4_t) {vget_lane_f16(__s2_266, __p3_266), vget_lane_f16(__s2_266, __p3_266), vget_lane_f16(__s2_266, __p3_266), vget_lane_f16(__s2_266, __p3_266)}); \71944 __ret_266 = vfmlal_high_f16(__s0_266, __s1_266, (float16x4_t) {vget_lane_f16(__s2_266, __p3_266), vget_lane_f16(__s2_266, __p3_266), vget_lane_f16(__s2_266, __p3_266), vget_lane_f16(__s2_266, __p3_266)}); \
71945 __ret_266; \71945 __ret_266; \
71946})71946})
71947#else71947#else
71948#define vfmlal_lane_high_u32(__p0_267, __p1_267, __p2_267, __p3_267) __extension__ ({ \71948#define vfmlal_lane_high_f16(__p0_267, __p1_267, __p2_267, __p3_267) __extension__ ({ \
71949 float32x2_t __s0_267 = __p0_267; \71949 float32x2_t __s0_267 = __p0_267; \
71950 float16x4_t __s1_267 = __p1_267; \71950 float16x4_t __s1_267 = __p1_267; \
71951 float16x4_t __s2_267 = __p2_267; \71951 float16x4_t __s2_267 = __p2_267; \
...@@ -71953,23 +71953,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -71953,23 +71953,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
71953 float16x4_t __rev1_267; __rev1_267 = __builtin_shufflevector(__s1_267, __s1_267, 3, 2, 1, 0); \71953 float16x4_t __rev1_267; __rev1_267 = __builtin_shufflevector(__s1_267, __s1_267, 3, 2, 1, 0); \
71954 float16x4_t __rev2_267; __rev2_267 = __builtin_shufflevector(__s2_267, __s2_267, 3, 2, 1, 0); \71954 float16x4_t __rev2_267; __rev2_267 = __builtin_shufflevector(__s2_267, __s2_267, 3, 2, 1, 0); \
71955 float32x2_t __ret_267; \71955 float32x2_t __ret_267; \
71956 __ret_267 = __noswap_vfmlal_high_u32(__rev0_267, __rev1_267, (float16x4_t) {__noswap_vget_lane_f16(__rev2_267, __p3_267), __noswap_vget_lane_f16(__rev2_267, __p3_267), __noswap_vget_lane_f16(__rev2_267, __p3_267), __noswap_vget_lane_f16(__rev2_267, __p3_267)}); \71956 __ret_267 = __noswap_vfmlal_high_f16(__rev0_267, __rev1_267, (float16x4_t) {__noswap_vget_lane_f16(__rev2_267, __p3_267), __noswap_vget_lane_f16(__rev2_267, __p3_267), __noswap_vget_lane_f16(__rev2_267, __p3_267), __noswap_vget_lane_f16(__rev2_267, __p3_267)}); \
71957 __ret_267 = __builtin_shufflevector(__ret_267, __ret_267, 1, 0); \71957 __ret_267 = __builtin_shufflevector(__ret_267, __ret_267, 1, 0); \
71958 __ret_267; \71958 __ret_267; \
71959})71959})
71960#endif71960#endif
7196171961
71962#ifdef __LITTLE_ENDIAN__71962#ifdef __LITTLE_ENDIAN__
71963#define vfmlalq_lane_low_u32(__p0_268, __p1_268, __p2_268, __p3_268) __extension__ ({ \71963#define vfmlalq_lane_low_f16(__p0_268, __p1_268, __p2_268, __p3_268) __extension__ ({ \
71964 float32x4_t __s0_268 = __p0_268; \71964 float32x4_t __s0_268 = __p0_268; \
71965 float16x8_t __s1_268 = __p1_268; \71965 float16x8_t __s1_268 = __p1_268; \
71966 float16x4_t __s2_268 = __p2_268; \71966 float16x4_t __s2_268 = __p2_268; \
71967 float32x4_t __ret_268; \71967 float32x4_t __ret_268; \
71968 __ret_268 = vfmlalq_low_u32(__s0_268, __s1_268, (float16x8_t) {vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268)}); \71968 __ret_268 = vfmlalq_low_f16(__s0_268, __s1_268, (float16x8_t) {vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268)}); \
71969 __ret_268; \71969 __ret_268; \
71970})71970})
71971#else71971#else
71972#define vfmlalq_lane_low_u32(__p0_269, __p1_269, __p2_269, __p3_269) __extension__ ({ \71972#define vfmlalq_lane_low_f16(__p0_269, __p1_269, __p2_269, __p3_269) __extension__ ({ \
71973 float32x4_t __s0_269 = __p0_269; \71973 float32x4_t __s0_269 = __p0_269; \
71974 float16x8_t __s1_269 = __p1_269; \71974 float16x8_t __s1_269 = __p1_269; \
71975 float16x4_t __s2_269 = __p2_269; \71975 float16x4_t __s2_269 = __p2_269; \
...@@ -71977,23 +71977,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -71977,23 +71977,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
71977 float16x8_t __rev1_269; __rev1_269 = __builtin_shufflevector(__s1_269, __s1_269, 7, 6, 5, 4, 3, 2, 1, 0); \71977 float16x8_t __rev1_269; __rev1_269 = __builtin_shufflevector(__s1_269, __s1_269, 7, 6, 5, 4, 3, 2, 1, 0); \
71978 float16x4_t __rev2_269; __rev2_269 = __builtin_shufflevector(__s2_269, __s2_269, 3, 2, 1, 0); \71978 float16x4_t __rev2_269; __rev2_269 = __builtin_shufflevector(__s2_269, __s2_269, 3, 2, 1, 0); \
71979 float32x4_t __ret_269; \71979 float32x4_t __ret_269; \
71980 __ret_269 = __noswap_vfmlalq_low_u32(__rev0_269, __rev1_269, (float16x8_t) {__noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269)}); \71980 __ret_269 = __noswap_vfmlalq_low_f16(__rev0_269, __rev1_269, (float16x8_t) {__noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269)}); \
71981 __ret_269 = __builtin_shufflevector(__ret_269, __ret_269, 3, 2, 1, 0); \71981 __ret_269 = __builtin_shufflevector(__ret_269, __ret_269, 3, 2, 1, 0); \
71982 __ret_269; \71982 __ret_269; \
71983})71983})
71984#endif71984#endif
7198571985
71986#ifdef __LITTLE_ENDIAN__71986#ifdef __LITTLE_ENDIAN__
71987#define vfmlal_lane_low_u32(__p0_270, __p1_270, __p2_270, __p3_270) __extension__ ({ \71987#define vfmlal_lane_low_f16(__p0_270, __p1_270, __p2_270, __p3_270) __extension__ ({ \
71988 float32x2_t __s0_270 = __p0_270; \71988 float32x2_t __s0_270 = __p0_270; \
71989 float16x4_t __s1_270 = __p1_270; \71989 float16x4_t __s1_270 = __p1_270; \
71990 float16x4_t __s2_270 = __p2_270; \71990 float16x4_t __s2_270 = __p2_270; \
71991 float32x2_t __ret_270; \71991 float32x2_t __ret_270; \
71992 __ret_270 = vfmlal_low_u32(__s0_270, __s1_270, (float16x4_t) {vget_lane_f16(__s2_270, __p3_270), vget_lane_f16(__s2_270, __p3_270), vget_lane_f16(__s2_270, __p3_270), vget_lane_f16(__s2_270, __p3_270)}); \71992 __ret_270 = vfmlal_low_f16(__s0_270, __s1_270, (float16x4_t) {vget_lane_f16(__s2_270, __p3_270), vget_lane_f16(__s2_270, __p3_270), vget_lane_f16(__s2_270, __p3_270), vget_lane_f16(__s2_270, __p3_270)}); \
71993 __ret_270; \71993 __ret_270; \
71994})71994})
71995#else71995#else
71996#define vfmlal_lane_low_u32(__p0_271, __p1_271, __p2_271, __p3_271) __extension__ ({ \71996#define vfmlal_lane_low_f16(__p0_271, __p1_271, __p2_271, __p3_271) __extension__ ({ \
71997 float32x2_t __s0_271 = __p0_271; \71997 float32x2_t __s0_271 = __p0_271; \
71998 float16x4_t __s1_271 = __p1_271; \71998 float16x4_t __s1_271 = __p1_271; \
71999 float16x4_t __s2_271 = __p2_271; \71999 float16x4_t __s2_271 = __p2_271; \
...@@ -72001,23 +72001,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -72001,23 +72001,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
72001 float16x4_t __rev1_271; __rev1_271 = __builtin_shufflevector(__s1_271, __s1_271, 3, 2, 1, 0); \72001 float16x4_t __rev1_271; __rev1_271 = __builtin_shufflevector(__s1_271, __s1_271, 3, 2, 1, 0); \
72002 float16x4_t __rev2_271; __rev2_271 = __builtin_shufflevector(__s2_271, __s2_271, 3, 2, 1, 0); \72002 float16x4_t __rev2_271; __rev2_271 = __builtin_shufflevector(__s2_271, __s2_271, 3, 2, 1, 0); \
72003 float32x2_t __ret_271; \72003 float32x2_t __ret_271; \
72004 __ret_271 = __noswap_vfmlal_low_u32(__rev0_271, __rev1_271, (float16x4_t) {__noswap_vget_lane_f16(__rev2_271, __p3_271), __noswap_vget_lane_f16(__rev2_271, __p3_271), __noswap_vget_lane_f16(__rev2_271, __p3_271), __noswap_vget_lane_f16(__rev2_271, __p3_271)}); \72004 __ret_271 = __noswap_vfmlal_low_f16(__rev0_271, __rev1_271, (float16x4_t) {__noswap_vget_lane_f16(__rev2_271, __p3_271), __noswap_vget_lane_f16(__rev2_271, __p3_271), __noswap_vget_lane_f16(__rev2_271, __p3_271), __noswap_vget_lane_f16(__rev2_271, __p3_271)}); \
72005 __ret_271 = __builtin_shufflevector(__ret_271, __ret_271, 1, 0); \72005 __ret_271 = __builtin_shufflevector(__ret_271, __ret_271, 1, 0); \
72006 __ret_271; \72006 __ret_271; \
72007})72007})
72008#endif72008#endif
7200972009
72010#ifdef __LITTLE_ENDIAN__72010#ifdef __LITTLE_ENDIAN__
72011#define vfmlalq_laneq_high_u32(__p0_272, __p1_272, __p2_272, __p3_272) __extension__ ({ \72011#define vfmlalq_laneq_high_f16(__p0_272, __p1_272, __p2_272, __p3_272) __extension__ ({ \
72012 float32x4_t __s0_272 = __p0_272; \72012 float32x4_t __s0_272 = __p0_272; \
72013 float16x8_t __s1_272 = __p1_272; \72013 float16x8_t __s1_272 = __p1_272; \
72014 float16x8_t __s2_272 = __p2_272; \72014 float16x8_t __s2_272 = __p2_272; \
72015 float32x4_t __ret_272; \72015 float32x4_t __ret_272; \
72016 __ret_272 = vfmlalq_high_u32(__s0_272, __s1_272, (float16x8_t) {vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272)}); \72016 __ret_272 = vfmlalq_high_f16(__s0_272, __s1_272, (float16x8_t) {vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272)}); \
72017 __ret_272; \72017 __ret_272; \
72018})72018})
72019#else72019#else
72020#define vfmlalq_laneq_high_u32(__p0_273, __p1_273, __p2_273, __p3_273) __extension__ ({ \72020#define vfmlalq_laneq_high_f16(__p0_273, __p1_273, __p2_273, __p3_273) __extension__ ({ \
72021 float32x4_t __s0_273 = __p0_273; \72021 float32x4_t __s0_273 = __p0_273; \
72022 float16x8_t __s1_273 = __p1_273; \72022 float16x8_t __s1_273 = __p1_273; \
72023 float16x8_t __s2_273 = __p2_273; \72023 float16x8_t __s2_273 = __p2_273; \
...@@ -72025,23 +72025,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -72025,23 +72025,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
72025 float16x8_t __rev1_273; __rev1_273 = __builtin_shufflevector(__s1_273, __s1_273, 7, 6, 5, 4, 3, 2, 1, 0); \72025 float16x8_t __rev1_273; __rev1_273 = __builtin_shufflevector(__s1_273, __s1_273, 7, 6, 5, 4, 3, 2, 1, 0); \
72026 float16x8_t __rev2_273; __rev2_273 = __builtin_shufflevector(__s2_273, __s2_273, 7, 6, 5, 4, 3, 2, 1, 0); \72026 float16x8_t __rev2_273; __rev2_273 = __builtin_shufflevector(__s2_273, __s2_273, 7, 6, 5, 4, 3, 2, 1, 0); \
72027 float32x4_t __ret_273; \72027 float32x4_t __ret_273; \
72028 __ret_273 = __noswap_vfmlalq_high_u32(__rev0_273, __rev1_273, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273)}); \72028 __ret_273 = __noswap_vfmlalq_high_f16(__rev0_273, __rev1_273, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273)}); \
72029 __ret_273 = __builtin_shufflevector(__ret_273, __ret_273, 3, 2, 1, 0); \72029 __ret_273 = __builtin_shufflevector(__ret_273, __ret_273, 3, 2, 1, 0); \
72030 __ret_273; \72030 __ret_273; \
72031})72031})
72032#endif72032#endif
7203372033
72034#ifdef __LITTLE_ENDIAN__72034#ifdef __LITTLE_ENDIAN__
72035#define vfmlal_laneq_high_u32(__p0_274, __p1_274, __p2_274, __p3_274) __extension__ ({ \72035#define vfmlal_laneq_high_f16(__p0_274, __p1_274, __p2_274, __p3_274) __extension__ ({ \
72036 float32x2_t __s0_274 = __p0_274; \72036 float32x2_t __s0_274 = __p0_274; \
72037 float16x4_t __s1_274 = __p1_274; \72037 float16x4_t __s1_274 = __p1_274; \
72038 float16x8_t __s2_274 = __p2_274; \72038 float16x8_t __s2_274 = __p2_274; \
72039 float32x2_t __ret_274; \72039 float32x2_t __ret_274; \
72040 __ret_274 = vfmlal_high_u32(__s0_274, __s1_274, (float16x4_t) {vgetq_lane_f16(__s2_274, __p3_274), vgetq_lane_f16(__s2_274, __p3_274), vgetq_lane_f16(__s2_274, __p3_274), vgetq_lane_f16(__s2_274, __p3_274)}); \72040 __ret_274 = vfmlal_high_f16(__s0_274, __s1_274, (float16x4_t) {vgetq_lane_f16(__s2_274, __p3_274), vgetq_lane_f16(__s2_274, __p3_274), vgetq_lane_f16(__s2_274, __p3_274), vgetq_lane_f16(__s2_274, __p3_274)}); \
72041 __ret_274; \72041 __ret_274; \
72042})72042})
72043#else72043#else
72044#define vfmlal_laneq_high_u32(__p0_275, __p1_275, __p2_275, __p3_275) __extension__ ({ \72044#define vfmlal_laneq_high_f16(__p0_275, __p1_275, __p2_275, __p3_275) __extension__ ({ \
72045 float32x2_t __s0_275 = __p0_275; \72045 float32x2_t __s0_275 = __p0_275; \
72046 float16x4_t __s1_275 = __p1_275; \72046 float16x4_t __s1_275 = __p1_275; \
72047 float16x8_t __s2_275 = __p2_275; \72047 float16x8_t __s2_275 = __p2_275; \
...@@ -72049,23 +72049,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -72049,23 +72049,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
72049 float16x4_t __rev1_275; __rev1_275 = __builtin_shufflevector(__s1_275, __s1_275, 3, 2, 1, 0); \72049 float16x4_t __rev1_275; __rev1_275 = __builtin_shufflevector(__s1_275, __s1_275, 3, 2, 1, 0); \
72050 float16x8_t __rev2_275; __rev2_275 = __builtin_shufflevector(__s2_275, __s2_275, 7, 6, 5, 4, 3, 2, 1, 0); \72050 float16x8_t __rev2_275; __rev2_275 = __builtin_shufflevector(__s2_275, __s2_275, 7, 6, 5, 4, 3, 2, 1, 0); \
72051 float32x2_t __ret_275; \72051 float32x2_t __ret_275; \
72052 __ret_275 = __noswap_vfmlal_high_u32(__rev0_275, __rev1_275, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_275, __p3_275), __noswap_vgetq_lane_f16(__rev2_275, __p3_275), __noswap_vgetq_lane_f16(__rev2_275, __p3_275), __noswap_vgetq_lane_f16(__rev2_275, __p3_275)}); \72052 __ret_275 = __noswap_vfmlal_high_f16(__rev0_275, __rev1_275, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_275, __p3_275), __noswap_vgetq_lane_f16(__rev2_275, __p3_275), __noswap_vgetq_lane_f16(__rev2_275, __p3_275), __noswap_vgetq_lane_f16(__rev2_275, __p3_275)}); \
72053 __ret_275 = __builtin_shufflevector(__ret_275, __ret_275, 1, 0); \72053 __ret_275 = __builtin_shufflevector(__ret_275, __ret_275, 1, 0); \
72054 __ret_275; \72054 __ret_275; \
72055})72055})
72056#endif72056#endif
7205772057
72058#ifdef __LITTLE_ENDIAN__72058#ifdef __LITTLE_ENDIAN__
72059#define vfmlalq_laneq_low_u32(__p0_276, __p1_276, __p2_276, __p3_276) __extension__ ({ \72059#define vfmlalq_laneq_low_f16(__p0_276, __p1_276, __p2_276, __p3_276) __extension__ ({ \
72060 float32x4_t __s0_276 = __p0_276; \72060 float32x4_t __s0_276 = __p0_276; \
72061 float16x8_t __s1_276 = __p1_276; \72061 float16x8_t __s1_276 = __p1_276; \
72062 float16x8_t __s2_276 = __p2_276; \72062 float16x8_t __s2_276 = __p2_276; \
72063 float32x4_t __ret_276; \72063 float32x4_t __ret_276; \
72064 __ret_276 = vfmlalq_low_u32(__s0_276, __s1_276, (float16x8_t) {vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276)}); \72064 __ret_276 = vfmlalq_low_f16(__s0_276, __s1_276, (float16x8_t) {vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276)}); \
72065 __ret_276; \72065 __ret_276; \
72066})72066})
72067#else72067#else
72068#define vfmlalq_laneq_low_u32(__p0_277, __p1_277, __p2_277, __p3_277) __extension__ ({ \72068#define vfmlalq_laneq_low_f16(__p0_277, __p1_277, __p2_277, __p3_277) __extension__ ({ \
72069 float32x4_t __s0_277 = __p0_277; \72069 float32x4_t __s0_277 = __p0_277; \
72070 float16x8_t __s1_277 = __p1_277; \72070 float16x8_t __s1_277 = __p1_277; \
72071 float16x8_t __s2_277 = __p2_277; \72071 float16x8_t __s2_277 = __p2_277; \
...@@ -72073,23 +72073,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -72073,23 +72073,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
72073 float16x8_t __rev1_277; __rev1_277 = __builtin_shufflevector(__s1_277, __s1_277, 7, 6, 5, 4, 3, 2, 1, 0); \72073 float16x8_t __rev1_277; __rev1_277 = __builtin_shufflevector(__s1_277, __s1_277, 7, 6, 5, 4, 3, 2, 1, 0); \
72074 float16x8_t __rev2_277; __rev2_277 = __builtin_shufflevector(__s2_277, __s2_277, 7, 6, 5, 4, 3, 2, 1, 0); \72074 float16x8_t __rev2_277; __rev2_277 = __builtin_shufflevector(__s2_277, __s2_277, 7, 6, 5, 4, 3, 2, 1, 0); \
72075 float32x4_t __ret_277; \72075 float32x4_t __ret_277; \
72076 __ret_277 = __noswap_vfmlalq_low_u32(__rev0_277, __rev1_277, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277)}); \72076 __ret_277 = __noswap_vfmlalq_low_f16(__rev0_277, __rev1_277, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277)}); \
72077 __ret_277 = __builtin_shufflevector(__ret_277, __ret_277, 3, 2, 1, 0); \72077 __ret_277 = __builtin_shufflevector(__ret_277, __ret_277, 3, 2, 1, 0); \
72078 __ret_277; \72078 __ret_277; \
72079})72079})
72080#endif72080#endif
7208172081
72082#ifdef __LITTLE_ENDIAN__72082#ifdef __LITTLE_ENDIAN__
72083#define vfmlal_laneq_low_u32(__p0_278, __p1_278, __p2_278, __p3_278) __extension__ ({ \72083#define vfmlal_laneq_low_f16(__p0_278, __p1_278, __p2_278, __p3_278) __extension__ ({ \
72084 float32x2_t __s0_278 = __p0_278; \72084 float32x2_t __s0_278 = __p0_278; \
72085 float16x4_t __s1_278 = __p1_278; \72085 float16x4_t __s1_278 = __p1_278; \
72086 float16x8_t __s2_278 = __p2_278; \72086 float16x8_t __s2_278 = __p2_278; \
72087 float32x2_t __ret_278; \72087 float32x2_t __ret_278; \
72088 __ret_278 = vfmlal_low_u32(__s0_278, __s1_278, (float16x4_t) {vgetq_lane_f16(__s2_278, __p3_278), vgetq_lane_f16(__s2_278, __p3_278), vgetq_lane_f16(__s2_278, __p3_278), vgetq_lane_f16(__s2_278, __p3_278)}); \72088 __ret_278 = vfmlal_low_f16(__s0_278, __s1_278, (float16x4_t) {vgetq_lane_f16(__s2_278, __p3_278), vgetq_lane_f16(__s2_278, __p3_278), vgetq_lane_f16(__s2_278, __p3_278), vgetq_lane_f16(__s2_278, __p3_278)}); \
72089 __ret_278; \72089 __ret_278; \
72090})72090})
72091#else72091#else
72092#define vfmlal_laneq_low_u32(__p0_279, __p1_279, __p2_279, __p3_279) __extension__ ({ \72092#define vfmlal_laneq_low_f16(__p0_279, __p1_279, __p2_279, __p3_279) __extension__ ({ \
72093 float32x2_t __s0_279 = __p0_279; \72093 float32x2_t __s0_279 = __p0_279; \
72094 float16x4_t __s1_279 = __p1_279; \72094 float16x4_t __s1_279 = __p1_279; \
72095 float16x8_t __s2_279 = __p2_279; \72095 float16x8_t __s2_279 = __p2_279; \
...@@ -72097,23 +72097,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -72097,23 +72097,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
72097 float16x4_t __rev1_279; __rev1_279 = __builtin_shufflevector(__s1_279, __s1_279, 3, 2, 1, 0); \72097 float16x4_t __rev1_279; __rev1_279 = __builtin_shufflevector(__s1_279, __s1_279, 3, 2, 1, 0); \
72098 float16x8_t __rev2_279; __rev2_279 = __builtin_shufflevector(__s2_279, __s2_279, 7, 6, 5, 4, 3, 2, 1, 0); \72098 float16x8_t __rev2_279; __rev2_279 = __builtin_shufflevector(__s2_279, __s2_279, 7, 6, 5, 4, 3, 2, 1, 0); \
72099 float32x2_t __ret_279; \72099 float32x2_t __ret_279; \
72100 __ret_279 = __noswap_vfmlal_low_u32(__rev0_279, __rev1_279, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_279, __p3_279), __noswap_vgetq_lane_f16(__rev2_279, __p3_279), __noswap_vgetq_lane_f16(__rev2_279, __p3_279), __noswap_vgetq_lane_f16(__rev2_279, __p3_279)}); \72100 __ret_279 = __noswap_vfmlal_low_f16(__rev0_279, __rev1_279, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_279, __p3_279), __noswap_vgetq_lane_f16(__rev2_279, __p3_279), __noswap_vgetq_lane_f16(__rev2_279, __p3_279), __noswap_vgetq_lane_f16(__rev2_279, __p3_279)}); \
72101 __ret_279 = __builtin_shufflevector(__ret_279, __ret_279, 1, 0); \72101 __ret_279 = __builtin_shufflevector(__ret_279, __ret_279, 1, 0); \
72102 __ret_279; \72102 __ret_279; \
72103})72103})
72104#endif72104#endif
7210572105
72106#ifdef __LITTLE_ENDIAN__72106#ifdef __LITTLE_ENDIAN__
72107#define vfmlslq_lane_high_u32(__p0_280, __p1_280, __p2_280, __p3_280) __extension__ ({ \72107#define vfmlslq_lane_high_f16(__p0_280, __p1_280, __p2_280, __p3_280) __extension__ ({ \
72108 float32x4_t __s0_280 = __p0_280; \72108 float32x4_t __s0_280 = __p0_280; \
72109 float16x8_t __s1_280 = __p1_280; \72109 float16x8_t __s1_280 = __p1_280; \
72110 float16x4_t __s2_280 = __p2_280; \72110 float16x4_t __s2_280 = __p2_280; \
72111 float32x4_t __ret_280; \72111 float32x4_t __ret_280; \
72112 __ret_280 = vfmlslq_high_u32(__s0_280, __s1_280, (float16x8_t) {vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280)}); \72112 __ret_280 = vfmlslq_high_f16(__s0_280, __s1_280, (float16x8_t) {vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280)}); \
72113 __ret_280; \72113 __ret_280; \
72114})72114})
72115#else72115#else
72116#define vfmlslq_lane_high_u32(__p0_281, __p1_281, __p2_281, __p3_281) __extension__ ({ \72116#define vfmlslq_lane_high_f16(__p0_281, __p1_281, __p2_281, __p3_281) __extension__ ({ \
72117 float32x4_t __s0_281 = __p0_281; \72117 float32x4_t __s0_281 = __p0_281; \
72118 float16x8_t __s1_281 = __p1_281; \72118 float16x8_t __s1_281 = __p1_281; \
72119 float16x4_t __s2_281 = __p2_281; \72119 float16x4_t __s2_281 = __p2_281; \
...@@ -72121,23 +72121,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -72121,23 +72121,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
72121 float16x8_t __rev1_281; __rev1_281 = __builtin_shufflevector(__s1_281, __s1_281, 7, 6, 5, 4, 3, 2, 1, 0); \72121 float16x8_t __rev1_281; __rev1_281 = __builtin_shufflevector(__s1_281, __s1_281, 7, 6, 5, 4, 3, 2, 1, 0); \
72122 float16x4_t __rev2_281; __rev2_281 = __builtin_shufflevector(__s2_281, __s2_281, 3, 2, 1, 0); \72122 float16x4_t __rev2_281; __rev2_281 = __builtin_shufflevector(__s2_281, __s2_281, 3, 2, 1, 0); \
72123 float32x4_t __ret_281; \72123 float32x4_t __ret_281; \
72124 __ret_281 = __noswap_vfmlslq_high_u32(__rev0_281, __rev1_281, (float16x8_t) {__noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281)}); \72124 __ret_281 = __noswap_vfmlslq_high_f16(__rev0_281, __rev1_281, (float16x8_t) {__noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281)}); \
72125 __ret_281 = __builtin_shufflevector(__ret_281, __ret_281, 3, 2, 1, 0); \72125 __ret_281 = __builtin_shufflevector(__ret_281, __ret_281, 3, 2, 1, 0); \
72126 __ret_281; \72126 __ret_281; \
72127})72127})
72128#endif72128#endif
7212972129
72130#ifdef __LITTLE_ENDIAN__72130#ifdef __LITTLE_ENDIAN__
72131#define vfmlsl_lane_high_u32(__p0_282, __p1_282, __p2_282, __p3_282) __extension__ ({ \72131#define vfmlsl_lane_high_f16(__p0_282, __p1_282, __p2_282, __p3_282) __extension__ ({ \
72132 float32x2_t __s0_282 = __p0_282; \72132 float32x2_t __s0_282 = __p0_282; \
72133 float16x4_t __s1_282 = __p1_282; \72133 float16x4_t __s1_282 = __p1_282; \
72134 float16x4_t __s2_282 = __p2_282; \72134 float16x4_t __s2_282 = __p2_282; \
72135 float32x2_t __ret_282; \72135 float32x2_t __ret_282; \
72136 __ret_282 = vfmlsl_high_u32(__s0_282, __s1_282, (float16x4_t) {vget_lane_f16(__s2_282, __p3_282), vget_lane_f16(__s2_282, __p3_282), vget_lane_f16(__s2_282, __p3_282), vget_lane_f16(__s2_282, __p3_282)}); \72136 __ret_282 = vfmlsl_high_f16(__s0_282, __s1_282, (float16x4_t) {vget_lane_f16(__s2_282, __p3_282), vget_lane_f16(__s2_282, __p3_282), vget_lane_f16(__s2_282, __p3_282), vget_lane_f16(__s2_282, __p3_282)}); \
72137 __ret_282; \72137 __ret_282; \
72138})72138})
72139#else72139#else
72140#define vfmlsl_lane_high_u32(__p0_283, __p1_283, __p2_283, __p3_283) __extension__ ({ \72140#define vfmlsl_lane_high_f16(__p0_283, __p1_283, __p2_283, __p3_283) __extension__ ({ \
72141 float32x2_t __s0_283 = __p0_283; \72141 float32x2_t __s0_283 = __p0_283; \
72142 float16x4_t __s1_283 = __p1_283; \72142 float16x4_t __s1_283 = __p1_283; \
72143 float16x4_t __s2_283 = __p2_283; \72143 float16x4_t __s2_283 = __p2_283; \
...@@ -72145,23 +72145,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -72145,23 +72145,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
72145 float16x4_t __rev1_283; __rev1_283 = __builtin_shufflevector(__s1_283, __s1_283, 3, 2, 1, 0); \72145 float16x4_t __rev1_283; __rev1_283 = __builtin_shufflevector(__s1_283, __s1_283, 3, 2, 1, 0); \
72146 float16x4_t __rev2_283; __rev2_283 = __builtin_shufflevector(__s2_283, __s2_283, 3, 2, 1, 0); \72146 float16x4_t __rev2_283; __rev2_283 = __builtin_shufflevector(__s2_283, __s2_283, 3, 2, 1, 0); \
72147 float32x2_t __ret_283; \72147 float32x2_t __ret_283; \
72148 __ret_283 = __noswap_vfmlsl_high_u32(__rev0_283, __rev1_283, (float16x4_t) {__noswap_vget_lane_f16(__rev2_283, __p3_283), __noswap_vget_lane_f16(__rev2_283, __p3_283), __noswap_vget_lane_f16(__rev2_283, __p3_283), __noswap_vget_lane_f16(__rev2_283, __p3_283)}); \72148 __ret_283 = __noswap_vfmlsl_high_f16(__rev0_283, __rev1_283, (float16x4_t) {__noswap_vget_lane_f16(__rev2_283, __p3_283), __noswap_vget_lane_f16(__rev2_283, __p3_283), __noswap_vget_lane_f16(__rev2_283, __p3_283), __noswap_vget_lane_f16(__rev2_283, __p3_283)}); \
72149 __ret_283 = __builtin_shufflevector(__ret_283, __ret_283, 1, 0); \72149 __ret_283 = __builtin_shufflevector(__ret_283, __ret_283, 1, 0); \
72150 __ret_283; \72150 __ret_283; \
72151})72151})
72152#endif72152#endif
7215372153
72154#ifdef __LITTLE_ENDIAN__72154#ifdef __LITTLE_ENDIAN__
72155#define vfmlslq_lane_low_u32(__p0_284, __p1_284, __p2_284, __p3_284) __extension__ ({ \72155#define vfmlslq_lane_low_f16(__p0_284, __p1_284, __p2_284, __p3_284) __extension__ ({ \
72156 float32x4_t __s0_284 = __p0_284; \72156 float32x4_t __s0_284 = __p0_284; \
72157 float16x8_t __s1_284 = __p1_284; \72157 float16x8_t __s1_284 = __p1_284; \
72158 float16x4_t __s2_284 = __p2_284; \72158 float16x4_t __s2_284 = __p2_284; \
72159 float32x4_t __ret_284; \72159 float32x4_t __ret_284; \
72160 __ret_284 = vfmlslq_low_u32(__s0_284, __s1_284, (float16x8_t) {vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284)}); \72160 __ret_284 = vfmlslq_low_f16(__s0_284, __s1_284, (float16x8_t) {vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284)}); \
72161 __ret_284; \72161 __ret_284; \
72162})72162})
72163#else72163#else
72164#define vfmlslq_lane_low_u32(__p0_285, __p1_285, __p2_285, __p3_285) __extension__ ({ \72164#define vfmlslq_lane_low_f16(__p0_285, __p1_285, __p2_285, __p3_285) __extension__ ({ \
72165 float32x4_t __s0_285 = __p0_285; \72165 float32x4_t __s0_285 = __p0_285; \
72166 float16x8_t __s1_285 = __p1_285; \72166 float16x8_t __s1_285 = __p1_285; \
72167 float16x4_t __s2_285 = __p2_285; \72167 float16x4_t __s2_285 = __p2_285; \
...@@ -72169,23 +72169,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -72169,23 +72169,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
72169 float16x8_t __rev1_285; __rev1_285 = __builtin_shufflevector(__s1_285, __s1_285, 7, 6, 5, 4, 3, 2, 1, 0); \72169 float16x8_t __rev1_285; __rev1_285 = __builtin_shufflevector(__s1_285, __s1_285, 7, 6, 5, 4, 3, 2, 1, 0); \
72170 float16x4_t __rev2_285; __rev2_285 = __builtin_shufflevector(__s2_285, __s2_285, 3, 2, 1, 0); \72170 float16x4_t __rev2_285; __rev2_285 = __builtin_shufflevector(__s2_285, __s2_285, 3, 2, 1, 0); \
72171 float32x4_t __ret_285; \72171 float32x4_t __ret_285; \
72172 __ret_285 = __noswap_vfmlslq_low_u32(__rev0_285, __rev1_285, (float16x8_t) {__noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285)}); \72172 __ret_285 = __noswap_vfmlslq_low_f16(__rev0_285, __rev1_285, (float16x8_t) {__noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285)}); \
72173 __ret_285 = __builtin_shufflevector(__ret_285, __ret_285, 3, 2, 1, 0); \72173 __ret_285 = __builtin_shufflevector(__ret_285, __ret_285, 3, 2, 1, 0); \
72174 __ret_285; \72174 __ret_285; \
72175})72175})
72176#endif72176#endif
7217772177
72178#ifdef __LITTLE_ENDIAN__72178#ifdef __LITTLE_ENDIAN__
72179#define vfmlsl_lane_low_u32(__p0_286, __p1_286, __p2_286, __p3_286) __extension__ ({ \72179#define vfmlsl_lane_low_f16(__p0_286, __p1_286, __p2_286, __p3_286) __extension__ ({ \
72180 float32x2_t __s0_286 = __p0_286; \72180 float32x2_t __s0_286 = __p0_286; \
72181 float16x4_t __s1_286 = __p1_286; \72181 float16x4_t __s1_286 = __p1_286; \
72182 float16x4_t __s2_286 = __p2_286; \72182 float16x4_t __s2_286 = __p2_286; \
72183 float32x2_t __ret_286; \72183 float32x2_t __ret_286; \
72184 __ret_286 = vfmlsl_low_u32(__s0_286, __s1_286, (float16x4_t) {vget_lane_f16(__s2_286, __p3_286), vget_lane_f16(__s2_286, __p3_286), vget_lane_f16(__s2_286, __p3_286), vget_lane_f16(__s2_286, __p3_286)}); \72184 __ret_286 = vfmlsl_low_f16(__s0_286, __s1_286, (float16x4_t) {vget_lane_f16(__s2_286, __p3_286), vget_lane_f16(__s2_286, __p3_286), vget_lane_f16(__s2_286, __p3_286), vget_lane_f16(__s2_286, __p3_286)}); \
72185 __ret_286; \72185 __ret_286; \
72186})72186})
72187#else72187#else
72188#define vfmlsl_lane_low_u32(__p0_287, __p1_287, __p2_287, __p3_287) __extension__ ({ \72188#define vfmlsl_lane_low_f16(__p0_287, __p1_287, __p2_287, __p3_287) __extension__ ({ \
72189 float32x2_t __s0_287 = __p0_287; \72189 float32x2_t __s0_287 = __p0_287; \
72190 float16x4_t __s1_287 = __p1_287; \72190 float16x4_t __s1_287 = __p1_287; \
72191 float16x4_t __s2_287 = __p2_287; \72191 float16x4_t __s2_287 = __p2_287; \
...@@ -72193,23 +72193,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -72193,23 +72193,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
72193 float16x4_t __rev1_287; __rev1_287 = __builtin_shufflevector(__s1_287, __s1_287, 3, 2, 1, 0); \72193 float16x4_t __rev1_287; __rev1_287 = __builtin_shufflevector(__s1_287, __s1_287, 3, 2, 1, 0); \
72194 float16x4_t __rev2_287; __rev2_287 = __builtin_shufflevector(__s2_287, __s2_287, 3, 2, 1, 0); \72194 float16x4_t __rev2_287; __rev2_287 = __builtin_shufflevector(__s2_287, __s2_287, 3, 2, 1, 0); \
72195 float32x2_t __ret_287; \72195 float32x2_t __ret_287; \
72196 __ret_287 = __noswap_vfmlsl_low_u32(__rev0_287, __rev1_287, (float16x4_t) {__noswap_vget_lane_f16(__rev2_287, __p3_287), __noswap_vget_lane_f16(__rev2_287, __p3_287), __noswap_vget_lane_f16(__rev2_287, __p3_287), __noswap_vget_lane_f16(__rev2_287, __p3_287)}); \72196 __ret_287 = __noswap_vfmlsl_low_f16(__rev0_287, __rev1_287, (float16x4_t) {__noswap_vget_lane_f16(__rev2_287, __p3_287), __noswap_vget_lane_f16(__rev2_287, __p3_287), __noswap_vget_lane_f16(__rev2_287, __p3_287), __noswap_vget_lane_f16(__rev2_287, __p3_287)}); \
72197 __ret_287 = __builtin_shufflevector(__ret_287, __ret_287, 1, 0); \72197 __ret_287 = __builtin_shufflevector(__ret_287, __ret_287, 1, 0); \
72198 __ret_287; \72198 __ret_287; \
72199})72199})
72200#endif72200#endif
7220172201
72202#ifdef __LITTLE_ENDIAN__72202#ifdef __LITTLE_ENDIAN__
72203#define vfmlslq_laneq_high_u32(__p0_288, __p1_288, __p2_288, __p3_288) __extension__ ({ \72203#define vfmlslq_laneq_high_f16(__p0_288, __p1_288, __p2_288, __p3_288) __extension__ ({ \
72204 float32x4_t __s0_288 = __p0_288; \72204 float32x4_t __s0_288 = __p0_288; \
72205 float16x8_t __s1_288 = __p1_288; \72205 float16x8_t __s1_288 = __p1_288; \
72206 float16x8_t __s2_288 = __p2_288; \72206 float16x8_t __s2_288 = __p2_288; \
72207 float32x4_t __ret_288; \72207 float32x4_t __ret_288; \
72208 __ret_288 = vfmlslq_high_u32(__s0_288, __s1_288, (float16x8_t) {vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288)}); \72208 __ret_288 = vfmlslq_high_f16(__s0_288, __s1_288, (float16x8_t) {vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288)}); \
72209 __ret_288; \72209 __ret_288; \
72210})72210})
72211#else72211#else
72212#define vfmlslq_laneq_high_u32(__p0_289, __p1_289, __p2_289, __p3_289) __extension__ ({ \72212#define vfmlslq_laneq_high_f16(__p0_289, __p1_289, __p2_289, __p3_289) __extension__ ({ \
72213 float32x4_t __s0_289 = __p0_289; \72213 float32x4_t __s0_289 = __p0_289; \
72214 float16x8_t __s1_289 = __p1_289; \72214 float16x8_t __s1_289 = __p1_289; \
72215 float16x8_t __s2_289 = __p2_289; \72215 float16x8_t __s2_289 = __p2_289; \
...@@ -72217,23 +72217,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -72217,23 +72217,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
72217 float16x8_t __rev1_289; __rev1_289 = __builtin_shufflevector(__s1_289, __s1_289, 7, 6, 5, 4, 3, 2, 1, 0); \72217 float16x8_t __rev1_289; __rev1_289 = __builtin_shufflevector(__s1_289, __s1_289, 7, 6, 5, 4, 3, 2, 1, 0); \
72218 float16x8_t __rev2_289; __rev2_289 = __builtin_shufflevector(__s2_289, __s2_289, 7, 6, 5, 4, 3, 2, 1, 0); \72218 float16x8_t __rev2_289; __rev2_289 = __builtin_shufflevector(__s2_289, __s2_289, 7, 6, 5, 4, 3, 2, 1, 0); \
72219 float32x4_t __ret_289; \72219 float32x4_t __ret_289; \
72220 __ret_289 = __noswap_vfmlslq_high_u32(__rev0_289, __rev1_289, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289)}); \72220 __ret_289 = __noswap_vfmlslq_high_f16(__rev0_289, __rev1_289, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289)}); \
72221 __ret_289 = __builtin_shufflevector(__ret_289, __ret_289, 3, 2, 1, 0); \72221 __ret_289 = __builtin_shufflevector(__ret_289, __ret_289, 3, 2, 1, 0); \
72222 __ret_289; \72222 __ret_289; \
72223})72223})
72224#endif72224#endif
7222572225
72226#ifdef __LITTLE_ENDIAN__72226#ifdef __LITTLE_ENDIAN__
72227#define vfmlsl_laneq_high_u32(__p0_290, __p1_290, __p2_290, __p3_290) __extension__ ({ \72227#define vfmlsl_laneq_high_f16(__p0_290, __p1_290, __p2_290, __p3_290) __extension__ ({ \
72228 float32x2_t __s0_290 = __p0_290; \72228 float32x2_t __s0_290 = __p0_290; \
72229 float16x4_t __s1_290 = __p1_290; \72229 float16x4_t __s1_290 = __p1_290; \
72230 float16x8_t __s2_290 = __p2_290; \72230 float16x8_t __s2_290 = __p2_290; \
72231 float32x2_t __ret_290; \72231 float32x2_t __ret_290; \
72232 __ret_290 = vfmlsl_high_u32(__s0_290, __s1_290, (float16x4_t) {vgetq_lane_f16(__s2_290, __p3_290), vgetq_lane_f16(__s2_290, __p3_290), vgetq_lane_f16(__s2_290, __p3_290), vgetq_lane_f16(__s2_290, __p3_290)}); \72232 __ret_290 = vfmlsl_high_f16(__s0_290, __s1_290, (float16x4_t) {vgetq_lane_f16(__s2_290, __p3_290), vgetq_lane_f16(__s2_290, __p3_290), vgetq_lane_f16(__s2_290, __p3_290), vgetq_lane_f16(__s2_290, __p3_290)}); \
72233 __ret_290; \72233 __ret_290; \
72234})72234})
72235#else72235#else
72236#define vfmlsl_laneq_high_u32(__p0_291, __p1_291, __p2_291, __p3_291) __extension__ ({ \72236#define vfmlsl_laneq_high_f16(__p0_291, __p1_291, __p2_291, __p3_291) __extension__ ({ \
72237 float32x2_t __s0_291 = __p0_291; \72237 float32x2_t __s0_291 = __p0_291; \
72238 float16x4_t __s1_291 = __p1_291; \72238 float16x4_t __s1_291 = __p1_291; \
72239 float16x8_t __s2_291 = __p2_291; \72239 float16x8_t __s2_291 = __p2_291; \
...@@ -72241,23 +72241,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -72241,23 +72241,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
72241 float16x4_t __rev1_291; __rev1_291 = __builtin_shufflevector(__s1_291, __s1_291, 3, 2, 1, 0); \72241 float16x4_t __rev1_291; __rev1_291 = __builtin_shufflevector(__s1_291, __s1_291, 3, 2, 1, 0); \
72242 float16x8_t __rev2_291; __rev2_291 = __builtin_shufflevector(__s2_291, __s2_291, 7, 6, 5, 4, 3, 2, 1, 0); \72242 float16x8_t __rev2_291; __rev2_291 = __builtin_shufflevector(__s2_291, __s2_291, 7, 6, 5, 4, 3, 2, 1, 0); \
72243 float32x2_t __ret_291; \72243 float32x2_t __ret_291; \
72244 __ret_291 = __noswap_vfmlsl_high_u32(__rev0_291, __rev1_291, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_291, __p3_291), __noswap_vgetq_lane_f16(__rev2_291, __p3_291), __noswap_vgetq_lane_f16(__rev2_291, __p3_291), __noswap_vgetq_lane_f16(__rev2_291, __p3_291)}); \72244 __ret_291 = __noswap_vfmlsl_high_f16(__rev0_291, __rev1_291, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_291, __p3_291), __noswap_vgetq_lane_f16(__rev2_291, __p3_291), __noswap_vgetq_lane_f16(__rev2_291, __p3_291), __noswap_vgetq_lane_f16(__rev2_291, __p3_291)}); \
72245 __ret_291 = __builtin_shufflevector(__ret_291, __ret_291, 1, 0); \72245 __ret_291 = __builtin_shufflevector(__ret_291, __ret_291, 1, 0); \
72246 __ret_291; \72246 __ret_291; \
72247})72247})
72248#endif72248#endif
7224972249
72250#ifdef __LITTLE_ENDIAN__72250#ifdef __LITTLE_ENDIAN__
72251#define vfmlslq_laneq_low_u32(__p0_292, __p1_292, __p2_292, __p3_292) __extension__ ({ \72251#define vfmlslq_laneq_low_f16(__p0_292, __p1_292, __p2_292, __p3_292) __extension__ ({ \
72252 float32x4_t __s0_292 = __p0_292; \72252 float32x4_t __s0_292 = __p0_292; \
72253 float16x8_t __s1_292 = __p1_292; \72253 float16x8_t __s1_292 = __p1_292; \
72254 float16x8_t __s2_292 = __p2_292; \72254 float16x8_t __s2_292 = __p2_292; \
72255 float32x4_t __ret_292; \72255 float32x4_t __ret_292; \
72256 __ret_292 = vfmlslq_low_u32(__s0_292, __s1_292, (float16x8_t) {vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292)}); \72256 __ret_292 = vfmlslq_low_f16(__s0_292, __s1_292, (float16x8_t) {vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292)}); \
72257 __ret_292; \72257 __ret_292; \
72258})72258})
72259#else72259#else
72260#define vfmlslq_laneq_low_u32(__p0_293, __p1_293, __p2_293, __p3_293) __extension__ ({ \72260#define vfmlslq_laneq_low_f16(__p0_293, __p1_293, __p2_293, __p3_293) __extension__ ({ \
72261 float32x4_t __s0_293 = __p0_293; \72261 float32x4_t __s0_293 = __p0_293; \
72262 float16x8_t __s1_293 = __p1_293; \72262 float16x8_t __s1_293 = __p1_293; \
72263 float16x8_t __s2_293 = __p2_293; \72263 float16x8_t __s2_293 = __p2_293; \
...@@ -72265,23 +72265,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -72265,23 +72265,23 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
72265 float16x8_t __rev1_293; __rev1_293 = __builtin_shufflevector(__s1_293, __s1_293, 7, 6, 5, 4, 3, 2, 1, 0); \72265 float16x8_t __rev1_293; __rev1_293 = __builtin_shufflevector(__s1_293, __s1_293, 7, 6, 5, 4, 3, 2, 1, 0); \
72266 float16x8_t __rev2_293; __rev2_293 = __builtin_shufflevector(__s2_293, __s2_293, 7, 6, 5, 4, 3, 2, 1, 0); \72266 float16x8_t __rev2_293; __rev2_293 = __builtin_shufflevector(__s2_293, __s2_293, 7, 6, 5, 4, 3, 2, 1, 0); \
72267 float32x4_t __ret_293; \72267 float32x4_t __ret_293; \
72268 __ret_293 = __noswap_vfmlslq_low_u32(__rev0_293, __rev1_293, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293)}); \72268 __ret_293 = __noswap_vfmlslq_low_f16(__rev0_293, __rev1_293, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293)}); \
72269 __ret_293 = __builtin_shufflevector(__ret_293, __ret_293, 3, 2, 1, 0); \72269 __ret_293 = __builtin_shufflevector(__ret_293, __ret_293, 3, 2, 1, 0); \
72270 __ret_293; \72270 __ret_293; \
72271})72271})
72272#endif72272#endif
7227372273
72274#ifdef __LITTLE_ENDIAN__72274#ifdef __LITTLE_ENDIAN__
72275#define vfmlsl_laneq_low_u32(__p0_294, __p1_294, __p2_294, __p3_294) __extension__ ({ \72275#define vfmlsl_laneq_low_f16(__p0_294, __p1_294, __p2_294, __p3_294) __extension__ ({ \
72276 float32x2_t __s0_294 = __p0_294; \72276 float32x2_t __s0_294 = __p0_294; \
72277 float16x4_t __s1_294 = __p1_294; \72277 float16x4_t __s1_294 = __p1_294; \
72278 float16x8_t __s2_294 = __p2_294; \72278 float16x8_t __s2_294 = __p2_294; \
72279 float32x2_t __ret_294; \72279 float32x2_t __ret_294; \
72280 __ret_294 = vfmlsl_low_u32(__s0_294, __s1_294, (float16x4_t) {vgetq_lane_f16(__s2_294, __p3_294), vgetq_lane_f16(__s2_294, __p3_294), vgetq_lane_f16(__s2_294, __p3_294), vgetq_lane_f16(__s2_294, __p3_294)}); \72280 __ret_294 = vfmlsl_low_f16(__s0_294, __s1_294, (float16x4_t) {vgetq_lane_f16(__s2_294, __p3_294), vgetq_lane_f16(__s2_294, __p3_294), vgetq_lane_f16(__s2_294, __p3_294), vgetq_lane_f16(__s2_294, __p3_294)}); \
72281 __ret_294; \72281 __ret_294; \
72282})72282})
72283#else72283#else
72284#define vfmlsl_laneq_low_u32(__p0_295, __p1_295, __p2_295, __p3_295) __extension__ ({ \72284#define vfmlsl_laneq_low_f16(__p0_295, __p1_295, __p2_295, __p3_295) __extension__ ({ \
72285 float32x2_t __s0_295 = __p0_295; \72285 float32x2_t __s0_295 = __p0_295; \
72286 float16x4_t __s1_295 = __p1_295; \72286 float16x4_t __s1_295 = __p1_295; \
72287 float16x8_t __s2_295 = __p2_295; \72287 float16x8_t __s2_295 = __p2_295; \
...@@ -72289,7 +72289,7 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in...@@ -72289,7 +72289,7 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in
72289 float16x4_t __rev1_295; __rev1_295 = __builtin_shufflevector(__s1_295, __s1_295, 3, 2, 1, 0); \72289 float16x4_t __rev1_295; __rev1_295 = __builtin_shufflevector(__s1_295, __s1_295, 3, 2, 1, 0); \
72290 float16x8_t __rev2_295; __rev2_295 = __builtin_shufflevector(__s2_295, __s2_295, 7, 6, 5, 4, 3, 2, 1, 0); \72290 float16x8_t __rev2_295; __rev2_295 = __builtin_shufflevector(__s2_295, __s2_295, 7, 6, 5, 4, 3, 2, 1, 0); \
72291 float32x2_t __ret_295; \72291 float32x2_t __ret_295; \
72292 __ret_295 = __noswap_vfmlsl_low_u32(__rev0_295, __rev1_295, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_295, __p3_295), __noswap_vgetq_lane_f16(__rev2_295, __p3_295), __noswap_vgetq_lane_f16(__rev2_295, __p3_295), __noswap_vgetq_lane_f16(__rev2_295, __p3_295)}); \72292 __ret_295 = __noswap_vfmlsl_low_f16(__rev0_295, __rev1_295, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_295, __p3_295), __noswap_vgetq_lane_f16(__rev2_295, __p3_295), __noswap_vgetq_lane_f16(__rev2_295, __p3_295), __noswap_vgetq_lane_f16(__rev2_295, __p3_295)}); \
72293 __ret_295 = __builtin_shufflevector(__ret_295, __ret_295, 1, 0); \72293 __ret_295 = __builtin_shufflevector(__ret_295, __ret_295, 1, 0); \
72294 __ret_295; \72294 __ret_295; \
72295})72295})
lib/include/armintr.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- armintr.h - ARM Windows intrinsics -------------------------------===1/*===---- armintr.h - ARM Windows intrinsics -------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/avx2intrin.h+5-27
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- avx2intrin.h - AVX2 intrinsics -----------------------------------===1/*===---- avx2intrin.h - AVX2 intrinsics -----------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -146,21 +132,13 @@ _mm256_andnot_si256(__m256i __a, __m256i __b)...@@ -146,21 +132,13 @@ _mm256_andnot_si256(__m256i __a, __m256i __b)
146static __inline__ __m256i __DEFAULT_FN_ATTRS256132static __inline__ __m256i __DEFAULT_FN_ATTRS256
147_mm256_avg_epu8(__m256i __a, __m256i __b)133_mm256_avg_epu8(__m256i __a, __m256i __b)
148{134{
149 typedef unsigned short __v32hu __attribute__((__vector_size__(64)));135 return (__m256i)__builtin_ia32_pavgb256((__v32qi)__a, (__v32qi)__b);
150 return (__m256i)__builtin_convertvector(
151 ((__builtin_convertvector((__v32qu)__a, __v32hu) +
152 __builtin_convertvector((__v32qu)__b, __v32hu)) + 1)
153 >> 1, __v32qu);
154}136}
155137
156static __inline__ __m256i __DEFAULT_FN_ATTRS256138static __inline__ __m256i __DEFAULT_FN_ATTRS256
157_mm256_avg_epu16(__m256i __a, __m256i __b)139_mm256_avg_epu16(__m256i __a, __m256i __b)
158{140{
159 typedef unsigned int __v16su __attribute__((__vector_size__(64)));141 return (__m256i)__builtin_ia32_pavgw256((__v16hi)__a, (__v16hi)__b);
160 return (__m256i)__builtin_convertvector(
161 ((__builtin_convertvector((__v16hu)__a, __v16su) +
162 __builtin_convertvector((__v16hu)__b, __v16su)) + 1)
163 >> 1, __v16hu);
164}142}
165143
166static __inline__ __m256i __DEFAULT_FN_ATTRS256144static __inline__ __m256i __DEFAULT_FN_ATTRS256
lib/include/avx512bf16intrin.h created+279
...@@ -0,0 +1,279 @@
1/*===------------ avx512bf16intrin.h - AVX512_BF16 intrinsics --------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error "Never use <avx512bf16intrin.h> directly; include <immintrin.h> instead."
11#endif
12
13#ifndef __AVX512BF16INTRIN_H
14#define __AVX512BF16INTRIN_H
15
16typedef short __m512bh __attribute__((__vector_size__(64), __aligned__(64)));
17typedef short __m256bh __attribute__((__vector_size__(32), __aligned__(32)));
18typedef unsigned short __bfloat16;
19
20#define __DEFAULT_FN_ATTRS512 \
21 __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16"), \
22 __min_vector_width__(512)))
23#define __DEFAULT_FN_ATTRS \
24 __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16")))
25
26/// Convert One BF16 Data to One Single Float Data.
27///
28/// \headerfile <x86intrin.h>
29///
30/// This intrinsic does not correspond to a specific instruction.
31///
32/// \param __A
33/// A bfloat data.
34/// \returns A float data whose sign field and exponent field keep unchanged,
35/// and fraction field is extended to 23 bits.
36static __inline__ float __DEFAULT_FN_ATTRS _mm_cvtsbh_ss(__bfloat16 __A) {
37 return __builtin_ia32_cvtsbf162ss_32(__A);
38}
39
40/// Convert Two Packed Single Data to One Packed BF16 Data.
41///
42/// \headerfile <x86intrin.h>
43///
44/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
45///
46/// \param __A
47/// A 512-bit vector of [16 x float].
48/// \param __B
49/// A 512-bit vector of [16 x float].
50/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from
51/// conversion of __B, and higher 256 bits come from conversion of __A.
52static __inline__ __m512bh __DEFAULT_FN_ATTRS512
53_mm512_cvtne2ps_pbh(__m512 __A, __m512 __B) {
54 return (__m512bh)__builtin_ia32_cvtne2ps2bf16_512((__v16sf) __A,
55 (__v16sf) __B);
56}
57
58/// Convert Two Packed Single Data to One Packed BF16 Data.
59///
60/// \headerfile <x86intrin.h>
61///
62/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
63///
64/// \param __A
65/// A 512-bit vector of [16 x float].
66/// \param __B
67/// A 512-bit vector of [16 x float].
68/// \param __W
69/// A 512-bit vector of [32 x bfloat].
70/// \param __U
71/// A 32-bit mask value specifying what is chosen for each element.
72/// A 1 means conversion of __A or __B. A 0 means element from __W.
73/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from
74/// conversion of __B, and higher 256 bits come from conversion of __A.
75static __inline__ __m512bh __DEFAULT_FN_ATTRS512
76_mm512_mask_cvtne2ps_pbh(__m512bh __W, __mmask32 __U, __m512 __A, __m512 __B) {
77 return (__m512bh)__builtin_ia32_selectw_512((__mmask32)__U,
78 (__v32hi)_mm512_cvtne2ps_pbh(__A, __B),
79 (__v32hi)__W);
80}
81
82/// Convert Two Packed Single Data to One Packed BF16 Data.
83///
84/// \headerfile <x86intrin.h>
85///
86/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
87///
88/// \param __A
89/// A 512-bit vector of [16 x float].
90/// \param __B
91/// A 512-bit vector of [16 x float].
92/// \param __U
93/// A 32-bit mask value specifying what is chosen for each element.
94/// A 1 means conversion of __A or __B. A 0 means element is zero.
95/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from
96/// conversion of __B, and higher 256 bits come from conversion of __A.
97static __inline__ __m512bh __DEFAULT_FN_ATTRS512
98_mm512_maskz_cvtne2ps_pbh(__mmask32 __U, __m512 __A, __m512 __B) {
99 return (__m512bh)__builtin_ia32_selectw_512((__mmask32)__U,
100 (__v32hi)_mm512_cvtne2ps_pbh(__A, __B),
101 (__v32hi)_mm512_setzero_si512());
102}
103
104/// Convert Packed Single Data to Packed BF16 Data.
105///
106/// \headerfile <x86intrin.h>
107///
108/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
109///
110/// \param __A
111/// A 512-bit vector of [16 x float].
112/// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.
113static __inline__ __m256bh __DEFAULT_FN_ATTRS512
114_mm512_cvtneps_pbh(__m512 __A) {
115 return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,
116 (__v16hi)_mm256_undefined_si256(),
117 (__mmask16)-1);
118}
119
120/// Convert Packed Single Data to Packed BF16 Data.
121///
122/// \headerfile <x86intrin.h>
123///
124/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
125///
126/// \param __A
127/// A 512-bit vector of [16 x float].
128/// \param __W
129/// A 256-bit vector of [16 x bfloat].
130/// \param __U
131/// A 16-bit mask value specifying what is chosen for each element.
132/// A 1 means conversion of __A. A 0 means element from __W.
133/// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.
134static __inline__ __m256bh __DEFAULT_FN_ATTRS512
135_mm512_mask_cvtneps_pbh(__m256bh __W, __mmask16 __U, __m512 __A) {
136 return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,
137 (__v16hi)__W,
138 (__mmask16)__U);
139}
140
141/// Convert Packed Single Data to Packed BF16 Data.
142///
143/// \headerfile <x86intrin.h>
144///
145/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
146///
147/// \param __A
148/// A 512-bit vector of [16 x float].
149/// \param __U
150/// A 16-bit mask value specifying what is chosen for each element.
151/// A 1 means conversion of __A. A 0 means element is zero.
152/// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.
153static __inline__ __m256bh __DEFAULT_FN_ATTRS512
154_mm512_maskz_cvtneps_pbh(__mmask16 __U, __m512 __A) {
155 return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,
156 (__v16hi)_mm256_setzero_si256(),
157 (__mmask16)__U);
158}
159
160/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
161///
162/// \headerfile <x86intrin.h>
163///
164/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
165///
166/// \param __A
167/// A 512-bit vector of [32 x bfloat].
168/// \param __B
169/// A 512-bit vector of [32 x bfloat].
170/// \param __D
171/// A 512-bit vector of [16 x float].
172/// \returns A 512-bit vector of [16 x float] comes from Dot Product of
173/// __A, __B and __D
174static __inline__ __m512 __DEFAULT_FN_ATTRS512
175_mm512_dpbf16_ps(__m512 __D, __m512bh __A, __m512bh __B) {
176 return (__m512)__builtin_ia32_dpbf16ps_512((__v16sf) __D,
177 (__v16si) __A,
178 (__v16si) __B);
179}
180
181/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
182///
183/// \headerfile <x86intrin.h>
184///
185/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
186///
187/// \param __A
188/// A 512-bit vector of [32 x bfloat].
189/// \param __B
190/// A 512-bit vector of [32 x bfloat].
191/// \param __D
192/// A 512-bit vector of [16 x float].
193/// \param __U
194/// A 16-bit mask value specifying what is chosen for each element.
195/// A 1 means __A and __B's dot product accumulated with __D. A 0 means __D.
196/// \returns A 512-bit vector of [16 x float] comes from Dot Product of
197/// __A, __B and __D
198static __inline__ __m512 __DEFAULT_FN_ATTRS512
199_mm512_mask_dpbf16_ps(__m512 __D, __mmask16 __U, __m512bh __A, __m512bh __B) {
200 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
201 (__v16sf)_mm512_dpbf16_ps(__D, __A, __B),
202 (__v16sf)__D);
203}
204
205/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
206///
207/// \headerfile <x86intrin.h>
208///
209/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
210///
211/// \param __A
212/// A 512-bit vector of [32 x bfloat].
213/// \param __B
214/// A 512-bit vector of [32 x bfloat].
215/// \param __D
216/// A 512-bit vector of [16 x float].
217/// \param __U
218/// A 16-bit mask value specifying what is chosen for each element.
219/// A 1 means __A and __B's dot product accumulated with __D. A 0 means 0.
220/// \returns A 512-bit vector of [16 x float] comes from Dot Product of
221/// __A, __B and __D
222static __inline__ __m512 __DEFAULT_FN_ATTRS512
223_mm512_maskz_dpbf16_ps(__mmask16 __U, __m512 __D, __m512bh __A, __m512bh __B) {
224 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
225 (__v16sf)_mm512_dpbf16_ps(__D, __A, __B),
226 (__v16sf)_mm512_setzero_si512());
227}
228
229/// Convert Packed BF16 Data to Packed float Data.
230///
231/// \headerfile <x86intrin.h>
232///
233/// \param __A
234/// A 256-bit vector of [16 x bfloat].
235/// \returns A 512-bit vector of [16 x float] come from convertion of __A
236static __inline__ __m512 __DEFAULT_FN_ATTRS512 _mm512_cvtpbh_ps(__m256bh __A) {
237 return _mm512_castsi512_ps((__m512i)_mm512_slli_epi32(
238 (__m512i)_mm512_cvtepi16_epi32((__m256i)__A), 16));
239}
240
241/// Convert Packed BF16 Data to Packed float Data using zeroing mask.
242///
243/// \headerfile <x86intrin.h>
244///
245/// \param __U
246/// A 16-bit mask. Elements are zeroed out when the corresponding mask
247/// bit is not set.
248/// \param __A
249/// A 256-bit vector of [16 x bfloat].
250/// \returns A 512-bit vector of [16 x float] come from convertion of __A
251static __inline__ __m512 __DEFAULT_FN_ATTRS512
252_mm512_maskz_cvtpbh_ps(__mmask16 __U, __m256bh __A) {
253 return _mm512_castsi512_ps((__m512i)_mm512_slli_epi32(
254 (__m512i)_mm512_maskz_cvtepi16_epi32((__mmask16)__U, (__m256i)__A), 16));
255}
256
257/// Convert Packed BF16 Data to Packed float Data using merging mask.
258///
259/// \headerfile <x86intrin.h>
260///
261/// \param __S
262/// A 512-bit vector of [16 x float]. Elements are copied from __S when
263/// the corresponding mask bit is not set.
264/// \param __U
265/// A 16-bit mask.
266/// \param __A
267/// A 256-bit vector of [16 x bfloat].
268/// \returns A 512-bit vector of [16 x float] come from convertion of __A
269static __inline__ __m512 __DEFAULT_FN_ATTRS512
270_mm512_mask_cvtpbh_ps(__m512 __S, __mmask16 __U, __m256bh __A) {
271 return _mm512_castsi512_ps((__m512i)_mm512_mask_slli_epi32(
272 (__m512i)__S, (__mmask16)__U,
273 (__m512i)_mm512_cvtepi16_epi32((__m256i)__A), 16));
274}
275
276#undef __DEFAULT_FN_ATTRS
277#undef __DEFAULT_FN_ATTRS512
278
279#endif
lib/include/avx512bitalgintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===------------- avx512bitalgintrin.h - BITALG intrinsics ------------------===1/*===------------- avx512bitalgintrin.h - BITALG intrinsics ------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/avx512bwintrin.h+11-33
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===------------- avx512bwintrin.h - AVX512BW intrinsics ------------------===1/*===------------- avx512bwintrin.h - AVX512BW intrinsics ------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
...@@ -719,11 +705,7 @@ _mm512_maskz_adds_epu16 (__mmask32 __U, __m512i __A, __m512i __B)...@@ -719,11 +705,7 @@ _mm512_maskz_adds_epu16 (__mmask32 __U, __m512i __A, __m512i __B)
719static __inline__ __m512i __DEFAULT_FN_ATTRS512705static __inline__ __m512i __DEFAULT_FN_ATTRS512
720_mm512_avg_epu8 (__m512i __A, __m512i __B)706_mm512_avg_epu8 (__m512i __A, __m512i __B)
721{707{
722 typedef unsigned short __v64hu __attribute__((__vector_size__(128)));708 return (__m512i)__builtin_ia32_pavgb512((__v64qi)__A, (__v64qi)__B);
723 return (__m512i)__builtin_convertvector(
724 ((__builtin_convertvector((__v64qu) __A, __v64hu) +
725 __builtin_convertvector((__v64qu) __B, __v64hu)) + 1)
726 >> 1, __v64qu);
727}709}
728710
729static __inline__ __m512i __DEFAULT_FN_ATTRS512711static __inline__ __m512i __DEFAULT_FN_ATTRS512
...@@ -746,11 +728,7 @@ _mm512_maskz_avg_epu8 (__mmask64 __U, __m512i __A, __m512i __B)...@@ -746,11 +728,7 @@ _mm512_maskz_avg_epu8 (__mmask64 __U, __m512i __A, __m512i __B)
746static __inline__ __m512i __DEFAULT_FN_ATTRS512728static __inline__ __m512i __DEFAULT_FN_ATTRS512
747_mm512_avg_epu16 (__m512i __A, __m512i __B)729_mm512_avg_epu16 (__m512i __A, __m512i __B)
748{730{
749 typedef unsigned int __v32su __attribute__((__vector_size__(128)));731 return (__m512i)__builtin_ia32_pavgw512((__v32hi)__A, (__v32hi)__B);
750 return (__m512i)__builtin_convertvector(
751 ((__builtin_convertvector((__v32hu) __A, __v32su) +
752 __builtin_convertvector((__v32hu) __B, __v32su)) + 1)
753 >> 1, __v32hu);
754}732}
755733
756static __inline__ __m512i __DEFAULT_FN_ATTRS512734static __inline__ __m512i __DEFAULT_FN_ATTRS512
...@@ -1733,14 +1711,14 @@ _mm512_maskz_set1_epi8 (__mmask64 __M, char __A)...@@ -1733,14 +1711,14 @@ _mm512_maskz_set1_epi8 (__mmask64 __M, char __A)
1733 (__v64qi) _mm512_setzero_si512());1711 (__v64qi) _mm512_setzero_si512());
1734}1712}
17351713
1736static __inline__ __mmask64 __DEFAULT_FN_ATTRS5121714static __inline__ __mmask64 __DEFAULT_FN_ATTRS
1737_mm512_kunpackd (__mmask64 __A, __mmask64 __B)1715_mm512_kunpackd (__mmask64 __A, __mmask64 __B)
1738{1716{
1739 return (__mmask64) __builtin_ia32_kunpckdi ((__mmask64) __A,1717 return (__mmask64) __builtin_ia32_kunpckdi ((__mmask64) __A,
1740 (__mmask64) __B);1718 (__mmask64) __B);
1741}1719}
17421720
1743static __inline__ __mmask32 __DEFAULT_FN_ATTRS5121721static __inline__ __mmask32 __DEFAULT_FN_ATTRS
1744_mm512_kunpackw (__mmask32 __A, __mmask32 __B)1722_mm512_kunpackw (__mmask32 __A, __mmask32 __B)
1745{1723{
1746 return (__mmask32) __builtin_ia32_kunpcksi ((__mmask32) __A,1724 return (__mmask32) __builtin_ia32_kunpcksi ((__mmask32) __A,
...@@ -1751,7 +1729,7 @@ static __inline __m512i __DEFAULT_FN_ATTRS512...@@ -1751,7 +1729,7 @@ static __inline __m512i __DEFAULT_FN_ATTRS512
1751_mm512_loadu_epi16 (void const *__P)1729_mm512_loadu_epi16 (void const *__P)
1752{1730{
1753 struct __loadu_epi16 {1731 struct __loadu_epi16 {
1754 __m512i __v;1732 __m512i_u __v;
1755 } __attribute__((__packed__, __may_alias__));1733 } __attribute__((__packed__, __may_alias__));
1756 return ((struct __loadu_epi16*)__P)->__v;1734 return ((struct __loadu_epi16*)__P)->__v;
1757}1735}
...@@ -1777,7 +1755,7 @@ static __inline __m512i __DEFAULT_FN_ATTRS512...@@ -1777,7 +1755,7 @@ static __inline __m512i __DEFAULT_FN_ATTRS512
1777_mm512_loadu_epi8 (void const *__P)1755_mm512_loadu_epi8 (void const *__P)
1778{1756{
1779 struct __loadu_epi8 {1757 struct __loadu_epi8 {
1780 __m512i __v;1758 __m512i_u __v;
1781 } __attribute__((__packed__, __may_alias__));1759 } __attribute__((__packed__, __may_alias__));
1782 return ((struct __loadu_epi8*)__P)->__v;1760 return ((struct __loadu_epi8*)__P)->__v;
1783}1761}
...@@ -1803,7 +1781,7 @@ static __inline void __DEFAULT_FN_ATTRS512...@@ -1803,7 +1781,7 @@ static __inline void __DEFAULT_FN_ATTRS512
1803_mm512_storeu_epi16 (void *__P, __m512i __A)1781_mm512_storeu_epi16 (void *__P, __m512i __A)
1804{1782{
1805 struct __storeu_epi16 {1783 struct __storeu_epi16 {
1806 __m512i __v;1784 __m512i_u __v;
1807 } __attribute__((__packed__, __may_alias__));1785 } __attribute__((__packed__, __may_alias__));
1808 ((struct __storeu_epi16*)__P)->__v = __A;1786 ((struct __storeu_epi16*)__P)->__v = __A;
1809}1787}
...@@ -1820,7 +1798,7 @@ static __inline void __DEFAULT_FN_ATTRS512...@@ -1820,7 +1798,7 @@ static __inline void __DEFAULT_FN_ATTRS512
1820_mm512_storeu_epi8 (void *__P, __m512i __A)1798_mm512_storeu_epi8 (void *__P, __m512i __A)
1821{1799{
1822 struct __storeu_epi8 {1800 struct __storeu_epi8 {
1823 __m512i __v;1801 __m512i_u __v;
1824 } __attribute__((__packed__, __may_alias__));1802 } __attribute__((__packed__, __may_alias__));
1825 ((struct __storeu_epi8*)__P)->__v = __A;1803 ((struct __storeu_epi8*)__P)->__v = __A;
1826}1804}
lib/include/avx512cdintrin.h+17-35
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===------------- avx512cdintrin.h - AVX512CD intrinsics ------------------===1/*===------------- avx512cdintrin.h - AVX512CD intrinsics ------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
...@@ -34,49 +20,45 @@...@@ -34,49 +20,45 @@
34static __inline__ __m512i __DEFAULT_FN_ATTRS20static __inline__ __m512i __DEFAULT_FN_ATTRS
35_mm512_conflict_epi64 (__m512i __A)21_mm512_conflict_epi64 (__m512i __A)
36{22{
37 return (__m512i) __builtin_ia32_vpconflictdi_512_mask ((__v8di) __A,23 return (__m512i) __builtin_ia32_vpconflictdi_512 ((__v8di) __A);
38 (__v8di) _mm512_setzero_si512 (),
39 (__mmask8) -1);
40}24}
4125
42static __inline__ __m512i __DEFAULT_FN_ATTRS26static __inline__ __m512i __DEFAULT_FN_ATTRS
43_mm512_mask_conflict_epi64 (__m512i __W, __mmask8 __U, __m512i __A)27_mm512_mask_conflict_epi64 (__m512i __W, __mmask8 __U, __m512i __A)
44{28{
45 return (__m512i) __builtin_ia32_vpconflictdi_512_mask ((__v8di) __A,29 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
46 (__v8di) __W,30 (__v8di)_mm512_conflict_epi64(__A),
47 (__mmask8) __U);31 (__v8di)__W);
48}32}
4933
50static __inline__ __m512i __DEFAULT_FN_ATTRS34static __inline__ __m512i __DEFAULT_FN_ATTRS
51_mm512_maskz_conflict_epi64 (__mmask8 __U, __m512i __A)35_mm512_maskz_conflict_epi64 (__mmask8 __U, __m512i __A)
52{36{
53 return (__m512i) __builtin_ia32_vpconflictdi_512_mask ((__v8di) __A,37 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
54 (__v8di) _mm512_setzero_si512 (),38 (__v8di)_mm512_conflict_epi64(__A),
55 (__mmask8) __U);39 (__v8di)_mm512_setzero_si512 ());
56}40}
5741
58static __inline__ __m512i __DEFAULT_FN_ATTRS42static __inline__ __m512i __DEFAULT_FN_ATTRS
59_mm512_conflict_epi32 (__m512i __A)43_mm512_conflict_epi32 (__m512i __A)
60{44{
61 return (__m512i) __builtin_ia32_vpconflictsi_512_mask ((__v16si) __A,45 return (__m512i) __builtin_ia32_vpconflictsi_512 ((__v16si) __A);
62 (__v16si) _mm512_setzero_si512 (),
63 (__mmask16) -1);
64}46}
6547
66static __inline__ __m512i __DEFAULT_FN_ATTRS48static __inline__ __m512i __DEFAULT_FN_ATTRS
67_mm512_mask_conflict_epi32 (__m512i __W, __mmask16 __U, __m512i __A)49_mm512_mask_conflict_epi32 (__m512i __W, __mmask16 __U, __m512i __A)
68{50{
69 return (__m512i) __builtin_ia32_vpconflictsi_512_mask ((__v16si) __A,51 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
70 (__v16si) __W,52 (__v16si)_mm512_conflict_epi32(__A),
71 (__mmask16) __U);53 (__v16si)__W);
72}54}
7355
74static __inline__ __m512i __DEFAULT_FN_ATTRS56static __inline__ __m512i __DEFAULT_FN_ATTRS
75_mm512_maskz_conflict_epi32 (__mmask16 __U, __m512i __A)57_mm512_maskz_conflict_epi32 (__mmask16 __U, __m512i __A)
76{58{
77 return (__m512i) __builtin_ia32_vpconflictsi_512_mask ((__v16si) __A,59 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
78 (__v16si) _mm512_setzero_si512 (),60 (__v16si)_mm512_conflict_epi32(__A),
79 (__mmask16) __U);61 (__v16si)_mm512_setzero_si512());
80}62}
8163
82static __inline__ __m512i __DEFAULT_FN_ATTRS64static __inline__ __m512i __DEFAULT_FN_ATTRS
lib/include/avx512dqintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- avx512dqintrin.h - AVX512DQ intrinsics ---------------------------===1/*===---- avx512dqintrin.h - AVX512DQ intrinsics ---------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/avx512erintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- avx512erintrin.h - AVX512ER intrinsics ---------------------------===1/*===---- avx512erintrin.h - AVX512ER intrinsics ---------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/avx512fintrin.h+47-68
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- avx512fintrin.h - AVX512F intrinsics -----------------------------===1/*===---- avx512fintrin.h - AVX512F intrinsics -----------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -40,9 +26,13 @@ typedef unsigned short __v32hu __attribute__((__vector_size__(64)));...@@ -40,9 +26,13 @@ typedef unsigned short __v32hu __attribute__((__vector_size__(64)));
40typedef unsigned long long __v8du __attribute__((__vector_size__(64)));26typedef unsigned long long __v8du __attribute__((__vector_size__(64)));
41typedef unsigned int __v16su __attribute__((__vector_size__(64)));27typedef unsigned int __v16su __attribute__((__vector_size__(64)));
4228
43typedef float __m512 __attribute__((__vector_size__(64)));29typedef float __m512 __attribute__((__vector_size__(64), __aligned__(64)));
44typedef double __m512d __attribute__((__vector_size__(64)));30typedef double __m512d __attribute__((__vector_size__(64), __aligned__(64)));
45typedef long long __m512i __attribute__((__vector_size__(64)));31typedef long long __m512i __attribute__((__vector_size__(64), __aligned__(64)));
32
33typedef float __m512_u __attribute__((__vector_size__(64), __aligned__(1)));
34typedef double __m512d_u __attribute__((__vector_size__(64), __aligned__(1)));
35typedef long long __m512i_u __attribute__((__vector_size__(64), __aligned__(1)));
4636
47typedef unsigned char __mmask8;37typedef unsigned char __mmask8;
48typedef unsigned short __mmask16;38typedef unsigned short __mmask16;
...@@ -1991,12 +1981,12 @@ _mm512_maskz_add_ps(__mmask16 __U, __m512 __A, __m512 __B) {...@@ -1991,12 +1981,12 @@ _mm512_maskz_add_ps(__mmask16 __U, __m512 __A, __m512 __B) {
1991#define _mm512_mask_add_round_pd(W, U, A, B, R) \1981#define _mm512_mask_add_round_pd(W, U, A, B, R) \
1992 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \1982 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
1993 (__v8df)_mm512_add_round_pd((A), (B), (R)), \1983 (__v8df)_mm512_add_round_pd((A), (B), (R)), \
1994 (__v8df)(__m512d)(W));1984 (__v8df)(__m512d)(W))
19951985
1996#define _mm512_maskz_add_round_pd(U, A, B, R) \1986#define _mm512_maskz_add_round_pd(U, A, B, R) \
1997 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \1987 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
1998 (__v8df)_mm512_add_round_pd((A), (B), (R)), \1988 (__v8df)_mm512_add_round_pd((A), (B), (R)), \
1999 (__v8df)_mm512_setzero_pd());1989 (__v8df)_mm512_setzero_pd())
20001990
2001#define _mm512_add_round_ps(A, B, R) \1991#define _mm512_add_round_ps(A, B, R) \
2002 (__m512)__builtin_ia32_addps512((__v16sf)(__m512)(A), \1992 (__m512)__builtin_ia32_addps512((__v16sf)(__m512)(A), \
...@@ -2005,12 +1995,12 @@ _mm512_maskz_add_ps(__mmask16 __U, __m512 __A, __m512 __B) {...@@ -2005,12 +1995,12 @@ _mm512_maskz_add_ps(__mmask16 __U, __m512 __A, __m512 __B) {
2005#define _mm512_mask_add_round_ps(W, U, A, B, R) \1995#define _mm512_mask_add_round_ps(W, U, A, B, R) \
2006 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \1996 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2007 (__v16sf)_mm512_add_round_ps((A), (B), (R)), \1997 (__v16sf)_mm512_add_round_ps((A), (B), (R)), \
2008 (__v16sf)(__m512)(W));1998 (__v16sf)(__m512)(W))
20091999
2010#define _mm512_maskz_add_round_ps(U, A, B, R) \2000#define _mm512_maskz_add_round_ps(U, A, B, R) \
2011 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \2001 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2012 (__v16sf)_mm512_add_round_ps((A), (B), (R)), \2002 (__v16sf)_mm512_add_round_ps((A), (B), (R)), \
2013 (__v16sf)_mm512_setzero_ps());2003 (__v16sf)_mm512_setzero_ps())
20142004
2015static __inline__ __m128 __DEFAULT_FN_ATTRS1282005static __inline__ __m128 __DEFAULT_FN_ATTRS128
2016_mm_mask_sub_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {2006_mm_mask_sub_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
...@@ -2106,12 +2096,12 @@ _mm512_maskz_sub_ps(__mmask16 __U, __m512 __A, __m512 __B) {...@@ -2106,12 +2096,12 @@ _mm512_maskz_sub_ps(__mmask16 __U, __m512 __A, __m512 __B) {
2106#define _mm512_mask_sub_round_pd(W, U, A, B, R) \2096#define _mm512_mask_sub_round_pd(W, U, A, B, R) \
2107 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \2097 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
2108 (__v8df)_mm512_sub_round_pd((A), (B), (R)), \2098 (__v8df)_mm512_sub_round_pd((A), (B), (R)), \
2109 (__v8df)(__m512d)(W));2099 (__v8df)(__m512d)(W))
21102100
2111#define _mm512_maskz_sub_round_pd(U, A, B, R) \2101#define _mm512_maskz_sub_round_pd(U, A, B, R) \
2112 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \2102 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
2113 (__v8df)_mm512_sub_round_pd((A), (B), (R)), \2103 (__v8df)_mm512_sub_round_pd((A), (B), (R)), \
2114 (__v8df)_mm512_setzero_pd());2104 (__v8df)_mm512_setzero_pd())
21152105
2116#define _mm512_sub_round_ps(A, B, R) \2106#define _mm512_sub_round_ps(A, B, R) \
2117 (__m512)__builtin_ia32_subps512((__v16sf)(__m512)(A), \2107 (__m512)__builtin_ia32_subps512((__v16sf)(__m512)(A), \
...@@ -2120,12 +2110,12 @@ _mm512_maskz_sub_ps(__mmask16 __U, __m512 __A, __m512 __B) {...@@ -2120,12 +2110,12 @@ _mm512_maskz_sub_ps(__mmask16 __U, __m512 __A, __m512 __B) {
2120#define _mm512_mask_sub_round_ps(W, U, A, B, R) \2110#define _mm512_mask_sub_round_ps(W, U, A, B, R) \
2121 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \2111 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2122 (__v16sf)_mm512_sub_round_ps((A), (B), (R)), \2112 (__v16sf)_mm512_sub_round_ps((A), (B), (R)), \
2123 (__v16sf)(__m512)(W));2113 (__v16sf)(__m512)(W))
21242114
2125#define _mm512_maskz_sub_round_ps(U, A, B, R) \2115#define _mm512_maskz_sub_round_ps(U, A, B, R) \
2126 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \2116 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2127 (__v16sf)_mm512_sub_round_ps((A), (B), (R)), \2117 (__v16sf)_mm512_sub_round_ps((A), (B), (R)), \
2128 (__v16sf)_mm512_setzero_ps());2118 (__v16sf)_mm512_setzero_ps())
21292119
2130static __inline__ __m128 __DEFAULT_FN_ATTRS1282120static __inline__ __m128 __DEFAULT_FN_ATTRS128
2131_mm_mask_mul_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {2121_mm_mask_mul_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
...@@ -2221,12 +2211,12 @@ _mm512_maskz_mul_ps(__mmask16 __U, __m512 __A, __m512 __B) {...@@ -2221,12 +2211,12 @@ _mm512_maskz_mul_ps(__mmask16 __U, __m512 __A, __m512 __B) {
2221#define _mm512_mask_mul_round_pd(W, U, A, B, R) \2211#define _mm512_mask_mul_round_pd(W, U, A, B, R) \
2222 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \2212 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
2223 (__v8df)_mm512_mul_round_pd((A), (B), (R)), \2213 (__v8df)_mm512_mul_round_pd((A), (B), (R)), \
2224 (__v8df)(__m512d)(W));2214 (__v8df)(__m512d)(W))
22252215
2226#define _mm512_maskz_mul_round_pd(U, A, B, R) \2216#define _mm512_maskz_mul_round_pd(U, A, B, R) \
2227 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \2217 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
2228 (__v8df)_mm512_mul_round_pd((A), (B), (R)), \2218 (__v8df)_mm512_mul_round_pd((A), (B), (R)), \
2229 (__v8df)_mm512_setzero_pd());2219 (__v8df)_mm512_setzero_pd())
22302220
2231#define _mm512_mul_round_ps(A, B, R) \2221#define _mm512_mul_round_ps(A, B, R) \
2232 (__m512)__builtin_ia32_mulps512((__v16sf)(__m512)(A), \2222 (__m512)__builtin_ia32_mulps512((__v16sf)(__m512)(A), \
...@@ -2235,12 +2225,12 @@ _mm512_maskz_mul_ps(__mmask16 __U, __m512 __A, __m512 __B) {...@@ -2235,12 +2225,12 @@ _mm512_maskz_mul_ps(__mmask16 __U, __m512 __A, __m512 __B) {
2235#define _mm512_mask_mul_round_ps(W, U, A, B, R) \2225#define _mm512_mask_mul_round_ps(W, U, A, B, R) \
2236 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \2226 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2237 (__v16sf)_mm512_mul_round_ps((A), (B), (R)), \2227 (__v16sf)_mm512_mul_round_ps((A), (B), (R)), \
2238 (__v16sf)(__m512)(W));2228 (__v16sf)(__m512)(W))
22392229
2240#define _mm512_maskz_mul_round_ps(U, A, B, R) \2230#define _mm512_maskz_mul_round_ps(U, A, B, R) \
2241 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \2231 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2242 (__v16sf)_mm512_mul_round_ps((A), (B), (R)), \2232 (__v16sf)_mm512_mul_round_ps((A), (B), (R)), \
2243 (__v16sf)_mm512_setzero_ps());2233 (__v16sf)_mm512_setzero_ps())
22442234
2245static __inline__ __m128 __DEFAULT_FN_ATTRS1282235static __inline__ __m128 __DEFAULT_FN_ATTRS128
2246_mm_mask_div_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {2236_mm_mask_div_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
...@@ -2349,12 +2339,12 @@ _mm512_maskz_div_ps(__mmask16 __U, __m512 __A, __m512 __B) {...@@ -2349,12 +2339,12 @@ _mm512_maskz_div_ps(__mmask16 __U, __m512 __A, __m512 __B) {
2349#define _mm512_mask_div_round_pd(W, U, A, B, R) \2339#define _mm512_mask_div_round_pd(W, U, A, B, R) \
2350 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \2340 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
2351 (__v8df)_mm512_div_round_pd((A), (B), (R)), \2341 (__v8df)_mm512_div_round_pd((A), (B), (R)), \
2352 (__v8df)(__m512d)(W));2342 (__v8df)(__m512d)(W))
23532343
2354#define _mm512_maskz_div_round_pd(U, A, B, R) \2344#define _mm512_maskz_div_round_pd(U, A, B, R) \
2355 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \2345 (__m512d)__builtin_ia32_selectpd_512((__mmask8)(U), \
2356 (__v8df)_mm512_div_round_pd((A), (B), (R)), \2346 (__v8df)_mm512_div_round_pd((A), (B), (R)), \
2357 (__v8df)_mm512_setzero_pd());2347 (__v8df)_mm512_setzero_pd())
23582348
2359#define _mm512_div_round_ps(A, B, R) \2349#define _mm512_div_round_ps(A, B, R) \
2360 (__m512)__builtin_ia32_divps512((__v16sf)(__m512)(A), \2350 (__m512)__builtin_ia32_divps512((__v16sf)(__m512)(A), \
...@@ -2363,12 +2353,12 @@ _mm512_maskz_div_ps(__mmask16 __U, __m512 __A, __m512 __B) {...@@ -2363,12 +2353,12 @@ _mm512_maskz_div_ps(__mmask16 __U, __m512 __A, __m512 __B) {
2363#define _mm512_mask_div_round_ps(W, U, A, B, R) \2353#define _mm512_mask_div_round_ps(W, U, A, B, R) \
2364 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \2354 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2365 (__v16sf)_mm512_div_round_ps((A), (B), (R)), \2355 (__v16sf)_mm512_div_round_ps((A), (B), (R)), \
2366 (__v16sf)(__m512)(W));2356 (__v16sf)(__m512)(W))
23672357
2368#define _mm512_maskz_div_round_ps(U, A, B, R) \2358#define _mm512_maskz_div_round_ps(U, A, B, R) \
2369 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \2359 (__m512)__builtin_ia32_selectps_512((__mmask16)(U), \
2370 (__v16sf)_mm512_div_round_ps((A), (B), (R)), \2360 (__v16sf)_mm512_div_round_ps((A), (B), (R)), \
2371 (__v16sf)_mm512_setzero_ps());2361 (__v16sf)_mm512_setzero_ps())
23722362
2373#define _mm512_roundscale_ps(A, B) \2363#define _mm512_roundscale_ps(A, B) \
2374 (__m512)__builtin_ia32_rndscaleps_mask((__v16sf)(__m512)(A), (int)(B), \2364 (__m512)__builtin_ia32_rndscaleps_mask((__v16sf)(__m512)(A), (int)(B), \
...@@ -3789,20 +3779,9 @@ _mm512_mask_cvtpd_pslo (__m512 __W, __mmask8 __U,__m512d __A)...@@ -3789,20 +3779,9 @@ _mm512_mask_cvtpd_pslo (__m512 __W, __mmask8 __U,__m512d __A)
3789 (__v16hi)_mm256_setzero_si256(), \3779 (__v16hi)_mm256_setzero_si256(), \
3790 (__mmask16)(W))3780 (__mmask16)(W))
37913781
3792#define _mm512_cvtps_ph(A, I) \3782#define _mm512_cvtps_ph _mm512_cvt_roundps_ph
3793 (__m256i)__builtin_ia32_vcvtps2ph512_mask((__v16sf)(__m512)(A), (int)(I), \3783#define _mm512_mask_cvtps_ph _mm512_mask_cvt_roundps_ph
3794 (__v16hi)_mm256_setzero_si256(), \3784#define _mm512_maskz_cvtps_ph _mm512_maskz_cvt_roundps_ph
3795 (__mmask16)-1)
3796
3797#define _mm512_mask_cvtps_ph(U, W, A, I) \
3798 (__m256i)__builtin_ia32_vcvtps2ph512_mask((__v16sf)(__m512)(A), (int)(I), \
3799 (__v16hi)(__m256i)(U), \
3800 (__mmask16)(W))
3801
3802#define _mm512_maskz_cvtps_ph(W, A, I) \
3803 (__m256i)__builtin_ia32_vcvtps2ph512_mask((__v16sf)(__m512)(A), (int)(I), \
3804 (__v16hi)_mm256_setzero_si256(), \
3805 (__mmask16)(W))
38063785
3807#define _mm512_cvt_roundph_ps(A, R) \3786#define _mm512_cvt_roundph_ps(A, R) \
3808 (__m512)__builtin_ia32_vcvtph2ps512_mask((__v16hi)(__m256i)(A), \3787 (__m512)__builtin_ia32_vcvtph2ps512_mask((__v16hi)(__m256i)(A), \
...@@ -4324,7 +4303,7 @@ static __inline __m512i __DEFAULT_FN_ATTRS512...@@ -4324,7 +4303,7 @@ static __inline __m512i __DEFAULT_FN_ATTRS512
4324_mm512_loadu_si512 (void const *__P)4303_mm512_loadu_si512 (void const *__P)
4325{4304{
4326 struct __loadu_si512 {4305 struct __loadu_si512 {
4327 __m512i __v;4306 __m512i_u __v;
4328 } __attribute__((__packed__, __may_alias__));4307 } __attribute__((__packed__, __may_alias__));
4329 return ((struct __loadu_si512*)__P)->__v;4308 return ((struct __loadu_si512*)__P)->__v;
4330}4309}
...@@ -4333,7 +4312,7 @@ static __inline __m512i __DEFAULT_FN_ATTRS512...@@ -4333,7 +4312,7 @@ static __inline __m512i __DEFAULT_FN_ATTRS512
4333_mm512_loadu_epi32 (void const *__P)4312_mm512_loadu_epi32 (void const *__P)
4334{4313{
4335 struct __loadu_epi32 {4314 struct __loadu_epi32 {
4336 __m512i __v;4315 __m512i_u __v;
4337 } __attribute__((__packed__, __may_alias__));4316 } __attribute__((__packed__, __may_alias__));
4338 return ((struct __loadu_epi32*)__P)->__v;4317 return ((struct __loadu_epi32*)__P)->__v;
4339}4318}
...@@ -4360,7 +4339,7 @@ static __inline __m512i __DEFAULT_FN_ATTRS512...@@ -4360,7 +4339,7 @@ static __inline __m512i __DEFAULT_FN_ATTRS512
4360_mm512_loadu_epi64 (void const *__P)4339_mm512_loadu_epi64 (void const *__P)
4361{4340{
4362 struct __loadu_epi64 {4341 struct __loadu_epi64 {
4363 __m512i __v;4342 __m512i_u __v;
4364 } __attribute__((__packed__, __may_alias__));4343 } __attribute__((__packed__, __may_alias__));
4365 return ((struct __loadu_epi64*)__P)->__v;4344 return ((struct __loadu_epi64*)__P)->__v;
4366}4345}
...@@ -4420,7 +4399,7 @@ static __inline __m512d __DEFAULT_FN_ATTRS512...@@ -4420,7 +4399,7 @@ static __inline __m512d __DEFAULT_FN_ATTRS512
4420_mm512_loadu_pd(void const *__p)4399_mm512_loadu_pd(void const *__p)
4421{4400{
4422 struct __loadu_pd {4401 struct __loadu_pd {
4423 __m512d __v;4402 __m512d_u __v;
4424 } __attribute__((__packed__, __may_alias__));4403 } __attribute__((__packed__, __may_alias__));
4425 return ((struct __loadu_pd*)__p)->__v;4404 return ((struct __loadu_pd*)__p)->__v;
4426}4405}
...@@ -4429,7 +4408,7 @@ static __inline __m512 __DEFAULT_FN_ATTRS512...@@ -4429,7 +4408,7 @@ static __inline __m512 __DEFAULT_FN_ATTRS512
4429_mm512_loadu_ps(void const *__p)4408_mm512_loadu_ps(void const *__p)
4430{4409{
4431 struct __loadu_ps {4410 struct __loadu_ps {
4432 __m512 __v;4411 __m512_u __v;
4433 } __attribute__((__packed__, __may_alias__));4412 } __attribute__((__packed__, __may_alias__));
4434 return ((struct __loadu_ps*)__p)->__v;4413 return ((struct __loadu_ps*)__p)->__v;
4435}4414}
...@@ -4504,7 +4483,7 @@ static __inline void __DEFAULT_FN_ATTRS512...@@ -4504,7 +4483,7 @@ static __inline void __DEFAULT_FN_ATTRS512
4504_mm512_storeu_epi64 (void *__P, __m512i __A)4483_mm512_storeu_epi64 (void *__P, __m512i __A)
4505{4484{
4506 struct __storeu_epi64 {4485 struct __storeu_epi64 {
4507 __m512i __v;4486 __m512i_u __v;
4508 } __attribute__((__packed__, __may_alias__));4487 } __attribute__((__packed__, __may_alias__));
4509 ((struct __storeu_epi64*)__P)->__v = __A;4488 ((struct __storeu_epi64*)__P)->__v = __A;
4510}4489}
...@@ -4520,7 +4499,7 @@ static __inline void __DEFAULT_FN_ATTRS512...@@ -4520,7 +4499,7 @@ static __inline void __DEFAULT_FN_ATTRS512
4520_mm512_storeu_si512 (void *__P, __m512i __A)4499_mm512_storeu_si512 (void *__P, __m512i __A)
4521{4500{
4522 struct __storeu_si512 {4501 struct __storeu_si512 {
4523 __m512i __v;4502 __m512i_u __v;
4524 } __attribute__((__packed__, __may_alias__));4503 } __attribute__((__packed__, __may_alias__));
4525 ((struct __storeu_si512*)__P)->__v = __A;4504 ((struct __storeu_si512*)__P)->__v = __A;
4526}4505}
...@@ -4529,7 +4508,7 @@ static __inline void __DEFAULT_FN_ATTRS512...@@ -4529,7 +4508,7 @@ static __inline void __DEFAULT_FN_ATTRS512
4529_mm512_storeu_epi32 (void *__P, __m512i __A)4508_mm512_storeu_epi32 (void *__P, __m512i __A)
4530{4509{
4531 struct __storeu_epi32 {4510 struct __storeu_epi32 {
4532 __m512i __v;4511 __m512i_u __v;
4533 } __attribute__((__packed__, __may_alias__));4512 } __attribute__((__packed__, __may_alias__));
4534 ((struct __storeu_epi32*)__P)->__v = __A;4513 ((struct __storeu_epi32*)__P)->__v = __A;
4535}4514}
...@@ -4551,7 +4530,7 @@ static __inline void __DEFAULT_FN_ATTRS512...@@ -4551,7 +4530,7 @@ static __inline void __DEFAULT_FN_ATTRS512
4551_mm512_storeu_pd(void *__P, __m512d __A)4530_mm512_storeu_pd(void *__P, __m512d __A)
4552{4531{
4553 struct __storeu_pd {4532 struct __storeu_pd {
4554 __m512d __v;4533 __m512d_u __v;
4555 } __attribute__((__packed__, __may_alias__));4534 } __attribute__((__packed__, __may_alias__));
4556 ((struct __storeu_pd*)__P)->__v = __A;4535 ((struct __storeu_pd*)__P)->__v = __A;
4557}4536}
...@@ -4567,7 +4546,7 @@ static __inline void __DEFAULT_FN_ATTRS512...@@ -4567,7 +4546,7 @@ static __inline void __DEFAULT_FN_ATTRS512
4567_mm512_storeu_ps(void *__P, __m512 __A)4546_mm512_storeu_ps(void *__P, __m512 __A)
4568{4547{
4569 struct __storeu_ps {4548 struct __storeu_ps {
4570 __m512 __v;4549 __m512_u __v;
4571 } __attribute__((__packed__, __may_alias__));4550 } __attribute__((__packed__, __may_alias__));
4572 ((struct __storeu_ps*)__P)->__v = __A;4551 ((struct __storeu_ps*)__P)->__v = __A;
4573}4552}
...@@ -9329,7 +9308,7 @@ _mm512_mask_abs_pd(__m512d __W, __mmask8 __K, __m512d __A)...@@ -9329,7 +9308,7 @@ _mm512_mask_abs_pd(__m512d __W, __mmask8 __K, __m512d __A)
9329 __v2du __t6 = __t4 op __t5; \9308 __v2du __t6 = __t4 op __t5; \
9330 __v2du __t7 = __builtin_shufflevector(__t6, __t6, 1, 0); \9309 __v2du __t7 = __builtin_shufflevector(__t6, __t6, 1, 0); \
9331 __v2du __t8 = __t6 op __t7; \9310 __v2du __t8 = __t6 op __t7; \
9332 return __t8[0];9311 return __t8[0]
93339312
9334static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_add_epi64(__m512i __W) {9313static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_add_epi64(__m512i __W) {
9335 _mm512_mask_reduce_operator(+);9314 _mm512_mask_reduce_operator(+);
...@@ -9381,7 +9360,7 @@ _mm512_mask_reduce_or_epi64(__mmask8 __M, __m512i __W) {...@@ -9381,7 +9360,7 @@ _mm512_mask_reduce_or_epi64(__mmask8 __M, __m512i __W) {
9381 __m128d __t6 = __t4 op __t5; \9360 __m128d __t6 = __t4 op __t5; \
9382 __m128d __t7 = __builtin_shufflevector(__t6, __t6, 1, 0); \9361 __m128d __t7 = __builtin_shufflevector(__t6, __t6, 1, 0); \
9383 __m128d __t8 = __t6 op __t7; \9362 __m128d __t8 = __t6 op __t7; \
9384 return __t8[0];9363 return __t8[0]
93859364
9386static __inline__ double __DEFAULT_FN_ATTRS512 _mm512_reduce_add_pd(__m512d __W) {9365static __inline__ double __DEFAULT_FN_ATTRS512 _mm512_reduce_add_pd(__m512d __W) {
9387 _mm512_mask_reduce_operator(+);9366 _mm512_mask_reduce_operator(+);
...@@ -9415,7 +9394,7 @@ _mm512_mask_reduce_mul_pd(__mmask8 __M, __m512d __W) {...@@ -9415,7 +9394,7 @@ _mm512_mask_reduce_mul_pd(__mmask8 __M, __m512d __W) {
9415 __v4su __t8 = __t6 op __t7; \9394 __v4su __t8 = __t6 op __t7; \
9416 __v4su __t9 = __builtin_shufflevector(__t8, __t8, 1, 0, 3, 2); \9395 __v4su __t9 = __builtin_shufflevector(__t8, __t8, 1, 0, 3, 2); \
9417 __v4su __t10 = __t8 op __t9; \9396 __v4su __t10 = __t8 op __t9; \
9418 return __t10[0];9397 return __t10[0]
94199398
9420static __inline__ int __DEFAULT_FN_ATTRS5129399static __inline__ int __DEFAULT_FN_ATTRS512
9421_mm512_reduce_add_epi32(__m512i __W) {9400_mm512_reduce_add_epi32(__m512i __W) {
...@@ -9473,7 +9452,7 @@ _mm512_mask_reduce_or_epi32(__mmask16 __M, __m512i __W) {...@@ -9473,7 +9452,7 @@ _mm512_mask_reduce_or_epi32(__mmask16 __M, __m512i __W) {
9473 __m128 __t8 = __t6 op __t7; \9452 __m128 __t8 = __t6 op __t7; \
9474 __m128 __t9 = __builtin_shufflevector(__t8, __t8, 1, 0, 3, 2); \9453 __m128 __t9 = __builtin_shufflevector(__t8, __t8, 1, 0, 3, 2); \
9475 __m128 __t10 = __t8 op __t9; \9454 __m128 __t10 = __t8 op __t9; \
9476 return __t10[0];9455 return __t10[0]
94779456
9478static __inline__ float __DEFAULT_FN_ATTRS5129457static __inline__ float __DEFAULT_FN_ATTRS512
9479_mm512_reduce_add_ps(__m512 __W) {9458_mm512_reduce_add_ps(__m512 __W) {
...@@ -9505,7 +9484,7 @@ _mm512_mask_reduce_mul_ps(__mmask16 __M, __m512 __W) {...@@ -9505,7 +9484,7 @@ _mm512_mask_reduce_mul_ps(__mmask16 __M, __m512 __W) {
9505 __m512i __t4 = _mm512_##op(__t2, __t3); \9484 __m512i __t4 = _mm512_##op(__t2, __t3); \
9506 __m512i __t5 = (__m512i)__builtin_shufflevector((__v8di)__t4, (__v8di)__t4, 1, 0, 3, 2, 5, 4, 7, 6); \9485 __m512i __t5 = (__m512i)__builtin_shufflevector((__v8di)__t4, (__v8di)__t4, 1, 0, 3, 2, 5, 4, 7, 6); \
9507 __v8di __t6 = (__v8di)_mm512_##op(__t4, __t5); \9486 __v8di __t6 = (__v8di)_mm512_##op(__t4, __t5); \
9508 return __t6[0];9487 return __t6[0]
95099488
9510static __inline__ long long __DEFAULT_FN_ATTRS5129489static __inline__ long long __DEFAULT_FN_ATTRS512
9511_mm512_reduce_max_epi64(__m512i __V) {9490_mm512_reduce_max_epi64(__m512i __V) {
...@@ -9563,7 +9542,7 @@ _mm512_mask_reduce_min_epu64(__mmask8 __M, __m512i __V) {...@@ -9563,7 +9542,7 @@ _mm512_mask_reduce_min_epu64(__mmask8 __M, __m512i __V) {
9563 __m128i __t8 = _mm_##op(__t6, __t7); \9542 __m128i __t8 = _mm_##op(__t6, __t7); \
9564 __m128i __t9 = (__m128i)__builtin_shufflevector((__v4si)__t8, (__v4si)__t8, 1, 0, 3, 2); \9543 __m128i __t9 = (__m128i)__builtin_shufflevector((__v4si)__t8, (__v4si)__t8, 1, 0, 3, 2); \
9565 __v4si __t10 = (__v4si)_mm_##op(__t8, __t9); \9544 __v4si __t10 = (__v4si)_mm_##op(__t8, __t9); \
9566 return __t10[0];9545 return __t10[0]
95679546
9568static __inline__ int __DEFAULT_FN_ATTRS5129547static __inline__ int __DEFAULT_FN_ATTRS512
9569_mm512_reduce_max_epi32(__m512i __V) {9548_mm512_reduce_max_epi32(__m512i __V) {
...@@ -9619,7 +9598,7 @@ _mm512_mask_reduce_min_epu32(__mmask16 __M, __m512i __V) {...@@ -9619,7 +9598,7 @@ _mm512_mask_reduce_min_epu32(__mmask16 __M, __m512i __V) {
9619 __m128d __t6 = _mm_##op(__t4, __t5); \9598 __m128d __t6 = _mm_##op(__t4, __t5); \
9620 __m128d __t7 = __builtin_shufflevector(__t6, __t6, 1, 0); \9599 __m128d __t7 = __builtin_shufflevector(__t6, __t6, 1, 0); \
9621 __m128d __t8 = _mm_##op(__t6, __t7); \9600 __m128d __t8 = _mm_##op(__t6, __t7); \
9622 return __t8[0];9601 return __t8[0]
96239602
9624static __inline__ double __DEFAULT_FN_ATTRS5129603static __inline__ double __DEFAULT_FN_ATTRS512
9625_mm512_reduce_max_pd(__m512d __V) {9604_mm512_reduce_max_pd(__m512d __V) {
...@@ -9655,7 +9634,7 @@ _mm512_mask_reduce_min_pd(__mmask8 __M, __m512d __V) {...@@ -9655,7 +9634,7 @@ _mm512_mask_reduce_min_pd(__mmask8 __M, __m512d __V) {
9655 __m128 __t8 = _mm_##op(__t6, __t7); \9634 __m128 __t8 = _mm_##op(__t6, __t7); \
9656 __m128 __t9 = __builtin_shufflevector(__t8, __t8, 1, 0, 3, 2); \9635 __m128 __t9 = __builtin_shufflevector(__t8, __t8, 1, 0, 3, 2); \
9657 __m128 __t10 = _mm_##op(__t8, __t9); \9636 __m128 __t10 = _mm_##op(__t8, __t9); \
9658 return __t10[0];9637 return __t10[0]
96599638
9660static __inline__ float __DEFAULT_FN_ATTRS5129639static __inline__ float __DEFAULT_FN_ATTRS512
9661_mm512_reduce_max_ps(__m512 __V) {9640_mm512_reduce_max_ps(__m512 __V) {
lib/include/avx512ifmaintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===------------- avx512ifmaintrin.h - IFMA intrinsics ------------------===1/*===------------- avx512ifmaintrin.h - IFMA intrinsics ------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/avx512ifmavlintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===------------- avx512ifmavlintrin.h - IFMA intrinsics ------------------===1/*===------------- avx512ifmavlintrin.h - IFMA intrinsics ------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/avx512pfintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===------------- avx512pfintrin.h - PF intrinsics ------------------------===1/*===------------- avx512pfintrin.h - PF intrinsics ------------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/avx512vbmi2intrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===------------- avx512vbmi2intrin.h - VBMI2 intrinsics ------------------===1/*===------------- avx512vbmi2intrin.h - VBMI2 intrinsics ------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/avx512vbmiintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===------------- avx512vbmiintrin.h - VBMI intrinsics ------------------===1/*===------------- avx512vbmiintrin.h - VBMI intrinsics ------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/avx512vbmivlintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===------------- avx512vbmivlintrin.h - VBMI intrinsics ------------------===1/*===------------- avx512vbmivlintrin.h - VBMI intrinsics ------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/avx512vlbf16intrin.h created+474
...@@ -0,0 +1,474 @@
1/*===--------- avx512vlbf16intrin.h - AVX512_BF16 intrinsics ---------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error "Never use <avx512vlbf16intrin.h> directly; include <immintrin.h> instead."
11#endif
12
13#ifndef __AVX512VLBF16INTRIN_H
14#define __AVX512VLBF16INTRIN_H
15
16typedef short __m128bh __attribute__((__vector_size__(16), __aligned__(16)));
17
18#define __DEFAULT_FN_ATTRS128 \
19 __attribute__((__always_inline__, __nodebug__, \
20 __target__("avx512vl, avx512bf16"), __min_vector_width__(128)))
21#define __DEFAULT_FN_ATTRS256 \
22 __attribute__((__always_inline__, __nodebug__, \
23 __target__("avx512vl, avx512bf16"), __min_vector_width__(256)))
24
25/// Convert Two Packed Single Data to One Packed BF16 Data.
26///
27/// \headerfile <x86intrin.h>
28///
29/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
30///
31/// \param __A
32/// A 128-bit vector of [4 x float].
33/// \param __B
34/// A 128-bit vector of [4 x float].
35/// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
36/// conversion of __B, and higher 64 bits come from conversion of __A.
37static __inline__ __m128bh __DEFAULT_FN_ATTRS128
38_mm_cvtne2ps_pbh(__m128 __A, __m128 __B) {
39 return (__m128bh)__builtin_ia32_cvtne2ps2bf16_128((__v4sf) __A,
40 (__v4sf) __B);
41}
42
43/// Convert Two Packed Single Data to One Packed BF16 Data.
44///
45/// \headerfile <x86intrin.h>
46///
47/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
48///
49/// \param __A
50/// A 128-bit vector of [4 x float].
51/// \param __B
52/// A 128-bit vector of [4 x float].
53/// \param __W
54/// A 128-bit vector of [8 x bfloat].
55/// \param __U
56/// A 8-bit mask value specifying what is chosen for each element.
57/// A 1 means conversion of __A or __B. A 0 means element from __W.
58/// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
59/// conversion of __B, and higher 64 bits come from conversion of __A.
60static __inline__ __m128bh __DEFAULT_FN_ATTRS128
61_mm_mask_cvtne2ps_pbh(__m128bh __W, __mmask8 __U, __m128 __A, __m128 __B) {
62 return (__m128bh)__builtin_ia32_selectw_128((__mmask8)__U,
63 (__v8hi)_mm_cvtne2ps_pbh(__A, __B),
64 (__v8hi)__W);
65}
66
67/// Convert Two Packed Single Data to One Packed BF16 Data.
68///
69/// \headerfile <x86intrin.h>
70///
71/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
72///
73/// \param __A
74/// A 128-bit vector of [4 x float].
75/// \param __B
76/// A 128-bit vector of [4 x float].
77/// \param __U
78/// A 8-bit mask value specifying what is chosen for each element.
79/// A 1 means conversion of __A or __B. A 0 means element is zero.
80/// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
81/// conversion of __B, and higher 64 bits come from conversion of __A.
82static __inline__ __m128bh __DEFAULT_FN_ATTRS128
83_mm_maskz_cvtne2ps_pbh(__mmask8 __U, __m128 __A, __m128 __B) {
84 return (__m128bh)__builtin_ia32_selectw_128((__mmask8)__U,
85 (__v8hi)_mm_cvtne2ps_pbh(__A, __B),
86 (__v8hi)_mm_setzero_si128());
87}
88
89/// Convert Two Packed Single Data to One Packed BF16 Data.
90///
91/// \headerfile <x86intrin.h>
92///
93/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
94///
95/// \param __A
96/// A 256-bit vector of [8 x float].
97/// \param __B
98/// A 256-bit vector of [8 x float].
99/// \returns A 256-bit vector of [16 x bfloat] whose lower 128 bits come from
100/// conversion of __B, and higher 128 bits come from conversion of __A.
101static __inline__ __m256bh __DEFAULT_FN_ATTRS256
102_mm256_cvtne2ps_pbh(__m256 __A, __m256 __B) {
103 return (__m256bh)__builtin_ia32_cvtne2ps2bf16_256((__v8sf) __A,
104 (__v8sf) __B);
105}
106
107/// Convert Two Packed Single Data to One Packed BF16 Data.
108///
109/// \headerfile <x86intrin.h>
110///
111/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
112///
113/// \param __A
114/// A 256-bit vector of [8 x float].
115/// \param __B
116/// A 256-bit vector of [8 x float].
117/// \param __W
118/// A 256-bit vector of [16 x bfloat].
119/// \param __U
120/// A 16-bit mask value specifying what is chosen for each element.
121/// A 1 means conversion of __A or __B. A 0 means element from __W.
122/// \returns A 256-bit vector of [16 x bfloat] whose lower 128 bits come from
123/// conversion of __B, and higher 128 bits come from conversion of __A.
124static __inline__ __m256bh __DEFAULT_FN_ATTRS256
125_mm256_mask_cvtne2ps_pbh(__m256bh __W, __mmask16 __U, __m256 __A, __m256 __B) {
126 return (__m256bh)__builtin_ia32_selectw_256((__mmask16)__U,
127 (__v16hi)_mm256_cvtne2ps_pbh(__A, __B),
128 (__v16hi)__W);
129}
130
131/// Convert Two Packed Single Data to One Packed BF16 Data.
132///
133/// \headerfile <x86intrin.h>
134///
135/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
136///
137/// \param __A
138/// A 256-bit vector of [8 x float].
139/// \param __B
140/// A 256-bit vector of [8 x float].
141/// \param __U
142/// A 16-bit mask value specifying what is chosen for each element.
143/// A 1 means conversion of __A or __B. A 0 means element is zero.
144/// \returns A 256-bit vector of [16 x bfloat] whose lower 128 bits come from
145/// conversion of __B, and higher 128 bits come from conversion of __A.
146static __inline__ __m256bh __DEFAULT_FN_ATTRS256
147_mm256_maskz_cvtne2ps_pbh(__mmask16 __U, __m256 __A, __m256 __B) {
148 return (__m256bh)__builtin_ia32_selectw_256((__mmask16)__U,
149 (__v16hi)_mm256_cvtne2ps_pbh(__A, __B),
150 (__v16hi)_mm256_setzero_si256());
151}
152
153/// Convert Packed Single Data to Packed BF16 Data.
154///
155/// \headerfile <x86intrin.h>
156///
157/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
158///
159/// \param __A
160/// A 128-bit vector of [4 x float].
161/// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
162/// conversion of __A, and higher 64 bits are 0.
163static __inline__ __m128bh __DEFAULT_FN_ATTRS128
164_mm_cvtneps_pbh(__m128 __A) {
165 return (__m128bh)__builtin_ia32_cvtneps2bf16_128_mask((__v4sf) __A,
166 (__v8hi)_mm_undefined_si128(),
167 (__mmask8)-1);
168}
169
170/// Convert Packed Single Data to Packed BF16 Data.
171///
172/// \headerfile <x86intrin.h>
173///
174/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
175///
176/// \param __A
177/// A 128-bit vector of [4 x float].
178/// \param __W
179/// A 128-bit vector of [8 x bfloat].
180/// \param __U
181/// A 4-bit mask value specifying what is chosen for each element.
182/// A 1 means conversion of __A. A 0 means element from __W.
183/// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
184/// conversion of __A, and higher 64 bits are 0.
185static __inline__ __m128bh __DEFAULT_FN_ATTRS128
186_mm_mask_cvtneps_pbh(__m128bh __W, __mmask8 __U, __m128 __A) {
187 return (__m128bh)__builtin_ia32_cvtneps2bf16_128_mask((__v4sf) __A,
188 (__v8hi)__W,
189 (__mmask8)__U);
190}
191
192/// Convert Packed Single Data to Packed BF16 Data.
193///
194/// \headerfile <x86intrin.h>
195///
196/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
197///
198/// \param __A
199/// A 128-bit vector of [4 x float].
200/// \param __U
201/// A 4-bit mask value specifying what is chosen for each element.
202/// A 1 means conversion of __A. A 0 means element is zero.
203/// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
204/// conversion of __A, and higher 64 bits are 0.
205static __inline__ __m128bh __DEFAULT_FN_ATTRS128
206_mm_maskz_cvtneps_pbh(__mmask8 __U, __m128 __A) {
207 return (__m128bh)__builtin_ia32_cvtneps2bf16_128_mask((__v4sf) __A,
208 (__v8hi)_mm_setzero_si128(),
209 (__mmask8)__U);
210}
211
212/// Convert Packed Single Data to Packed BF16 Data.
213///
214/// \headerfile <x86intrin.h>
215///
216/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
217///
218/// \param __A
219/// A 256-bit vector of [8 x float].
220/// \returns A 128-bit vector of [8 x bfloat] comes from conversion of __A.
221static __inline__ __m128bh __DEFAULT_FN_ATTRS256
222_mm256_cvtneps_pbh(__m256 __A) {
223 return (__m128bh)__builtin_ia32_cvtneps2bf16_256_mask((__v8sf)__A,
224 (__v8hi)_mm_undefined_si128(),
225 (__mmask8)-1);
226}
227
228/// Convert Packed Single Data to Packed BF16 Data.
229///
230/// \headerfile <x86intrin.h>
231///
232/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
233///
234/// \param __A
235/// A 256-bit vector of [8 x float].
236/// \param __W
237/// A 256-bit vector of [8 x bfloat].
238/// \param __U
239/// A 8-bit mask value specifying what is chosen for each element.
240/// A 1 means conversion of __A. A 0 means element from __W.
241/// \returns A 128-bit vector of [8 x bfloat] comes from conversion of __A.
242static __inline__ __m128bh __DEFAULT_FN_ATTRS256
243_mm256_mask_cvtneps_pbh(__m128bh __W, __mmask8 __U, __m256 __A) {
244 return (__m128bh)__builtin_ia32_cvtneps2bf16_256_mask((__v8sf)__A,
245 (__v8hi)__W,
246 (__mmask8)__U);
247}
248
249/// Convert Packed Single Data to Packed BF16 Data.
250///
251/// \headerfile <x86intrin.h>
252///
253/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
254///
255/// \param __A
256/// A 256-bit vector of [8 x float].
257/// \param __U
258/// A 8-bit mask value specifying what is chosen for each element.
259/// A 1 means conversion of __A. A 0 means element is zero.
260/// \returns A 128-bit vector of [8 x bfloat] comes from conversion of __A.
261static __inline__ __m128bh __DEFAULT_FN_ATTRS256
262_mm256_maskz_cvtneps_pbh(__mmask8 __U, __m256 __A) {
263 return (__m128bh)__builtin_ia32_cvtneps2bf16_256_mask((__v8sf)__A,
264 (__v8hi)_mm_setzero_si128(),
265 (__mmask8)__U);
266}
267
268/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
269///
270/// \headerfile <x86intrin.h>
271///
272/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
273///
274/// \param __A
275/// A 128-bit vector of [8 x bfloat].
276/// \param __B
277/// A 128-bit vector of [8 x bfloat].
278/// \param __D
279/// A 128-bit vector of [4 x float].
280/// \returns A 128-bit vector of [4 x float] comes from Dot Product of
281/// __A, __B and __D
282static __inline__ __m128 __DEFAULT_FN_ATTRS128
283_mm_dpbf16_ps(__m128 __D, __m128bh __A, __m128bh __B) {
284 return (__m128)__builtin_ia32_dpbf16ps_128((__v4sf)__D,
285 (__v4si)__A,
286 (__v4si)__B);
287}
288
289/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
290///
291/// \headerfile <x86intrin.h>
292///
293/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
294///
295/// \param __A
296/// A 128-bit vector of [8 x bfloat].
297/// \param __B
298/// A 128-bit vector of [8 x bfloat].
299/// \param __D
300/// A 128-bit vector of [4 x float].
301/// \param __U
302/// A 8-bit mask value specifying what is chosen for each element.
303/// A 1 means __A and __B's dot product accumulated with __D. A 0 means __D.
304/// \returns A 128-bit vector of [4 x float] comes from Dot Product of
305/// __A, __B and __D
306static __inline__ __m128 __DEFAULT_FN_ATTRS128
307_mm_mask_dpbf16_ps(__m128 __D, __mmask8 __U, __m128bh __A, __m128bh __B) {
308 return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
309 (__v4sf)_mm_dpbf16_ps(__D, __A, __B),
310 (__v4sf)__D);
311}
312
313/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
314///
315/// \headerfile <x86intrin.h>
316///
317/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
318///
319/// \param __A
320/// A 128-bit vector of [8 x bfloat].
321/// \param __B
322/// A 128-bit vector of [8 x bfloat].
323/// \param __D
324/// A 128-bit vector of [4 x float].
325/// \param __U
326/// A 8-bit mask value specifying what is chosen for each element.
327/// A 1 means __A and __B's dot product accumulated with __D. A 0 means 0.
328/// \returns A 128-bit vector of [4 x float] comes from Dot Product of
329/// __A, __B and __D
330static __inline__ __m128 __DEFAULT_FN_ATTRS128
331_mm_maskz_dpbf16_ps(__mmask8 __U, __m128 __D, __m128bh __A, __m128bh __B) {
332 return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
333 (__v4sf)_mm_dpbf16_ps(__D, __A, __B),
334 (__v4sf)_mm_setzero_si128());
335}
336
337/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
338///
339/// \headerfile <x86intrin.h>
340///
341/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
342///
343/// \param __A
344/// A 256-bit vector of [16 x bfloat].
345/// \param __B
346/// A 256-bit vector of [16 x bfloat].
347/// \param __D
348/// A 256-bit vector of [8 x float].
349/// \returns A 256-bit vector of [8 x float] comes from Dot Product of
350/// __A, __B and __D
351static __inline__ __m256 __DEFAULT_FN_ATTRS256
352_mm256_dpbf16_ps(__m256 __D, __m256bh __A, __m256bh __B) {
353 return (__m256)__builtin_ia32_dpbf16ps_256((__v8sf)__D,
354 (__v8si)__A,
355 (__v8si)__B);
356}
357
358/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
359///
360/// \headerfile <x86intrin.h>
361///
362/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
363///
364/// \param __A
365/// A 256-bit vector of [16 x bfloat].
366/// \param __B
367/// A 256-bit vector of [16 x bfloat].
368/// \param __D
369/// A 256-bit vector of [8 x float].
370/// \param __U
371/// A 16-bit mask value specifying what is chosen for each element.
372/// A 1 means __A and __B's dot product accumulated with __D. A 0 means __D.
373/// \returns A 256-bit vector of [8 x float] comes from Dot Product of
374/// __A, __B and __D
375static __inline__ __m256 __DEFAULT_FN_ATTRS256
376_mm256_mask_dpbf16_ps(__m256 __D, __mmask8 __U, __m256bh __A, __m256bh __B) {
377 return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
378 (__v8sf)_mm256_dpbf16_ps(__D, __A, __B),
379 (__v8sf)__D);
380}
381
382/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
383///
384/// \headerfile <x86intrin.h>
385///
386/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
387///
388/// \param __A
389/// A 256-bit vector of [16 x bfloat].
390/// \param __B
391/// A 256-bit vector of [16 x bfloat].
392/// \param __D
393/// A 256-bit vector of [8 x float].
394/// \param __U
395/// A 8-bit mask value specifying what is chosen for each element.
396/// A 1 means __A and __B's dot product accumulated with __D. A 0 means 0.
397/// \returns A 256-bit vector of [8 x float] comes from Dot Product of
398/// __A, __B and __D
399static __inline__ __m256 __DEFAULT_FN_ATTRS256
400_mm256_maskz_dpbf16_ps(__mmask8 __U, __m256 __D, __m256bh __A, __m256bh __B) {
401 return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
402 (__v8sf)_mm256_dpbf16_ps(__D, __A, __B),
403 (__v8sf)_mm256_setzero_si256());
404}
405
406/// Convert One Single float Data to One BF16 Data.
407///
408/// \headerfile <x86intrin.h>
409///
410/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
411///
412/// \param __A
413/// A float data.
414/// \returns A bf16 data whose sign field and exponent field keep unchanged,
415/// and fraction field is truncated to 7 bits.
416static __inline__ __bfloat16 __DEFAULT_FN_ATTRS128 _mm_cvtness_sbh(float __A) {
417 __v4sf __V = {__A, 0, 0, 0};
418 __v8hi __R = __builtin_ia32_cvtneps2bf16_128_mask(
419 (__v4sf)__V, (__v8hi)_mm_undefined_si128(), (__mmask8)-1);
420 return __R[0];
421}
422
423/// Convert Packed BF16 Data to Packed float Data.
424///
425/// \headerfile <x86intrin.h>
426///
427/// \param __A
428/// A 128-bit vector of [8 x bfloat].
429/// \returns A 256-bit vector of [8 x float] come from convertion of __A
430static __inline__ __m256 __DEFAULT_FN_ATTRS256 _mm256_cvtpbh_ps(__m128bh __A) {
431 return _mm256_castsi256_ps((__m256i)_mm256_slli_epi32(
432 (__m256i)_mm256_cvtepi16_epi32((__m128i)__A), 16));
433}
434
435/// Convert Packed BF16 Data to Packed float Data using zeroing mask.
436///
437/// \headerfile <x86intrin.h>
438///
439/// \param __U
440/// A 8-bit mask. Elements are zeroed out when the corresponding mask
441/// bit is not set.
442/// \param __A
443/// A 128-bit vector of [8 x bfloat].
444/// \returns A 256-bit vector of [8 x float] come from convertion of __A
445static __inline__ __m256 __DEFAULT_FN_ATTRS256
446_mm256_maskz_cvtpbh_ps(__mmask8 __U, __m128bh __A) {
447 return _mm256_castsi256_ps((__m256i)_mm256_slli_epi32(
448 (__m256i)_mm256_maskz_cvtepi16_epi32((__mmask8)__U, (__m128i)__A), 16));
449}
450
451/// Convert Packed BF16 Data to Packed float Data using merging mask.
452///
453/// \headerfile <x86intrin.h>
454///
455/// \param __S
456/// A 256-bit vector of [8 x float]. Elements are copied from __S when
457/// the corresponding mask bit is not set.
458/// \param __U
459/// A 8-bit mask. Elements are zeroed out when the corresponding mask
460/// bit is not set.
461/// \param __A
462/// A 128-bit vector of [8 x bfloat].
463/// \returns A 256-bit vector of [8 x float] come from convertion of __A
464static __inline__ __m256 __DEFAULT_FN_ATTRS256
465_mm256_mask_cvtpbh_ps(__m256 __S, __mmask8 __U, __m128bh __A) {
466 return _mm256_castsi256_ps((__m256i)_mm256_mask_slli_epi32(
467 (__m256i)__S, (__mmask8)__U, (__m256i)_mm256_cvtepi16_epi32((__m128i)__A),
468 16));
469}
470
471#undef __DEFAULT_FN_ATTRS128
472#undef __DEFAULT_FN_ATTRS256
473
474#endif
lib/include/avx512vlbitalgintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===---- avx512vlbitalgintrin.h - BITALG intrinsics -----------------------===1/*===---- avx512vlbitalgintrin.h - BITALG intrinsics -----------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/avx512vlbwintrin.h+11-25
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- avx512vlbwintrin.h - AVX512VL and AVX512BW intrinsics ------------===1/*===---- avx512vlbwintrin.h - AVX512VL and AVX512BW intrinsics ------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -2301,7 +2287,7 @@ static __inline __m128i __DEFAULT_FN_ATTRS128...@@ -2301,7 +2287,7 @@ static __inline __m128i __DEFAULT_FN_ATTRS128
2301_mm_loadu_epi16 (void const *__P)2287_mm_loadu_epi16 (void const *__P)
2302{2288{
2303 struct __loadu_epi16 {2289 struct __loadu_epi16 {
2304 __m128i __v;2290 __m128i_u __v;
2305 } __attribute__((__packed__, __may_alias__));2291 } __attribute__((__packed__, __may_alias__));
2306 return ((struct __loadu_epi16*)__P)->__v;2292 return ((struct __loadu_epi16*)__P)->__v;
2307}2293}
...@@ -2327,7 +2313,7 @@ static __inline __m256i __DEFAULT_FN_ATTRS256...@@ -2327,7 +2313,7 @@ static __inline __m256i __DEFAULT_FN_ATTRS256
2327_mm256_loadu_epi16 (void const *__P)2313_mm256_loadu_epi16 (void const *__P)
2328{2314{
2329 struct __loadu_epi16 {2315 struct __loadu_epi16 {
2330 __m256i __v;2316 __m256i_u __v;
2331 } __attribute__((__packed__, __may_alias__));2317 } __attribute__((__packed__, __may_alias__));
2332 return ((struct __loadu_epi16*)__P)->__v;2318 return ((struct __loadu_epi16*)__P)->__v;
2333}2319}
...@@ -2353,7 +2339,7 @@ static __inline __m128i __DEFAULT_FN_ATTRS128...@@ -2353,7 +2339,7 @@ static __inline __m128i __DEFAULT_FN_ATTRS128
2353_mm_loadu_epi8 (void const *__P)2339_mm_loadu_epi8 (void const *__P)
2354{2340{
2355 struct __loadu_epi8 {2341 struct __loadu_epi8 {
2356 __m128i __v;2342 __m128i_u __v;
2357 } __attribute__((__packed__, __may_alias__));2343 } __attribute__((__packed__, __may_alias__));
2358 return ((struct __loadu_epi8*)__P)->__v;2344 return ((struct __loadu_epi8*)__P)->__v;
2359}2345}
...@@ -2379,7 +2365,7 @@ static __inline __m256i __DEFAULT_FN_ATTRS256...@@ -2379,7 +2365,7 @@ static __inline __m256i __DEFAULT_FN_ATTRS256
2379_mm256_loadu_epi8 (void const *__P)2365_mm256_loadu_epi8 (void const *__P)
2380{2366{
2381 struct __loadu_epi8 {2367 struct __loadu_epi8 {
2382 __m256i __v;2368 __m256i_u __v;
2383 } __attribute__((__packed__, __may_alias__));2369 } __attribute__((__packed__, __may_alias__));
2384 return ((struct __loadu_epi8*)__P)->__v;2370 return ((struct __loadu_epi8*)__P)->__v;
2385}2371}
...@@ -2405,7 +2391,7 @@ static __inline void __DEFAULT_FN_ATTRS128...@@ -2405,7 +2391,7 @@ static __inline void __DEFAULT_FN_ATTRS128
2405_mm_storeu_epi16 (void *__P, __m128i __A)2391_mm_storeu_epi16 (void *__P, __m128i __A)
2406{2392{
2407 struct __storeu_epi16 {2393 struct __storeu_epi16 {
2408 __m128i __v;2394 __m128i_u __v;
2409 } __attribute__((__packed__, __may_alias__));2395 } __attribute__((__packed__, __may_alias__));
2410 ((struct __storeu_epi16*)__P)->__v = __A;2396 ((struct __storeu_epi16*)__P)->__v = __A;
2411}2397}
...@@ -2422,7 +2408,7 @@ static __inline void __DEFAULT_FN_ATTRS256...@@ -2422,7 +2408,7 @@ static __inline void __DEFAULT_FN_ATTRS256
2422_mm256_storeu_epi16 (void *__P, __m256i __A)2408_mm256_storeu_epi16 (void *__P, __m256i __A)
2423{2409{
2424 struct __storeu_epi16 {2410 struct __storeu_epi16 {
2425 __m256i __v;2411 __m256i_u __v;
2426 } __attribute__((__packed__, __may_alias__));2412 } __attribute__((__packed__, __may_alias__));
2427 ((struct __storeu_epi16*)__P)->__v = __A;2413 ((struct __storeu_epi16*)__P)->__v = __A;
2428}2414}
...@@ -2439,7 +2425,7 @@ static __inline void __DEFAULT_FN_ATTRS128...@@ -2439,7 +2425,7 @@ static __inline void __DEFAULT_FN_ATTRS128
2439_mm_storeu_epi8 (void *__P, __m128i __A)2425_mm_storeu_epi8 (void *__P, __m128i __A)
2440{2426{
2441 struct __storeu_epi8 {2427 struct __storeu_epi8 {
2442 __m128i __v;2428 __m128i_u __v;
2443 } __attribute__((__packed__, __may_alias__));2429 } __attribute__((__packed__, __may_alias__));
2444 ((struct __storeu_epi8*)__P)->__v = __A;2430 ((struct __storeu_epi8*)__P)->__v = __A;
2445}2431}
...@@ -2456,7 +2442,7 @@ static __inline void __DEFAULT_FN_ATTRS256...@@ -2456,7 +2442,7 @@ static __inline void __DEFAULT_FN_ATTRS256
2456_mm256_storeu_epi8 (void *__P, __m256i __A)2442_mm256_storeu_epi8 (void *__P, __m256i __A)
2457{2443{
2458 struct __storeu_epi8 {2444 struct __storeu_epi8 {
2459 __m256i __v;2445 __m256i_u __v;
2460 } __attribute__((__packed__, __may_alias__));2446 } __attribute__((__packed__, __may_alias__));
2461 ((struct __storeu_epi8*)__P)->__v = __A;2447 ((struct __storeu_epi8*)__P)->__v = __A;
2462}2448}
lib/include/avx512vlcdintrin.h+31-55
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- avx512vlcdintrin.h - AVX512VL and AVX512CD intrinsics ------------===1/*===---- avx512vlcdintrin.h - AVX512VL and AVX512CD intrinsics ------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -60,99 +46,89 @@ _mm256_broadcastmw_epi32 (__mmask16 __A)...@@ -60,99 +46,89 @@ _mm256_broadcastmw_epi32 (__mmask16 __A)
60static __inline__ __m128i __DEFAULT_FN_ATTRS12846static __inline__ __m128i __DEFAULT_FN_ATTRS128
61_mm_conflict_epi64 (__m128i __A)47_mm_conflict_epi64 (__m128i __A)
62{48{
63 return (__m128i) __builtin_ia32_vpconflictdi_128_mask ((__v2di) __A,49 return (__m128i) __builtin_ia32_vpconflictdi_128 ((__v2di) __A);
64 (__v2di) _mm_undefined_si128 (),
65 (__mmask8) -1);
66}50}
6751
68static __inline__ __m128i __DEFAULT_FN_ATTRS12852static __inline__ __m128i __DEFAULT_FN_ATTRS128
69_mm_mask_conflict_epi64 (__m128i __W, __mmask8 __U, __m128i __A)53_mm_mask_conflict_epi64 (__m128i __W, __mmask8 __U, __m128i __A)
70{54{
71 return (__m128i) __builtin_ia32_vpconflictdi_128_mask ((__v2di) __A,55 return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
72 (__v2di) __W,56 (__v2di)_mm_conflict_epi64(__A),
73 (__mmask8) __U);57 (__v2di)__W);
74}58}
7559
76static __inline__ __m128i __DEFAULT_FN_ATTRS12860static __inline__ __m128i __DEFAULT_FN_ATTRS128
77_mm_maskz_conflict_epi64 (__mmask8 __U, __m128i __A)61_mm_maskz_conflict_epi64 (__mmask8 __U, __m128i __A)
78{62{
79 return (__m128i) __builtin_ia32_vpconflictdi_128_mask ((__v2di) __A,63 return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
80 (__v2di)64 (__v2di)_mm_conflict_epi64(__A),
81 _mm_setzero_si128 (),65 (__v2di)_mm_setzero_si128());
82 (__mmask8) __U);
83}66}
8467
85static __inline__ __m256i __DEFAULT_FN_ATTRS25668static __inline__ __m256i __DEFAULT_FN_ATTRS256
86_mm256_conflict_epi64 (__m256i __A)69_mm256_conflict_epi64 (__m256i __A)
87{70{
88 return (__m256i) __builtin_ia32_vpconflictdi_256_mask ((__v4di) __A,71 return (__m256i) __builtin_ia32_vpconflictdi_256 ((__v4di) __A);
89 (__v4di) _mm256_undefined_si256 (),
90 (__mmask8) -1);
91}72}
9273
93static __inline__ __m256i __DEFAULT_FN_ATTRS25674static __inline__ __m256i __DEFAULT_FN_ATTRS256
94_mm256_mask_conflict_epi64 (__m256i __W, __mmask8 __U, __m256i __A)75_mm256_mask_conflict_epi64 (__m256i __W, __mmask8 __U, __m256i __A)
95{76{
96 return (__m256i) __builtin_ia32_vpconflictdi_256_mask ((__v4di) __A,77 return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
97 (__v4di) __W,78 (__v4di)_mm256_conflict_epi64(__A),
98 (__mmask8) __U);79 (__v4di)__W);
99}80}
10081
101static __inline__ __m256i __DEFAULT_FN_ATTRS25682static __inline__ __m256i __DEFAULT_FN_ATTRS256
102_mm256_maskz_conflict_epi64 (__mmask8 __U, __m256i __A)83_mm256_maskz_conflict_epi64 (__mmask8 __U, __m256i __A)
103{84{
104 return (__m256i) __builtin_ia32_vpconflictdi_256_mask ((__v4di) __A,85 return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
105 (__v4di) _mm256_setzero_si256 (),86 (__v4di)_mm256_conflict_epi64(__A),
106 (__mmask8) __U);87 (__v4di)_mm256_setzero_si256());
107}88}
10889
109static __inline__ __m128i __DEFAULT_FN_ATTRS12890static __inline__ __m128i __DEFAULT_FN_ATTRS128
110_mm_conflict_epi32 (__m128i __A)91_mm_conflict_epi32 (__m128i __A)
111{92{
112 return (__m128i) __builtin_ia32_vpconflictsi_128_mask ((__v4si) __A,93 return (__m128i) __builtin_ia32_vpconflictsi_128 ((__v4si) __A);
113 (__v4si) _mm_undefined_si128 (),
114 (__mmask8) -1);
115}94}
11695
117static __inline__ __m128i __DEFAULT_FN_ATTRS12896static __inline__ __m128i __DEFAULT_FN_ATTRS128
118_mm_mask_conflict_epi32 (__m128i __W, __mmask8 __U, __m128i __A)97_mm_mask_conflict_epi32 (__m128i __W, __mmask8 __U, __m128i __A)
119{98{
120 return (__m128i) __builtin_ia32_vpconflictsi_128_mask ((__v4si) __A,99 return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
121 (__v4si) __W,100 (__v4si)_mm_conflict_epi32(__A),
122 (__mmask8) __U);101 (__v4si)__W);
123}102}
124103
125static __inline__ __m128i __DEFAULT_FN_ATTRS128104static __inline__ __m128i __DEFAULT_FN_ATTRS128
126_mm_maskz_conflict_epi32 (__mmask8 __U, __m128i __A)105_mm_maskz_conflict_epi32 (__mmask8 __U, __m128i __A)
127{106{
128 return (__m128i) __builtin_ia32_vpconflictsi_128_mask ((__v4si) __A,107 return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
129 (__v4si) _mm_setzero_si128 (),108 (__v4si)_mm_conflict_epi32(__A),
130 (__mmask8) __U);109 (__v4si)_mm_setzero_si128());
131}110}
132111
133static __inline__ __m256i __DEFAULT_FN_ATTRS256112static __inline__ __m256i __DEFAULT_FN_ATTRS256
134_mm256_conflict_epi32 (__m256i __A)113_mm256_conflict_epi32 (__m256i __A)
135{114{
136 return (__m256i) __builtin_ia32_vpconflictsi_256_mask ((__v8si) __A,115 return (__m256i) __builtin_ia32_vpconflictsi_256 ((__v8si) __A);
137 (__v8si) _mm256_undefined_si256 (),
138 (__mmask8) -1);
139}116}
140117
141static __inline__ __m256i __DEFAULT_FN_ATTRS256118static __inline__ __m256i __DEFAULT_FN_ATTRS256
142_mm256_mask_conflict_epi32 (__m256i __W, __mmask8 __U, __m256i __A)119_mm256_mask_conflict_epi32 (__m256i __W, __mmask8 __U, __m256i __A)
143{120{
144 return (__m256i) __builtin_ia32_vpconflictsi_256_mask ((__v8si) __A,121 return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
145 (__v8si) __W,122 (__v8si)_mm256_conflict_epi32(__A),
146 (__mmask8) __U);123 (__v8si)__W);
147}124}
148125
149static __inline__ __m256i __DEFAULT_FN_ATTRS256126static __inline__ __m256i __DEFAULT_FN_ATTRS256
150_mm256_maskz_conflict_epi32 (__mmask8 __U, __m256i __A)127_mm256_maskz_conflict_epi32 (__mmask8 __U, __m256i __A)
151{128{
152 return (__m256i) __builtin_ia32_vpconflictsi_256_mask ((__v8si) __A,129 return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
153 (__v8si)130 (__v8si)_mm256_conflict_epi32(__A),
154 _mm256_setzero_si256 (),131 (__v8si)_mm256_setzero_si256());
155 (__mmask8) __U);
156}132}
157133
158static __inline__ __m128i __DEFAULT_FN_ATTRS128134static __inline__ __m128i __DEFAULT_FN_ATTRS128
lib/include/avx512vldqintrin.h+17-35
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- avx512vldqintrin.h - AVX512VL and AVX512DQ intrinsics ------------===1/*===---- avx512vldqintrin.h - AVX512VL and AVX512DQ intrinsics ------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -523,23 +509,21 @@ _mm_maskz_cvtepi64_ps (__mmask8 __U, __m128i __A) {...@@ -523,23 +509,21 @@ _mm_maskz_cvtepi64_ps (__mmask8 __U, __m128i __A) {
523509
524static __inline__ __m128 __DEFAULT_FN_ATTRS256510static __inline__ __m128 __DEFAULT_FN_ATTRS256
525_mm256_cvtepi64_ps (__m256i __A) {511_mm256_cvtepi64_ps (__m256i __A) {
526 return (__m128) __builtin_ia32_cvtqq2ps256_mask ((__v4di) __A,512 return (__m128)__builtin_convertvector((__v4di)__A, __v4sf);
527 (__v4sf) _mm_setzero_ps(),
528 (__mmask8) -1);
529}513}
530514
531static __inline__ __m128 __DEFAULT_FN_ATTRS256515static __inline__ __m128 __DEFAULT_FN_ATTRS256
532_mm256_mask_cvtepi64_ps (__m128 __W, __mmask8 __U, __m256i __A) {516_mm256_mask_cvtepi64_ps (__m128 __W, __mmask8 __U, __m256i __A) {
533 return (__m128) __builtin_ia32_cvtqq2ps256_mask ((__v4di) __A,517 return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
534 (__v4sf) __W,518 (__v4sf)_mm256_cvtepi64_ps(__A),
535 (__mmask8) __U);519 (__v4sf)__W);
536}520}
537521
538static __inline__ __m128 __DEFAULT_FN_ATTRS256522static __inline__ __m128 __DEFAULT_FN_ATTRS256
539_mm256_maskz_cvtepi64_ps (__mmask8 __U, __m256i __A) {523_mm256_maskz_cvtepi64_ps (__mmask8 __U, __m256i __A) {
540 return (__m128) __builtin_ia32_cvtqq2ps256_mask ((__v4di) __A,524 return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
541 (__v4sf) _mm_setzero_ps(),525 (__v4sf)_mm256_cvtepi64_ps(__A),
542 (__mmask8) __U);526 (__v4sf)_mm_setzero_ps());
543}527}
544528
545static __inline__ __m128i __DEFAULT_FN_ATTRS128529static __inline__ __m128i __DEFAULT_FN_ATTRS128
...@@ -771,23 +755,21 @@ _mm_maskz_cvtepu64_ps (__mmask8 __U, __m128i __A) {...@@ -771,23 +755,21 @@ _mm_maskz_cvtepu64_ps (__mmask8 __U, __m128i __A) {
771755
772static __inline__ __m128 __DEFAULT_FN_ATTRS256756static __inline__ __m128 __DEFAULT_FN_ATTRS256
773_mm256_cvtepu64_ps (__m256i __A) {757_mm256_cvtepu64_ps (__m256i __A) {
774 return (__m128) __builtin_ia32_cvtuqq2ps256_mask ((__v4di) __A,758 return (__m128)__builtin_convertvector((__v4du)__A, __v4sf);
775 (__v4sf) _mm_setzero_ps(),
776 (__mmask8) -1);
777}759}
778760
779static __inline__ __m128 __DEFAULT_FN_ATTRS256761static __inline__ __m128 __DEFAULT_FN_ATTRS256
780_mm256_mask_cvtepu64_ps (__m128 __W, __mmask8 __U, __m256i __A) {762_mm256_mask_cvtepu64_ps (__m128 __W, __mmask8 __U, __m256i __A) {
781 return (__m128) __builtin_ia32_cvtuqq2ps256_mask ((__v4di) __A,763 return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
782 (__v4sf) __W,764 (__v4sf)_mm256_cvtepu64_ps(__A),
783 (__mmask8) __U);765 (__v4sf)__W);
784}766}
785767
786static __inline__ __m128 __DEFAULT_FN_ATTRS256768static __inline__ __m128 __DEFAULT_FN_ATTRS256
787_mm256_maskz_cvtepu64_ps (__mmask8 __U, __m256i __A) {769_mm256_maskz_cvtepu64_ps (__mmask8 __U, __m256i __A) {
788 return (__m128) __builtin_ia32_cvtuqq2ps256_mask ((__v4di) __A,770 return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
789 (__v4sf) _mm_setzero_ps(),771 (__v4sf)_mm256_cvtepu64_ps(__A),
790 (__mmask8) __U);772 (__v4sf)_mm_setzero_ps());
791}773}
792774
793#define _mm_range_pd(A, B, C) \775#define _mm_range_pd(A, B, C) \
lib/include/avx512vlintrin.h+19-58
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- avx512vlintrin.h - AVX512VL intrinsics ---------------------------===1/*===---- avx512vlintrin.h - AVX512VL intrinsics ---------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -5513,7 +5499,7 @@ static __inline __m128i __DEFAULT_FN_ATTRS128...@@ -5513,7 +5499,7 @@ static __inline __m128i __DEFAULT_FN_ATTRS128
5513_mm_loadu_epi64 (void const *__P)5499_mm_loadu_epi64 (void const *__P)
5514{5500{
5515 struct __loadu_epi64 {5501 struct __loadu_epi64 {
5516 __m128i __v;5502 __m128i_u __v;
5517 } __attribute__((__packed__, __may_alias__));5503 } __attribute__((__packed__, __may_alias__));
5518 return ((struct __loadu_epi64*)__P)->__v;5504 return ((struct __loadu_epi64*)__P)->__v;
5519}5505}
...@@ -5539,7 +5525,7 @@ static __inline __m256i __DEFAULT_FN_ATTRS256...@@ -5539,7 +5525,7 @@ static __inline __m256i __DEFAULT_FN_ATTRS256
5539_mm256_loadu_epi64 (void const *__P)5525_mm256_loadu_epi64 (void const *__P)
5540{5526{
5541 struct __loadu_epi64 {5527 struct __loadu_epi64 {
5542 __m256i __v;5528 __m256i_u __v;
5543 } __attribute__((__packed__, __may_alias__));5529 } __attribute__((__packed__, __may_alias__));
5544 return ((struct __loadu_epi64*)__P)->__v;5530 return ((struct __loadu_epi64*)__P)->__v;
5545}5531}
...@@ -5565,7 +5551,7 @@ static __inline __m128i __DEFAULT_FN_ATTRS128...@@ -5565,7 +5551,7 @@ static __inline __m128i __DEFAULT_FN_ATTRS128
5565_mm_loadu_epi32 (void const *__P)5551_mm_loadu_epi32 (void const *__P)
5566{5552{
5567 struct __loadu_epi32 {5553 struct __loadu_epi32 {
5568 __m128i __v;5554 __m128i_u __v;
5569 } __attribute__((__packed__, __may_alias__));5555 } __attribute__((__packed__, __may_alias__));
5570 return ((struct __loadu_epi32*)__P)->__v;5556 return ((struct __loadu_epi32*)__P)->__v;
5571}5557}
...@@ -5591,7 +5577,7 @@ static __inline __m256i __DEFAULT_FN_ATTRS256...@@ -5591,7 +5577,7 @@ static __inline __m256i __DEFAULT_FN_ATTRS256
5591_mm256_loadu_epi32 (void const *__P)5577_mm256_loadu_epi32 (void const *__P)
5592{5578{
5593 struct __loadu_epi32 {5579 struct __loadu_epi32 {
5594 __m256i __v;5580 __m256i_u __v;
5595 } __attribute__((__packed__, __may_alias__));5581 } __attribute__((__packed__, __may_alias__));
5596 return ((struct __loadu_epi32*)__P)->__v;5582 return ((struct __loadu_epi32*)__P)->__v;
5597}5583}
...@@ -5717,7 +5703,7 @@ static __inline void __DEFAULT_FN_ATTRS128...@@ -5717,7 +5703,7 @@ static __inline void __DEFAULT_FN_ATTRS128
5717_mm_storeu_epi64 (void *__P, __m128i __A)5703_mm_storeu_epi64 (void *__P, __m128i __A)
5718{5704{
5719 struct __storeu_epi64 {5705 struct __storeu_epi64 {
5720 __m128i __v;5706 __m128i_u __v;
5721 } __attribute__((__packed__, __may_alias__));5707 } __attribute__((__packed__, __may_alias__));
5722 ((struct __storeu_epi64*)__P)->__v = __A;5708 ((struct __storeu_epi64*)__P)->__v = __A;
5723}5709}
...@@ -5734,7 +5720,7 @@ static __inline void __DEFAULT_FN_ATTRS256...@@ -5734,7 +5720,7 @@ static __inline void __DEFAULT_FN_ATTRS256
5734_mm256_storeu_epi64 (void *__P, __m256i __A)5720_mm256_storeu_epi64 (void *__P, __m256i __A)
5735{5721{
5736 struct __storeu_epi64 {5722 struct __storeu_epi64 {
5737 __m256i __v;5723 __m256i_u __v;
5738 } __attribute__((__packed__, __may_alias__));5724 } __attribute__((__packed__, __may_alias__));
5739 ((struct __storeu_epi64*)__P)->__v = __A;5725 ((struct __storeu_epi64*)__P)->__v = __A;
5740}5726}
...@@ -5751,7 +5737,7 @@ static __inline void __DEFAULT_FN_ATTRS128...@@ -5751,7 +5737,7 @@ static __inline void __DEFAULT_FN_ATTRS128
5751_mm_storeu_epi32 (void *__P, __m128i __A)5737_mm_storeu_epi32 (void *__P, __m128i __A)
5752{5738{
5753 struct __storeu_epi32 {5739 struct __storeu_epi32 {
5754 __m128i __v;5740 __m128i_u __v;
5755 } __attribute__((__packed__, __may_alias__));5741 } __attribute__((__packed__, __may_alias__));
5756 ((struct __storeu_epi32*)__P)->__v = __A;5742 ((struct __storeu_epi32*)__P)->__v = __A;
5757}5743}
...@@ -5768,7 +5754,7 @@ static __inline void __DEFAULT_FN_ATTRS256...@@ -5768,7 +5754,7 @@ static __inline void __DEFAULT_FN_ATTRS256
5768_mm256_storeu_epi32 (void *__P, __m256i __A)5754_mm256_storeu_epi32 (void *__P, __m256i __A)
5769{5755{
5770 struct __storeu_epi32 {5756 struct __storeu_epi32 {
5771 __m256i __v;5757 __m256i_u __v;
5772 } __attribute__((__packed__, __may_alias__));5758 } __attribute__((__packed__, __may_alias__));
5773 ((struct __storeu_epi32*)__P)->__v = __A;5759 ((struct __storeu_epi32*)__P)->__v = __A;
5774}5760}
...@@ -7000,7 +6986,7 @@ _mm_mask_cvtsepi32_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A)...@@ -7000,7 +6986,7 @@ _mm_mask_cvtsepi32_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A)
7000 __builtin_ia32_pmovsdb128mem_mask ((__v16qi *) __P, (__v4si) __A, __M);6986 __builtin_ia32_pmovsdb128mem_mask ((__v16qi *) __P, (__v4si) __A, __M);
7001}6987}
70026988
7003static __inline__ __m128i __DEFAULT_FN_ATTRS1286989static __inline__ __m128i __DEFAULT_FN_ATTRS256
7004_mm256_cvtsepi32_epi8 (__m256i __A)6990_mm256_cvtsepi32_epi8 (__m256i __A)
7005{6991{
7006 return (__m128i) __builtin_ia32_pmovsdb256_mask ((__v8si) __A,6992 return (__m128i) __builtin_ia32_pmovsdb256_mask ((__v8si) __A,
...@@ -7023,7 +7009,7 @@ _mm256_maskz_cvtsepi32_epi8 (__mmask8 __M, __m256i __A)...@@ -7023,7 +7009,7 @@ _mm256_maskz_cvtsepi32_epi8 (__mmask8 __M, __m256i __A)
7023 __M);7009 __M);
7024}7010}
70257011
7026static __inline__ void __DEFAULT_FN_ATTRS1287012static __inline__ void __DEFAULT_FN_ATTRS256
7027_mm256_mask_cvtsepi32_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A)7013_mm256_mask_cvtsepi32_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A)
7028{7014{
7029 __builtin_ia32_pmovsdb256mem_mask ((__v16qi *) __P, (__v8si) __A, __M);7015 __builtin_ia32_pmovsdb256mem_mask ((__v16qi *) __P, (__v8si) __A, __M);
...@@ -7581,7 +7567,7 @@ _mm_maskz_cvtepi32_epi8 (__mmask8 __M, __m128i __A)...@@ -7581,7 +7567,7 @@ _mm_maskz_cvtepi32_epi8 (__mmask8 __M, __m128i __A)
7581 __M);7567 __M);
7582}7568}
75837569
7584static __inline__ void __DEFAULT_FN_ATTRS2567570static __inline__ void __DEFAULT_FN_ATTRS128
7585_mm_mask_cvtepi32_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A)7571_mm_mask_cvtepi32_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A)
7586{7572{
7587 __builtin_ia32_pmovdb128mem_mask ((__v16qi *) __P, (__v4si) __A, __M);7573 __builtin_ia32_pmovdb128mem_mask ((__v16qi *) __P, (__v4si) __A, __M);
...@@ -8425,22 +8411,6 @@ _mm256_maskz_cvtph_ps (__mmask8 __U, __m128i __A)...@@ -8425,22 +8411,6 @@ _mm256_maskz_cvtph_ps (__mmask8 __U, __m128i __A)
8425 (__mmask8) __U);8411 (__mmask8) __U);
8426}8412}
84278413
8428static __inline __m128i __DEFAULT_FN_ATTRS128
8429_mm_mask_cvtps_ph (__m128i __W, __mmask8 __U, __m128 __A)
8430{
8431 return (__m128i) __builtin_ia32_vcvtps2ph_mask ((__v4sf) __A, _MM_FROUND_CUR_DIRECTION,
8432 (__v8hi) __W,
8433 (__mmask8) __U);
8434}
8435
8436static __inline __m128i __DEFAULT_FN_ATTRS128
8437_mm_maskz_cvtps_ph (__mmask8 __U, __m128 __A)
8438{
8439 return (__m128i) __builtin_ia32_vcvtps2ph_mask ((__v4sf) __A, _MM_FROUND_CUR_DIRECTION,
8440 (__v8hi) _mm_setzero_si128 (),
8441 (__mmask8) __U);
8442}
8443
8444#define _mm_mask_cvt_roundps_ph(W, U, A, I) \8414#define _mm_mask_cvt_roundps_ph(W, U, A, I) \
8445 (__m128i)__builtin_ia32_vcvtps2ph_mask((__v4sf)(__m128)(A), (int)(I), \8415 (__m128i)__builtin_ia32_vcvtps2ph_mask((__v4sf)(__m128)(A), (int)(I), \
8446 (__v8hi)(__m128i)(W), \8416 (__v8hi)(__m128i)(W), \
...@@ -8451,21 +8421,9 @@ _mm_maskz_cvtps_ph (__mmask8 __U, __m128 __A)...@@ -8451,21 +8421,9 @@ _mm_maskz_cvtps_ph (__mmask8 __U, __m128 __A)
8451 (__v8hi)_mm_setzero_si128(), \8421 (__v8hi)_mm_setzero_si128(), \
8452 (__mmask8)(U))8422 (__mmask8)(U))
84538423
8454static __inline __m128i __DEFAULT_FN_ATTRS2568424#define _mm_mask_cvtps_ph _mm_mask_cvt_roundps_ph
8455_mm256_mask_cvtps_ph (__m128i __W, __mmask8 __U, __m256 __A)8425#define _mm_maskz_cvtps_ph _mm_maskz_cvt_roundps_ph
8456{
8457 return (__m128i) __builtin_ia32_vcvtps2ph256_mask ((__v8sf) __A, _MM_FROUND_CUR_DIRECTION,
8458 (__v8hi) __W,
8459 (__mmask8) __U);
8460}
84618426
8462static __inline __m128i __DEFAULT_FN_ATTRS256
8463_mm256_maskz_cvtps_ph ( __mmask8 __U, __m256 __A)
8464{
8465 return (__m128i) __builtin_ia32_vcvtps2ph256_mask ((__v8sf) __A, _MM_FROUND_CUR_DIRECTION,
8466 (__v8hi) _mm_setzero_si128(),
8467 (__mmask8) __U);
8468}
8469#define _mm256_mask_cvt_roundps_ph(W, U, A, I) \8427#define _mm256_mask_cvt_roundps_ph(W, U, A, I) \
8470 (__m128i)__builtin_ia32_vcvtps2ph256_mask((__v8sf)(__m256)(A), (int)(I), \8428 (__m128i)__builtin_ia32_vcvtps2ph256_mask((__v8sf)(__m256)(A), (int)(I), \
8471 (__v8hi)(__m128i)(W), \8429 (__v8hi)(__m128i)(W), \
...@@ -8476,6 +8434,9 @@ _mm256_maskz_cvtps_ph ( __mmask8 __U, __m256 __A)...@@ -8476,6 +8434,9 @@ _mm256_maskz_cvtps_ph ( __mmask8 __U, __m256 __A)
8476 (__v8hi)_mm_setzero_si128(), \8434 (__v8hi)_mm_setzero_si128(), \
8477 (__mmask8)(U))8435 (__mmask8)(U))
84788436
8437#define _mm256_mask_cvtps_ph _mm256_mask_cvt_roundps_ph
8438#define _mm256_maskz_cvtps_ph _mm256_maskz_cvt_roundps_ph
8439
84798440
8480#undef __DEFAULT_FN_ATTRS1288441#undef __DEFAULT_FN_ATTRS128
8481#undef __DEFAULT_FN_ATTRS2568442#undef __DEFAULT_FN_ATTRS256
lib/include/avx512vlvbmi2intrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===------------- avx512vlvbmi2intrin.h - VBMI2 intrinsics -----------------===1/*===------------- avx512vlvbmi2intrin.h - VBMI2 intrinsics -----------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/avx512vlvnniintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===------------- avx512vlvnniintrin.h - VNNI intrinsics ------------------===1/*===------------- avx512vlvnniintrin.h - VNNI intrinsics ------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/avx512vlvp2intersectintrin.h created+121
...@@ -0,0 +1,121 @@
1/*===------ avx512vlvp2intersectintrin.h - VL VP2INTERSECT intrinsics ------===
2 *
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 * of this software and associated documentation files (the "Software"), to deal
6 * in the Software without restriction, including without limitation the rights
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *
22 *===-----------------------------------------------------------------------===
23 */
24#ifndef __IMMINTRIN_H
25#error "Never use <avx512vlvp2intersectintrin.h> directly; include <immintrin.h> instead."
26#endif
27
28#ifndef _AVX512VLVP2INTERSECT_H
29#define _AVX512VLVP2INTERSECT_H
30
31#define __DEFAULT_FN_ATTRS128 \
32 __attribute__((__always_inline__, __nodebug__, __target__("avx512vl,avx512vp2intersect"), \
33 __min_vector_width__(128)))
34
35#define __DEFAULT_FN_ATTRS256 \
36 __attribute__((__always_inline__, __nodebug__, __target__("avx512vl,avx512vp2intersect"), \
37 __min_vector_width__(256)))
38/// Store, in an even/odd pair of mask registers, the indicators of the
39/// locations of value matches between dwords in operands __a and __b.
40///
41/// \headerfile <x86intrin.h>
42///
43/// This intrinsic corresponds to the <c> VP2INTERSECTD </c> instruction.
44///
45/// \param __a
46/// A 256-bit vector of [8 x i32].
47/// \param __b
48/// A 256-bit vector of [8 x i32]
49/// \param __m0
50/// A pointer point to 8-bit mask
51/// \param __m1
52/// A pointer point to 8-bit mask
53static __inline__ void __DEFAULT_FN_ATTRS256
54_mm256_2intersect_epi32(__m256i __a, __m256i __b, __mmask8 *__m0, __mmask8 *__m1) {
55 __builtin_ia32_vp2intersect_d_256((__v8si)__a, (__v8si)__b, __m0, __m1);
56}
57
58/// Store, in an even/odd pair of mask registers, the indicators of the
59/// locations of value matches between quadwords in operands __a and __b.
60///
61/// \headerfile <x86intrin.h>
62///
63/// This intrinsic corresponds to the <c> VP2INTERSECTQ </c> instruction.
64///
65/// \param __a
66/// A 256-bit vector of [4 x i64].
67/// \param __b
68/// A 256-bit vector of [4 x i64]
69/// \param __m0
70/// A pointer point to 8-bit mask
71/// \param __m1
72/// A pointer point to 8-bit mask
73static __inline__ void __DEFAULT_FN_ATTRS256
74_mm256_2intersect_epi64(__m256i __a, __m256i __b, __mmask8 *__m0, __mmask8 *__m1) {
75 __builtin_ia32_vp2intersect_q_256((__v4di)__a, (__v4di)__b, __m0, __m1);
76}
77
78/// Store, in an even/odd pair of mask registers, the indicators of the
79/// locations of value matches between dwords in operands __a and __b.
80///
81/// \headerfile <x86intrin.h>
82///
83/// This intrinsic corresponds to the <c> VP2INTERSECTD </c> instruction.
84///
85/// \param __a
86/// A 128-bit vector of [4 x i32].
87/// \param __b
88/// A 128-bit vector of [4 x i32]
89/// \param __m0
90/// A pointer point to 8-bit mask
91/// \param __m1
92/// A pointer point to 8-bit mask
93static __inline__ void __DEFAULT_FN_ATTRS128
94_mm_2intersect_epi32(__m128i __a, __m128i __b, __mmask8 *__m0, __mmask8 *__m1) {
95 __builtin_ia32_vp2intersect_d_128((__v4si)__a, (__v4si)__b, __m0, __m1);
96}
97
98/// Store, in an even/odd pair of mask registers, the indicators of the
99/// locations of value matches between quadwords in operands __a and __b.
100///
101/// \headerfile <x86intrin.h>
102///
103/// This intrinsic corresponds to the <c> VP2INTERSECTQ </c> instruction.
104///
105/// \param __a
106/// A 128-bit vector of [2 x i64].
107/// \param __b
108/// A 128-bit vector of [2 x i64]
109/// \param __m0
110/// A pointer point to 8-bit mask
111/// \param __m1
112/// A pointer point to 8-bit mask
113static __inline__ void __DEFAULT_FN_ATTRS128
114_mm_2intersect_epi64(__m128i __a, __m128i __b, __mmask8 *__m0, __mmask8 *__m1) {
115 __builtin_ia32_vp2intersect_q_128((__v2di)__a, (__v2di)__b, __m0, __m1);
116}
117
118#undef __DEFAULT_FN_ATTRS128
119#undef __DEFAULT_FN_ATTRS256
120
121#endif
lib/include/avx512vnniintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===------------- avx512vnniintrin.h - VNNI intrinsics ------------------===1/*===------------- avx512vnniintrin.h - VNNI intrinsics ------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/avx512vp2intersectintrin.h created+77
...@@ -0,0 +1,77 @@
1/*===------- avx512vpintersectintrin.h - VP2INTERSECT intrinsics ------------===
2 *
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 * of this software and associated documentation files (the "Software"), to deal
6 * in the Software without restriction, including without limitation the rights
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *
22 *===-----------------------------------------------------------------------===
23 */
24#ifndef __IMMINTRIN_H
25#error "Never use <avx512vp2intersect.h> directly; include <immintrin.h> instead."
26#endif
27
28#ifndef _AVX512VP2INTERSECT_H
29#define _AVX512VP2INTERSECT_H
30
31#define __DEFAULT_FN_ATTRS \
32 __attribute__((__always_inline__, __nodebug__, __target__("avx512vp2intersect"), \
33 __min_vector_width__(512)))
34
35/// Store, in an even/odd pair of mask registers, the indicators of the
36/// locations of value matches between dwords in operands __a and __b.
37///
38/// \headerfile <x86intrin.h>
39///
40/// This intrinsic corresponds to the <c> VP2INTERSECTD </c> instruction.
41///
42/// \param __a
43/// A 512-bit vector of [16 x i32].
44/// \param __b
45/// A 512-bit vector of [16 x i32]
46/// \param __m0
47/// A pointer point to 16-bit mask
48/// \param __m1
49/// A pointer point to 16-bit mask
50static __inline__ void __DEFAULT_FN_ATTRS
51_mm512_2intersect_epi32(__m512i __a, __m512i __b, __mmask16 *__m0, __mmask16 *__m1) {
52 __builtin_ia32_vp2intersect_d_512((__v16si)__a, (__v16si)__b, __m0, __m1);
53}
54
55/// Store, in an even/odd pair of mask registers, the indicators of the
56/// locations of value matches between quadwords in operands __a and __b.
57///
58/// \headerfile <x86intrin.h>
59///
60/// This intrinsic corresponds to the <c> VP2INTERSECTQ </c> instruction.
61///
62/// \param __a
63/// A 512-bit vector of [8 x i64].
64/// \param __b
65/// A 512-bit vector of [8 x i64]
66/// \param __m0
67/// A pointer point to 8-bit mask
68/// \param __m1
69/// A pointer point to 8-bit mask
70static __inline__ void __DEFAULT_FN_ATTRS
71_mm512_2intersect_epi64(__m512i __a, __m512i __b, __mmask8 *__m0, __mmask8 *__m1) {
72 __builtin_ia32_vp2intersect_q_512((__v8di)__a, (__v8di)__b, __m0, __m1);
73}
74
75#undef __DEFAULT_FN_ATTRS
76
77#endif
lib/include/avx512vpopcntdqintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===----- avx512vpopcntdqintrin.h - AVX512VPOPCNTDQ intrinsics-------------===1/*===----- avx512vpopcntdqintrin.h - AVX512VPOPCNTDQ intrinsics-------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/avx512vpopcntdqvlintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===---- avx512vpopcntdqintrin.h - AVX512VPOPCNTDQ intrinsics -------------===1/*===---- avx512vpopcntdqintrin.h - AVX512VPOPCNTDQ intrinsics -------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/avxintrin.h+20-30
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- avxintrin.h - AVX intrinsics -------------------------------------===1/*===---- avxintrin.h - AVX intrinsics -------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -45,9 +31,13 @@ typedef unsigned char __v32qu __attribute__ ((__vector_size__ (32)));...@@ -45,9 +31,13 @@ typedef unsigned char __v32qu __attribute__ ((__vector_size__ (32)));
45 * appear in the interface though. */31 * appear in the interface though. */
46typedef signed char __v32qs __attribute__((__vector_size__(32)));32typedef signed char __v32qs __attribute__((__vector_size__(32)));
4733
48typedef float __m256 __attribute__ ((__vector_size__ (32)));34typedef float __m256 __attribute__ ((__vector_size__ (32), __aligned__(32)));
49typedef double __m256d __attribute__((__vector_size__(32)));35typedef double __m256d __attribute__((__vector_size__(32), __aligned__(32)));
50typedef long long __m256i __attribute__((__vector_size__(32)));36typedef long long __m256i __attribute__((__vector_size__(32), __aligned__(32)));
37
38typedef float __m256_u __attribute__ ((__vector_size__ (32), __aligned__(1)));
39typedef double __m256d_u __attribute__((__vector_size__(32), __aligned__(1)));
40typedef long long __m256i_u __attribute__((__vector_size__(32), __aligned__(1)));
5141
52/* Define the default attributes for the functions in this file. */42/* Define the default attributes for the functions in this file. */
53#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("avx"), __min_vector_width__(256)))43#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("avx"), __min_vector_width__(256)))
...@@ -3113,7 +3103,7 @@ static __inline __m256d __DEFAULT_FN_ATTRS...@@ -3113,7 +3103,7 @@ static __inline __m256d __DEFAULT_FN_ATTRS
3113_mm256_loadu_pd(double const *__p)3103_mm256_loadu_pd(double const *__p)
3114{3104{
3115 struct __loadu_pd {3105 struct __loadu_pd {
3116 __m256d __v;3106 __m256d_u __v;
3117 } __attribute__((__packed__, __may_alias__));3107 } __attribute__((__packed__, __may_alias__));
3118 return ((struct __loadu_pd*)__p)->__v;3108 return ((struct __loadu_pd*)__p)->__v;
3119}3109}
...@@ -3133,7 +3123,7 @@ static __inline __m256 __DEFAULT_FN_ATTRS...@@ -3133,7 +3123,7 @@ static __inline __m256 __DEFAULT_FN_ATTRS
3133_mm256_loadu_ps(float const *__p)3123_mm256_loadu_ps(float const *__p)
3134{3124{
3135 struct __loadu_ps {3125 struct __loadu_ps {
3136 __m256 __v;3126 __m256_u __v;
3137 } __attribute__((__packed__, __may_alias__));3127 } __attribute__((__packed__, __may_alias__));
3138 return ((struct __loadu_ps*)__p)->__v;3128 return ((struct __loadu_ps*)__p)->__v;
3139}3129}
...@@ -3166,10 +3156,10 @@ _mm256_load_si256(__m256i const *__p)...@@ -3166,10 +3156,10 @@ _mm256_load_si256(__m256i const *__p)
3166/// A pointer to a 256-bit integer vector containing integer values.3156/// A pointer to a 256-bit integer vector containing integer values.
3167/// \returns A 256-bit integer vector containing the moved values.3157/// \returns A 256-bit integer vector containing the moved values.
3168static __inline __m256i __DEFAULT_FN_ATTRS3158static __inline __m256i __DEFAULT_FN_ATTRS
3169_mm256_loadu_si256(__m256i const *__p)3159_mm256_loadu_si256(__m256i_u const *__p)
3170{3160{
3171 struct __loadu_si256 {3161 struct __loadu_si256 {
3172 __m256i __v;3162 __m256i_u __v;
3173 } __attribute__((__packed__, __may_alias__));3163 } __attribute__((__packed__, __may_alias__));
3174 return ((struct __loadu_si256*)__p)->__v;3164 return ((struct __loadu_si256*)__p)->__v;
3175}3165}
...@@ -3246,7 +3236,7 @@ static __inline void __DEFAULT_FN_ATTRS...@@ -3246,7 +3236,7 @@ static __inline void __DEFAULT_FN_ATTRS
3246_mm256_storeu_pd(double *__p, __m256d __a)3236_mm256_storeu_pd(double *__p, __m256d __a)
3247{3237{
3248 struct __storeu_pd {3238 struct __storeu_pd {
3249 __m256d __v;3239 __m256d_u __v;
3250 } __attribute__((__packed__, __may_alias__));3240 } __attribute__((__packed__, __may_alias__));
3251 ((struct __storeu_pd*)__p)->__v = __a;3241 ((struct __storeu_pd*)__p)->__v = __a;
3252}3242}
...@@ -3266,7 +3256,7 @@ static __inline void __DEFAULT_FN_ATTRS...@@ -3266,7 +3256,7 @@ static __inline void __DEFAULT_FN_ATTRS
3266_mm256_storeu_ps(float *__p, __m256 __a)3256_mm256_storeu_ps(float *__p, __m256 __a)
3267{3257{
3268 struct __storeu_ps {3258 struct __storeu_ps {
3269 __m256 __v;3259 __m256_u __v;
3270 } __attribute__((__packed__, __may_alias__));3260 } __attribute__((__packed__, __may_alias__));
3271 ((struct __storeu_ps*)__p)->__v = __a;3261 ((struct __storeu_ps*)__p)->__v = __a;
3272}3262}
...@@ -3301,10 +3291,10 @@ _mm256_store_si256(__m256i *__p, __m256i __a)...@@ -3301,10 +3291,10 @@ _mm256_store_si256(__m256i *__p, __m256i __a)
3301/// \param __a3291/// \param __a
3302/// A 256-bit integer vector containing the values to be moved.3292/// A 256-bit integer vector containing the values to be moved.
3303static __inline void __DEFAULT_FN_ATTRS3293static __inline void __DEFAULT_FN_ATTRS
3304_mm256_storeu_si256(__m256i *__p, __m256i __a)3294_mm256_storeu_si256(__m256i_u *__p, __m256i __a)
3305{3295{
3306 struct __storeu_si256 {3296 struct __storeu_si256 {
3307 __m256i __v;3297 __m256i_u __v;
3308 } __attribute__((__packed__, __may_alias__));3298 } __attribute__((__packed__, __may_alias__));
3309 ((struct __storeu_si256*)__p)->__v = __a;3299 ((struct __storeu_si256*)__p)->__v = __a;
3310}3300}
...@@ -4834,7 +4824,7 @@ _mm256_loadu2_m128d(double const *__addr_hi, double const *__addr_lo)...@@ -4834,7 +4824,7 @@ _mm256_loadu2_m128d(double const *__addr_hi, double const *__addr_lo)
4834/// address of the memory location does not have to be aligned.4824/// address of the memory location does not have to be aligned.
4835/// \returns A 256-bit integer vector containing the concatenated result.4825/// \returns A 256-bit integer vector containing the concatenated result.
4836static __inline __m256i __DEFAULT_FN_ATTRS4826static __inline __m256i __DEFAULT_FN_ATTRS
4837_mm256_loadu2_m128i(__m128i const *__addr_hi, __m128i const *__addr_lo)4827_mm256_loadu2_m128i(__m128i_u const *__addr_hi, __m128i_u const *__addr_lo)
4838{4828{
4839 __m256i __v256 = _mm256_castsi128_si256(_mm_loadu_si128(__addr_lo));4829 __m256i __v256 = _mm256_castsi128_si256(_mm_loadu_si128(__addr_lo));
4840 return _mm256_insertf128_si256(__v256, _mm_loadu_si128(__addr_hi), 1);4830 return _mm256_insertf128_si256(__v256, _mm_loadu_si128(__addr_hi), 1);
...@@ -4918,7 +4908,7 @@ _mm256_storeu2_m128d(double *__addr_hi, double *__addr_lo, __m256d __a)...@@ -4918,7 +4908,7 @@ _mm256_storeu2_m128d(double *__addr_hi, double *__addr_lo, __m256d __a)
4918/// \param __a4908/// \param __a
4919/// A 256-bit integer vector.4909/// A 256-bit integer vector.
4920static __inline void __DEFAULT_FN_ATTRS4910static __inline void __DEFAULT_FN_ATTRS
4921_mm256_storeu2_m128i(__m128i *__addr_hi, __m128i *__addr_lo, __m256i __a)4911_mm256_storeu2_m128i(__m128i_u *__addr_hi, __m128i_u *__addr_lo, __m256i __a)
4922{4912{
4923 __m128i __v128;4913 __m128i __v128;
49244914
lib/include/bmi2intrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- bmi2intrin.h - BMI2 intrinsics -----------------------------------===1/*===---- bmi2intrin.h - BMI2 intrinsics -----------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/bmiintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- bmiintrin.h - BMI intrinsics -------------------------------------===1/*===---- bmiintrin.h - BMI intrinsics -------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/cetintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- cetintrin.h - CET intrinsic --------------------------------------===1/*===---- cetintrin.h - CET intrinsic --------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/cldemoteintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- cldemoteintrin.h - CLDEMOTE intrinsic ----------------------------===1/*===---- cldemoteintrin.h - CLDEMOTE intrinsic ----------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/clflushoptintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- clflushoptintrin.h - CLFLUSHOPT intrinsic ------------------------===1/*===---- clflushoptintrin.h - CLFLUSHOPT intrinsic ------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/clwbintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- clwbintrin.h - CLWB intrinsic ------------------------------------===1/*===---- clwbintrin.h - CLWB intrinsic ------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/clzerointrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===----------------------- clzerointrin.h - CLZERO ----------------------===1/*===----------------------- clzerointrin.h - CLZERO ----------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/cpuid.h+7-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- cpuid.h - X86 cpu model detection --------------------------------===1/*===---- cpuid.h - X86 cpu model detection --------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -191,6 +177,7 @@...@@ -191,6 +177,7 @@
191#define bit_CLDEMOTE 0x02000000177#define bit_CLDEMOTE 0x02000000
192#define bit_MOVDIRI 0x08000000178#define bit_MOVDIRI 0x08000000
193#define bit_MOVDIR64B 0x10000000179#define bit_MOVDIR64B 0x10000000
180#define bit_ENQCMD 0x20000000
194181
195/* Features in %edx for leaf 7 sub-leaf 0 */182/* Features in %edx for leaf 7 sub-leaf 0 */
196#define bit_AVX5124VNNIW 0x00000004183#define bit_AVX5124VNNIW 0x00000004
...@@ -198,6 +185,9 @@...@@ -198,6 +185,9 @@
198#define bit_PCONFIG 0x00040000185#define bit_PCONFIG 0x00040000
199#define bit_IBT 0x00100000186#define bit_IBT 0x00100000
200187
188/* Features in %eax for leaf 7 sub-leaf 1 */
189#define bit_AVX512BF16 0x00000020
190
201/* Features in %eax for leaf 13 sub-leaf 1 */191/* Features in %eax for leaf 13 sub-leaf 1 */
202#define bit_XSAVEOPT 0x00000001192#define bit_XSAVEOPT 0x00000001
203#define bit_XSAVEC 0x00000002193#define bit_XSAVEC 0x00000002
lib/include/emmintrin.h+18-37
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- emmintrin.h - SSE2 intrinsics ------------------------------------===1/*===---- emmintrin.h - SSE2 intrinsics ------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -26,8 +12,11 @@...@@ -26,8 +12,11 @@
2612
27#include <xmmintrin.h>13#include <xmmintrin.h>
2814
29typedef double __m128d __attribute__((__vector_size__(16)));15typedef double __m128d __attribute__((__vector_size__(16), __aligned__(16)));
30typedef long long __m128i __attribute__((__vector_size__(16)));16typedef long long __m128i __attribute__((__vector_size__(16), __aligned__(16)));
17
18typedef double __m128d_u __attribute__((__vector_size__(16), __aligned__(1)));
19typedef long long __m128i_u __attribute__((__vector_size__(16), __aligned__(1)));
3120
32/* Type defines. */21/* Type defines. */
33typedef double __v2df __attribute__ ((__vector_size__ (16)));22typedef double __v2df __attribute__ ((__vector_size__ (16)));
...@@ -1652,7 +1641,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS...@@ -1652,7 +1641,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS
1652_mm_loadu_pd(double const *__dp)1641_mm_loadu_pd(double const *__dp)
1653{1642{
1654 struct __loadu_pd {1643 struct __loadu_pd {
1655 __m128d __v;1644 __m128d_u __v;
1656 } __attribute__((__packed__, __may_alias__));1645 } __attribute__((__packed__, __may_alias__));
1657 return ((struct __loadu_pd*)__dp)->__v;1646 return ((struct __loadu_pd*)__dp)->__v;
1658}1647}
...@@ -2042,7 +2031,7 @@ static __inline__ void __DEFAULT_FN_ATTRS...@@ -2042,7 +2031,7 @@ static __inline__ void __DEFAULT_FN_ATTRS
2042_mm_storeu_pd(double *__dp, __m128d __a)2031_mm_storeu_pd(double *__dp, __m128d __a)
2043{2032{
2044 struct __storeu_pd {2033 struct __storeu_pd {
2045 __m128d __v;2034 __m128d_u __v;
2046 } __attribute__((__packed__, __may_alias__));2035 } __attribute__((__packed__, __may_alias__));
2047 ((struct __storeu_pd*)__dp)->__v = __a;2036 ((struct __storeu_pd*)__dp)->__v = __a;
2048}2037}
...@@ -2316,11 +2305,7 @@ _mm_adds_epu16(__m128i __a, __m128i __b)...@@ -2316,11 +2305,7 @@ _mm_adds_epu16(__m128i __a, __m128i __b)
2316static __inline__ __m128i __DEFAULT_FN_ATTRS2305static __inline__ __m128i __DEFAULT_FN_ATTRS
2317_mm_avg_epu8(__m128i __a, __m128i __b)2306_mm_avg_epu8(__m128i __a, __m128i __b)
2318{2307{
2319 typedef unsigned short __v16hu __attribute__ ((__vector_size__ (32)));2308 return (__m128i)__builtin_ia32_pavgb128((__v16qi)__a, (__v16qi)__b);
2320 return (__m128i)__builtin_convertvector(
2321 ((__builtin_convertvector((__v16qu)__a, __v16hu) +
2322 __builtin_convertvector((__v16qu)__b, __v16hu)) + 1)
2323 >> 1, __v16qu);
2324}2309}
23252310
2326/// Computes the rounded avarages of corresponding elements of two2311/// Computes the rounded avarages of corresponding elements of two
...@@ -2340,11 +2325,7 @@ _mm_avg_epu8(__m128i __a, __m128i __b)...@@ -2340,11 +2325,7 @@ _mm_avg_epu8(__m128i __a, __m128i __b)
2340static __inline__ __m128i __DEFAULT_FN_ATTRS2325static __inline__ __m128i __DEFAULT_FN_ATTRS
2341_mm_avg_epu16(__m128i __a, __m128i __b)2326_mm_avg_epu16(__m128i __a, __m128i __b)
2342{2327{
2343 typedef unsigned int __v8su __attribute__ ((__vector_size__ (32)));2328 return (__m128i)__builtin_ia32_pavgw128((__v8hi)__a, (__v8hi)__b);
2344 return (__m128i)__builtin_convertvector(
2345 ((__builtin_convertvector((__v8hu)__a, __v8su) +
2346 __builtin_convertvector((__v8hu)__b, __v8su)) + 1)
2347 >> 1, __v8hu);
2348}2329}
23492330
2350/// Multiplies the corresponding elements of two 128-bit signed [8 x i16]2331/// Multiplies the corresponding elements of two 128-bit signed [8 x i16]
...@@ -3564,10 +3545,10 @@ _mm_load_si128(__m128i const *__p)...@@ -3564,10 +3545,10 @@ _mm_load_si128(__m128i const *__p)
3564/// A pointer to a memory location containing integer values.3545/// A pointer to a memory location containing integer values.
3565/// \returns A 128-bit integer vector containing the moved values.3546/// \returns A 128-bit integer vector containing the moved values.
3566static __inline__ __m128i __DEFAULT_FN_ATTRS3547static __inline__ __m128i __DEFAULT_FN_ATTRS
3567_mm_loadu_si128(__m128i const *__p)3548_mm_loadu_si128(__m128i_u const *__p)
3568{3549{
3569 struct __loadu_si128 {3550 struct __loadu_si128 {
3570 __m128i __v;3551 __m128i_u __v;
3571 } __attribute__((__packed__, __may_alias__));3552 } __attribute__((__packed__, __may_alias__));
3572 return ((struct __loadu_si128*)__p)->__v;3553 return ((struct __loadu_si128*)__p)->__v;
3573}3554}
...@@ -3585,7 +3566,7 @@ _mm_loadu_si128(__m128i const *__p)...@@ -3585,7 +3566,7 @@ _mm_loadu_si128(__m128i const *__p)
3585/// \returns A 128-bit vector of [2 x i64]. The lower order bits contain the3566/// \returns A 128-bit vector of [2 x i64]. The lower order bits contain the
3586/// moved value. The higher order bits are cleared.3567/// moved value. The higher order bits are cleared.
3587static __inline__ __m128i __DEFAULT_FN_ATTRS3568static __inline__ __m128i __DEFAULT_FN_ATTRS
3588_mm_loadl_epi64(__m128i const *__p)3569_mm_loadl_epi64(__m128i_u const *__p)
3589{3570{
3590 struct __mm_loadl_epi64_struct {3571 struct __mm_loadl_epi64_struct {
3591 long long __u;3572 long long __u;
...@@ -4027,10 +4008,10 @@ _mm_store_si128(__m128i *__p, __m128i __b)...@@ -4027,10 +4008,10 @@ _mm_store_si128(__m128i *__p, __m128i __b)
4027/// \param __b4008/// \param __b
4028/// A 128-bit integer vector containing the values to be moved.4009/// A 128-bit integer vector containing the values to be moved.
4029static __inline__ void __DEFAULT_FN_ATTRS4010static __inline__ void __DEFAULT_FN_ATTRS
4030_mm_storeu_si128(__m128i *__p, __m128i __b)4011_mm_storeu_si128(__m128i_u *__p, __m128i __b)
4031{4012{
4032 struct __storeu_si128 {4013 struct __storeu_si128 {
4033 __m128i __v;4014 __m128i_u __v;
4034 } __attribute__((__packed__, __may_alias__));4015 } __attribute__((__packed__, __may_alias__));
4035 ((struct __storeu_si128*)__p)->__v = __b;4016 ((struct __storeu_si128*)__p)->__v = __b;
4036}4017}
...@@ -4139,7 +4120,7 @@ _mm_maskmoveu_si128(__m128i __d, __m128i __n, char *__p)...@@ -4139,7 +4120,7 @@ _mm_maskmoveu_si128(__m128i __d, __m128i __n, char *__p)
4139/// A 128-bit integer vector of [2 x i64]. The lower 64 bits contain the4120/// A 128-bit integer vector of [2 x i64]. The lower 64 bits contain the
4140/// value to be stored.4121/// value to be stored.
4141static __inline__ void __DEFAULT_FN_ATTRS4122static __inline__ void __DEFAULT_FN_ATTRS
4142_mm_storel_epi64(__m128i *__p, __m128i __a)4123_mm_storel_epi64(__m128i_u *__p, __m128i __a)
4143{4124{
4144 struct __mm_storel_epi64_struct {4125 struct __mm_storel_epi64_struct {
4145 long long __u;4126 long long __u;
lib/include/enqcmdintrin.h created+63
...@@ -0,0 +1,63 @@
1/*===------------------ enqcmdintrin.h - enqcmd intrinsics -----------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error "Never use <enqcmdintrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifndef __ENQCMDINTRIN_H
15#define __ENQCMDINTRIN_H
16
17/* Define the default attributes for the functions in this file */
18#define _DEFAULT_FN_ATTRS \
19 __attribute__((__always_inline__, __nodebug__, __target__("enqcmd")))
20
21/// Reads 64-byte command pointed by \a __src, formats 64-byte enqueue store
22/// data, and performs 64-byte enqueue store to memory pointed by \a __dst.
23/// This intrinsics may only be used in User mode.
24///
25/// \headerfile <x86intrin.h>
26///
27/// This intrinsics corresponds to the <c> ENQCMD </c> instruction.
28///
29/// \param __dst
30/// Pointer to the destination of the enqueue store.
31/// \param __src
32/// Pointer to 64-byte command data.
33/// \returns If the command data is successfully written to \a __dst then 0 is
34/// returned. Otherwise 1 is returned.
35static __inline__ int _DEFAULT_FN_ATTRS
36_enqcmd (void *__dst, const void *__src)
37{
38 return __builtin_ia32_enqcmd(__dst, __src);
39}
40
41/// Reads 64-byte command pointed by \a __src, formats 64-byte enqueue store
42/// data, and performs 64-byte enqueue store to memory pointed by \a __dst
43/// This intrinsic may only be used in Privileged mode.
44///
45/// \headerfile <x86intrin.h>
46///
47/// This intrinsics corresponds to the <c> ENQCMDS </c> instruction.
48///
49/// \param __dst
50/// Pointer to the destination of the enqueue store.
51/// \param __src
52/// Pointer to 64-byte command data.
53/// \returns If the command data is successfully written to \a __dst then 0 is
54/// returned. Otherwise 1 is returned.
55static __inline__ int _DEFAULT_FN_ATTRS
56_enqcmds (void *__dst, const void *__src)
57{
58 return __builtin_ia32_enqcmds(__dst, __src);
59}
60
61#undef _DEFAULT_FN_ATTRS
62
63#endif /* __ENQCMDINTRIN_H */
lib/include/f16cintrin.h+6-20
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- f16cintrin.h - F16C intrinsics -----------------------------------===1/*===---- f16cintrin.h - F16C intrinsics -----------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -52,9 +38,9 @@...@@ -52,9 +38,9 @@
52static __inline float __DEFAULT_FN_ATTRS12838static __inline float __DEFAULT_FN_ATTRS128
53_cvtsh_ss(unsigned short __a)39_cvtsh_ss(unsigned short __a)
54{40{
55 __v8hi v = {(short)__a, 0, 0, 0, 0, 0, 0, 0};41 __v8hi __v = {(short)__a, 0, 0, 0, 0, 0, 0, 0};
56 __v4sf r = __builtin_ia32_vcvtph2ps(v);42 __v4sf __r = __builtin_ia32_vcvtph2ps(__v);
57 return r[0];43 return __r[0];
58}44}
5945
60/// Converts a 32-bit single-precision float value to a 16-bit46/// Converts a 32-bit single-precision float value to a 16-bit
lib/include/float.h+7-21
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- float.h - Characteristics of floating point types ----------------===1/*===---- float.h - Characteristics of floating point types ----------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -51,7 +37,7 @@...@@ -51,7 +37,7 @@
51# undef FLT_MANT_DIG37# undef FLT_MANT_DIG
52# undef DBL_MANT_DIG38# undef DBL_MANT_DIG
53# undef LDBL_MANT_DIG39# undef LDBL_MANT_DIG
54# if __STDC_VERSION__ >= 199901L || !defined(__STRICT_ANSI__)40# if __STDC_VERSION__ >= 199901L || !defined(__STRICT_ANSI__) || __cplusplus >= 201103L
55# undef DECIMAL_DIG41# undef DECIMAL_DIG
56# endif42# endif
57# undef FLT_DIG43# undef FLT_DIG
...@@ -78,7 +64,7 @@...@@ -78,7 +64,7 @@
78# undef FLT_MIN64# undef FLT_MIN
79# undef DBL_MIN65# undef DBL_MIN
80# undef LDBL_MIN66# undef LDBL_MIN
81# if __STDC_VERSION__ >= 201112L || !defined(__STRICT_ANSI__)67# if __STDC_VERSION__ >= 201112L || !defined(__STRICT_ANSI__) || __cplusplus >= 201703L
82# undef FLT_TRUE_MIN68# undef FLT_TRUE_MIN
83# undef DBL_TRUE_MIN69# undef DBL_TRUE_MIN
84# undef LDBL_TRUE_MIN70# undef LDBL_TRUE_MIN
...@@ -101,7 +87,7 @@...@@ -101,7 +87,7 @@
101#define DBL_MANT_DIG __DBL_MANT_DIG__87#define DBL_MANT_DIG __DBL_MANT_DIG__
102#define LDBL_MANT_DIG __LDBL_MANT_DIG__88#define LDBL_MANT_DIG __LDBL_MANT_DIG__
10389
104#if __STDC_VERSION__ >= 199901L || !defined(__STRICT_ANSI__)90#if __STDC_VERSION__ >= 199901L || !defined(__STRICT_ANSI__) || __cplusplus >= 201103L
105# define DECIMAL_DIG __DECIMAL_DIG__91# define DECIMAL_DIG __DECIMAL_DIG__
106#endif92#endif
10793
...@@ -137,7 +123,7 @@...@@ -137,7 +123,7 @@
137#define DBL_MIN __DBL_MIN__123#define DBL_MIN __DBL_MIN__
138#define LDBL_MIN __LDBL_MIN__124#define LDBL_MIN __LDBL_MIN__
139125
140#if __STDC_VERSION__ >= 201112L || !defined(__STRICT_ANSI__)126#if __STDC_VERSION__ >= 201112L || !defined(__STRICT_ANSI__) || __cplusplus >= 201703L
141# define FLT_TRUE_MIN __FLT_DENORM_MIN__127# define FLT_TRUE_MIN __FLT_DENORM_MIN__
142# define DBL_TRUE_MIN __DBL_DENORM_MIN__128# define DBL_TRUE_MIN __DBL_DENORM_MIN__
143# define LDBL_TRUE_MIN __LDBL_DENORM_MIN__129# define LDBL_TRUE_MIN __LDBL_DENORM_MIN__
lib/include/fma4intrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- fma4intrin.h - FMA4 intrinsics -----------------------------------===1/*===---- fma4intrin.h - FMA4 intrinsics -----------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/fmaintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- fmaintrin.h - FMA intrinsics -------------------------------------===1/*===---- fmaintrin.h - FMA intrinsics -------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/fxsrintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- fxsrintrin.h - FXSR intrinsic ------------------------------------===1/*===---- fxsrintrin.h - FXSR intrinsic ------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/gfniintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===----------------- gfniintrin.h - GFNI intrinsics ----------------------===1/*===----------------- gfniintrin.h - GFNI intrinsics ----------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/htmintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- htmintrin.h - Standard header for PowerPC HTM ---------------===*\1/*===---- htmintrin.h - Standard header for PowerPC HTM ---------------===*\
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21\*===----------------------------------------------------------------------===*/7\*===----------------------------------------------------------------------===*/
228
lib/include/htmxlintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- htmxlintrin.h - XL compiler HTM execution intrinsics-------------===*\1/*===---- htmxlintrin.h - XL compiler HTM execution intrinsics-------------===*\
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21\*===----------------------------------------------------------------------===*/7\*===----------------------------------------------------------------------===*/
228
lib/include/ia32intrin.h+303-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/* ===-------- ia32intrin.h ---------------------------------------------------===1/* ===-------- ia32intrin.h ---------------------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -28,6 +14,160 @@...@@ -28,6 +14,160 @@
28#ifndef __IA32INTRIN_H14#ifndef __IA32INTRIN_H
29#define __IA32INTRIN_H15#define __IA32INTRIN_H
3016
17/** Find the first set bit starting from the lsb. Result is undefined if
18 * input is 0.
19 *
20 * \headerfile <x86intrin.h>
21 *
22 * This intrinsic corresponds to the <c> BSF </c> instruction or the
23 * <c> TZCNT </c> instruction.
24 *
25 * \param __A
26 * A 32-bit integer operand.
27 * \returns A 32-bit integer containing the bit number.
28 */
29static __inline__ int __attribute__((__always_inline__, __nodebug__))
30__bsfd(int __A) {
31 return __builtin_ctz(__A);
32}
33
34/** Find the first set bit starting from the msb. Result is undefined if
35 * input is 0.
36 *
37 * \headerfile <x86intrin.h>
38 *
39 * This intrinsic corresponds to the <c> BSR </c> instruction or the
40 * <c> LZCNT </c> instruction and an <c> XOR </c>.
41 *
42 * \param __A
43 * A 32-bit integer operand.
44 * \returns A 32-bit integer containing the bit number.
45 */
46static __inline__ int __attribute__((__always_inline__, __nodebug__))
47__bsrd(int __A) {
48 return 31 - __builtin_clz(__A);
49}
50
51/** Swaps the bytes in the input. Converting little endian to big endian or
52 * vice versa.
53 *
54 * \headerfile <x86intrin.h>
55 *
56 * This intrinsic corresponds to the <c> BSWAP </c> instruction.
57 *
58 * \param __A
59 * A 32-bit integer operand.
60 * \returns A 32-bit integer containing the swapped bytes.
61 */
62static __inline__ int __attribute__((__always_inline__, __nodebug__))
63__bswapd(int __A) {
64 return __builtin_bswap32(__A);
65}
66
67static __inline__ int __attribute__((__always_inline__, __nodebug__))
68_bswap(int __A) {
69 return __builtin_bswap32(__A);
70}
71
72#define _bit_scan_forward(A) __bsfd((A))
73#define _bit_scan_reverse(A) __bsrd((A))
74
75#ifdef __x86_64__
76/** Find the first set bit starting from the lsb. Result is undefined if
77 * input is 0.
78 *
79 * \headerfile <x86intrin.h>
80 *
81 * This intrinsic corresponds to the <c> BSF </c> instruction or the
82 * <c> TZCNT </c> instruction.
83 *
84 * \param __A
85 * A 64-bit integer operand.
86 * \returns A 32-bit integer containing the bit number.
87 */
88static __inline__ int __attribute__((__always_inline__, __nodebug__))
89__bsfq(long long __A) {
90 return __builtin_ctzll(__A);
91}
92
93/** Find the first set bit starting from the msb. Result is undefined if
94 * input is 0.
95 *
96 * \headerfile <x86intrin.h>
97 *
98 * This intrinsic corresponds to the <c> BSR </c> instruction or the
99 * <c> LZCNT </c> instruction and an <c> XOR </c>.
100 *
101 * \param __A
102 * A 64-bit integer operand.
103 * \returns A 32-bit integer containing the bit number.
104 */
105static __inline__ int __attribute__((__always_inline__, __nodebug__))
106__bsrq(long long __A) {
107 return 63 - __builtin_clzll(__A);
108}
109
110/** Swaps the bytes in the input. Converting little endian to big endian or
111 * vice versa.
112 *
113 * \headerfile <x86intrin.h>
114 *
115 * This intrinsic corresponds to the <c> BSWAP </c> instruction.
116 *
117 * \param __A
118 * A 64-bit integer operand.
119 * \returns A 64-bit integer containing the swapped bytes.
120 */
121static __inline__ long long __attribute__((__always_inline__, __nodebug__))
122__bswapq(long long __A) {
123 return __builtin_bswap64(__A);
124}
125
126#define _bswap64(A) __bswapq((A))
127#endif
128
129/** Counts the number of bits in the source operand having a value of 1.
130 *
131 * \headerfile <x86intrin.h>
132 *
133 * This intrinsic corresponds to the <c> POPCNT </c> instruction or a
134 * a sequence of arithmetic and logic ops to calculate it.
135 *
136 * \param __A
137 * An unsigned 32-bit integer operand.
138 * \returns A 32-bit integer containing the number of bits with value 1 in the
139 * source operand.
140 */
141static __inline__ int __attribute__((__always_inline__, __nodebug__))
142__popcntd(unsigned int __A)
143{
144 return __builtin_popcount(__A);
145}
146
147#define _popcnt32(A) __popcntd((A))
148
149#ifdef __x86_64__
150/** Counts the number of bits in the source operand having a value of 1.
151 *
152 * \headerfile <x86intrin.h>
153 *
154 * This intrinsic corresponds to the <c> POPCNT </c> instruction or a
155 * a sequence of arithmetic and logic ops to calculate it.
156 *
157 * \param __A
158 * An unsigned 64-bit integer operand.
159 * \returns A 64-bit integer containing the number of bits with value 1 in the
160 * source operand.
161 */
162static __inline__ long long __attribute__((__always_inline__, __nodebug__))
163__popcntq(unsigned long long __A)
164{
165 return __builtin_popcountll(__A);
166}
167
168#define _popcnt64(A) __popcntq((A))
169#endif /* __x86_64__ */
170
31#ifdef __x86_64__171#ifdef __x86_64__
32static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__))172static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__))
33__readeflags(void)173__readeflags(void)
...@@ -55,6 +195,92 @@ __writeeflags(unsigned int __f)...@@ -55,6 +195,92 @@ __writeeflags(unsigned int __f)
55}195}
56#endif /* !__x86_64__ */196#endif /* !__x86_64__ */
57197
198/** Adds the unsigned integer operand to the CRC-32C checksum of the
199 * unsigned char operand.
200 *
201 * \headerfile <x86intrin.h>
202 *
203 * This intrinsic corresponds to the <c> CRC32B </c> instruction.
204 *
205 * \param __C
206 * An unsigned integer operand to add to the CRC-32C checksum of operand
207 * \a __D.
208 * \param __D
209 * An unsigned 8-bit integer operand used to compute the CRC-32C checksum.
210 * \returns The result of adding operand \a __C to the CRC-32C checksum of
211 * operand \a __D.
212 */
213static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("sse4.2")))
214__crc32b(unsigned int __C, unsigned char __D)
215{
216 return __builtin_ia32_crc32qi(__C, __D);
217}
218
219/** Adds the unsigned integer operand to the CRC-32C checksum of the
220 * unsigned short operand.
221 *
222 * \headerfile <x86intrin.h>
223 *
224 * This intrinsic corresponds to the <c> CRC32W </c> instruction.
225 *
226 * \param __C
227 * An unsigned integer operand to add to the CRC-32C checksum of operand
228 * \a __D.
229 * \param __D
230 * An unsigned 16-bit integer operand used to compute the CRC-32C checksum.
231 * \returns The result of adding operand \a __C to the CRC-32C checksum of
232 * operand \a __D.
233 */
234static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("sse4.2")))
235__crc32w(unsigned int __C, unsigned short __D)
236{
237 return __builtin_ia32_crc32hi(__C, __D);
238}
239
240/** Adds the unsigned integer operand to the CRC-32C checksum of the
241 * second unsigned integer operand.
242 *
243 * \headerfile <x86intrin.h>
244 *
245 * This intrinsic corresponds to the <c> CRC32D </c> instruction.
246 *
247 * \param __C
248 * An unsigned integer operand to add to the CRC-32C checksum of operand
249 * \a __D.
250 * \param __D
251 * An unsigned 32-bit integer operand used to compute the CRC-32C checksum.
252 * \returns The result of adding operand \a __C to the CRC-32C checksum of
253 * operand \a __D.
254 */
255static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("sse4.2")))
256__crc32d(unsigned int __C, unsigned int __D)
257{
258 return __builtin_ia32_crc32si(__C, __D);
259}
260
261#ifdef __x86_64__
262/** Adds the unsigned integer operand to the CRC-32C checksum of the
263 * unsigned 64-bit integer operand.
264 *
265 * \headerfile <x86intrin.h>
266 *
267 * This intrinsic corresponds to the <c> CRC32Q </c> instruction.
268 *
269 * \param __C
270 * An unsigned integer operand to add to the CRC-32C checksum of operand
271 * \a __D.
272 * \param __D
273 * An unsigned 64-bit integer operand used to compute the CRC-32C checksum.
274 * \returns The result of adding operand \a __C to the CRC-32C checksum of
275 * operand \a __D.
276 */
277static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("sse4.2")))
278__crc32q(unsigned long long __C, unsigned long long __D)
279{
280 return __builtin_ia32_crc32di(__C, __D);
281}
282#endif /* __x86_64__ */
283
58static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__))284static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__))
59__rdpmc(int __A) {285__rdpmc(int __A) {
60 return __builtin_ia32_rdpmc(__A);286 return __builtin_ia32_rdpmc(__A);
...@@ -75,4 +301,64 @@ _wbinvd(void) {...@@ -75,4 +301,64 @@ _wbinvd(void) {
75 __builtin_ia32_wbinvd();301 __builtin_ia32_wbinvd();
76}302}
77303
304static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__))
305__rolb(unsigned char __X, int __C) {
306 return __builtin_rotateleft8(__X, __C);
307}
308
309static __inline__ unsigned char __attribute__((__always_inline__, __nodebug__))
310__rorb(unsigned char __X, int __C) {
311 return __builtin_rotateright8(__X, __C);
312}
313
314static __inline__ unsigned short __attribute__((__always_inline__, __nodebug__))
315__rolw(unsigned short __X, int __C) {
316 return __builtin_rotateleft16(__X, __C);
317}
318
319static __inline__ unsigned short __attribute__((__always_inline__, __nodebug__))
320__rorw(unsigned short __X, int __C) {
321 return __builtin_rotateright16(__X, __C);
322}
323
324static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
325__rold(unsigned int __X, int __C) {
326 return __builtin_rotateleft32(__X, __C);
327}
328
329static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
330__rord(unsigned int __X, int __C) {
331 return __builtin_rotateright32(__X, __C);
332}
333
334#ifdef __x86_64__
335static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__))
336__rolq(unsigned long long __X, int __C) {
337 return __builtin_rotateleft64(__X, __C);
338}
339
340static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__))
341__rorq(unsigned long long __X, int __C) {
342 return __builtin_rotateright64(__X, __C);
343}
344#endif /* __x86_64__ */
345
346#ifndef _MSC_VER
347/* These are already provided as builtins for MSVC. */
348/* Select the correct function based on the size of long. */
349#ifdef __LP64__
350#define _lrotl(a,b) __rolq((a), (b))
351#define _lrotr(a,b) __rorq((a), (b))
352#else
353#define _lrotl(a,b) __rold((a), (b))
354#define _lrotr(a,b) __rord((a), (b))
355#endif
356#define _rotl(a,b) __rold((a), (b))
357#define _rotr(a,b) __rord((a), (b))
358#endif // _MSC_VER
359
360/* These are not builtins so need to be provided in all modes. */
361#define _rotwl(a,b) __rolw((a), (b))
362#define _rotwr(a,b) __rorw((a), (b))
363
78#endif /* __IA32INTRIN_H */364#endif /* __IA32INTRIN_H */
lib/include/immintrin.h+29-33
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- immintrin.h - Intel intrinsics -----------------------------------===1/*===---- immintrin.h - Intel intrinsics -----------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -195,6 +181,15 @@...@@ -195,6 +181,15 @@
195#include <avx512pfintrin.h>181#include <avx512pfintrin.h>
196#endif182#endif
197183
184#if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512BF16__)
185#include <avx512bf16intrin.h>
186#endif
187
188#if !defined(_MSC_VER) || __has_feature(modules) || \
189 (defined(__AVX512VL__) && defined(__AVX512BF16__))
190#include <avx512vlbf16intrin.h>
191#endif
192
198#if !defined(_MSC_VER) || __has_feature(modules) || defined(__PKU__)193#if !defined(_MSC_VER) || __has_feature(modules) || defined(__PKU__)
199#include <pkuintrin.h>194#include <pkuintrin.h>
200#endif195#endif
...@@ -241,18 +236,6 @@ _rdrand64_step(unsigned long long *__p)...@@ -241,18 +236,6 @@ _rdrand64_step(unsigned long long *__p)
241#endif236#endif
242#endif /* __RDRND__ */237#endif /* __RDRND__ */
243238
244/* __bit_scan_forward */
245static __inline__ int __attribute__((__always_inline__, __nodebug__))
246_bit_scan_forward(int __A) {
247 return __builtin_ctz(__A);
248}
249
250/* __bit_scan_reverse */
251static __inline__ int __attribute__((__always_inline__, __nodebug__))
252_bit_scan_reverse(int __A) {
253 return 31 - __builtin_clz(__A);
254}
255
256#if !defined(_MSC_VER) || __has_feature(modules) || defined(__FSGSBASE__)239#if !defined(_MSC_VER) || __has_feature(modules) || defined(__FSGSBASE__)
257#ifdef __x86_64__240#ifdef __x86_64__
258static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))241static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
...@@ -378,9 +361,8 @@ _storebe_i64(void * __P, long long __D) {...@@ -378,9 +361,8 @@ _storebe_i64(void * __P, long long __D) {
378#include <fxsrintrin.h>361#include <fxsrintrin.h>
379#endif362#endif
380363
381#if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVE__)364/* No feature check desired due to internal MSC_VER checks */
382#include <xsaveintrin.h>365#include <xsaveintrin.h>
383#endif
384366
385#if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVEOPT__)367#if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVEOPT__)
386#include <xsaveoptintrin.h>368#include <xsaveoptintrin.h>
...@@ -439,7 +421,21 @@ _storebe_i64(void * __P, long long __D) {...@@ -439,7 +421,21 @@ _storebe_i64(void * __P, long long __D) {
439#include <invpcidintrin.h>421#include <invpcidintrin.h>
440#endif422#endif
441423
442#ifdef _MSC_VER424#if !defined(_MSC_VER) || __has_feature(modules) || \
425 defined(__AVX512VP2INTERSECT__)
426#include <avx512vp2intersectintrin.h>
427#endif
428
429#if !defined(_MSC_VER) || __has_feature(modules) || \
430 (defined(__AVX512VL__) && defined(__AVX512VP2INTERSECT__))
431#include <avx512vlvp2intersectintrin.h>
432#endif
433
434#if !defined(_MSC_VER) || __has_feature(modules) || defined(__ENQCMD__)
435#include <enqcmdintrin.h>
436#endif
437
438#if defined(_MSC_VER) && __has_extension(gnu_asm)
443/* Define the default attributes for these intrinsics */439/* Define the default attributes for these intrinsics */
444#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))440#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
445#ifdef __cplusplus441#ifdef __cplusplus
...@@ -521,6 +517,6 @@ _InterlockedCompareExchange64_HLERelease(__int64 volatile *_Destination,...@@ -521,6 +517,6 @@ _InterlockedCompareExchange64_HLERelease(__int64 volatile *_Destination,
521517
522#undef __DEFAULT_FN_ATTRS518#undef __DEFAULT_FN_ATTRS
523519
524#endif /* _MSC_VER */520#endif /* defined(_MSC_VER) && __has_extension(gnu_asm) */
525521
526#endif /* __IMMINTRIN_H */522#endif /* __IMMINTRIN_H */
lib/include/intrin.h+6-36
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/* ===-------- intrin.h ---------------------------------------------------===1/* ===-------- intrin.h ---------------------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -200,10 +186,6 @@ __attribute__((__deprecated__("use other intrinsics or C++11 atomics instead")))...@@ -200,10 +186,6 @@ __attribute__((__deprecated__("use other intrinsics or C++11 atomics instead")))
200_WriteBarrier(void);186_WriteBarrier(void);
201unsigned __int32 xbegin(void);187unsigned __int32 xbegin(void);
202void _xend(void);188void _xend(void);
203static __inline__
204#define _XCR_XFEATURE_ENABLED_MASK 0
205unsigned __int64 __cdecl _xgetbv(unsigned int);
206void __cdecl _xsetbv(unsigned int, unsigned __int64);
207189
208/* These additional intrinsics are turned on in x64/amd64/x86_64 mode. */190/* These additional intrinsics are turned on in x64/amd64/x86_64 mode. */
209#ifdef __x86_64__191#ifdef __x86_64__
...@@ -539,12 +521,6 @@ __cpuidex(int __info[4], int __level, int __ecx) {...@@ -539,12 +521,6 @@ __cpuidex(int __info[4], int __level, int __ecx) {
539 __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3])521 __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3])
540 : "a"(__level), "c"(__ecx));522 : "a"(__level), "c"(__ecx));
541}523}
542static __inline__ unsigned __int64 __cdecl __DEFAULT_FN_ATTRS
543_xgetbv(unsigned int __xcr_no) {
544 unsigned int __eax, __edx;
545 __asm__ ("xgetbv" : "=a" (__eax), "=d" (__edx) : "c" (__xcr_no));
546 return ((unsigned __int64)__edx << 32) | __eax;
547}
548static __inline__ void __DEFAULT_FN_ATTRS524static __inline__ void __DEFAULT_FN_ATTRS
549__halt(void) {525__halt(void) {
550 __asm__ volatile ("hlt");526 __asm__ volatile ("hlt");
...@@ -567,15 +543,9 @@ long _InterlockedAdd(long volatile *Addend, long Value);...@@ -567,15 +543,9 @@ long _InterlockedAdd(long volatile *Addend, long Value);
567__int64 _ReadStatusReg(int);543__int64 _ReadStatusReg(int);
568void _WriteStatusReg(int, __int64);544void _WriteStatusReg(int, __int64);
569545
570static inline unsigned short _byteswap_ushort (unsigned short val) {546unsigned short __cdecl _byteswap_ushort(unsigned short val);
571 return __builtin_bswap16(val);547unsigned long __cdecl _byteswap_ulong (unsigned long val);
572}548unsigned __int64 __cdecl _byteswap_uint64(unsigned __int64 val);
573static inline unsigned long _byteswap_ulong (unsigned long val) {
574 return __builtin_bswap32(val);
575}
576static inline unsigned __int64 _byteswap_uint64 (unsigned __int64 val) {
577 return __builtin_bswap64(val);
578}
579#endif549#endif
580550
581/*----------------------------------------------------------------------------*\551/*----------------------------------------------------------------------------*\
lib/include/inttypes.h+8-17
...@@ -1,27 +1,18 @@...@@ -1,27 +1,18 @@
1/*===---- inttypes.h - Standard header for integer printf macros ----------===*\1/*===---- inttypes.h - Standard header for integer printf macros ----------===*\
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21\*===----------------------------------------------------------------------===*/7\*===----------------------------------------------------------------------===*/
228
23#ifndef __CLANG_INTTYPES_H9#ifndef __CLANG_INTTYPES_H
10// AIX system headers need inttypes.h to be re-enterable while _STD_TYPES_T
11// is defined until an inclusion of it without _STD_TYPES_T occurs, in which
12// case the header guard macro is defined.
13#if !defined(_AIX) || !defined(_STD_TYPES_T)
24#define __CLANG_INTTYPES_H14#define __CLANG_INTTYPES_H
15#endif
2516
26#if defined(_MSC_VER) && _MSC_VER < 180017#if defined(_MSC_VER) && _MSC_VER < 1800
27#error MSVC does not have inttypes.h prior to Visual Studio 201318#error MSVC does not have inttypes.h prior to Visual Studio 2013
lib/include/invpcidintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===------------- invpcidintrin.h - INVPCID intrinsic ---------------------===1/*===------------- invpcidintrin.h - INVPCID intrinsic ---------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/iso646.h+3-19
...@@ -1,24 +1,8 @@...@@ -1,24 +1,8 @@
1/*===---- iso646.h - Standard header for alternate spellings of operators---===1/*===---- iso646.h - Standard header for alternate spellings of operators---===
2 *2 *
3 * Copyright (c) 2008 Eli Friedman3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 *4 * See https://llvm.org/LICENSE.txt for license information.
5 * Permission is hereby granted, free of charge, to any person obtaining a copy5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * of this software and associated documentation files (the "Software"), to deal
7 * in the Software without restriction, including without limitation the rights
8 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 * copies of the Software, and to permit persons to whom the Software is
10 * furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 * THE SOFTWARE.
22 *6 *
23 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
24 */8 */
lib/include/limits.h+3-19
...@@ -1,24 +1,8 @@...@@ -1,24 +1,8 @@
1/*===---- limits.h - Standard header for integer sizes --------------------===*\1/*===---- limits.h - Standard header for integer sizes --------------------===*\
2 *2 *
3 * Copyright (c) 2009 Chris Lattner3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 *4 * See https://llvm.org/LICENSE.txt for license information.
5 * Permission is hereby granted, free of charge, to any person obtaining a copy5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * of this software and associated documentation files (the "Software"), to deal
7 * in the Software without restriction, including without limitation the rights
8 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 * copies of the Software, and to permit persons to whom the Software is
10 * furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 * THE SOFTWARE.
22 *6 *
23\*===----------------------------------------------------------------------===*/7\*===----------------------------------------------------------------------===*/
248
lib/include/lwpintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- lwpintrin.h - LWP intrinsics -------------------------------------===1/*===---- lwpintrin.h - LWP intrinsics -------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/lzcntintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- lzcntintrin.h - LZCNT intrinsics ---------------------------------===1/*===---- lzcntintrin.h - LZCNT intrinsics ---------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/mm3dnow.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- mm3dnow.h - 3DNow! intrinsics ------------------------------------===1/*===---- mm3dnow.h - 3DNow! intrinsics ------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/mm_malloc.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- mm_malloc.h - Allocating and Freeing Aligned Memory Blocks -------===1/*===---- mm_malloc.h - Allocating and Freeing Aligned Memory Blocks -------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/mmintrin.h+4-18
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- mmintrin.h - MMX intrinsics --------------------------------------===1/*===---- mmintrin.h - MMX intrinsics --------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -24,7 +10,7 @@...@@ -24,7 +10,7 @@
24#ifndef __MMINTRIN_H10#ifndef __MMINTRIN_H
25#define __MMINTRIN_H11#define __MMINTRIN_H
2612
27typedef long long __m64 __attribute__((__vector_size__(8)));13typedef long long __m64 __attribute__((__vector_size__(8), __aligned__(8)));
2814
29typedef long long __v1di __attribute__((__vector_size__(8)));15typedef long long __v1di __attribute__((__vector_size__(8)));
30typedef int __v2si __attribute__((__vector_size__(8)));16typedef int __v2si __attribute__((__vector_size__(8)));
lib/include/module.modulemap+4-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- module.modulemap - intrinsics module map -------------------------===1/*===---- module.modulemap - intrinsics module map -------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -168,4 +154,5 @@ module _Builtin_stddef_max_align_t [system] [extern_c] {...@@ -168,4 +154,5 @@ module _Builtin_stddef_max_align_t [system] [extern_c] {
168module opencl_c {154module opencl_c {
169 requires opencl155 requires opencl
170 header "opencl-c.h"156 header "opencl-c.h"
157 header "opencl-c-base.h"
171}158}
lib/include/movdirintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===------------------------- movdirintrin.h ------------------------------===1/*===------------------------- movdirintrin.h ------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/msa.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- msa.h - MIPS MSA intrinsics --------------------------------------===1/*===---- msa.h - MIPS MSA intrinsics --------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/mwaitxintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- mwaitxintrin.h - MONITORX/MWAITX intrinsics ----------------------===1/*===---- mwaitxintrin.h - MONITORX/MWAITX intrinsics ----------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/nmmintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- nmmintrin.h - SSE4 intrinsics ------------------------------------===1/*===---- nmmintrin.h - SSE4 intrinsics ------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/opencl-c-base.h created+578
...@@ -0,0 +1,578 @@
1//===----- opencl-c-base.h - OpenCL C language base definitions -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef _OPENCL_BASE_H_
10#define _OPENCL_BASE_H_
11
12// built-in scalar data types:
13
14/**
15 * An unsigned 8-bit integer.
16 */
17typedef unsigned char uchar;
18
19/**
20 * An unsigned 16-bit integer.
21 */
22typedef unsigned short ushort;
23
24/**
25 * An unsigned 32-bit integer.
26 */
27typedef unsigned int uint;
28
29/**
30 * An unsigned 64-bit integer.
31 */
32typedef unsigned long ulong;
33
34/**
35 * The unsigned integer type of the result of the sizeof operator. This
36 * is a 32-bit unsigned integer if CL_DEVICE_ADDRESS_BITS
37 * defined in table 4.3 is 32-bits and is a 64-bit unsigned integer if
38 * CL_DEVICE_ADDRESS_BITS is 64-bits.
39 */
40typedef __SIZE_TYPE__ size_t;
41
42/**
43 * A signed integer type that is the result of subtracting two pointers.
44 * This is a 32-bit signed integer if CL_DEVICE_ADDRESS_BITS
45 * defined in table 4.3 is 32-bits and is a 64-bit signed integer if
46 * CL_DEVICE_ADDRESS_BITS is 64-bits.
47 */
48typedef __PTRDIFF_TYPE__ ptrdiff_t;
49
50/**
51 * A signed integer type with the property that any valid pointer to
52 * void can be converted to this type, then converted back to pointer
53 * to void, and the result will compare equal to the original pointer.
54 */
55typedef __INTPTR_TYPE__ intptr_t;
56
57/**
58 * An unsigned integer type with the property that any valid pointer to
59 * void can be converted to this type, then converted back to pointer
60 * to void, and the result will compare equal to the original pointer.
61 */
62typedef __UINTPTR_TYPE__ uintptr_t;
63
64// built-in vector data types:
65typedef char char2 __attribute__((ext_vector_type(2)));
66typedef char char3 __attribute__((ext_vector_type(3)));
67typedef char char4 __attribute__((ext_vector_type(4)));
68typedef char char8 __attribute__((ext_vector_type(8)));
69typedef char char16 __attribute__((ext_vector_type(16)));
70typedef uchar uchar2 __attribute__((ext_vector_type(2)));
71typedef uchar uchar3 __attribute__((ext_vector_type(3)));
72typedef uchar uchar4 __attribute__((ext_vector_type(4)));
73typedef uchar uchar8 __attribute__((ext_vector_type(8)));
74typedef uchar uchar16 __attribute__((ext_vector_type(16)));
75typedef short short2 __attribute__((ext_vector_type(2)));
76typedef short short3 __attribute__((ext_vector_type(3)));
77typedef short short4 __attribute__((ext_vector_type(4)));
78typedef short short8 __attribute__((ext_vector_type(8)));
79typedef short short16 __attribute__((ext_vector_type(16)));
80typedef ushort ushort2 __attribute__((ext_vector_type(2)));
81typedef ushort ushort3 __attribute__((ext_vector_type(3)));
82typedef ushort ushort4 __attribute__((ext_vector_type(4)));
83typedef ushort ushort8 __attribute__((ext_vector_type(8)));
84typedef ushort ushort16 __attribute__((ext_vector_type(16)));
85typedef int int2 __attribute__((ext_vector_type(2)));
86typedef int int3 __attribute__((ext_vector_type(3)));
87typedef int int4 __attribute__((ext_vector_type(4)));
88typedef int int8 __attribute__((ext_vector_type(8)));
89typedef int int16 __attribute__((ext_vector_type(16)));
90typedef uint uint2 __attribute__((ext_vector_type(2)));
91typedef uint uint3 __attribute__((ext_vector_type(3)));
92typedef uint uint4 __attribute__((ext_vector_type(4)));
93typedef uint uint8 __attribute__((ext_vector_type(8)));
94typedef uint uint16 __attribute__((ext_vector_type(16)));
95typedef long long2 __attribute__((ext_vector_type(2)));
96typedef long long3 __attribute__((ext_vector_type(3)));
97typedef long long4 __attribute__((ext_vector_type(4)));
98typedef long long8 __attribute__((ext_vector_type(8)));
99typedef long long16 __attribute__((ext_vector_type(16)));
100typedef ulong ulong2 __attribute__((ext_vector_type(2)));
101typedef ulong ulong3 __attribute__((ext_vector_type(3)));
102typedef ulong ulong4 __attribute__((ext_vector_type(4)));
103typedef ulong ulong8 __attribute__((ext_vector_type(8)));
104typedef ulong ulong16 __attribute__((ext_vector_type(16)));
105typedef float float2 __attribute__((ext_vector_type(2)));
106typedef float float3 __attribute__((ext_vector_type(3)));
107typedef float float4 __attribute__((ext_vector_type(4)));
108typedef float float8 __attribute__((ext_vector_type(8)));
109typedef float float16 __attribute__((ext_vector_type(16)));
110#ifdef cl_khr_fp16
111#pragma OPENCL EXTENSION cl_khr_fp16 : enable
112typedef half half2 __attribute__((ext_vector_type(2)));
113typedef half half3 __attribute__((ext_vector_type(3)));
114typedef half half4 __attribute__((ext_vector_type(4)));
115typedef half half8 __attribute__((ext_vector_type(8)));
116typedef half half16 __attribute__((ext_vector_type(16)));
117#endif
118#ifdef cl_khr_fp64
119#if __OPENCL_C_VERSION__ < CL_VERSION_1_2
120#pragma OPENCL EXTENSION cl_khr_fp64 : enable
121#endif
122typedef double double2 __attribute__((ext_vector_type(2)));
123typedef double double3 __attribute__((ext_vector_type(3)));
124typedef double double4 __attribute__((ext_vector_type(4)));
125typedef double double8 __attribute__((ext_vector_type(8)));
126typedef double double16 __attribute__((ext_vector_type(16)));
127#endif
128
129#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
130#define NULL ((void*)0)
131#endif
132
133/**
134 * Value of maximum non-infinite single-precision floating-point
135 * number.
136 */
137#define MAXFLOAT 0x1.fffffep127f
138
139/**
140 * A positive float constant expression. HUGE_VALF evaluates
141 * to +infinity. Used as an error value returned by the built-in
142 * math functions.
143 */
144#define HUGE_VALF (__builtin_huge_valf())
145
146/**
147 * A positive double constant expression. HUGE_VAL evaluates
148 * to +infinity. Used as an error value returned by the built-in
149 * math functions.
150 */
151#define HUGE_VAL (__builtin_huge_val())
152
153/**
154 * A constant expression of type float representing positive or
155 * unsigned infinity.
156 */
157#define INFINITY (__builtin_inff())
158
159/**
160 * A constant expression of type float representing a quiet NaN.
161 */
162#define NAN as_float(INT_MAX)
163
164#define FP_ILOGB0 INT_MIN
165#define FP_ILOGBNAN INT_MAX
166
167#define FLT_DIG 6
168#define FLT_MANT_DIG 24
169#define FLT_MAX_10_EXP +38
170#define FLT_MAX_EXP +128
171#define FLT_MIN_10_EXP -37
172#define FLT_MIN_EXP -125
173#define FLT_RADIX 2
174#define FLT_MAX 0x1.fffffep127f
175#define FLT_MIN 0x1.0p-126f
176#define FLT_EPSILON 0x1.0p-23f
177
178#define M_E_F 2.71828182845904523536028747135266250f
179#define M_LOG2E_F 1.44269504088896340735992468100189214f
180#define M_LOG10E_F 0.434294481903251827651128918916605082f
181#define M_LN2_F 0.693147180559945309417232121458176568f
182#define M_LN10_F 2.30258509299404568401799145468436421f
183#define M_PI_F 3.14159265358979323846264338327950288f
184#define M_PI_2_F 1.57079632679489661923132169163975144f
185#define M_PI_4_F 0.785398163397448309615660845819875721f
186#define M_1_PI_F 0.318309886183790671537767526745028724f
187#define M_2_PI_F 0.636619772367581343075535053490057448f
188#define M_2_SQRTPI_F 1.12837916709551257389615890312154517f
189#define M_SQRT2_F 1.41421356237309504880168872420969808f
190#define M_SQRT1_2_F 0.707106781186547524400844362104849039f
191
192#define DBL_DIG 15
193#define DBL_MANT_DIG 53
194#define DBL_MAX_10_EXP +308
195#define DBL_MAX_EXP +1024
196#define DBL_MIN_10_EXP -307
197#define DBL_MIN_EXP -1021
198#define DBL_RADIX 2
199#define DBL_MAX 0x1.fffffffffffffp1023
200#define DBL_MIN 0x1.0p-1022
201#define DBL_EPSILON 0x1.0p-52
202
203#define M_E 0x1.5bf0a8b145769p+1
204#define M_LOG2E 0x1.71547652b82fep+0
205#define M_LOG10E 0x1.bcb7b1526e50ep-2
206#define M_LN2 0x1.62e42fefa39efp-1
207#define M_LN10 0x1.26bb1bbb55516p+1
208#define M_PI 0x1.921fb54442d18p+1
209#define M_PI_2 0x1.921fb54442d18p+0
210#define M_PI_4 0x1.921fb54442d18p-1
211#define M_1_PI 0x1.45f306dc9c883p-2
212#define M_2_PI 0x1.45f306dc9c883p-1
213#define M_2_SQRTPI 0x1.20dd750429b6dp+0
214#define M_SQRT2 0x1.6a09e667f3bcdp+0
215#define M_SQRT1_2 0x1.6a09e667f3bcdp-1
216
217#ifdef cl_khr_fp16
218
219#define HALF_DIG 3
220#define HALF_MANT_DIG 11
221#define HALF_MAX_10_EXP +4
222#define HALF_MAX_EXP +16
223#define HALF_MIN_10_EXP -4
224#define HALF_MIN_EXP -13
225#define HALF_RADIX 2
226#define HALF_MAX ((0x1.ffcp15h))
227#define HALF_MIN ((0x1.0p-14h))
228#define HALF_EPSILON ((0x1.0p-10h))
229
230#define M_E_H 2.71828182845904523536028747135266250h
231#define M_LOG2E_H 1.44269504088896340735992468100189214h
232#define M_LOG10E_H 0.434294481903251827651128918916605082h
233#define M_LN2_H 0.693147180559945309417232121458176568h
234#define M_LN10_H 2.30258509299404568401799145468436421h
235#define M_PI_H 3.14159265358979323846264338327950288h
236#define M_PI_2_H 1.57079632679489661923132169163975144h
237#define M_PI_4_H 0.785398163397448309615660845819875721h
238#define M_1_PI_H 0.318309886183790671537767526745028724h
239#define M_2_PI_H 0.636619772367581343075535053490057448h
240#define M_2_SQRTPI_H 1.12837916709551257389615890312154517h
241#define M_SQRT2_H 1.41421356237309504880168872420969808h
242#define M_SQRT1_2_H 0.707106781186547524400844362104849039h
243
244#endif //cl_khr_fp16
245
246#define CHAR_BIT 8
247#define SCHAR_MAX 127
248#define SCHAR_MIN (-128)
249#define UCHAR_MAX 255
250#define CHAR_MAX SCHAR_MAX
251#define CHAR_MIN SCHAR_MIN
252#define USHRT_MAX 65535
253#define SHRT_MAX 32767
254#define SHRT_MIN (-32768)
255#define UINT_MAX 0xffffffff
256#define INT_MAX 2147483647
257#define INT_MIN (-2147483647-1)
258#define ULONG_MAX 0xffffffffffffffffUL
259#define LONG_MAX 0x7fffffffffffffffL
260#define LONG_MIN (-0x7fffffffffffffffL-1)
261
262// OpenCL v1.1 s6.11.8, v1.2 s6.12.8, v2.0 s6.13.8 - Synchronization Functions
263
264// Flag type and values for barrier, mem_fence, read_mem_fence, write_mem_fence
265typedef uint cl_mem_fence_flags;
266
267/**
268 * Queue a memory fence to ensure correct
269 * ordering of memory operations to local memory
270 */
271#define CLK_LOCAL_MEM_FENCE 0x01
272
273/**
274 * Queue a memory fence to ensure correct
275 * ordering of memory operations to global memory
276 */
277#define CLK_GLOBAL_MEM_FENCE 0x02
278
279#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
280
281typedef enum memory_scope {
282 memory_scope_work_item = __OPENCL_MEMORY_SCOPE_WORK_ITEM,
283 memory_scope_work_group = __OPENCL_MEMORY_SCOPE_WORK_GROUP,
284 memory_scope_device = __OPENCL_MEMORY_SCOPE_DEVICE,
285 memory_scope_all_svm_devices = __OPENCL_MEMORY_SCOPE_ALL_SVM_DEVICES,
286#if defined(cl_intel_subgroups) || defined(cl_khr_subgroups)
287 memory_scope_sub_group = __OPENCL_MEMORY_SCOPE_SUB_GROUP
288#endif
289} memory_scope;
290
291#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0
292
293#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
294/**
295 * Queue a memory fence to ensure correct ordering of memory
296 * operations between work-items of a work-group to
297 * image memory.
298 */
299#define CLK_IMAGE_MEM_FENCE 0x04
300
301#ifndef ATOMIC_VAR_INIT
302#define ATOMIC_VAR_INIT(x) (x)
303#endif //ATOMIC_VAR_INIT
304#define ATOMIC_FLAG_INIT 0
305
306// enum values aligned with what clang uses in EmitAtomicExpr()
307typedef enum memory_order
308{
309 memory_order_relaxed = __ATOMIC_RELAXED,
310 memory_order_acquire = __ATOMIC_ACQUIRE,
311 memory_order_release = __ATOMIC_RELEASE,
312 memory_order_acq_rel = __ATOMIC_ACQ_REL,
313 memory_order_seq_cst = __ATOMIC_SEQ_CST
314} memory_order;
315
316#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0
317
318// OpenCL v1.1 s6.11.3, v1.2 s6.12.14, v2.0 s6.13.14 - Image Read and Write Functions
319
320// These values need to match the runtime equivalent
321//
322// Addressing Mode.
323//
324#define CLK_ADDRESS_NONE 0
325#define CLK_ADDRESS_CLAMP_TO_EDGE 2
326#define CLK_ADDRESS_CLAMP 4
327#define CLK_ADDRESS_REPEAT 6
328#define CLK_ADDRESS_MIRRORED_REPEAT 8
329
330//
331// Coordination Normalization
332//
333#define CLK_NORMALIZED_COORDS_FALSE 0
334#define CLK_NORMALIZED_COORDS_TRUE 1
335
336//
337// Filtering Mode.
338//
339#define CLK_FILTER_NEAREST 0x10
340#define CLK_FILTER_LINEAR 0x20
341
342#ifdef cl_khr_gl_msaa_sharing
343#pragma OPENCL EXTENSION cl_khr_gl_msaa_sharing : enable
344#endif //cl_khr_gl_msaa_sharing
345
346//
347// Channel Datatype.
348//
349#define CLK_SNORM_INT8 0x10D0
350#define CLK_SNORM_INT16 0x10D1
351#define CLK_UNORM_INT8 0x10D2
352#define CLK_UNORM_INT16 0x10D3
353#define CLK_UNORM_SHORT_565 0x10D4
354#define CLK_UNORM_SHORT_555 0x10D5
355#define CLK_UNORM_INT_101010 0x10D6
356#define CLK_SIGNED_INT8 0x10D7
357#define CLK_SIGNED_INT16 0x10D8
358#define CLK_SIGNED_INT32 0x10D9
359#define CLK_UNSIGNED_INT8 0x10DA
360#define CLK_UNSIGNED_INT16 0x10DB
361#define CLK_UNSIGNED_INT32 0x10DC
362#define CLK_HALF_FLOAT 0x10DD
363#define CLK_FLOAT 0x10DE
364#define CLK_UNORM_INT24 0x10DF
365
366// Channel order, numbering must be aligned with cl_channel_order in cl.h
367//
368#define CLK_R 0x10B0
369#define CLK_A 0x10B1
370#define CLK_RG 0x10B2
371#define CLK_RA 0x10B3
372#define CLK_RGB 0x10B4
373#define CLK_RGBA 0x10B5
374#define CLK_BGRA 0x10B6
375#define CLK_ARGB 0x10B7
376#define CLK_INTENSITY 0x10B8
377#define CLK_LUMINANCE 0x10B9
378#define CLK_Rx 0x10BA
379#define CLK_RGx 0x10BB
380#define CLK_RGBx 0x10BC
381#define CLK_DEPTH 0x10BD
382#define CLK_DEPTH_STENCIL 0x10BE
383#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
384#define CLK_sRGB 0x10BF
385#define CLK_sRGBx 0x10C0
386#define CLK_sRGBA 0x10C1
387#define CLK_sBGRA 0x10C2
388#define CLK_ABGR 0x10C3
389#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0
390
391// OpenCL v2.0 s6.13.16 - Pipe Functions
392#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
393#define CLK_NULL_RESERVE_ID (__builtin_astype(((void*)(__SIZE_MAX__)), reserve_id_t))
394#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0
395
396
397// OpenCL v2.0 s6.13.17 - Enqueue Kernels
398#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
399
400#define CL_COMPLETE 0x0
401#define CL_RUNNING 0x1
402#define CL_SUBMITTED 0x2
403#define CL_QUEUED 0x3
404
405#define CLK_SUCCESS 0
406#define CLK_ENQUEUE_FAILURE -101
407#define CLK_INVALID_QUEUE -102
408#define CLK_INVALID_NDRANGE -160
409#define CLK_INVALID_EVENT_WAIT_LIST -57
410#define CLK_DEVICE_QUEUE_FULL -161
411#define CLK_INVALID_ARG_SIZE -51
412#define CLK_EVENT_ALLOCATION_FAILURE -100
413#define CLK_OUT_OF_RESOURCES -5
414
415#define CLK_NULL_QUEUE 0
416#define CLK_NULL_EVENT (__builtin_astype(((void*)(__SIZE_MAX__)), clk_event_t))
417
418// execution model related definitions
419#define CLK_ENQUEUE_FLAGS_NO_WAIT 0x0
420#define CLK_ENQUEUE_FLAGS_WAIT_KERNEL 0x1
421#define CLK_ENQUEUE_FLAGS_WAIT_WORK_GROUP 0x2
422
423typedef int kernel_enqueue_flags_t;
424typedef int clk_profiling_info;
425
426// Profiling info name (see capture_event_profiling_info)
427#define CLK_PROFILING_COMMAND_EXEC_TIME 0x1
428
429#define MAX_WORK_DIM 3
430
431typedef struct {
432 unsigned int workDimension;
433 size_t globalWorkOffset[MAX_WORK_DIM];
434 size_t globalWorkSize[MAX_WORK_DIM];
435 size_t localWorkSize[MAX_WORK_DIM];
436} ndrange_t;
437
438#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0
439
440#ifdef cl_intel_device_side_avc_motion_estimation
441#pragma OPENCL EXTENSION cl_intel_device_side_avc_motion_estimation : begin
442
443#define CLK_AVC_ME_MAJOR_16x16_INTEL 0x0
444#define CLK_AVC_ME_MAJOR_16x8_INTEL 0x1
445#define CLK_AVC_ME_MAJOR_8x16_INTEL 0x2
446#define CLK_AVC_ME_MAJOR_8x8_INTEL 0x3
447
448#define CLK_AVC_ME_MINOR_8x8_INTEL 0x0
449#define CLK_AVC_ME_MINOR_8x4_INTEL 0x1
450#define CLK_AVC_ME_MINOR_4x8_INTEL 0x2
451#define CLK_AVC_ME_MINOR_4x4_INTEL 0x3
452
453#define CLK_AVC_ME_MAJOR_FORWARD_INTEL 0x0
454#define CLK_AVC_ME_MAJOR_BACKWARD_INTEL 0x1
455#define CLK_AVC_ME_MAJOR_BIDIRECTIONAL_INTEL 0x2
456
457#define CLK_AVC_ME_PARTITION_MASK_ALL_INTEL 0x0
458#define CLK_AVC_ME_PARTITION_MASK_16x16_INTEL 0x7E
459#define CLK_AVC_ME_PARTITION_MASK_16x8_INTEL 0x7D
460#define CLK_AVC_ME_PARTITION_MASK_8x16_INTEL 0x7B
461#define CLK_AVC_ME_PARTITION_MASK_8x8_INTEL 0x77
462#define CLK_AVC_ME_PARTITION_MASK_8x4_INTEL 0x6F
463#define CLK_AVC_ME_PARTITION_MASK_4x8_INTEL 0x5F
464#define CLK_AVC_ME_PARTITION_MASK_4x4_INTEL 0x3F
465
466#define CLK_AVC_ME_SLICE_TYPE_PRED_INTEL 0x0
467#define CLK_AVC_ME_SLICE_TYPE_BPRED_INTEL 0x1
468#define CLK_AVC_ME_SLICE_TYPE_INTRA_INTEL 0x2
469
470#define CLK_AVC_ME_SEARCH_WINDOW_EXHAUSTIVE_INTEL 0x0
471#define CLK_AVC_ME_SEARCH_WINDOW_SMALL_INTEL 0x1
472#define CLK_AVC_ME_SEARCH_WINDOW_TINY_INTEL 0x2
473#define CLK_AVC_ME_SEARCH_WINDOW_EXTRA_TINY_INTEL 0x3
474#define CLK_AVC_ME_SEARCH_WINDOW_DIAMOND_INTEL 0x4
475#define CLK_AVC_ME_SEARCH_WINDOW_LARGE_DIAMOND_INTEL 0x5
476#define CLK_AVC_ME_SEARCH_WINDOW_RESERVED0_INTEL 0x6
477#define CLK_AVC_ME_SEARCH_WINDOW_RESERVED1_INTEL 0x7
478#define CLK_AVC_ME_SEARCH_WINDOW_CUSTOM_INTEL 0x8
479
480#define CLK_AVC_ME_SAD_ADJUST_MODE_NONE_INTEL 0x0
481#define CLK_AVC_ME_SAD_ADJUST_MODE_HAAR_INTEL 0x2
482
483#define CLK_AVC_ME_SUBPIXEL_MODE_INTEGER_INTEL 0x0
484#define CLK_AVC_ME_SUBPIXEL_MODE_HPEL_INTEL 0x1
485#define CLK_AVC_ME_SUBPIXEL_MODE_QPEL_INTEL 0x3
486
487#define CLK_AVC_ME_COST_PRECISION_QPEL_INTEL 0x0
488#define CLK_AVC_ME_COST_PRECISION_HPEL_INTEL 0x1
489#define CLK_AVC_ME_COST_PRECISION_PEL_INTEL 0x2
490#define CLK_AVC_ME_COST_PRECISION_DPEL_INTEL 0x3
491
492#define CLK_AVC_ME_BIDIR_WEIGHT_QUARTER_INTEL 0x10
493#define CLK_AVC_ME_BIDIR_WEIGHT_THIRD_INTEL 0x15
494#define CLK_AVC_ME_BIDIR_WEIGHT_HALF_INTEL 0x20
495#define CLK_AVC_ME_BIDIR_WEIGHT_TWO_THIRD_INTEL 0x2B
496#define CLK_AVC_ME_BIDIR_WEIGHT_THREE_QUARTER_INTEL 0x30
497
498#define CLK_AVC_ME_BORDER_REACHED_LEFT_INTEL 0x0
499#define CLK_AVC_ME_BORDER_REACHED_RIGHT_INTEL 0x2
500#define CLK_AVC_ME_BORDER_REACHED_TOP_INTEL 0x4
501#define CLK_AVC_ME_BORDER_REACHED_BOTTOM_INTEL 0x8
502
503#define CLK_AVC_ME_INTRA_16x16_INTEL 0x0
504#define CLK_AVC_ME_INTRA_8x8_INTEL 0x1
505#define CLK_AVC_ME_INTRA_4x4_INTEL 0x2
506
507#define CLK_AVC_ME_SKIP_BLOCK_PARTITION_16x16_INTEL 0x0
508#define CLK_AVC_ME_SKIP_BLOCK_PARTITION_8x8_INTEL 0x4000
509
510#define CLK_AVC_ME_SKIP_BLOCK_16x16_FORWARD_ENABLE_INTEL (0x1 << 24)
511#define CLK_AVC_ME_SKIP_BLOCK_16x16_BACKWARD_ENABLE_INTEL (0x2 << 24)
512#define CLK_AVC_ME_SKIP_BLOCK_16x16_DUAL_ENABLE_INTEL (0x3 << 24)
513#define CLK_AVC_ME_SKIP_BLOCK_8x8_FORWARD_ENABLE_INTEL (0x55 << 24)
514#define CLK_AVC_ME_SKIP_BLOCK_8x8_BACKWARD_ENABLE_INTEL (0xAA << 24)
515#define CLK_AVC_ME_SKIP_BLOCK_8x8_DUAL_ENABLE_INTEL (0xFF << 24)
516#define CLK_AVC_ME_SKIP_BLOCK_8x8_0_FORWARD_ENABLE_INTEL (0x1 << 24)
517#define CLK_AVC_ME_SKIP_BLOCK_8x8_0_BACKWARD_ENABLE_INTEL (0x2 << 24)
518#define CLK_AVC_ME_SKIP_BLOCK_8x8_1_FORWARD_ENABLE_INTEL (0x1 << 26)
519#define CLK_AVC_ME_SKIP_BLOCK_8x8_1_BACKWARD_ENABLE_INTEL (0x2 << 26)
520#define CLK_AVC_ME_SKIP_BLOCK_8x8_2_FORWARD_ENABLE_INTEL (0x1 << 28)
521#define CLK_AVC_ME_SKIP_BLOCK_8x8_2_BACKWARD_ENABLE_INTEL (0x2 << 28)
522#define CLK_AVC_ME_SKIP_BLOCK_8x8_3_FORWARD_ENABLE_INTEL (0x1 << 30)
523#define CLK_AVC_ME_SKIP_BLOCK_8x8_3_BACKWARD_ENABLE_INTEL (0x2 << 30)
524
525#define CLK_AVC_ME_BLOCK_BASED_SKIP_4x4_INTEL 0x00
526#define CLK_AVC_ME_BLOCK_BASED_SKIP_8x8_INTEL 0x80
527
528#define CLK_AVC_ME_INTRA_LUMA_PARTITION_MASK_ALL_INTEL 0x0
529#define CLK_AVC_ME_INTRA_LUMA_PARTITION_MASK_16x16_INTEL 0x6
530#define CLK_AVC_ME_INTRA_LUMA_PARTITION_MASK_8x8_INTEL 0x5
531#define CLK_AVC_ME_INTRA_LUMA_PARTITION_MASK_4x4_INTEL 0x3
532
533#define CLK_AVC_ME_INTRA_NEIGHBOR_LEFT_MASK_ENABLE_INTEL 0x60
534#define CLK_AVC_ME_INTRA_NEIGHBOR_UPPER_MASK_ENABLE_INTEL 0x10
535#define CLK_AVC_ME_INTRA_NEIGHBOR_UPPER_RIGHT_MASK_ENABLE_INTEL 0x8
536#define CLK_AVC_ME_INTRA_NEIGHBOR_UPPER_LEFT_MASK_ENABLE_INTEL 0x4
537
538#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_VERTICAL_INTEL 0x0
539#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_HORIZONTAL_INTEL 0x1
540#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_DC_INTEL 0x2
541#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_DIAGONAL_DOWN_LEFT_INTEL 0x3
542#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_DIAGONAL_DOWN_RIGHT_INTEL 0x4
543#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_PLANE_INTEL 0x4
544#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_VERTICAL_RIGHT_INTEL 0x5
545#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_HORIZONTAL_DOWN_INTEL 0x6
546#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_VERTICAL_LEFT_INTEL 0x7
547#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_HORIZONTAL_UP_INTEL 0x8
548#define CLK_AVC_ME_CHROMA_PREDICTOR_MODE_DC_INTEL 0x0
549#define CLK_AVC_ME_CHROMA_PREDICTOR_MODE_HORIZONTAL_INTEL 0x1
550#define CLK_AVC_ME_CHROMA_PREDICTOR_MODE_VERTICAL_INTEL 0x2
551#define CLK_AVC_ME_CHROMA_PREDICTOR_MODE_PLANE_INTEL 0x3
552
553#define CLK_AVC_ME_FRAME_FORWARD_INTEL 0x1
554#define CLK_AVC_ME_FRAME_BACKWARD_INTEL 0x2
555#define CLK_AVC_ME_FRAME_DUAL_INTEL 0x3
556
557#define CLK_AVC_ME_INTERLACED_SCAN_TOP_FIELD_INTEL 0x0
558#define CLK_AVC_ME_INTERLACED_SCAN_BOTTOM_FIELD_INTEL 0x1
559
560#define CLK_AVC_ME_INITIALIZE_INTEL 0x0
561
562#define CLK_AVC_IME_PAYLOAD_INITIALIZE_INTEL 0x0
563#define CLK_AVC_REF_PAYLOAD_INITIALIZE_INTEL 0x0
564#define CLK_AVC_SIC_PAYLOAD_INITIALIZE_INTEL 0x0
565
566#define CLK_AVC_IME_RESULT_INITIALIZE_INTEL 0x0
567#define CLK_AVC_REF_RESULT_INITIALIZE_INTEL 0x0
568#define CLK_AVC_SIC_RESULT_INITIALIZE_INTEL 0x0
569
570#define CLK_AVC_IME_RESULT_SINGLE_REFERENCE_STREAMOUT_INITIALIZE_INTEL 0x0
571#define CLK_AVC_IME_RESULT_SINGLE_REFERENCE_STREAMIN_INITIALIZE_INTEL 0x0
572#define CLK_AVC_IME_RESULT_DUAL_REFERENCE_STREAMOUT_INITIALIZE_INTEL 0x0
573#define CLK_AVC_IME_RESULT_DUAL_REFERENCE_STREAMIN_INITIALIZE_INTEL 0x0
574
575#pragma OPENCL EXTENSION cl_intel_device_side_avc_motion_estimation : end
576#endif // cl_intel_device_side_avc_motion_estimation
577
578#endif //_OPENCL_BASE_H_
lib/include/opencl-c.h+79-619
...@@ -1,15 +1,16 @@...@@ -1,15 +1,16 @@
1//===--- opencl-c.h - OpenCL C language builtin function header -----------===//1//===--- opencl-c.h - OpenCL C language builtin function header -----------===//
2//2//
3// The LLVM Compiler Infrastructure3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4//4// See https://llvm.org/LICENSE.txt for license information.
5// This file is distributed under the University of Illinois Open Source5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6// License. See LICENSE.TXT for details.
7//6//
8//===----------------------------------------------------------------------===//7//===----------------------------------------------------------------------===//
98
10#ifndef _OPENCL_H_9#ifndef _OPENCL_H_
11#define _OPENCL_H_10#define _OPENCL_H_
1211
12#include "opencl-c-base.h"
13
13#if __OPENCL_C_VERSION__ >= CL_VERSION_2_014#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
14#ifndef cl_khr_depth_images15#ifndef cl_khr_depth_images
15#define cl_khr_depth_images16#define cl_khr_depth_images
...@@ -23,9 +24,6 @@...@@ -23,9 +24,6 @@
23#endif //__OPENCL_C_VERSION__ < CL_VERSION_2_024#endif //__OPENCL_C_VERSION__ < CL_VERSION_2_0
2425
25#if __OPENCL_C_VERSION__ >= CL_VERSION_1_226#if __OPENCL_C_VERSION__ >= CL_VERSION_1_2
26#ifndef cl_intel_planar_yuv
27#define cl_intel_planar_yuv
28#endif // cl_intel_planar_yuv
29#pragma OPENCL EXTENSION cl_intel_planar_yuv : begin27#pragma OPENCL EXTENSION cl_intel_planar_yuv : begin
30#pragma OPENCL EXTENSION cl_intel_planar_yuv : end28#pragma OPENCL EXTENSION cl_intel_planar_yuv : end
31#endif // __OPENCL_C_VERSION__ >= CL_VERSION_1_229#endif // __OPENCL_C_VERSION__ >= CL_VERSION_1_2
...@@ -37,255 +35,6 @@...@@ -37,255 +35,6 @@
37#define __purefn __attribute__((pure))35#define __purefn __attribute__((pure))
38#define __cnfn __attribute__((const))36#define __cnfn __attribute__((const))
3937
40// built-in scalar data types:
41
42/**
43 * An unsigned 8-bit integer.
44 */
45typedef unsigned char uchar;
46
47/**
48 * An unsigned 16-bit integer.
49 */
50typedef unsigned short ushort;
51
52/**
53 * An unsigned 32-bit integer.
54 */
55typedef unsigned int uint;
56
57/**
58 * An unsigned 64-bit integer.
59 */
60typedef unsigned long ulong;
61
62/**
63 * The unsigned integer type of the result of the sizeof operator. This
64 * is a 32-bit unsigned integer if CL_DEVICE_ADDRESS_BITS
65 * defined in table 4.3 is 32-bits and is a 64-bit unsigned integer if
66 * CL_DEVICE_ADDRESS_BITS is 64-bits.
67 */
68typedef __SIZE_TYPE__ size_t;
69
70/**
71 * A signed integer type that is the result of subtracting two pointers.
72 * This is a 32-bit signed integer if CL_DEVICE_ADDRESS_BITS
73 * defined in table 4.3 is 32-bits and is a 64-bit signed integer if
74 * CL_DEVICE_ADDRESS_BITS is 64-bits.
75 */
76typedef __PTRDIFF_TYPE__ ptrdiff_t;
77
78/**
79* A signed integer type with the property that any valid pointer to
80* void can be converted to this type, then converted back to pointer
81* to void, and the result will compare equal to the original pointer.
82*/
83typedef __INTPTR_TYPE__ intptr_t;
84
85/**
86* An unsigned integer type with the property that any valid pointer to
87* void can be converted to this type, then converted back to pointer
88* to void, and the result will compare equal to the original pointer.
89*/
90typedef __UINTPTR_TYPE__ uintptr_t;
91
92// built-in vector data types:
93typedef char char2 __attribute__((ext_vector_type(2)));
94typedef char char3 __attribute__((ext_vector_type(3)));
95typedef char char4 __attribute__((ext_vector_type(4)));
96typedef char char8 __attribute__((ext_vector_type(8)));
97typedef char char16 __attribute__((ext_vector_type(16)));
98typedef uchar uchar2 __attribute__((ext_vector_type(2)));
99typedef uchar uchar3 __attribute__((ext_vector_type(3)));
100typedef uchar uchar4 __attribute__((ext_vector_type(4)));
101typedef uchar uchar8 __attribute__((ext_vector_type(8)));
102typedef uchar uchar16 __attribute__((ext_vector_type(16)));
103typedef short short2 __attribute__((ext_vector_type(2)));
104typedef short short3 __attribute__((ext_vector_type(3)));
105typedef short short4 __attribute__((ext_vector_type(4)));
106typedef short short8 __attribute__((ext_vector_type(8)));
107typedef short short16 __attribute__((ext_vector_type(16)));
108typedef ushort ushort2 __attribute__((ext_vector_type(2)));
109typedef ushort ushort3 __attribute__((ext_vector_type(3)));
110typedef ushort ushort4 __attribute__((ext_vector_type(4)));
111typedef ushort ushort8 __attribute__((ext_vector_type(8)));
112typedef ushort ushort16 __attribute__((ext_vector_type(16)));
113typedef int int2 __attribute__((ext_vector_type(2)));
114typedef int int3 __attribute__((ext_vector_type(3)));
115typedef int int4 __attribute__((ext_vector_type(4)));
116typedef int int8 __attribute__((ext_vector_type(8)));
117typedef int int16 __attribute__((ext_vector_type(16)));
118typedef uint uint2 __attribute__((ext_vector_type(2)));
119typedef uint uint3 __attribute__((ext_vector_type(3)));
120typedef uint uint4 __attribute__((ext_vector_type(4)));
121typedef uint uint8 __attribute__((ext_vector_type(8)));
122typedef uint uint16 __attribute__((ext_vector_type(16)));
123typedef long long2 __attribute__((ext_vector_type(2)));
124typedef long long3 __attribute__((ext_vector_type(3)));
125typedef long long4 __attribute__((ext_vector_type(4)));
126typedef long long8 __attribute__((ext_vector_type(8)));
127typedef long long16 __attribute__((ext_vector_type(16)));
128typedef ulong ulong2 __attribute__((ext_vector_type(2)));
129typedef ulong ulong3 __attribute__((ext_vector_type(3)));
130typedef ulong ulong4 __attribute__((ext_vector_type(4)));
131typedef ulong ulong8 __attribute__((ext_vector_type(8)));
132typedef ulong ulong16 __attribute__((ext_vector_type(16)));
133typedef float float2 __attribute__((ext_vector_type(2)));
134typedef float float3 __attribute__((ext_vector_type(3)));
135typedef float float4 __attribute__((ext_vector_type(4)));
136typedef float float8 __attribute__((ext_vector_type(8)));
137typedef float float16 __attribute__((ext_vector_type(16)));
138#ifdef cl_khr_fp16
139#pragma OPENCL EXTENSION cl_khr_fp16 : enable
140typedef half half2 __attribute__((ext_vector_type(2)));
141typedef half half3 __attribute__((ext_vector_type(3)));
142typedef half half4 __attribute__((ext_vector_type(4)));
143typedef half half8 __attribute__((ext_vector_type(8)));
144typedef half half16 __attribute__((ext_vector_type(16)));
145#endif
146#ifdef cl_khr_fp64
147#if __OPENCL_C_VERSION__ < CL_VERSION_1_2
148#pragma OPENCL EXTENSION cl_khr_fp64 : enable
149#endif
150typedef double double2 __attribute__((ext_vector_type(2)));
151typedef double double3 __attribute__((ext_vector_type(3)));
152typedef double double4 __attribute__((ext_vector_type(4)));
153typedef double double8 __attribute__((ext_vector_type(8)));
154typedef double double16 __attribute__((ext_vector_type(16)));
155#endif
156
157#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
158#define NULL ((void*)0)
159#endif
160
161/**
162 * Value of maximum non-infinite single-precision floating-point
163 * number.
164 */
165#define MAXFLOAT 0x1.fffffep127f
166
167/**
168 * A positive float constant expression. HUGE_VALF evaluates
169 * to +infinity. Used as an error value returned by the built-in
170 * math functions.
171 */
172#define HUGE_VALF (__builtin_huge_valf())
173
174/**
175 * A positive double constant expression. HUGE_VAL evaluates
176 * to +infinity. Used as an error value returned by the built-in
177 * math functions.
178 */
179#define HUGE_VAL (__builtin_huge_val())
180
181/**
182 * A constant expression of type float representing positive or
183 * unsigned infinity.
184 */
185#define INFINITY (__builtin_inff())
186
187/**
188 * A constant expression of type float representing a quiet NaN.
189 */
190#define NAN as_float(INT_MAX)
191
192#define FP_ILOGB0 INT_MIN
193#define FP_ILOGBNAN INT_MAX
194
195#define FLT_DIG 6
196#define FLT_MANT_DIG 24
197#define FLT_MAX_10_EXP +38
198#define FLT_MAX_EXP +128
199#define FLT_MIN_10_EXP -37
200#define FLT_MIN_EXP -125
201#define FLT_RADIX 2
202#define FLT_MAX 0x1.fffffep127f
203#define FLT_MIN 0x1.0p-126f
204#define FLT_EPSILON 0x1.0p-23f
205
206#define M_E_F 2.71828182845904523536028747135266250f
207#define M_LOG2E_F 1.44269504088896340735992468100189214f
208#define M_LOG10E_F 0.434294481903251827651128918916605082f
209#define M_LN2_F 0.693147180559945309417232121458176568f
210#define M_LN10_F 2.30258509299404568401799145468436421f
211#define M_PI_F 3.14159265358979323846264338327950288f
212#define M_PI_2_F 1.57079632679489661923132169163975144f
213#define M_PI_4_F 0.785398163397448309615660845819875721f
214#define M_1_PI_F 0.318309886183790671537767526745028724f
215#define M_2_PI_F 0.636619772367581343075535053490057448f
216#define M_2_SQRTPI_F 1.12837916709551257389615890312154517f
217#define M_SQRT2_F 1.41421356237309504880168872420969808f
218#define M_SQRT1_2_F 0.707106781186547524400844362104849039f
219
220#define DBL_DIG 15
221#define DBL_MANT_DIG 53
222#define DBL_MAX_10_EXP +308
223#define DBL_MAX_EXP +1024
224#define DBL_MIN_10_EXP -307
225#define DBL_MIN_EXP -1021
226#define DBL_RADIX 2
227#define DBL_MAX 0x1.fffffffffffffp1023
228#define DBL_MIN 0x1.0p-1022
229#define DBL_EPSILON 0x1.0p-52
230
231#define M_E 0x1.5bf0a8b145769p+1
232#define M_LOG2E 0x1.71547652b82fep+0
233#define M_LOG10E 0x1.bcb7b1526e50ep-2
234#define M_LN2 0x1.62e42fefa39efp-1
235#define M_LN10 0x1.26bb1bbb55516p+1
236#define M_PI 0x1.921fb54442d18p+1
237#define M_PI_2 0x1.921fb54442d18p+0
238#define M_PI_4 0x1.921fb54442d18p-1
239#define M_1_PI 0x1.45f306dc9c883p-2
240#define M_2_PI 0x1.45f306dc9c883p-1
241#define M_2_SQRTPI 0x1.20dd750429b6dp+0
242#define M_SQRT2 0x1.6a09e667f3bcdp+0
243#define M_SQRT1_2 0x1.6a09e667f3bcdp-1
244
245#ifdef cl_khr_fp16
246
247#define HALF_DIG 3
248#define HALF_MANT_DIG 11
249#define HALF_MAX_10_EXP +4
250#define HALF_MAX_EXP +16
251#define HALF_MIN_10_EXP -4
252#define HALF_MIN_EXP -13
253#define HALF_RADIX 2
254#define HALF_MAX ((0x1.ffcp15h))
255#define HALF_MIN ((0x1.0p-14h))
256#define HALF_EPSILON ((0x1.0p-10h))
257
258#define M_E_H 2.71828182845904523536028747135266250h
259#define M_LOG2E_H 1.44269504088896340735992468100189214h
260#define M_LOG10E_H 0.434294481903251827651128918916605082h
261#define M_LN2_H 0.693147180559945309417232121458176568h
262#define M_LN10_H 2.30258509299404568401799145468436421h
263#define M_PI_H 3.14159265358979323846264338327950288h
264#define M_PI_2_H 1.57079632679489661923132169163975144h
265#define M_PI_4_H 0.785398163397448309615660845819875721h
266#define M_1_PI_H 0.318309886183790671537767526745028724h
267#define M_2_PI_H 0.636619772367581343075535053490057448h
268#define M_2_SQRTPI_H 1.12837916709551257389615890312154517h
269#define M_SQRT2_H 1.41421356237309504880168872420969808h
270#define M_SQRT1_2_H 0.707106781186547524400844362104849039h
271
272#endif //cl_khr_fp16
273
274#define CHAR_BIT 8
275#define SCHAR_MAX 127
276#define SCHAR_MIN (-128)
277#define UCHAR_MAX 255
278#define CHAR_MAX SCHAR_MAX
279#define CHAR_MIN SCHAR_MIN
280#define USHRT_MAX 65535
281#define SHRT_MAX 32767
282#define SHRT_MIN (-32768)
283#define UINT_MAX 0xffffffff
284#define INT_MAX 2147483647
285#define INT_MIN (-2147483647-1)
286#define ULONG_MAX 0xffffffffffffffffUL
287#define LONG_MAX 0x7fffffffffffffffL
288#define LONG_MIN (-0x7fffffffffffffffL-1)
28938
290// OpenCL v1.1/1.2/2.0 s6.2.3 - Explicit conversions39// OpenCL v1.1/1.2/2.0 s6.2.3 - Explicit conversions
29140
...@@ -9598,8 +9347,6 @@ long8 __ovld __cnfn clamp(long8 x, long8 minval, long8 maxval);...@@ -9598,8 +9347,6 @@ long8 __ovld __cnfn clamp(long8 x, long8 minval, long8 maxval);
9598ulong8 __ovld __cnfn clamp(ulong8 x, ulong8 minval, ulong8 maxval);9347ulong8 __ovld __cnfn clamp(ulong8 x, ulong8 minval, ulong8 maxval);
9599long16 __ovld __cnfn clamp(long16 x, long16 minval, long16 maxval);9348long16 __ovld __cnfn clamp(long16 x, long16 minval, long16 maxval);
9600ulong16 __ovld __cnfn clamp(ulong16 x, ulong16 minval, ulong16 maxval);9349ulong16 __ovld __cnfn clamp(ulong16 x, ulong16 minval, ulong16 maxval);
9601char __ovld __cnfn clamp(char x, char minval, char maxval);
9602uchar __ovld __cnfn clamp(uchar x, uchar minval, uchar maxval);
9603char2 __ovld __cnfn clamp(char2 x, char minval, char maxval);9350char2 __ovld __cnfn clamp(char2 x, char minval, char maxval);
9604uchar2 __ovld __cnfn clamp(uchar2 x, uchar minval, uchar maxval);9351uchar2 __ovld __cnfn clamp(uchar2 x, uchar minval, uchar maxval);
9605char3 __ovld __cnfn clamp(char3 x, char minval, char maxval);9352char3 __ovld __cnfn clamp(char3 x, char minval, char maxval);
...@@ -9610,8 +9357,6 @@ char8 __ovld __cnfn clamp(char8 x, char minval, char maxval);...@@ -9610,8 +9357,6 @@ char8 __ovld __cnfn clamp(char8 x, char minval, char maxval);
9610uchar8 __ovld __cnfn clamp(uchar8 x, uchar minval, uchar maxval);9357uchar8 __ovld __cnfn clamp(uchar8 x, uchar minval, uchar maxval);
9611char16 __ovld __cnfn clamp(char16 x, char minval, char maxval);9358char16 __ovld __cnfn clamp(char16 x, char minval, char maxval);
9612uchar16 __ovld __cnfn clamp(uchar16 x, uchar minval, uchar maxval);9359uchar16 __ovld __cnfn clamp(uchar16 x, uchar minval, uchar maxval);
9613short __ovld __cnfn clamp(short x, short minval, short maxval);
9614ushort __ovld __cnfn clamp(ushort x, ushort minval, ushort maxval);
9615short2 __ovld __cnfn clamp(short2 x, short minval, short maxval);9360short2 __ovld __cnfn clamp(short2 x, short minval, short maxval);
9616ushort2 __ovld __cnfn clamp(ushort2 x, ushort minval, ushort maxval);9361ushort2 __ovld __cnfn clamp(ushort2 x, ushort minval, ushort maxval);
9617short3 __ovld __cnfn clamp(short3 x, short minval, short maxval);9362short3 __ovld __cnfn clamp(short3 x, short minval, short maxval);
...@@ -9622,8 +9367,6 @@ short8 __ovld __cnfn clamp(short8 x, short minval, short maxval);...@@ -9622,8 +9367,6 @@ short8 __ovld __cnfn clamp(short8 x, short minval, short maxval);
9622ushort8 __ovld __cnfn clamp(ushort8 x, ushort minval, ushort maxval);9367ushort8 __ovld __cnfn clamp(ushort8 x, ushort minval, ushort maxval);
9623short16 __ovld __cnfn clamp(short16 x, short minval, short maxval);9368short16 __ovld __cnfn clamp(short16 x, short minval, short maxval);
9624ushort16 __ovld __cnfn clamp(ushort16 x, ushort minval, ushort maxval);9369ushort16 __ovld __cnfn clamp(ushort16 x, ushort minval, ushort maxval);
9625int __ovld __cnfn clamp(int x, int minval, int maxval);
9626uint __ovld __cnfn clamp(uint x, uint minval, uint maxval);
9627int2 __ovld __cnfn clamp(int2 x, int minval, int maxval);9370int2 __ovld __cnfn clamp(int2 x, int minval, int maxval);
9628uint2 __ovld __cnfn clamp(uint2 x, uint minval, uint maxval);9371uint2 __ovld __cnfn clamp(uint2 x, uint minval, uint maxval);
9629int3 __ovld __cnfn clamp(int3 x, int minval, int maxval);9372int3 __ovld __cnfn clamp(int3 x, int minval, int maxval);
...@@ -9634,8 +9377,6 @@ int8 __ovld __cnfn clamp(int8 x, int minval, int maxval);...@@ -9634,8 +9377,6 @@ int8 __ovld __cnfn clamp(int8 x, int minval, int maxval);
9634uint8 __ovld __cnfn clamp(uint8 x, uint minval, uint maxval);9377uint8 __ovld __cnfn clamp(uint8 x, uint minval, uint maxval);
9635int16 __ovld __cnfn clamp(int16 x, int minval, int maxval);9378int16 __ovld __cnfn clamp(int16 x, int minval, int maxval);
9636uint16 __ovld __cnfn clamp(uint16 x, uint minval, uint maxval);9379uint16 __ovld __cnfn clamp(uint16 x, uint minval, uint maxval);
9637long __ovld __cnfn clamp(long x, long minval, long maxval);
9638ulong __ovld __cnfn clamp(ulong x, ulong minval, ulong maxval);
9639long2 __ovld __cnfn clamp(long2 x, long minval, long maxval);9380long2 __ovld __cnfn clamp(long2 x, long minval, long maxval);
9640ulong2 __ovld __cnfn clamp(ulong2 x, ulong minval, ulong maxval);9381ulong2 __ovld __cnfn clamp(ulong2 x, ulong minval, ulong maxval);
9641long3 __ovld __cnfn clamp(long3 x, long minval, long maxval);9382long3 __ovld __cnfn clamp(long3 x, long minval, long maxval);
...@@ -9911,8 +9652,6 @@ long8 __ovld __cnfn max(long8 x, long8 y);...@@ -9911,8 +9652,6 @@ long8 __ovld __cnfn max(long8 x, long8 y);
9911ulong8 __ovld __cnfn max(ulong8 x, ulong8 y);9652ulong8 __ovld __cnfn max(ulong8 x, ulong8 y);
9912long16 __ovld __cnfn max(long16 x, long16 y);9653long16 __ovld __cnfn max(long16 x, long16 y);
9913ulong16 __ovld __cnfn max(ulong16 x, ulong16 y);9654ulong16 __ovld __cnfn max(ulong16 x, ulong16 y);
9914char __ovld __cnfn max(char x, char y);
9915uchar __ovld __cnfn max(uchar x, uchar y);
9916char2 __ovld __cnfn max(char2 x, char y);9655char2 __ovld __cnfn max(char2 x, char y);
9917uchar2 __ovld __cnfn max(uchar2 x, uchar y);9656uchar2 __ovld __cnfn max(uchar2 x, uchar y);
9918char3 __ovld __cnfn max(char3 x, char y);9657char3 __ovld __cnfn max(char3 x, char y);
...@@ -9923,8 +9662,6 @@ char8 __ovld __cnfn max(char8 x, char y);...@@ -9923,8 +9662,6 @@ char8 __ovld __cnfn max(char8 x, char y);
9923uchar8 __ovld __cnfn max(uchar8 x, uchar y);9662uchar8 __ovld __cnfn max(uchar8 x, uchar y);
9924char16 __ovld __cnfn max(char16 x, char y);9663char16 __ovld __cnfn max(char16 x, char y);
9925uchar16 __ovld __cnfn max(uchar16 x, uchar y);9664uchar16 __ovld __cnfn max(uchar16 x, uchar y);
9926short __ovld __cnfn max(short x, short y);
9927ushort __ovld __cnfn max(ushort x, ushort y);
9928short2 __ovld __cnfn max(short2 x, short y);9665short2 __ovld __cnfn max(short2 x, short y);
9929ushort2 __ovld __cnfn max(ushort2 x, ushort y);9666ushort2 __ovld __cnfn max(ushort2 x, ushort y);
9930short3 __ovld __cnfn max(short3 x, short y);9667short3 __ovld __cnfn max(short3 x, short y);
...@@ -9935,8 +9672,6 @@ short8 __ovld __cnfn max(short8 x, short y);...@@ -9935,8 +9672,6 @@ short8 __ovld __cnfn max(short8 x, short y);
9935ushort8 __ovld __cnfn max(ushort8 x, ushort y);9672ushort8 __ovld __cnfn max(ushort8 x, ushort y);
9936short16 __ovld __cnfn max(short16 x, short y);9673short16 __ovld __cnfn max(short16 x, short y);
9937ushort16 __ovld __cnfn max(ushort16 x, ushort y);9674ushort16 __ovld __cnfn max(ushort16 x, ushort y);
9938int __ovld __cnfn max(int x, int y);
9939uint __ovld __cnfn max(uint x, uint y);
9940int2 __ovld __cnfn max(int2 x, int y);9675int2 __ovld __cnfn max(int2 x, int y);
9941uint2 __ovld __cnfn max(uint2 x, uint y);9676uint2 __ovld __cnfn max(uint2 x, uint y);
9942int3 __ovld __cnfn max(int3 x, int y);9677int3 __ovld __cnfn max(int3 x, int y);
...@@ -9947,8 +9682,6 @@ int8 __ovld __cnfn max(int8 x, int y);...@@ -9947,8 +9682,6 @@ int8 __ovld __cnfn max(int8 x, int y);
9947uint8 __ovld __cnfn max(uint8 x, uint y);9682uint8 __ovld __cnfn max(uint8 x, uint y);
9948int16 __ovld __cnfn max(int16 x, int y);9683int16 __ovld __cnfn max(int16 x, int y);
9949uint16 __ovld __cnfn max(uint16 x, uint y);9684uint16 __ovld __cnfn max(uint16 x, uint y);
9950long __ovld __cnfn max(long x, long y);
9951ulong __ovld __cnfn max(ulong x, ulong y);
9952long2 __ovld __cnfn max(long2 x, long y);9685long2 __ovld __cnfn max(long2 x, long y);
9953ulong2 __ovld __cnfn max(ulong2 x, ulong y);9686ulong2 __ovld __cnfn max(ulong2 x, ulong y);
9954long3 __ovld __cnfn max(long3 x, long y);9687long3 __ovld __cnfn max(long3 x, long y);
...@@ -10011,8 +9744,6 @@ long8 __ovld __cnfn min(long8 x, long8 y);...@@ -10011,8 +9744,6 @@ long8 __ovld __cnfn min(long8 x, long8 y);
10011ulong8 __ovld __cnfn min(ulong8 x, ulong8 y);9744ulong8 __ovld __cnfn min(ulong8 x, ulong8 y);
10012long16 __ovld __cnfn min(long16 x, long16 y);9745long16 __ovld __cnfn min(long16 x, long16 y);
10013ulong16 __ovld __cnfn min(ulong16 x, ulong16 y);9746ulong16 __ovld __cnfn min(ulong16 x, ulong16 y);
10014char __ovld __cnfn min(char x, char y);
10015uchar __ovld __cnfn min(uchar x, uchar y);
10016char2 __ovld __cnfn min(char2 x, char y);9747char2 __ovld __cnfn min(char2 x, char y);
10017uchar2 __ovld __cnfn min(uchar2 x, uchar y);9748uchar2 __ovld __cnfn min(uchar2 x, uchar y);
10018char3 __ovld __cnfn min(char3 x, char y);9749char3 __ovld __cnfn min(char3 x, char y);
...@@ -10023,8 +9754,6 @@ char8 __ovld __cnfn min(char8 x, char y);...@@ -10023,8 +9754,6 @@ char8 __ovld __cnfn min(char8 x, char y);
10023uchar8 __ovld __cnfn min(uchar8 x, uchar y);9754uchar8 __ovld __cnfn min(uchar8 x, uchar y);
10024char16 __ovld __cnfn min(char16 x, char y);9755char16 __ovld __cnfn min(char16 x, char y);
10025uchar16 __ovld __cnfn min(uchar16 x, uchar y);9756uchar16 __ovld __cnfn min(uchar16 x, uchar y);
10026short __ovld __cnfn min(short x, short y);
10027ushort __ovld __cnfn min(ushort x, ushort y);
10028short2 __ovld __cnfn min(short2 x, short y);9757short2 __ovld __cnfn min(short2 x, short y);
10029ushort2 __ovld __cnfn min(ushort2 x, ushort y);9758ushort2 __ovld __cnfn min(ushort2 x, ushort y);
10030short3 __ovld __cnfn min(short3 x, short y);9759short3 __ovld __cnfn min(short3 x, short y);
...@@ -10035,8 +9764,6 @@ short8 __ovld __cnfn min(short8 x, short y);...@@ -10035,8 +9764,6 @@ short8 __ovld __cnfn min(short8 x, short y);
10035ushort8 __ovld __cnfn min(ushort8 x, ushort y);9764ushort8 __ovld __cnfn min(ushort8 x, ushort y);
10036short16 __ovld __cnfn min(short16 x, short y);9765short16 __ovld __cnfn min(short16 x, short y);
10037ushort16 __ovld __cnfn min(ushort16 x, ushort y);9766ushort16 __ovld __cnfn min(ushort16 x, ushort y);
10038int __ovld __cnfn min(int x, int y);
10039uint __ovld __cnfn min(uint x, uint y);
10040int2 __ovld __cnfn min(int2 x, int y);9767int2 __ovld __cnfn min(int2 x, int y);
10041uint2 __ovld __cnfn min(uint2 x, uint y);9768uint2 __ovld __cnfn min(uint2 x, uint y);
10042int3 __ovld __cnfn min(int3 x, int y);9769int3 __ovld __cnfn min(int3 x, int y);
...@@ -10047,8 +9774,6 @@ int8 __ovld __cnfn min(int8 x, int y);...@@ -10047,8 +9774,6 @@ int8 __ovld __cnfn min(int8 x, int y);
10047uint8 __ovld __cnfn min(uint8 x, uint y);9774uint8 __ovld __cnfn min(uint8 x, uint y);
10048int16 __ovld __cnfn min(int16 x, int y);9775int16 __ovld __cnfn min(int16 x, int y);
10049uint16 __ovld __cnfn min(uint16 x, uint y);9776uint16 __ovld __cnfn min(uint16 x, uint y);
10050long __ovld __cnfn min(long x, long y);
10051ulong __ovld __cnfn min(ulong x, ulong y);
10052long2 __ovld __cnfn min(long2 x, long y);9777long2 __ovld __cnfn min(long2 x, long y);
10053ulong2 __ovld __cnfn min(ulong2 x, ulong y);9778ulong2 __ovld __cnfn min(ulong2 x, ulong y);
10054long3 __ovld __cnfn min(long3 x, long y);9779long3 __ovld __cnfn min(long3 x, long y);
...@@ -10627,7 +10352,6 @@ half3 __ovld __cnfn step(half3 edge, half3 x);...@@ -10627,7 +10352,6 @@ half3 __ovld __cnfn step(half3 edge, half3 x);
10627half4 __ovld __cnfn step(half4 edge, half4 x);10352half4 __ovld __cnfn step(half4 edge, half4 x);
10628half8 __ovld __cnfn step(half8 edge, half8 x);10353half8 __ovld __cnfn step(half8 edge, half8 x);
10629half16 __ovld __cnfn step(half16 edge, half16 x);10354half16 __ovld __cnfn step(half16 edge, half16 x);
10630half __ovld __cnfn step(half edge, half x);
10631half2 __ovld __cnfn step(half edge, half2 x);10355half2 __ovld __cnfn step(half edge, half2 x);
10632half3 __ovld __cnfn step(half edge, half3 x);10356half3 __ovld __cnfn step(half edge, half3 x);
10633half4 __ovld __cnfn step(half edge, half4 x);10357half4 __ovld __cnfn step(half edge, half4 x);
...@@ -10679,7 +10403,6 @@ half3 __ovld __cnfn smoothstep(half3 edge0, half3 edge1, half3 x);...@@ -10679,7 +10403,6 @@ half3 __ovld __cnfn smoothstep(half3 edge0, half3 edge1, half3 x);
10679half4 __ovld __cnfn smoothstep(half4 edge0, half4 edge1, half4 x);10403half4 __ovld __cnfn smoothstep(half4 edge0, half4 edge1, half4 x);
10680half8 __ovld __cnfn smoothstep(half8 edge0, half8 edge1, half8 x);10404half8 __ovld __cnfn smoothstep(half8 edge0, half8 edge1, half8 x);
10681half16 __ovld __cnfn smoothstep(half16 edge0, half16 edge1, half16 x);10405half16 __ovld __cnfn smoothstep(half16 edge0, half16 edge1, half16 x);
10682half __ovld __cnfn smoothstep(half edge0, half edge1, half x);
10683half2 __ovld __cnfn smoothstep(half edge0, half edge1, half2 x);10406half2 __ovld __cnfn smoothstep(half edge0, half edge1, half2 x);
10684half3 __ovld __cnfn smoothstep(half edge0, half edge1, half3 x);10407half3 __ovld __cnfn smoothstep(half edge0, half edge1, half3 x);
10685half4 __ovld __cnfn smoothstep(half edge0, half edge1, half4 x);10408half4 __ovld __cnfn smoothstep(half edge0, half edge1, half4 x);
...@@ -12777,30 +12500,6 @@ void __ovld vstorea_half16_rtn(double16 data,size_t offset, __private half *p);...@@ -12777,30 +12500,6 @@ void __ovld vstorea_half16_rtn(double16 data,size_t offset, __private half *p);
1277712500
12778// OpenCL v1.1 s6.11.8, v1.2 s6.12.8, v2.0 s6.13.8 - Synchronization Functions12501// OpenCL v1.1 s6.11.8, v1.2 s6.12.8, v2.0 s6.13.8 - Synchronization Functions
1277912502
12780// Flag type and values for barrier, mem_fence, read_mem_fence, write_mem_fence
12781typedef uint cl_mem_fence_flags;
12782
12783/**
12784 * Queue a memory fence to ensure correct
12785 * ordering of memory operations to local memory
12786 */
12787#define CLK_LOCAL_MEM_FENCE 0x01
12788
12789/**
12790 * Queue a memory fence to ensure correct
12791 * ordering of memory operations to global memory
12792 */
12793#define CLK_GLOBAL_MEM_FENCE 0x02
12794
12795#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
12796/**
12797 * Queue a memory fence to ensure correct ordering of memory
12798 * operations between work-items of a work-group to
12799 * image memory.
12800 */
12801#define CLK_IMAGE_MEM_FENCE 0x04
12802#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0
12803
12804/**12503/**
12805 * All work-items in a work-group executing the kernel12504 * All work-items in a work-group executing the kernel
12806 * on a processor must execute this function before any12505 * on a processor must execute this function before any
...@@ -12834,17 +12533,6 @@ typedef uint cl_mem_fence_flags;...@@ -12834,17 +12533,6 @@ typedef uint cl_mem_fence_flags;
12834void __ovld __conv barrier(cl_mem_fence_flags flags);12533void __ovld __conv barrier(cl_mem_fence_flags flags);
1283512534
12836#if __OPENCL_C_VERSION__ >= CL_VERSION_2_012535#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
12837
12838typedef enum memory_scope {
12839 memory_scope_work_item = __OPENCL_MEMORY_SCOPE_WORK_ITEM,
12840 memory_scope_work_group = __OPENCL_MEMORY_SCOPE_WORK_GROUP,
12841 memory_scope_device = __OPENCL_MEMORY_SCOPE_DEVICE,
12842 memory_scope_all_svm_devices = __OPENCL_MEMORY_SCOPE_ALL_SVM_DEVICES,
12843#if defined(cl_intel_subgroups) || defined(cl_khr_subgroups)
12844 memory_scope_sub_group = __OPENCL_MEMORY_SCOPE_SUB_GROUP
12845#endif
12846} memory_scope;
12847
12848void __ovld __conv work_group_barrier(cl_mem_fence_flags flags, memory_scope scope);12536void __ovld __conv work_group_barrier(cl_mem_fence_flags flags, memory_scope scope);
12849void __ovld __conv work_group_barrier(cl_mem_fence_flags flags);12537void __ovld __conv work_group_barrier(cl_mem_fence_flags flags);
12850#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_012538#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0
...@@ -13341,6 +13029,10 @@ int __ovld atomic_add(volatile __global int *p, int val);...@@ -13341,6 +13029,10 @@ int __ovld atomic_add(volatile __global int *p, int val);
13341unsigned int __ovld atomic_add(volatile __global unsigned int *p, unsigned int val);13029unsigned int __ovld atomic_add(volatile __global unsigned int *p, unsigned int val);
13342int __ovld atomic_add(volatile __local int *p, int val);13030int __ovld atomic_add(volatile __local int *p, int val);
13343unsigned int __ovld atomic_add(volatile __local unsigned int *p, unsigned int val);13031unsigned int __ovld atomic_add(volatile __local unsigned int *p, unsigned int val);
13032#ifdef __OPENCL_CPP_VERSION__
13033int __ovld atomic_add(volatile int *p, int val);
13034unsigned int __ovld atomic_add(volatile unsigned int *p, unsigned int val);
13035#endif
1334413036
13345#if defined(cl_khr_global_int32_base_atomics)13037#if defined(cl_khr_global_int32_base_atomics)
13346int __ovld atom_add(volatile __global int *p, int val);13038int __ovld atom_add(volatile __global int *p, int val);
...@@ -13367,6 +13059,10 @@ int __ovld atomic_sub(volatile __global int *p, int val);...@@ -13367,6 +13059,10 @@ int __ovld atomic_sub(volatile __global int *p, int val);
13367unsigned int __ovld atomic_sub(volatile __global unsigned int *p, unsigned int val);13059unsigned int __ovld atomic_sub(volatile __global unsigned int *p, unsigned int val);
13368int __ovld atomic_sub(volatile __local int *p, int val);13060int __ovld atomic_sub(volatile __local int *p, int val);
13369unsigned int __ovld atomic_sub(volatile __local unsigned int *p, unsigned int val);13061unsigned int __ovld atomic_sub(volatile __local unsigned int *p, unsigned int val);
13062#ifdef __OPENCL_CPP_VERSION__
13063int __ovld atomic_sub(volatile int *p, int val);
13064unsigned int __ovld atomic_sub(volatile unsigned int *p, unsigned int val);
13065#endif
1337013066
13371#if defined(cl_khr_global_int32_base_atomics)13067#if defined(cl_khr_global_int32_base_atomics)
13372int __ovld atom_sub(volatile __global int *p, int val);13068int __ovld atom_sub(volatile __global int *p, int val);
...@@ -13395,6 +13091,11 @@ int __ovld atomic_xchg(volatile __local int *p, int val);...@@ -13395,6 +13091,11 @@ int __ovld atomic_xchg(volatile __local int *p, int val);
13395unsigned int __ovld atomic_xchg(volatile __local unsigned int *p, unsigned int val);13091unsigned int __ovld atomic_xchg(volatile __local unsigned int *p, unsigned int val);
13396float __ovld atomic_xchg(volatile __global float *p, float val);13092float __ovld atomic_xchg(volatile __global float *p, float val);
13397float __ovld atomic_xchg(volatile __local float *p, float val);13093float __ovld atomic_xchg(volatile __local float *p, float val);
13094#ifdef __OPENCL_CPP_VERSION__
13095int __ovld atomic_xchg(volatile int *p, int val);
13096unsigned int __ovld atomic_xchg(volatile unsigned int *p, unsigned int val);
13097float __ovld atomic_xchg(volatile float *p, float val);
13098#endif
1339813099
13399#if defined(cl_khr_global_int32_base_atomics)13100#if defined(cl_khr_global_int32_base_atomics)
13400int __ovld atom_xchg(volatile __global int *p, int val);13101int __ovld atom_xchg(volatile __global int *p, int val);
...@@ -13422,6 +13123,10 @@ int __ovld atomic_inc(volatile __global int *p);...@@ -13422,6 +13123,10 @@ int __ovld atomic_inc(volatile __global int *p);
13422unsigned int __ovld atomic_inc(volatile __global unsigned int *p);13123unsigned int __ovld atomic_inc(volatile __global unsigned int *p);
13423int __ovld atomic_inc(volatile __local int *p);13124int __ovld atomic_inc(volatile __local int *p);
13424unsigned int __ovld atomic_inc(volatile __local unsigned int *p);13125unsigned int __ovld atomic_inc(volatile __local unsigned int *p);
13126#ifdef __OPENCL_CPP_VERSION__
13127int __ovld atomic_inc(volatile int *p);
13128unsigned int __ovld atomic_inc(volatile unsigned int *p);
13129#endif
1342513130
13426#if defined(cl_khr_global_int32_base_atomics)13131#if defined(cl_khr_global_int32_base_atomics)
13427int __ovld atom_inc(volatile __global int *p);13132int __ovld atom_inc(volatile __global int *p);
...@@ -13449,6 +13154,10 @@ int __ovld atomic_dec(volatile __global int *p);...@@ -13449,6 +13154,10 @@ int __ovld atomic_dec(volatile __global int *p);
13449unsigned int __ovld atomic_dec(volatile __global unsigned int *p);13154unsigned int __ovld atomic_dec(volatile __global unsigned int *p);
13450int __ovld atomic_dec(volatile __local int *p);13155int __ovld atomic_dec(volatile __local int *p);
13451unsigned int __ovld atomic_dec(volatile __local unsigned int *p);13156unsigned int __ovld atomic_dec(volatile __local unsigned int *p);
13157#ifdef __OPENCL_CPP_VERSION__
13158int __ovld atomic_dec(volatile int *p);
13159unsigned int __ovld atomic_dec(volatile unsigned int *p);
13160#endif
1345213161
13453#if defined(cl_khr_global_int32_base_atomics)13162#if defined(cl_khr_global_int32_base_atomics)
13454int __ovld atom_dec(volatile __global int *p);13163int __ovld atom_dec(volatile __global int *p);
...@@ -13477,6 +13186,10 @@ int __ovld atomic_cmpxchg(volatile __global int *p, int cmp, int val);...@@ -13477,6 +13186,10 @@ int __ovld atomic_cmpxchg(volatile __global int *p, int cmp, int val);
13477unsigned int __ovld atomic_cmpxchg(volatile __global unsigned int *p, unsigned int cmp, unsigned int val);13186unsigned int __ovld atomic_cmpxchg(volatile __global unsigned int *p, unsigned int cmp, unsigned int val);
13478int __ovld atomic_cmpxchg(volatile __local int *p, int cmp, int val);13187int __ovld atomic_cmpxchg(volatile __local int *p, int cmp, int val);
13479unsigned int __ovld atomic_cmpxchg(volatile __local unsigned int *p, unsigned int cmp, unsigned int val);13188unsigned int __ovld atomic_cmpxchg(volatile __local unsigned int *p, unsigned int cmp, unsigned int val);
13189#ifdef __OPENCL_CPP_VERSION__
13190int __ovld atomic_cmpxchg(volatile int *p, int cmp, int val);
13191unsigned int __ovld atomic_cmpxchg(volatile unsigned int *p, unsigned int cmp, unsigned int val);
13192#endif
1348013193
13481#if defined(cl_khr_global_int32_base_atomics)13194#if defined(cl_khr_global_int32_base_atomics)
13482int __ovld atom_cmpxchg(volatile __global int *p, int cmp, int val);13195int __ovld atom_cmpxchg(volatile __global int *p, int cmp, int val);
...@@ -13505,6 +13218,10 @@ int __ovld atomic_min(volatile __global int *p, int val);...@@ -13505,6 +13218,10 @@ int __ovld atomic_min(volatile __global int *p, int val);
13505unsigned int __ovld atomic_min(volatile __global unsigned int *p, unsigned int val);13218unsigned int __ovld atomic_min(volatile __global unsigned int *p, unsigned int val);
13506int __ovld atomic_min(volatile __local int *p, int val);13219int __ovld atomic_min(volatile __local int *p, int val);
13507unsigned int __ovld atomic_min(volatile __local unsigned int *p, unsigned int val);13220unsigned int __ovld atomic_min(volatile __local unsigned int *p, unsigned int val);
13221#ifdef __OPENCL_CPP_VERSION__
13222int __ovld atomic_min(volatile int *p, int val);
13223unsigned int __ovld atomic_min(volatile unsigned int *p, unsigned int val);
13224#endif
1350813225
13509#if defined(cl_khr_global_int32_extended_atomics)13226#if defined(cl_khr_global_int32_extended_atomics)
13510int __ovld atom_min(volatile __global int *p, int val);13227int __ovld atom_min(volatile __global int *p, int val);
...@@ -13533,6 +13250,10 @@ int __ovld atomic_max(volatile __global int *p, int val);...@@ -13533,6 +13250,10 @@ int __ovld atomic_max(volatile __global int *p, int val);
13533unsigned int __ovld atomic_max(volatile __global unsigned int *p, unsigned int val);13250unsigned int __ovld atomic_max(volatile __global unsigned int *p, unsigned int val);
13534int __ovld atomic_max(volatile __local int *p, int val);13251int __ovld atomic_max(volatile __local int *p, int val);
13535unsigned int __ovld atomic_max(volatile __local unsigned int *p, unsigned int val);13252unsigned int __ovld atomic_max(volatile __local unsigned int *p, unsigned int val);
13253#ifdef __OPENCL_CPP_VERSION__
13254int __ovld atomic_max(volatile int *p, int val);
13255unsigned int __ovld atomic_max(volatile unsigned int *p, unsigned int val);
13256#endif
1353613257
13537#if defined(cl_khr_global_int32_extended_atomics)13258#if defined(cl_khr_global_int32_extended_atomics)
13538int __ovld atom_max(volatile __global int *p, int val);13259int __ovld atom_max(volatile __global int *p, int val);
...@@ -13560,6 +13281,10 @@ int __ovld atomic_and(volatile __global int *p, int val);...@@ -13560,6 +13281,10 @@ int __ovld atomic_and(volatile __global int *p, int val);
13560unsigned int __ovld atomic_and(volatile __global unsigned int *p, unsigned int val);13281unsigned int __ovld atomic_and(volatile __global unsigned int *p, unsigned int val);
13561int __ovld atomic_and(volatile __local int *p, int val);13282int __ovld atomic_and(volatile __local int *p, int val);
13562unsigned int __ovld atomic_and(volatile __local unsigned int *p, unsigned int val);13283unsigned int __ovld atomic_and(volatile __local unsigned int *p, unsigned int val);
13284#ifdef __OPENCL_CPP_VERSION__
13285int __ovld atomic_and(volatile int *p, int val);
13286unsigned int __ovld atomic_and(volatile unsigned int *p, unsigned int val);
13287#endif
1356313288
13564#if defined(cl_khr_global_int32_extended_atomics)13289#if defined(cl_khr_global_int32_extended_atomics)
13565int __ovld atom_and(volatile __global int *p, int val);13290int __ovld atom_and(volatile __global int *p, int val);
...@@ -13587,6 +13312,10 @@ int __ovld atomic_or(volatile __global int *p, int val);...@@ -13587,6 +13312,10 @@ int __ovld atomic_or(volatile __global int *p, int val);
13587unsigned int __ovld atomic_or(volatile __global unsigned int *p, unsigned int val);13312unsigned int __ovld atomic_or(volatile __global unsigned int *p, unsigned int val);
13588int __ovld atomic_or(volatile __local int *p, int val);13313int __ovld atomic_or(volatile __local int *p, int val);
13589unsigned int __ovld atomic_or(volatile __local unsigned int *p, unsigned int val);13314unsigned int __ovld atomic_or(volatile __local unsigned int *p, unsigned int val);
13315#ifdef __OPENCL_CPP_VERSION__
13316int __ovld atomic_or(volatile int *p, int val);
13317unsigned int __ovld atomic_or(volatile unsigned int *p, unsigned int val);
13318#endif
1359013319
13591#if defined(cl_khr_global_int32_extended_atomics)13320#if defined(cl_khr_global_int32_extended_atomics)
13592int __ovld atom_or(volatile __global int *p, int val);13321int __ovld atom_or(volatile __global int *p, int val);
...@@ -13614,6 +13343,10 @@ int __ovld atomic_xor(volatile __global int *p, int val);...@@ -13614,6 +13343,10 @@ int __ovld atomic_xor(volatile __global int *p, int val);
13614unsigned int __ovld atomic_xor(volatile __global unsigned int *p, unsigned int val);13343unsigned int __ovld atomic_xor(volatile __global unsigned int *p, unsigned int val);
13615int __ovld atomic_xor(volatile __local int *p, int val);13344int __ovld atomic_xor(volatile __local int *p, int val);
13616unsigned int __ovld atomic_xor(volatile __local unsigned int *p, unsigned int val);13345unsigned int __ovld atomic_xor(volatile __local unsigned int *p, unsigned int val);
13346#ifdef __OPENCL_CPP_VERSION__
13347int __ovld atomic_xor(volatile int *p, int val);
13348unsigned int __ovld atomic_xor(volatile unsigned int *p, unsigned int val);
13349#endif
1361713350
13618#if defined(cl_khr_global_int32_extended_atomics)13351#if defined(cl_khr_global_int32_extended_atomics)
13619int __ovld atom_xor(volatile __global int *p, int val);13352int __ovld atom_xor(volatile __global int *p, int val);
...@@ -13639,20 +13372,6 @@ unsigned long __ovld atom_xor(volatile __local unsigned long *p, unsigned long v...@@ -13639,20 +13372,6 @@ unsigned long __ovld atom_xor(volatile __local unsigned long *p, unsigned long v
13639// OpenCL v2.0 s6.13.11 - Atomics Functions13372// OpenCL v2.0 s6.13.11 - Atomics Functions
1364013373
13641#if __OPENCL_C_VERSION__ >= CL_VERSION_2_013374#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
13642#ifndef ATOMIC_VAR_INIT
13643#define ATOMIC_VAR_INIT(x) (x)
13644#endif //ATOMIC_VAR_INIT
13645#define ATOMIC_FLAG_INIT 0
13646
13647// enum values aligned with what clang uses in EmitAtomicExpr()
13648typedef enum memory_order
13649{
13650 memory_order_relaxed = __ATOMIC_RELAXED,
13651 memory_order_acquire = __ATOMIC_ACQUIRE,
13652 memory_order_release = __ATOMIC_RELEASE,
13653 memory_order_acq_rel = __ATOMIC_ACQ_REL,
13654 memory_order_seq_cst = __ATOMIC_SEQ_CST
13655} memory_order;
1365613375
13657// double atomics support requires extensions cl_khr_int64_base_atomics and cl_khr_int64_extended_atomics13376// double atomics support requires extensions cl_khr_int64_base_atomics and cl_khr_int64_extended_atomics
13658#if defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics)13377#if defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics)
...@@ -14470,33 +14189,11 @@ half16 __ovld __cnfn shuffle2(half16 x, half16 y, ushort16 mask);...@@ -14470,33 +14189,11 @@ half16 __ovld __cnfn shuffle2(half16 x, half16 y, ushort16 mask);
14470#if __OPENCL_C_VERSION__ >= CL_VERSION_1_214189#if __OPENCL_C_VERSION__ >= CL_VERSION_1_2
14471// OpenCL v1.2 s6.12.13, v2.0 s6.13.13 - printf14190// OpenCL v1.2 s6.12.13, v2.0 s6.13.13 - printf
1447214191
14473int printf(__constant const char* st, ...);14192int printf(__constant const char* st, ...) __attribute__((format(printf, 1, 2)));
14474#endif14193#endif
1447514194
14476// OpenCL v1.1 s6.11.3, v1.2 s6.12.14, v2.0 s6.13.14 - Image Read and Write Functions14195// OpenCL v1.1 s6.11.3, v1.2 s6.12.14, v2.0 s6.13.14 - Image Read and Write Functions
1447714196
14478// These values need to match the runtime equivalent
14479//
14480// Addressing Mode.
14481//
14482#define CLK_ADDRESS_NONE 0
14483#define CLK_ADDRESS_CLAMP_TO_EDGE 2
14484#define CLK_ADDRESS_CLAMP 4
14485#define CLK_ADDRESS_REPEAT 6
14486#define CLK_ADDRESS_MIRRORED_REPEAT 8
14487
14488//
14489// Coordination Normalization
14490//
14491#define CLK_NORMALIZED_COORDS_FALSE 0
14492#define CLK_NORMALIZED_COORDS_TRUE 1
14493
14494//
14495// Filtering Mode.
14496//
14497#define CLK_FILTER_NEAREST 0x10
14498#define CLK_FILTER_LINEAR 0x20
14499
14500#ifdef cl_khr_gl_msaa_sharing14197#ifdef cl_khr_gl_msaa_sharing
14501#pragma OPENCL EXTENSION cl_khr_gl_msaa_sharing : enable14198#pragma OPENCL EXTENSION cl_khr_gl_msaa_sharing : enable
14502#endif //cl_khr_gl_msaa_sharing14199#endif //cl_khr_gl_msaa_sharing
...@@ -14712,30 +14409,6 @@ float4 __purefn __ovld read_imagef(read_only image3d_t image, sampler_t sampler,...@@ -14712,30 +14409,6 @@ float4 __purefn __ovld read_imagef(read_only image3d_t image, sampler_t sampler,
14712int4 __purefn __ovld read_imagei(read_only image3d_t image, sampler_t sampler, float4 coord, float4 gradientX, float4 gradientY);14409int4 __purefn __ovld read_imagei(read_only image3d_t image, sampler_t sampler, float4 coord, float4 gradientX, float4 gradientY);
14713uint4 __purefn __ovld read_imageui(read_only image3d_t image, sampler_t sampler, float4 coord, float4 gradientX, float4 gradientY);14410uint4 __purefn __ovld read_imageui(read_only image3d_t image, sampler_t sampler, float4 coord, float4 gradientX, float4 gradientY);
1471414411
14715float4 __purefn __ovld read_imagef(read_only image1d_t image, sampler_t sampler, float coord, float lod);
14716int4 __purefn __ovld read_imagei(read_only image1d_t image, sampler_t sampler, float coord, float lod);
14717uint4 __purefn __ovld read_imageui(read_only image1d_t image, sampler_t sampler, float coord, float lod);
14718
14719float4 __purefn __ovld read_imagef(read_only image1d_array_t image_array, sampler_t sampler, float2 coord, float lod);
14720int4 __purefn __ovld read_imagei(read_only image1d_array_t image_array, sampler_t sampler, float2 coord, float lod);
14721uint4 __purefn __ovld read_imageui(read_only image1d_array_t image_array, sampler_t sampler, float2 coord, float lod);
14722
14723float4 __purefn __ovld read_imagef(read_only image2d_t image, sampler_t sampler, float2 coord, float lod);
14724int4 __purefn __ovld read_imagei(read_only image2d_t image, sampler_t sampler, float2 coord, float lod);
14725uint4 __purefn __ovld read_imageui(read_only image2d_t image, sampler_t sampler, float2 coord, float lod);
14726
14727float __purefn __ovld read_imagef(read_only image2d_depth_t image, sampler_t sampler, float2 coord, float lod);
14728
14729float4 __purefn __ovld read_imagef(read_only image2d_array_t image_array, sampler_t sampler, float4 coord, float lod);
14730int4 __purefn __ovld read_imagei(read_only image2d_array_t image_array, sampler_t sampler, float4 coord, float lod);
14731uint4 __purefn __ovld read_imageui(read_only image2d_array_t image_array, sampler_t sampler, float4 coord, float lod);
14732
14733float __purefn __ovld read_imagef(read_only image2d_array_depth_t image, sampler_t sampler, float4 coord, float lod);
14734
14735float4 __purefn __ovld read_imagef(read_only image3d_t image, sampler_t sampler, float4 coord, float lod);
14736int4 __purefn __ovld read_imagei(read_only image3d_t image, sampler_t sampler, float4 coord, float lod);
14737uint4 __purefn __ovld read_imageui(read_only image3d_t image, sampler_t sampler, float4 coord, float lod);
14738
14739#endif //cl_khr_mipmap_image14412#endif //cl_khr_mipmap_image
14740#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_014413#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0
1474114414
...@@ -14895,29 +14568,6 @@ float4 __purefn __ovld read_imagef(read_write image3d_t image, sampler_t sampler...@@ -14895,29 +14568,6 @@ float4 __purefn __ovld read_imagef(read_write image3d_t image, sampler_t sampler
14895int4 __purefn __ovld read_imagei(read_write image3d_t image, sampler_t sampler, float4 coord, float4 gradientX, float4 gradientY);14568int4 __purefn __ovld read_imagei(read_write image3d_t image, sampler_t sampler, float4 coord, float4 gradientX, float4 gradientY);
14896uint4 __purefn __ovld read_imageui(read_write image3d_t image, sampler_t sampler, float4 coord, float4 gradientX, float4 gradientY);14569uint4 __purefn __ovld read_imageui(read_write image3d_t image, sampler_t sampler, float4 coord, float4 gradientX, float4 gradientY);
1489714570
14898float4 __purefn __ovld read_imagef(read_write image1d_t image, sampler_t sampler, float coord, float lod);
14899int4 __purefn __ovld read_imagei(read_write image1d_t image, sampler_t sampler, float coord, float lod);
14900uint4 __purefn __ovld read_imageui(read_write image1d_t image, sampler_t sampler, float coord, float lod);
14901
14902float4 __purefn __ovld read_imagef(read_write image1d_array_t image_array, sampler_t sampler, float2 coord, float lod);
14903int4 __purefn __ovld read_imagei(read_write image1d_array_t image_array, sampler_t sampler, float2 coord, float lod);
14904uint4 __purefn __ovld read_imageui(read_write image1d_array_t image_array, sampler_t sampler, float2 coord, float lod);
14905
14906float4 __purefn __ovld read_imagef(read_write image2d_t image, sampler_t sampler, float2 coord, float lod);
14907int4 __purefn __ovld read_imagei(read_write image2d_t image, sampler_t sampler, float2 coord, float lod);
14908uint4 __purefn __ovld read_imageui(read_write image2d_t image, sampler_t sampler, float2 coord, float lod);
14909
14910float __purefn __ovld read_imagef(read_write image2d_depth_t image, sampler_t sampler, float2 coord, float lod);
14911
14912float4 __purefn __ovld read_imagef(read_write image2d_array_t image_array, sampler_t sampler, float4 coord, float lod);
14913int4 __purefn __ovld read_imagei(read_write image2d_array_t image_array, sampler_t sampler, float4 coord, float lod);
14914uint4 __purefn __ovld read_imageui(read_write image2d_array_t image_array, sampler_t sampler, float4 coord, float lod);
14915
14916float __purefn __ovld read_imagef(read_write image2d_array_depth_t image, sampler_t sampler, float4 coord, float lod);
14917
14918float4 __purefn __ovld read_imagef(read_write image3d_t image, sampler_t sampler, float4 coord, float lod);
14919int4 __purefn __ovld read_imagei(read_write image3d_t image, sampler_t sampler, float4 coord, float lod);
14920uint4 __purefn __ovld read_imageui(read_write image3d_t image, sampler_t sampler, float4 coord, float lod);
14921#endif //cl_khr_mipmap_image14571#endif //cl_khr_mipmap_image
14922#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_014572#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0
1492314573
...@@ -15332,26 +14982,6 @@ int __ovld get_image_num_mip_levels(read_write image2d_depth_t image);...@@ -15332,26 +14982,6 @@ int __ovld get_image_num_mip_levels(read_write image2d_depth_t image);
15332 * CLK_FLOAT14982 * CLK_FLOAT
15333 */14983 */
1533414984
15335//
15336// Channel Datatype.
15337//
15338#define CLK_SNORM_INT8 0x10D0
15339#define CLK_SNORM_INT16 0x10D1
15340#define CLK_UNORM_INT8 0x10D2
15341#define CLK_UNORM_INT16 0x10D3
15342#define CLK_UNORM_SHORT_565 0x10D4
15343#define CLK_UNORM_SHORT_555 0x10D5
15344#define CLK_UNORM_INT_101010 0x10D6
15345#define CLK_SIGNED_INT8 0x10D7
15346#define CLK_SIGNED_INT16 0x10D8
15347#define CLK_SIGNED_INT32 0x10D9
15348#define CLK_UNSIGNED_INT8 0x10DA
15349#define CLK_UNSIGNED_INT16 0x10DB
15350#define CLK_UNSIGNED_INT32 0x10DC
15351#define CLK_HALF_FLOAT 0x10DD
15352#define CLK_FLOAT 0x10DE
15353#define CLK_UNORM_INT24 0x10DF
15354
15355int __ovld __cnfn get_image_channel_data_type(read_only image1d_t image);14985int __ovld __cnfn get_image_channel_data_type(read_only image1d_t image);
15356int __ovld __cnfn get_image_channel_data_type(read_only image1d_buffer_t image);14986int __ovld __cnfn get_image_channel_data_type(read_only image1d_buffer_t image);
15357int __ovld __cnfn get_image_channel_data_type(read_only image2d_t image);14987int __ovld __cnfn get_image_channel_data_type(read_only image2d_t image);
...@@ -15423,30 +15053,6 @@ int __ovld __cnfn get_image_channel_data_type(read_write image2d_array_msaa_dept...@@ -15423,30 +15053,6 @@ int __ovld __cnfn get_image_channel_data_type(read_write image2d_array_msaa_dept
15423 * CLK_INTENSITY15053 * CLK_INTENSITY
15424 * CLK_LUMINANCE15054 * CLK_LUMINANCE
15425 */15055 */
15426// Channel order, numbering must be aligned with cl_channel_order in cl.h
15427//
15428#define CLK_R 0x10B0
15429#define CLK_A 0x10B1
15430#define CLK_RG 0x10B2
15431#define CLK_RA 0x10B3
15432#define CLK_RGB 0x10B4
15433#define CLK_RGBA 0x10B5
15434#define CLK_BGRA 0x10B6
15435#define CLK_ARGB 0x10B7
15436#define CLK_INTENSITY 0x10B8
15437#define CLK_LUMINANCE 0x10B9
15438#define CLK_Rx 0x10BA
15439#define CLK_RGx 0x10BB
15440#define CLK_RGBx 0x10BC
15441#define CLK_DEPTH 0x10BD
15442#define CLK_DEPTH_STENCIL 0x10BE
15443#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
15444#define CLK_sRGB 0x10BF
15445#define CLK_sRGBx 0x10C0
15446#define CLK_sRGBA 0x10C1
15447#define CLK_sBGRA 0x10C2
15448#define CLK_ABGR 0x10C3
15449#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0
1545015056
15451int __ovld __cnfn get_image_channel_order(read_only image1d_t image);15057int __ovld __cnfn get_image_channel_order(read_only image1d_t image);
15452int __ovld __cnfn get_image_channel_order(read_only image1d_buffer_t image);15058int __ovld __cnfn get_image_channel_order(read_only image1d_buffer_t image);
...@@ -15605,20 +15211,17 @@ size_t __ovld __cnfn get_image_array_size(read_write image2d_array_msaa_depth_t...@@ -15605,20 +15211,17 @@ size_t __ovld __cnfn get_image_array_size(read_write image2d_array_msaa_depth_t
15605#if defined(cl_khr_gl_msaa_sharing)15211#if defined(cl_khr_gl_msaa_sharing)
15606int __ovld get_image_num_samples(read_only image2d_msaa_t image);15212int __ovld get_image_num_samples(read_only image2d_msaa_t image);
15607int __ovld get_image_num_samples(read_only image2d_msaa_depth_t image);15213int __ovld get_image_num_samples(read_only image2d_msaa_depth_t image);
15608int __ovld get_image_num_samples(read_only image2d_array_msaa_depth_t image);
15609int __ovld get_image_num_samples(read_only image2d_array_msaa_t image);15214int __ovld get_image_num_samples(read_only image2d_array_msaa_t image);
15610int __ovld get_image_num_samples(read_only image2d_array_msaa_depth_t image);15215int __ovld get_image_num_samples(read_only image2d_array_msaa_depth_t image);
1561115216
15612int __ovld get_image_num_samples(write_only image2d_msaa_t image);15217int __ovld get_image_num_samples(write_only image2d_msaa_t image);
15613int __ovld get_image_num_samples(write_only image2d_msaa_depth_t image);15218int __ovld get_image_num_samples(write_only image2d_msaa_depth_t image);
15614int __ovld get_image_num_samples(write_only image2d_array_msaa_depth_t image);
15615int __ovld get_image_num_samples(write_only image2d_array_msaa_t image);15219int __ovld get_image_num_samples(write_only image2d_array_msaa_t image);
15616int __ovld get_image_num_samples(write_only image2d_array_msaa_depth_t image);15220int __ovld get_image_num_samples(write_only image2d_array_msaa_depth_t image);
1561715221
15618#if __OPENCL_C_VERSION__ >= CL_VERSION_2_015222#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
15619int __ovld get_image_num_samples(read_write image2d_msaa_t image);15223int __ovld get_image_num_samples(read_write image2d_msaa_t image);
15620int __ovld get_image_num_samples(read_write image2d_msaa_depth_t image);15224int __ovld get_image_num_samples(read_write image2d_msaa_depth_t image);
15621int __ovld get_image_num_samples(read_write image2d_array_msaa_depth_t image);
15622int __ovld get_image_num_samples(read_write image2d_array_msaa_t image);15225int __ovld get_image_num_samples(read_write image2d_array_msaa_t image);
15623int __ovld get_image_num_samples(read_write image2d_array_msaa_depth_t image);15226int __ovld get_image_num_samples(read_write image2d_array_msaa_depth_t image);
15624#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_015227#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0
...@@ -15728,7 +15331,6 @@ double __ovld __conv work_group_scan_inclusive_max(double x);...@@ -15728,7 +15331,6 @@ double __ovld __conv work_group_scan_inclusive_max(double x);
1572815331
15729// OpenCL v2.0 s6.13.16 - Pipe Functions15332// OpenCL v2.0 s6.13.16 - Pipe Functions
15730#if __OPENCL_C_VERSION__ >= CL_VERSION_2_015333#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
15731#define CLK_NULL_RESERVE_ID (__builtin_astype(((void*)(__SIZE_MAX__)), reserve_id_t))
15732bool __ovld is_valid_reserve_id(reserve_id_t reserve_id);15334bool __ovld is_valid_reserve_id(reserve_id_t reserve_id);
15733#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_015335#endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0
1573415336
...@@ -15736,44 +15338,6 @@ bool __ovld is_valid_reserve_id(reserve_id_t reserve_id);...@@ -15736,44 +15338,6 @@ bool __ovld is_valid_reserve_id(reserve_id_t reserve_id);
15736// OpenCL v2.0 s6.13.17 - Enqueue Kernels15338// OpenCL v2.0 s6.13.17 - Enqueue Kernels
15737#if __OPENCL_C_VERSION__ >= CL_VERSION_2_015339#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
1573815340
15739#define CL_COMPLETE 0x0
15740#define CL_RUNNING 0x1
15741#define CL_SUBMITTED 0x2
15742#define CL_QUEUED 0x3
15743
15744#define CLK_SUCCESS 0
15745#define CLK_ENQUEUE_FAILURE -101
15746#define CLK_INVALID_QUEUE -102
15747#define CLK_INVALID_NDRANGE -160
15748#define CLK_INVALID_EVENT_WAIT_LIST -57
15749#define CLK_DEVICE_QUEUE_FULL -161
15750#define CLK_INVALID_ARG_SIZE -51
15751#define CLK_EVENT_ALLOCATION_FAILURE -100
15752#define CLK_OUT_OF_RESOURCES -5
15753
15754#define CLK_NULL_QUEUE 0
15755#define CLK_NULL_EVENT (__builtin_astype(((void*)(__SIZE_MAX__)), clk_event_t))
15756
15757// execution model related definitions
15758#define CLK_ENQUEUE_FLAGS_NO_WAIT 0x0
15759#define CLK_ENQUEUE_FLAGS_WAIT_KERNEL 0x1
15760#define CLK_ENQUEUE_FLAGS_WAIT_WORK_GROUP 0x2
15761
15762typedef int kernel_enqueue_flags_t;
15763typedef int clk_profiling_info;
15764
15765// Profiling info name (see capture_event_profiling_info)
15766#define CLK_PROFILING_COMMAND_EXEC_TIME 0x1
15767
15768#define MAX_WORK_DIM 3
15769
15770typedef struct {
15771 unsigned int workDimension;
15772 size_t globalWorkOffset[MAX_WORK_DIM];
15773 size_t globalWorkSize[MAX_WORK_DIM];
15774 size_t localWorkSize[MAX_WORK_DIM];
15775} ndrange_t;
15776
15777ndrange_t __ovld ndrange_1D(size_t);15341ndrange_t __ovld ndrange_1D(size_t);
15778ndrange_t __ovld ndrange_1D(size_t, size_t);15342ndrange_t __ovld ndrange_1D(size_t, size_t);
15779ndrange_t __ovld ndrange_1D(size_t, size_t, size_t);15343ndrange_t __ovld ndrange_1D(size_t, size_t, size_t);
...@@ -16216,138 +15780,6 @@ void __ovld __conv intel_sub_group_block_write_us8( __global ushort* p, u...@@ -16216,138 +15780,6 @@ void __ovld __conv intel_sub_group_block_write_us8( __global ushort* p, u
16216#ifdef cl_intel_device_side_avc_motion_estimation15780#ifdef cl_intel_device_side_avc_motion_estimation
16217#pragma OPENCL EXTENSION cl_intel_device_side_avc_motion_estimation : begin15781#pragma OPENCL EXTENSION cl_intel_device_side_avc_motion_estimation : begin
1621815782
16219#define CLK_AVC_ME_MAJOR_16x16_INTEL 0x0
16220#define CLK_AVC_ME_MAJOR_16x8_INTEL 0x1
16221#define CLK_AVC_ME_MAJOR_8x16_INTEL 0x2
16222#define CLK_AVC_ME_MAJOR_8x8_INTEL 0x3
16223
16224#define CLK_AVC_ME_MINOR_8x8_INTEL 0x0
16225#define CLK_AVC_ME_MINOR_8x4_INTEL 0x1
16226#define CLK_AVC_ME_MINOR_4x8_INTEL 0x2
16227#define CLK_AVC_ME_MINOR_4x4_INTEL 0x3
16228
16229#define CLK_AVC_ME_MAJOR_FORWARD_INTEL 0x0
16230#define CLK_AVC_ME_MAJOR_BACKWARD_INTEL 0x1
16231#define CLK_AVC_ME_MAJOR_BIDIRECTIONAL_INTEL 0x2
16232
16233#define CLK_AVC_ME_PARTITION_MASK_ALL_INTEL 0x0
16234#define CLK_AVC_ME_PARTITION_MASK_16x16_INTEL 0x7E
16235#define CLK_AVC_ME_PARTITION_MASK_16x8_INTEL 0x7D
16236#define CLK_AVC_ME_PARTITION_MASK_8x16_INTEL 0x7B
16237#define CLK_AVC_ME_PARTITION_MASK_8x8_INTEL 0x77
16238#define CLK_AVC_ME_PARTITION_MASK_8x4_INTEL 0x6F
16239#define CLK_AVC_ME_PARTITION_MASK_4x8_INTEL 0x5F
16240#define CLK_AVC_ME_PARTITION_MASK_4x4_INTEL 0x3F
16241
16242#define CLK_AVC_ME_SLICE_TYPE_PRED_INTEL 0x0
16243#define CLK_AVC_ME_SLICE_TYPE_BPRED_INTEL 0x1
16244#define CLK_AVC_ME_SLICE_TYPE_INTRA_INTEL 0x2
16245
16246#define CLK_AVC_ME_SEARCH_WINDOW_EXHAUSTIVE_INTEL 0x0
16247#define CLK_AVC_ME_SEARCH_WINDOW_SMALL_INTEL 0x1
16248#define CLK_AVC_ME_SEARCH_WINDOW_TINY_INTEL 0x2
16249#define CLK_AVC_ME_SEARCH_WINDOW_EXTRA_TINY_INTEL 0x3
16250#define CLK_AVC_ME_SEARCH_WINDOW_DIAMOND_INTEL 0x4
16251#define CLK_AVC_ME_SEARCH_WINDOW_LARGE_DIAMOND_INTEL 0x5
16252#define CLK_AVC_ME_SEARCH_WINDOW_RESERVED0_INTEL 0x6
16253#define CLK_AVC_ME_SEARCH_WINDOW_RESERVED1_INTEL 0x7
16254#define CLK_AVC_ME_SEARCH_WINDOW_CUSTOM_INTEL 0x8
16255
16256#define CLK_AVC_ME_SAD_ADJUST_MODE_NONE_INTEL 0x0
16257#define CLK_AVC_ME_SAD_ADJUST_MODE_HAAR_INTEL 0x2
16258
16259#define CLK_AVC_ME_SUBPIXEL_MODE_INTEGER_INTEL 0x0
16260#define CLK_AVC_ME_SUBPIXEL_MODE_HPEL_INTEL 0x1
16261#define CLK_AVC_ME_SUBPIXEL_MODE_QPEL_INTEL 0x3
16262
16263#define CLK_AVC_ME_COST_PRECISION_QPEL_INTEL 0x0
16264#define CLK_AVC_ME_COST_PRECISION_HPEL_INTEL 0x1
16265#define CLK_AVC_ME_COST_PRECISION_PEL_INTEL 0x2
16266#define CLK_AVC_ME_COST_PRECISION_DPEL_INTEL 0x3
16267
16268#define CLK_AVC_ME_BIDIR_WEIGHT_QUARTER_INTEL 0x10
16269#define CLK_AVC_ME_BIDIR_WEIGHT_THIRD_INTEL 0x15
16270#define CLK_AVC_ME_BIDIR_WEIGHT_HALF_INTEL 0x20
16271#define CLK_AVC_ME_BIDIR_WEIGHT_TWO_THIRD_INTEL 0x2B
16272#define CLK_AVC_ME_BIDIR_WEIGHT_THREE_QUARTER_INTEL 0x30
16273
16274#define CLK_AVC_ME_BORDER_REACHED_LEFT_INTEL 0x0
16275#define CLK_AVC_ME_BORDER_REACHED_RIGHT_INTEL 0x2
16276#define CLK_AVC_ME_BORDER_REACHED_TOP_INTEL 0x4
16277#define CLK_AVC_ME_BORDER_REACHED_BOTTOM_INTEL 0x8
16278
16279#define CLK_AVC_ME_INTRA_16x16_INTEL 0x0
16280#define CLK_AVC_ME_INTRA_8x8_INTEL 0x1
16281#define CLK_AVC_ME_INTRA_4x4_INTEL 0x2
16282
16283#define CLK_AVC_ME_SKIP_BLOCK_PARTITION_16x16_INTEL 0x0
16284#define CLK_AVC_ME_SKIP_BLOCK_PARTITION_8x8_INTEL 0x4000
16285
16286#define CLK_AVC_ME_SKIP_BLOCK_16x16_FORWARD_ENABLE_INTEL (0x1 << 24)
16287#define CLK_AVC_ME_SKIP_BLOCK_16x16_BACKWARD_ENABLE_INTEL (0x2 << 24)
16288#define CLK_AVC_ME_SKIP_BLOCK_16x16_DUAL_ENABLE_INTEL (0x3 << 24)
16289#define CLK_AVC_ME_SKIP_BLOCK_8x8_FORWARD_ENABLE_INTEL (0x55 << 24)
16290#define CLK_AVC_ME_SKIP_BLOCK_8x8_BACKWARD_ENABLE_INTEL (0xAA << 24)
16291#define CLK_AVC_ME_SKIP_BLOCK_8x8_DUAL_ENABLE_INTEL (0xFF << 24)
16292#define CLK_AVC_ME_SKIP_BLOCK_8x8_0_FORWARD_ENABLE_INTEL (0x1 << 24)
16293#define CLK_AVC_ME_SKIP_BLOCK_8x8_0_BACKWARD_ENABLE_INTEL (0x2 << 24)
16294#define CLK_AVC_ME_SKIP_BLOCK_8x8_1_FORWARD_ENABLE_INTEL (0x1 << 26)
16295#define CLK_AVC_ME_SKIP_BLOCK_8x8_1_BACKWARD_ENABLE_INTEL (0x2 << 26)
16296#define CLK_AVC_ME_SKIP_BLOCK_8x8_2_FORWARD_ENABLE_INTEL (0x1 << 28)
16297#define CLK_AVC_ME_SKIP_BLOCK_8x8_2_BACKWARD_ENABLE_INTEL (0x2 << 28)
16298#define CLK_AVC_ME_SKIP_BLOCK_8x8_3_FORWARD_ENABLE_INTEL (0x1 << 30)
16299#define CLK_AVC_ME_SKIP_BLOCK_8x8_3_BACKWARD_ENABLE_INTEL (0x2 << 30)
16300
16301#define CLK_AVC_ME_BLOCK_BASED_SKIP_4x4_INTEL 0x00
16302#define CLK_AVC_ME_BLOCK_BASED_SKIP_8x8_INTEL 0x80
16303
16304#define CLK_AVC_ME_INTRA_LUMA_PARTITION_MASK_ALL_INTEL 0x0
16305#define CLK_AVC_ME_INTRA_LUMA_PARTITION_MASK_16x16_INTEL 0x6
16306#define CLK_AVC_ME_INTRA_LUMA_PARTITION_MASK_8x8_INTEL 0x5
16307#define CLK_AVC_ME_INTRA_LUMA_PARTITION_MASK_4x4_INTEL 0x3
16308
16309#define CLK_AVC_ME_INTRA_NEIGHBOR_LEFT_MASK_ENABLE_INTEL 0x60
16310#define CLK_AVC_ME_INTRA_NEIGHBOR_UPPER_MASK_ENABLE_INTEL 0x10
16311#define CLK_AVC_ME_INTRA_NEIGHBOR_UPPER_RIGHT_MASK_ENABLE_INTEL 0x8
16312#define CLK_AVC_ME_INTRA_NEIGHBOR_UPPER_LEFT_MASK_ENABLE_INTEL 0x4
16313
16314#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_VERTICAL_INTEL 0x0
16315#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_HORIZONTAL_INTEL 0x1
16316#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_DC_INTEL 0x2
16317#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_DIAGONAL_DOWN_LEFT_INTEL 0x3
16318#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_DIAGONAL_DOWN_RIGHT_INTEL 0x4
16319#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_PLANE_INTEL 0x4
16320#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_VERTICAL_RIGHT_INTEL 0x5
16321#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_HORIZONTAL_DOWN_INTEL 0x6
16322#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_VERTICAL_LEFT_INTEL 0x7
16323#define CLK_AVC_ME_LUMA_PREDICTOR_MODE_HORIZONTAL_UP_INTEL 0x8
16324#define CLK_AVC_ME_CHROMA_PREDICTOR_MODE_DC_INTEL 0x0
16325#define CLK_AVC_ME_CHROMA_PREDICTOR_MODE_HORIZONTAL_INTEL 0x1
16326#define CLK_AVC_ME_CHROMA_PREDICTOR_MODE_VERTICAL_INTEL 0x2
16327#define CLK_AVC_ME_CHROMA_PREDICTOR_MODE_PLANE_INTEL 0x3
16328
16329#define CLK_AVC_ME_FRAME_FORWARD_INTEL 0x1
16330#define CLK_AVC_ME_FRAME_BACKWARD_INTEL 0x2
16331#define CLK_AVC_ME_FRAME_DUAL_INTEL 0x3
16332
16333#define CLK_AVC_ME_INTERLACED_SCAN_TOP_FIELD_INTEL 0x0
16334#define CLK_AVC_ME_INTERLACED_SCAN_BOTTOM_FIELD_INTEL 0x1
16335
16336#define CLK_AVC_ME_INITIALIZE_INTEL 0x0
16337
16338#define CLK_AVC_IME_PAYLOAD_INITIALIZE_INTEL 0x0
16339#define CLK_AVC_REF_PAYLOAD_INITIALIZE_INTEL 0x0
16340#define CLK_AVC_SIC_PAYLOAD_INITIALIZE_INTEL 0x0
16341
16342#define CLK_AVC_IME_RESULT_INITIALIZE_INTEL 0x0
16343#define CLK_AVC_REF_RESULT_INITIALIZE_INTEL 0x0
16344#define CLK_AVC_SIC_RESULT_INITIALIZE_INTEL 0x0
16345
16346#define CLK_AVC_IME_RESULT_SINGLE_REFERENCE_STREAMOUT_INITIALIZE_INTEL 0x0
16347#define CLK_AVC_IME_RESULT_SINGLE_REFERENCE_STREAMIN_INITIALIZE_INTEL 0x0
16348#define CLK_AVC_IME_RESULT_DUAL_REFERENCE_STREAMOUT_INITIALIZE_INTEL 0x0
16349#define CLK_AVC_IME_RESULT_DUAL_REFERENCE_STREAMIN_INITIALIZE_INTEL 0x0
16350
16351// MCE built-in functions15783// MCE built-in functions
16352uchar __ovld15784uchar __ovld
16353intel_sub_group_avc_mce_get_default_inter_base_multi_reference_penalty(15785intel_sub_group_avc_mce_get_default_inter_base_multi_reference_penalty(
...@@ -17034,6 +16466,34 @@ uint8 __ovld amd_sadw(uint8 src0, uint8 src1, uint8 src2);...@@ -17034,6 +16466,34 @@ uint8 __ovld amd_sadw(uint8 src0, uint8 src1, uint8 src2);
17034uint16 __ovld amd_sadw(uint16 src0, uint16 src1, uint16 src2);16466uint16 __ovld amd_sadw(uint16 src0, uint16 src1, uint16 src2);
17035#endif // cl_amd_media_ops216467#endif // cl_amd_media_ops2
1703616468
16469#if defined(cl_arm_integer_dot_product_int8)
16470#pragma OPENCL EXTENSION cl_arm_integer_dot_product_int8 : begin
16471uint __ovld arm_dot(uchar4 a, uchar4 b);
16472int __ovld arm_dot(char4 a, char4 b);
16473#pragma OPENCL EXTENSION cl_arm_integer_dot_product_int8 : end
16474#endif // defined(cl_arm_integer_dot_product_int8)
16475
16476#if defined(cl_arm_integer_dot_product_accumulate_int8)
16477#pragma OPENCL EXTENSION cl_arm_integer_dot_product_accumulate_int8 : begin
16478uint __ovld arm_dot_acc(uchar4 a, uchar4 b, uint c);
16479int __ovld arm_dot_acc(char4 a, char4 b, int c);
16480#pragma OPENCL EXTENSION cl_arm_integer_dot_product_accumulate_int8 : end
16481#endif // defined(cl_arm_integer_dot_product_accumulate_int8)
16482
16483#if defined(cl_arm_integer_dot_product_accumulate_int16)
16484#pragma OPENCL EXTENSION cl_arm_integer_dot_product_accumulate_int16 : begin
16485uint __ovld arm_dot_acc(ushort2 a, ushort2 b, uint c);
16486int __ovld arm_dot_acc(short2 a, short2 b, int c);
16487#pragma OPENCL EXTENSION cl_arm_integer_dot_product_accumulate_int16 : end
16488#endif // defined(cl_arm_integer_dot_product_accumulate_int16)
16489
16490#if defined(cl_arm_integer_dot_product_accumulate_saturate_int8)
16491#pragma OPENCL EXTENSION cl_arm_integer_dot_product_accumulate_saturate_int8 : begin
16492uint __ovld arm_dot_acc_sat(uchar4 a, uchar4 b, uint c);
16493int __ovld arm_dot_acc_sat(char4 a, char4 b, int c);
16494#pragma OPENCL EXTENSION cl_arm_integer_dot_product_accumulate_saturate_int8 : end
16495#endif // defined(cl_arm_integer_dot_product_accumulate_saturate_int8)
16496
17037// Disable any extensions we may have enabled previously.16497// Disable any extensions we may have enabled previously.
17038#pragma OPENCL EXTENSION all : disable16498#pragma OPENCL EXTENSION all : disable
1703916499
lib/include/openmp_wrappers/__clang_openmp_math.h created+35
...@@ -0,0 +1,35 @@
1/*===---- __clang_openmp_math.h - OpenMP target math support ---------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10#if defined(__NVPTX__) && defined(_OPENMP)
11/// TODO:
12/// We are currently reusing the functionality of the Clang-CUDA code path
13/// as an alternative to the host declarations provided by math.h and cmath.
14/// This is suboptimal.
15///
16/// We should instead declare the device functions in a similar way, e.g.,
17/// through OpenMP 5.0 variants, and afterwards populate the module with the
18/// host declarations by unconditionally including the host math.h or cmath,
19/// respectively. This is actually what the Clang-CUDA code path does, using
20/// __device__ instead of variants to avoid redeclarations and get the desired
21/// overload resolution.
22
23#define __CUDA__
24
25#if defined(__cplusplus)
26 #include <__clang_cuda_cmath.h>
27#endif
28
29#undef __CUDA__
30
31/// Magic macro for stopping the math.h/cmath host header from being included.
32#define __CLANG_NO_HOST_MATH__
33
34#endif
35
lib/include/openmp_wrappers/__clang_openmp_math_declares.h created+33
...@@ -0,0 +1,33 @@
1/*===---- __clang_openmp_math_declares.h - OpenMP math declares ------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10#ifndef __CLANG_OPENMP_MATH_DECLARES_H__
11#define __CLANG_OPENMP_MATH_DECLARES_H__
12
13#ifndef _OPENMP
14#error "This file is for OpenMP compilation only."
15#endif
16
17#if defined(__NVPTX__) && defined(_OPENMP)
18
19#define __CUDA__
20
21#if defined(__cplusplus)
22 #include <__clang_cuda_math_forward_declares.h>
23#endif
24
25/// Include declarations for libdevice functions.
26#include <__clang_cuda_libdevice_declares.h>
27/// Provide definitions for these functions.
28#include <__clang_cuda_device_functions.h>
29
30#undef __CUDA__
31
32#endif
33#endif
lib/include/openmp_wrappers/cmath created+16
...@@ -0,0 +1,16 @@
1/*===-------------- cmath - Alternative cmath header -----------------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10#include <__clang_openmp_math.h>
11
12#ifndef __CLANG_NO_HOST_MATH__
13#include_next <cmath>
14#else
15#undef __CLANG_NO_HOST_MATH__
16#endif
lib/include/openmp_wrappers/math.h created+17
...@@ -0,0 +1,17 @@
1/*===------------- math.h - Alternative math.h header ----------------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10#include <__clang_openmp_math.h>
11
12#ifndef __CLANG_NO_HOST_MATH__
13#include_next <math.h>
14#else
15#undef __CLANG_NO_HOST_MATH__
16#endif
17
lib/include/pconfigintrin.h+7-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- pconfigintrin.h - X86 platform configuration ---------------------===1/*===---- pconfigintrin.h - X86 platform configuration ---------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -30,6 +16,8 @@...@@ -30,6 +16,8 @@
3016
31#define __PCONFIG_KEY_PROGRAM 0x0000000117#define __PCONFIG_KEY_PROGRAM 0x00000001
3218
19#if __has_extension(gnu_asm)
20
33/* Define the default attributes for the functions in this file. */21/* Define the default attributes for the functions in this file. */
34#define __DEFAULT_FN_ATTRS \22#define __DEFAULT_FN_ATTRS \
35 __attribute__((__always_inline__, __nodebug__, __target__("pconfig")))23 __attribute__((__always_inline__, __nodebug__, __target__("pconfig")))
...@@ -47,4 +35,6 @@ _pconfig_u32(unsigned int __leaf, __SIZE_TYPE__ __d[])...@@ -47,4 +35,6 @@ _pconfig_u32(unsigned int __leaf, __SIZE_TYPE__ __d[])
4735
48#undef __DEFAULT_FN_ATTRS36#undef __DEFAULT_FN_ATTRS
4937
38#endif /* __has_extension(gnu_asm) */
39
50#endif40#endif
lib/include/pkuintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===---- pkuintrin.h - PKU intrinsics -------------------------------------===1/*===---- pkuintrin.h - PKU intrinsics -------------------------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/pmmintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- pmmintrin.h - SSE3 intrinsics ------------------------------------===1/*===---- pmmintrin.h - SSE3 intrinsics ------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/popcntintrin.h+3-49
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- popcntintrin.h - POPCNT intrinsics -------------------------------===1/*===---- popcntintrin.h - POPCNT intrinsics -------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -43,22 +29,6 @@ _mm_popcnt_u32(unsigned int __A)...@@ -43,22 +29,6 @@ _mm_popcnt_u32(unsigned int __A)
43 return __builtin_popcount(__A);29 return __builtin_popcount(__A);
44}30}
4531
46/// Counts the number of bits in the source operand having a value of 1.
47///
48/// \headerfile <x86intrin.h>
49///
50/// This intrinsic corresponds to the <c> POPCNT </c> instruction.
51///
52/// \param __A
53/// A signed 32-bit integer operand.
54/// \returns A 32-bit integer containing the number of bits with value 1 in the
55/// source operand.
56static __inline__ int __DEFAULT_FN_ATTRS
57_popcnt32(int __A)
58{
59 return __builtin_popcount(__A);
60}
61
62#ifdef __x86_64__32#ifdef __x86_64__
63/// Counts the number of bits in the source operand having a value of 1.33/// Counts the number of bits in the source operand having a value of 1.
64///34///
...@@ -75,22 +45,6 @@ _mm_popcnt_u64(unsigned long long __A)...@@ -75,22 +45,6 @@ _mm_popcnt_u64(unsigned long long __A)
75{45{
76 return __builtin_popcountll(__A);46 return __builtin_popcountll(__A);
77}47}
78
79/// Counts the number of bits in the source operand having a value of 1.
80///
81/// \headerfile <x86intrin.h>
82///
83/// This intrinsic corresponds to the <c> POPCNT </c> instruction.
84///
85/// \param __A
86/// A signed 64-bit integer operand.
87/// \returns A 64-bit integer containing the number of bits with value 1 in the
88/// source operand.
89static __inline__ long long __DEFAULT_FN_ATTRS
90_popcnt64(long long __A)
91{
92 return __builtin_popcountll(__A);
93}
94#endif /* __x86_64__ */48#endif /* __x86_64__ */
9549
96#undef __DEFAULT_FN_ATTRS50#undef __DEFAULT_FN_ATTRS
lib/include/ppc_wrappers/emmintrin.h created+2318
...@@ -0,0 +1,2318 @@
1/*===---- emmintrin.h - Implementation of SSE2 intrinsics on PowerPC -------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10/* Implemented from the specification included in the Intel C++ Compiler
11 User Guide and Reference, version 9.0. */
12
13#ifndef NO_WARN_X86_INTRINSICS
14/* This header file is to help porting code using Intel intrinsics
15 explicitly from x86_64 to powerpc64/powerpc64le.
16
17 Since X86 SSE2 intrinsics mainly handles __m128i and __m128d type,
18 PowerPC VMX/VSX ISA is a good match for vector float SIMD operations.
19 However scalar float operations in vector (XMM) registers require
20 the POWER8 VSX ISA (2.07) level. There are differences for data
21 format and placement of float scalars in the vector register, which
22 require extra steps to match SSE2 scalar float semantics on POWER.
23
24 It should be noted that there's much difference between X86_64's
25 MXSCR and PowerISA's FPSCR/VSCR registers. It's recommended to use
26 portable <fenv.h> instead of access MXSCR directly.
27
28 Most SSE2 scalar float intrinsic operations can be performed more
29 efficiently as C language float scalar operations or optimized to
30 use vector SIMD operations. We recommend this for new applications.
31*/
32#error "Please read comment above. Use -DNO_WARN_X86_INTRINSICS to disable this error."
33#endif
34
35#ifndef EMMINTRIN_H_
36#define EMMINTRIN_H_
37
38#include <altivec.h>
39
40/* We need definitions from the SSE header files. */
41#include <xmmintrin.h>
42
43/* SSE2 */
44typedef __vector double __v2df;
45typedef __vector long long __v2di;
46typedef __vector unsigned long long __v2du;
47typedef __vector int __v4si;
48typedef __vector unsigned int __v4su;
49typedef __vector short __v8hi;
50typedef __vector unsigned short __v8hu;
51typedef __vector signed char __v16qi;
52typedef __vector unsigned char __v16qu;
53
54/* The Intel API is flexible enough that we must allow aliasing with other
55 vector types, and their scalar components. */
56typedef long long __m128i __attribute__ ((__vector_size__ (16), __may_alias__));
57typedef double __m128d __attribute__ ((__vector_size__ (16), __may_alias__));
58
59/* Unaligned version of the same types. */
60typedef long long __m128i_u __attribute__ ((__vector_size__ (16), __may_alias__, __aligned__ (1)));
61typedef double __m128d_u __attribute__ ((__vector_size__ (16), __may_alias__, __aligned__ (1)));
62
63/* Define two value permute mask. */
64#define _MM_SHUFFLE2(x,y) (((x) << 1) | (y))
65
66/* Create a vector with element 0 as F and the rest zero. */
67extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
68_mm_set_sd (double __F)
69{
70 return __extension__ (__m128d){ __F, 0.0 };
71}
72
73/* Create a vector with both elements equal to F. */
74extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
75_mm_set1_pd (double __F)
76{
77 return __extension__ (__m128d){ __F, __F };
78}
79
80extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
81_mm_set_pd1 (double __F)
82{
83 return _mm_set1_pd (__F);
84}
85
86/* Create a vector with the lower value X and upper value W. */
87extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
88_mm_set_pd (double __W, double __X)
89{
90 return __extension__ (__m128d){ __X, __W };
91}
92
93/* Create a vector with the lower value W and upper value X. */
94extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
95_mm_setr_pd (double __W, double __X)
96{
97 return __extension__ (__m128d){ __W, __X };
98}
99
100/* Create an undefined vector. */
101extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
102_mm_undefined_pd (void)
103{
104 __m128d __Y = __Y;
105 return __Y;
106}
107
108/* Create a vector of zeros. */
109extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
110_mm_setzero_pd (void)
111{
112 return (__m128d) vec_splats (0);
113}
114
115/* Sets the low DPFP value of A from the low value of B. */
116extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
117_mm_move_sd (__m128d __A, __m128d __B)
118{
119 __v2df result = (__v2df) __A;
120 result [0] = ((__v2df) __B)[0];
121 return (__m128d) result;
122}
123
124/* Load two DPFP values from P. The address must be 16-byte aligned. */
125extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
126_mm_load_pd (double const *__P)
127{
128 return ((__m128d)vec_ld(0, (__v16qu*)__P));
129}
130
131/* Load two DPFP values from P. The address need not be 16-byte aligned. */
132extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
133_mm_loadu_pd (double const *__P)
134{
135 return (vec_vsx_ld(0, __P));
136}
137
138/* Create a vector with all two elements equal to *P. */
139extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
140_mm_load1_pd (double const *__P)
141{
142 return (vec_splats (*__P));
143}
144
145/* Create a vector with element 0 as *P and the rest zero. */
146extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
147_mm_load_sd (double const *__P)
148{
149 return _mm_set_sd (*__P);
150}
151
152extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
153_mm_load_pd1 (double const *__P)
154{
155 return _mm_load1_pd (__P);
156}
157
158/* Load two DPFP values in reverse order. The address must be aligned. */
159extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
160_mm_loadr_pd (double const *__P)
161{
162 __v2df __tmp = _mm_load_pd (__P);
163 return (__m128d)vec_xxpermdi (__tmp, __tmp, 2);
164}
165
166/* Store two DPFP values. The address must be 16-byte aligned. */
167extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
168_mm_store_pd (double *__P, __m128d __A)
169{
170 vec_st((__v16qu)__A, 0, (__v16qu*)__P);
171}
172
173/* Store two DPFP values. The address need not be 16-byte aligned. */
174extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
175_mm_storeu_pd (double *__P, __m128d __A)
176{
177 *(__m128d_u *)__P = __A;
178}
179
180/* Stores the lower DPFP value. */
181extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
182_mm_store_sd (double *__P, __m128d __A)
183{
184 *__P = ((__v2df)__A)[0];
185}
186
187extern __inline double __attribute__((__gnu_inline__, __always_inline__, __artificial__))
188_mm_cvtsd_f64 (__m128d __A)
189{
190 return ((__v2df)__A)[0];
191}
192
193extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
194_mm_storel_pd (double *__P, __m128d __A)
195{
196 _mm_store_sd (__P, __A);
197}
198
199/* Stores the upper DPFP value. */
200extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
201_mm_storeh_pd (double *__P, __m128d __A)
202{
203 *__P = ((__v2df)__A)[1];
204}
205/* Store the lower DPFP value across two words.
206 The address must be 16-byte aligned. */
207extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
208_mm_store1_pd (double *__P, __m128d __A)
209{
210 _mm_store_pd (__P, vec_splat (__A, 0));
211}
212
213extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
214_mm_store_pd1 (double *__P, __m128d __A)
215{
216 _mm_store1_pd (__P, __A);
217}
218
219/* Store two DPFP values in reverse order. The address must be aligned. */
220extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
221_mm_storer_pd (double *__P, __m128d __A)
222{
223 _mm_store_pd (__P, vec_xxpermdi (__A, __A, 2));
224}
225
226/* Intel intrinsic. */
227extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
228_mm_cvtsi128_si64 (__m128i __A)
229{
230 return ((__v2di)__A)[0];
231}
232
233/* Microsoft intrinsic. */
234extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
235_mm_cvtsi128_si64x (__m128i __A)
236{
237 return ((__v2di)__A)[0];
238}
239
240extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
241_mm_add_pd (__m128d __A, __m128d __B)
242{
243 return (__m128d) ((__v2df)__A + (__v2df)__B);
244}
245
246/* Add the lower double-precision (64-bit) floating-point element in
247 a and b, store the result in the lower element of dst, and copy
248 the upper element from a to the upper element of dst. */
249extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
250_mm_add_sd (__m128d __A, __m128d __B)
251{
252 __A[0] = __A[0] + __B[0];
253 return (__A);
254}
255
256extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
257_mm_sub_pd (__m128d __A, __m128d __B)
258{
259 return (__m128d) ((__v2df)__A - (__v2df)__B);
260}
261
262extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
263_mm_sub_sd (__m128d __A, __m128d __B)
264{
265 __A[0] = __A[0] - __B[0];
266 return (__A);
267}
268
269extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
270_mm_mul_pd (__m128d __A, __m128d __B)
271{
272 return (__m128d) ((__v2df)__A * (__v2df)__B);
273}
274
275extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
276_mm_mul_sd (__m128d __A, __m128d __B)
277{
278 __A[0] = __A[0] * __B[0];
279 return (__A);
280}
281
282extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
283_mm_div_pd (__m128d __A, __m128d __B)
284{
285 return (__m128d) ((__v2df)__A / (__v2df)__B);
286}
287
288extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
289_mm_div_sd (__m128d __A, __m128d __B)
290{
291 __A[0] = __A[0] / __B[0];
292 return (__A);
293}
294
295extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
296_mm_sqrt_pd (__m128d __A)
297{
298 return (vec_sqrt (__A));
299}
300
301/* Return pair {sqrt (B[0]), A[1]}. */
302extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
303_mm_sqrt_sd (__m128d __A, __m128d __B)
304{
305 __v2df c;
306 c = vec_sqrt ((__v2df) _mm_set1_pd (__B[0]));
307 return (__m128d) _mm_setr_pd (c[0], __A[1]);
308}
309
310extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
311_mm_min_pd (__m128d __A, __m128d __B)
312{
313 return (vec_min (__A, __B));
314}
315
316extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
317_mm_min_sd (__m128d __A, __m128d __B)
318{
319 __v2df a, b, c;
320 a = vec_splats (__A[0]);
321 b = vec_splats (__B[0]);
322 c = vec_min (a, b);
323 return (__m128d) _mm_setr_pd (c[0], __A[1]);
324}
325
326extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
327_mm_max_pd (__m128d __A, __m128d __B)
328{
329 return (vec_max (__A, __B));
330}
331
332extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
333_mm_max_sd (__m128d __A, __m128d __B)
334{
335 __v2df a, b, c;
336 a = vec_splats (__A[0]);
337 b = vec_splats (__B[0]);
338 c = vec_max (a, b);
339 return (__m128d) _mm_setr_pd (c[0], __A[1]);
340}
341
342extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
343_mm_cmpeq_pd (__m128d __A, __m128d __B)
344{
345 return ((__m128d)vec_cmpeq ((__v2df) __A, (__v2df) __B));
346}
347
348extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
349_mm_cmplt_pd (__m128d __A, __m128d __B)
350{
351 return ((__m128d)vec_cmplt ((__v2df) __A, (__v2df) __B));
352}
353
354extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
355_mm_cmple_pd (__m128d __A, __m128d __B)
356{
357 return ((__m128d)vec_cmple ((__v2df) __A, (__v2df) __B));
358}
359
360extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
361_mm_cmpgt_pd (__m128d __A, __m128d __B)
362{
363 return ((__m128d)vec_cmpgt ((__v2df) __A, (__v2df) __B));
364}
365
366extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
367_mm_cmpge_pd (__m128d __A, __m128d __B)
368{
369 return ((__m128d)vec_cmpge ((__v2df) __A,(__v2df) __B));
370}
371
372extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
373_mm_cmpneq_pd (__m128d __A, __m128d __B)
374{
375 __v2df temp = (__v2df) vec_cmpeq ((__v2df) __A, (__v2df)__B);
376 return ((__m128d)vec_nor (temp, temp));
377}
378
379extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
380_mm_cmpnlt_pd (__m128d __A, __m128d __B)
381{
382 return ((__m128d)vec_cmpge ((__v2df) __A, (__v2df) __B));
383}
384
385extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
386_mm_cmpnle_pd (__m128d __A, __m128d __B)
387{
388 return ((__m128d)vec_cmpgt ((__v2df) __A, (__v2df) __B));
389}
390
391extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
392_mm_cmpngt_pd (__m128d __A, __m128d __B)
393{
394 return ((__m128d)vec_cmple ((__v2df) __A, (__v2df) __B));
395}
396
397extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
398_mm_cmpnge_pd (__m128d __A, __m128d __B)
399{
400 return ((__m128d)vec_cmplt ((__v2df) __A, (__v2df) __B));
401}
402
403extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
404_mm_cmpord_pd (__m128d __A, __m128d __B)
405{
406#if _ARCH_PWR8
407 __v2du c, d;
408 /* Compare against self will return false (0's) if NAN. */
409 c = (__v2du)vec_cmpeq (__A, __A);
410 d = (__v2du)vec_cmpeq (__B, __B);
411#else
412 __v2du a, b;
413 __v2du c, d;
414 const __v2du double_exp_mask = {0x7ff0000000000000, 0x7ff0000000000000};
415 a = (__v2du)vec_abs ((__v2df)__A);
416 b = (__v2du)vec_abs ((__v2df)__B);
417 c = (__v2du)vec_cmpgt (double_exp_mask, a);
418 d = (__v2du)vec_cmpgt (double_exp_mask, b);
419#endif
420 /* A != NAN and B != NAN. */
421 return ((__m128d)vec_and(c, d));
422}
423
424extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
425_mm_cmpunord_pd (__m128d __A, __m128d __B)
426{
427#if _ARCH_PWR8
428 __v2du c, d;
429 /* Compare against self will return false (0's) if NAN. */
430 c = (__v2du)vec_cmpeq ((__v2df)__A, (__v2df)__A);
431 d = (__v2du)vec_cmpeq ((__v2df)__B, (__v2df)__B);
432 /* A == NAN OR B == NAN converts too:
433 NOT(A != NAN) OR NOT(B != NAN). */
434 c = vec_nor (c, c);
435 return ((__m128d)vec_orc(c, d));
436#else
437 __v2du c, d;
438 /* Compare against self will return false (0's) if NAN. */
439 c = (__v2du)vec_cmpeq ((__v2df)__A, (__v2df)__A);
440 d = (__v2du)vec_cmpeq ((__v2df)__B, (__v2df)__B);
441 /* Convert the true ('1's) is NAN. */
442 c = vec_nor (c, c);
443 d = vec_nor (d, d);
444 return ((__m128d)vec_or(c, d));
445#endif
446}
447
448extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
449_mm_cmpeq_sd(__m128d __A, __m128d __B)
450{
451 __v2df a, b, c;
452 /* PowerISA VSX does not allow partial (for just lower double)
453 results. So to insure we don't generate spurious exceptions
454 (from the upper double values) we splat the lower double
455 before we do the operation. */
456 a = vec_splats (__A[0]);
457 b = vec_splats (__B[0]);
458 c = (__v2df) vec_cmpeq(a, b);
459 /* Then we merge the lower double result with the original upper
460 double from __A. */
461 return (__m128d) _mm_setr_pd (c[0], __A[1]);
462}
463
464extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
465_mm_cmplt_sd (__m128d __A, __m128d __B)
466{
467 __v2df a, b, c;
468 a = vec_splats (__A[0]);
469 b = vec_splats (__B[0]);
470 c = (__v2df) vec_cmplt(a, b);
471 return (__m128d) _mm_setr_pd (c[0], __A[1]);
472}
473
474extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
475_mm_cmple_sd (__m128d __A, __m128d __B)
476{
477 __v2df a, b, c;
478 a = vec_splats (__A[0]);
479 b = vec_splats (__B[0]);
480 c = (__v2df) vec_cmple(a, b);
481 return (__m128d) _mm_setr_pd (c[0], __A[1]);
482}
483
484extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
485_mm_cmpgt_sd (__m128d __A, __m128d __B)
486{
487 __v2df a, b, c;
488 a = vec_splats (__A[0]);
489 b = vec_splats (__B[0]);
490 c = (__v2df) vec_cmpgt(a, b);
491 return (__m128d) _mm_setr_pd (c[0], __A[1]);
492}
493
494extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
495_mm_cmpge_sd (__m128d __A, __m128d __B)
496{
497 __v2df a, b, c;
498 a = vec_splats (__A[0]);
499 b = vec_splats (__B[0]);
500 c = (__v2df) vec_cmpge(a, b);
501 return (__m128d) _mm_setr_pd (c[0], __A[1]);
502}
503
504extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
505_mm_cmpneq_sd (__m128d __A, __m128d __B)
506{
507 __v2df a, b, c;
508 a = vec_splats (__A[0]);
509 b = vec_splats (__B[0]);
510 c = (__v2df) vec_cmpeq(a, b);
511 c = vec_nor (c, c);
512 return (__m128d) _mm_setr_pd (c[0], __A[1]);
513}
514
515extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
516_mm_cmpnlt_sd (__m128d __A, __m128d __B)
517{
518 __v2df a, b, c;
519 a = vec_splats (__A[0]);
520 b = vec_splats (__B[0]);
521 /* Not less than is just greater than or equal. */
522 c = (__v2df) vec_cmpge(a, b);
523 return (__m128d) _mm_setr_pd (c[0], __A[1]);
524}
525
526extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
527_mm_cmpnle_sd (__m128d __A, __m128d __B)
528{
529 __v2df a, b, c;
530 a = vec_splats (__A[0]);
531 b = vec_splats (__B[0]);
532 /* Not less than or equal is just greater than. */
533 c = (__v2df) vec_cmpge(a, b);
534 return (__m128d) _mm_setr_pd (c[0], __A[1]);
535}
536
537extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
538_mm_cmpngt_sd (__m128d __A, __m128d __B)
539{
540 __v2df a, b, c;
541 a = vec_splats (__A[0]);
542 b = vec_splats (__B[0]);
543 /* Not greater than is just less than or equal. */
544 c = (__v2df) vec_cmple(a, b);
545 return (__m128d) _mm_setr_pd (c[0], __A[1]);
546}
547
548extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
549_mm_cmpnge_sd (__m128d __A, __m128d __B)
550{
551 __v2df a, b, c;
552 a = vec_splats (__A[0]);
553 b = vec_splats (__B[0]);
554 /* Not greater than or equal is just less than. */
555 c = (__v2df) vec_cmplt(a, b);
556 return (__m128d) _mm_setr_pd (c[0], __A[1]);
557}
558
559extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
560_mm_cmpord_sd (__m128d __A, __m128d __B)
561{
562 __v2df r;
563 r = (__v2df)_mm_cmpord_pd (vec_splats (__A[0]), vec_splats (__B[0]));
564 return (__m128d) _mm_setr_pd (r[0], ((__v2df)__A)[1]);
565}
566
567extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
568_mm_cmpunord_sd (__m128d __A, __m128d __B)
569{
570 __v2df r;
571 r = _mm_cmpunord_pd (vec_splats (__A[0]), vec_splats (__B[0]));
572 return (__m128d) _mm_setr_pd (r[0], __A[1]);
573}
574
575/* FIXME
576 The __mm_comi??_sd and __mm_ucomi??_sd implementations below are
577 exactly the same because GCC for PowerPC only generates unordered
578 compares (scalar and vector).
579 Technically __mm_comieq_sp et all should be using the ordered
580 compare and signal for QNaNs. The __mm_ucomieq_sd et all should
581 be OK. */
582extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
583_mm_comieq_sd (__m128d __A, __m128d __B)
584{
585 return (__A[0] == __B[0]);
586}
587
588extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
589_mm_comilt_sd (__m128d __A, __m128d __B)
590{
591 return (__A[0] < __B[0]);
592}
593
594extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
595_mm_comile_sd (__m128d __A, __m128d __B)
596{
597 return (__A[0] <= __B[0]);
598}
599
600extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
601_mm_comigt_sd (__m128d __A, __m128d __B)
602{
603 return (__A[0] > __B[0]);
604}
605
606extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
607_mm_comige_sd (__m128d __A, __m128d __B)
608{
609 return (__A[0] >= __B[0]);
610}
611
612extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
613_mm_comineq_sd (__m128d __A, __m128d __B)
614{
615 return (__A[0] != __B[0]);
616}
617
618extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
619_mm_ucomieq_sd (__m128d __A, __m128d __B)
620{
621 return (__A[0] == __B[0]);
622}
623
624extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
625_mm_ucomilt_sd (__m128d __A, __m128d __B)
626{
627 return (__A[0] < __B[0]);
628}
629
630extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
631_mm_ucomile_sd (__m128d __A, __m128d __B)
632{
633 return (__A[0] <= __B[0]);
634}
635
636extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
637_mm_ucomigt_sd (__m128d __A, __m128d __B)
638{
639 return (__A[0] > __B[0]);
640}
641
642extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
643_mm_ucomige_sd (__m128d __A, __m128d __B)
644{
645 return (__A[0] >= __B[0]);
646}
647
648extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
649_mm_ucomineq_sd (__m128d __A, __m128d __B)
650{
651 return (__A[0] != __B[0]);
652}
653
654/* Create a vector of Qi, where i is the element number. */
655extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
656_mm_set_epi64x (long long __q1, long long __q0)
657{
658 return __extension__ (__m128i)(__v2di){ __q0, __q1 };
659}
660
661extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
662_mm_set_epi64 (__m64 __q1, __m64 __q0)
663{
664 return _mm_set_epi64x ((long long)__q1, (long long)__q0);
665}
666
667extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
668_mm_set_epi32 (int __q3, int __q2, int __q1, int __q0)
669{
670 return __extension__ (__m128i)(__v4si){ __q0, __q1, __q2, __q3 };
671}
672
673extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
674_mm_set_epi16 (short __q7, short __q6, short __q5, short __q4,
675 short __q3, short __q2, short __q1, short __q0)
676{
677 return __extension__ (__m128i)(__v8hi){
678 __q0, __q1, __q2, __q3, __q4, __q5, __q6, __q7 };
679}
680
681extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
682_mm_set_epi8 (char __q15, char __q14, char __q13, char __q12,
683 char __q11, char __q10, char __q09, char __q08,
684 char __q07, char __q06, char __q05, char __q04,
685 char __q03, char __q02, char __q01, char __q00)
686{
687 return __extension__ (__m128i)(__v16qi){
688 __q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07,
689 __q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15
690 };
691}
692
693/* Set all of the elements of the vector to A. */
694extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
695_mm_set1_epi64x (long long __A)
696{
697 return _mm_set_epi64x (__A, __A);
698}
699
700extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
701_mm_set1_epi64 (__m64 __A)
702{
703 return _mm_set_epi64 (__A, __A);
704}
705
706extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
707_mm_set1_epi32 (int __A)
708{
709 return _mm_set_epi32 (__A, __A, __A, __A);
710}
711
712extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
713_mm_set1_epi16 (short __A)
714{
715 return _mm_set_epi16 (__A, __A, __A, __A, __A, __A, __A, __A);
716}
717
718extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
719_mm_set1_epi8 (char __A)
720{
721 return _mm_set_epi8 (__A, __A, __A, __A, __A, __A, __A, __A,
722 __A, __A, __A, __A, __A, __A, __A, __A);
723}
724
725/* Create a vector of Qi, where i is the element number.
726 The parameter order is reversed from the _mm_set_epi* functions. */
727extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
728_mm_setr_epi64 (__m64 __q0, __m64 __q1)
729{
730 return _mm_set_epi64 (__q1, __q0);
731}
732
733extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
734_mm_setr_epi32 (int __q0, int __q1, int __q2, int __q3)
735{
736 return _mm_set_epi32 (__q3, __q2, __q1, __q0);
737}
738
739extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
740_mm_setr_epi16 (short __q0, short __q1, short __q2, short __q3,
741 short __q4, short __q5, short __q6, short __q7)
742{
743 return _mm_set_epi16 (__q7, __q6, __q5, __q4, __q3, __q2, __q1, __q0);
744}
745
746extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
747_mm_setr_epi8 (char __q00, char __q01, char __q02, char __q03,
748 char __q04, char __q05, char __q06, char __q07,
749 char __q08, char __q09, char __q10, char __q11,
750 char __q12, char __q13, char __q14, char __q15)
751{
752 return _mm_set_epi8 (__q15, __q14, __q13, __q12, __q11, __q10, __q09, __q08,
753 __q07, __q06, __q05, __q04, __q03, __q02, __q01, __q00);
754}
755
756/* Create a vector with element 0 as *P and the rest zero. */
757extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
758_mm_load_si128 (__m128i const *__P)
759{
760 return *__P;
761}
762
763extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
764_mm_loadu_si128 (__m128i_u const *__P)
765{
766 return (__m128i) (vec_vsx_ld(0, (signed int const *)__P));
767}
768
769extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
770_mm_loadl_epi64 (__m128i_u const *__P)
771{
772 return _mm_set_epi64 ((__m64)0LL, *(__m64 *)__P);
773}
774
775extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
776_mm_store_si128 (__m128i *__P, __m128i __B)
777{
778 vec_st ((__v16qu) __B, 0, (__v16qu*)__P);
779}
780
781extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
782_mm_storeu_si128 (__m128i_u *__P, __m128i __B)
783{
784 *__P = __B;
785}
786
787extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
788_mm_storel_epi64 (__m128i_u *__P, __m128i __B)
789{
790 *(long long *)__P = ((__v2di)__B)[0];
791}
792
793extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
794_mm_movepi64_pi64 (__m128i_u __B)
795{
796 return (__m64) ((__v2di)__B)[0];
797}
798
799extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
800_mm_movpi64_epi64 (__m64 __A)
801{
802 return _mm_set_epi64 ((__m64)0LL, __A);
803}
804
805extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
806_mm_move_epi64 (__m128i __A)
807{
808 return _mm_set_epi64 ((__m64)0LL, (__m64)__A[0]);
809}
810
811/* Create an undefined vector. */
812extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
813_mm_undefined_si128 (void)
814{
815 __m128i __Y = __Y;
816 return __Y;
817}
818
819/* Create a vector of zeros. */
820extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
821_mm_setzero_si128 (void)
822{
823 return __extension__ (__m128i)(__v4si){ 0, 0, 0, 0 };
824}
825
826#ifdef _ARCH_PWR8
827extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
828_mm_cvtepi32_pd (__m128i __A)
829{
830 __v2di val;
831 /* For LE need to generate Vector Unpack Low Signed Word.
832 Which is generated from unpackh. */
833 val = (__v2di)vec_unpackh ((__v4si)__A);
834
835 return (__m128d)vec_ctf (val, 0);
836}
837#endif
838
839extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
840_mm_cvtepi32_ps (__m128i __A)
841{
842 return ((__m128)vec_ctf((__v4si)__A, 0));
843}
844
845extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
846_mm_cvtpd_epi32 (__m128d __A)
847{
848 __v2df rounded = vec_rint (__A);
849 __v4si result, temp;
850 const __v4si vzero =
851 { 0, 0, 0, 0 };
852
853 /* VSX Vector truncate Double-Precision to integer and Convert to
854 Signed Integer Word format with Saturate. */
855 __asm__(
856 "xvcvdpsxws %x0,%x1"
857 : "=wa" (temp)
858 : "wa" (rounded)
859 : );
860
861#ifdef _ARCH_PWR8
862 temp = vec_mergeo (temp, temp);
863 result = (__v4si) vec_vpkudum ((__vector long long) temp,
864 (__vector long long) vzero);
865#else
866 {
867 const __v16qu pkperm = {0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b,
868 0x14, 0x15, 0x16, 0x17, 0x1c, 0x1d, 0x1e, 0x1f };
869 result = (__v4si) vec_perm ((__v16qu) temp, (__v16qu) vzero, pkperm);
870 }
871#endif
872 return (__m128i) result;
873}
874
875extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
876_mm_cvtpd_pi32 (__m128d __A)
877{
878 __m128i result = _mm_cvtpd_epi32(__A);
879
880 return (__m64) result[0];
881}
882
883extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
884_mm_cvtpd_ps (__m128d __A)
885{
886 __v4sf result;
887 __v4si temp;
888 const __v4si vzero = { 0, 0, 0, 0 };
889
890 __asm__(
891 "xvcvdpsp %x0,%x1"
892 : "=wa" (temp)
893 : "wa" (__A)
894 : );
895
896#ifdef _ARCH_PWR8
897 temp = vec_mergeo (temp, temp);
898 result = (__v4sf) vec_vpkudum ((__vector long long) temp,
899 (__vector long long) vzero);
900#else
901 {
902 const __v16qu pkperm = {0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b,
903 0x14, 0x15, 0x16, 0x17, 0x1c, 0x1d, 0x1e, 0x1f };
904 result = (__v4sf) vec_perm ((__v16qu) temp, (__v16qu) vzero, pkperm);
905 }
906#endif
907 return ((__m128)result);
908}
909
910extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
911_mm_cvttpd_epi32 (__m128d __A)
912{
913 __v4si result;
914 __v4si temp;
915 const __v4si vzero = { 0, 0, 0, 0 };
916
917 /* VSX Vector truncate Double-Precision to integer and Convert to
918 Signed Integer Word format with Saturate. */
919 __asm__(
920 "xvcvdpsxws %x0,%x1"
921 : "=wa" (temp)
922 : "wa" (__A)
923 : );
924
925#ifdef _ARCH_PWR8
926 temp = vec_mergeo (temp, temp);
927 result = (__v4si) vec_vpkudum ((__vector long long) temp,
928 (__vector long long) vzero);
929#else
930 {
931 const __v16qu pkperm = {0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b,
932 0x14, 0x15, 0x16, 0x17, 0x1c, 0x1d, 0x1e, 0x1f };
933 result = (__v4si) vec_perm ((__v16qu) temp, (__v16qu) vzero, pkperm);
934 }
935#endif
936
937 return ((__m128i) result);
938}
939
940extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
941_mm_cvttpd_pi32 (__m128d __A)
942{
943 __m128i result = _mm_cvttpd_epi32 (__A);
944
945 return (__m64) result[0];
946}
947
948extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
949_mm_cvtsi128_si32 (__m128i __A)
950{
951 return ((__v4si)__A)[0];
952}
953
954#ifdef _ARCH_PWR8
955extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
956_mm_cvtpi32_pd (__m64 __A)
957{
958 __v4si temp;
959 __v2di tmp2;
960 __v2df result;
961
962 temp = (__v4si)vec_splats (__A);
963 tmp2 = (__v2di)vec_unpackl (temp);
964 result = vec_ctf ((__vector signed long long) tmp2, 0);
965 return (__m128d)result;
966}
967#endif
968
969extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
970_mm_cvtps_epi32 (__m128 __A)
971{
972 __v4sf rounded;
973 __v4si result;
974
975 rounded = vec_rint((__v4sf) __A);
976 result = vec_cts (rounded, 0);
977 return (__m128i) result;
978}
979
980extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
981_mm_cvttps_epi32 (__m128 __A)
982{
983 __v4si result;
984
985 result = vec_cts ((__v4sf) __A, 0);
986 return (__m128i) result;
987}
988
989extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
990_mm_cvtps_pd (__m128 __A)
991{
992 /* Check if vec_doubleh is defined by <altivec.h>. If so use that. */
993#ifdef vec_doubleh
994 return (__m128d) vec_doubleh ((__v4sf)__A);
995#else
996 /* Otherwise the compiler is not current and so need to generate the
997 equivalent code. */
998 __v4sf a = (__v4sf)__A;
999 __v4sf temp;
1000 __v2df result;
1001#ifdef __LITTLE_ENDIAN__
1002 /* The input float values are in elements {[0], [1]} but the convert
1003 instruction needs them in elements {[1], [3]}, So we use two
1004 shift left double vector word immediates to get the elements
1005 lined up. */
1006 temp = __builtin_vsx_xxsldwi (a, a, 3);
1007 temp = __builtin_vsx_xxsldwi (a, temp, 2);
1008#else
1009 /* The input float values are in elements {[0], [1]} but the convert
1010 instruction needs them in elements {[0], [2]}, So we use two
1011 shift left double vector word immediates to get the elements
1012 lined up. */
1013 temp = vec_vmrghw (a, a);
1014#endif
1015 __asm__(
1016 " xvcvspdp %x0,%x1"
1017 : "=wa" (result)
1018 : "wa" (temp)
1019 : );
1020 return (__m128d) result;
1021#endif
1022}
1023
1024extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1025_mm_cvtsd_si32 (__m128d __A)
1026{
1027 __v2df rounded = vec_rint((__v2df) __A);
1028 int result = ((__v2df)rounded)[0];
1029
1030 return result;
1031}
1032/* Intel intrinsic. */
1033extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1034_mm_cvtsd_si64 (__m128d __A)
1035{
1036 __v2df rounded = vec_rint ((__v2df) __A );
1037 long long result = ((__v2df) rounded)[0];
1038
1039 return result;
1040}
1041
1042/* Microsoft intrinsic. */
1043extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1044_mm_cvtsd_si64x (__m128d __A)
1045{
1046 return _mm_cvtsd_si64 ((__v2df)__A);
1047}
1048
1049extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1050_mm_cvttsd_si32 (__m128d __A)
1051{
1052 int result = ((__v2df)__A)[0];
1053
1054 return result;
1055}
1056
1057/* Intel intrinsic. */
1058extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1059_mm_cvttsd_si64 (__m128d __A)
1060{
1061 long long result = ((__v2df)__A)[0];
1062
1063 return result;
1064}
1065
1066/* Microsoft intrinsic. */
1067extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1068_mm_cvttsd_si64x (__m128d __A)
1069{
1070 return _mm_cvttsd_si64 (__A);
1071}
1072
1073extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1074_mm_cvtsd_ss (__m128 __A, __m128d __B)
1075{
1076 __v4sf result = (__v4sf)__A;
1077
1078#ifdef __LITTLE_ENDIAN__
1079 __v4sf temp_s;
1080 /* Copy double element[0] to element [1] for conversion. */
1081 __v2df temp_b = vec_splat((__v2df)__B, 0);
1082
1083 /* Pre-rotate __A left 3 (logically right 1) elements. */
1084 result = __builtin_vsx_xxsldwi (result, result, 3);
1085 /* Convert double to single float scalar in a vector. */
1086 __asm__(
1087 "xscvdpsp %x0,%x1"
1088 : "=wa" (temp_s)
1089 : "wa" (temp_b)
1090 : );
1091 /* Shift the resulting scalar into vector element [0]. */
1092 result = __builtin_vsx_xxsldwi (result, temp_s, 1);
1093#else
1094 result [0] = ((__v2df)__B)[0];
1095#endif
1096 return (__m128) result;
1097}
1098
1099extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1100_mm_cvtsi32_sd (__m128d __A, int __B)
1101{
1102 __v2df result = (__v2df)__A;
1103 double db = __B;
1104 result [0] = db;
1105 return (__m128d)result;
1106}
1107
1108/* Intel intrinsic. */
1109extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1110_mm_cvtsi64_sd (__m128d __A, long long __B)
1111{
1112 __v2df result = (__v2df)__A;
1113 double db = __B;
1114 result [0] = db;
1115 return (__m128d)result;
1116}
1117
1118/* Microsoft intrinsic. */
1119extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1120_mm_cvtsi64x_sd (__m128d __A, long long __B)
1121{
1122 return _mm_cvtsi64_sd (__A, __B);
1123}
1124
1125extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1126_mm_cvtss_sd (__m128d __A, __m128 __B)
1127{
1128#ifdef __LITTLE_ENDIAN__
1129 /* Use splat to move element [0] into position for the convert. */
1130 __v4sf temp = vec_splat ((__v4sf)__B, 0);
1131 __v2df res;
1132 /* Convert single float scalar to double in a vector. */
1133 __asm__(
1134 "xscvspdp %x0,%x1"
1135 : "=wa" (res)
1136 : "wa" (temp)
1137 : );
1138 return (__m128d) vec_mergel (res, (__v2df)__A);
1139#else
1140 __v2df res = (__v2df)__A;
1141 res [0] = ((__v4sf)__B) [0];
1142 return (__m128d) res;
1143#endif
1144}
1145
1146extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1147_mm_shuffle_pd(__m128d __A, __m128d __B, const int __mask)
1148{
1149 __vector double result;
1150 const int litmsk = __mask & 0x3;
1151
1152 if (litmsk == 0)
1153 result = vec_mergeh (__A, __B);
1154#if __GNUC__ < 6
1155 else if (litmsk == 1)
1156 result = vec_xxpermdi (__B, __A, 2);
1157 else if (litmsk == 2)
1158 result = vec_xxpermdi (__B, __A, 1);
1159#else
1160 else if (litmsk == 1)
1161 result = vec_xxpermdi (__A, __B, 2);
1162 else if (litmsk == 2)
1163 result = vec_xxpermdi (__A, __B, 1);
1164#endif
1165 else
1166 result = vec_mergel (__A, __B);
1167
1168 return result;
1169}
1170
1171extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1172_mm_unpackhi_pd (__m128d __A, __m128d __B)
1173{
1174 return (__m128d) vec_mergel ((__v2df)__A, (__v2df)__B);
1175}
1176
1177extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1178_mm_unpacklo_pd (__m128d __A, __m128d __B)
1179{
1180 return (__m128d) vec_mergeh ((__v2df)__A, (__v2df)__B);
1181}
1182
1183extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1184_mm_loadh_pd (__m128d __A, double const *__B)
1185{
1186 __v2df result = (__v2df)__A;
1187 result [1] = *__B;
1188 return (__m128d)result;
1189}
1190
1191extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1192_mm_loadl_pd (__m128d __A, double const *__B)
1193{
1194 __v2df result = (__v2df)__A;
1195 result [0] = *__B;
1196 return (__m128d)result;
1197}
1198
1199#ifdef _ARCH_PWR8
1200/* Intrinsic functions that require PowerISA 2.07 minimum. */
1201
1202/* Creates a 2-bit mask from the most significant bits of the DPFP values. */
1203extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1204_mm_movemask_pd (__m128d __A)
1205{
1206 __vector unsigned long long result;
1207 static const __vector unsigned int perm_mask =
1208 {
1209#ifdef __LITTLE_ENDIAN__
1210 0x80800040, 0x80808080, 0x80808080, 0x80808080
1211#else
1212 0x80808080, 0x80808080, 0x80808080, 0x80804000
1213#endif
1214 };
1215
1216 result = ((__vector unsigned long long)
1217 vec_vbpermq ((__vector unsigned char) __A,
1218 (__vector unsigned char) perm_mask));
1219
1220#ifdef __LITTLE_ENDIAN__
1221 return result[1];
1222#else
1223 return result[0];
1224#endif
1225}
1226#endif /* _ARCH_PWR8 */
1227
1228extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1229_mm_packs_epi16 (__m128i __A, __m128i __B)
1230{
1231 return (__m128i) vec_packs ((__v8hi) __A, (__v8hi)__B);
1232}
1233
1234extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1235_mm_packs_epi32 (__m128i __A, __m128i __B)
1236{
1237 return (__m128i) vec_packs ((__v4si)__A, (__v4si)__B);
1238}
1239
1240extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1241_mm_packus_epi16 (__m128i __A, __m128i __B)
1242{
1243 return (__m128i) vec_packsu ((__v8hi) __A, (__v8hi)__B);
1244}
1245
1246extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1247_mm_unpackhi_epi8 (__m128i __A, __m128i __B)
1248{
1249 return (__m128i) vec_mergel ((__v16qu)__A, (__v16qu)__B);
1250}
1251
1252extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1253_mm_unpackhi_epi16 (__m128i __A, __m128i __B)
1254{
1255 return (__m128i) vec_mergel ((__v8hu)__A, (__v8hu)__B);
1256}
1257
1258extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1259_mm_unpackhi_epi32 (__m128i __A, __m128i __B)
1260{
1261 return (__m128i) vec_mergel ((__v4su)__A, (__v4su)__B);
1262}
1263
1264extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1265_mm_unpackhi_epi64 (__m128i __A, __m128i __B)
1266{
1267 return (__m128i) vec_mergel ((__vector long long) __A,
1268 (__vector long long) __B);
1269}
1270
1271extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1272_mm_unpacklo_epi8 (__m128i __A, __m128i __B)
1273{
1274 return (__m128i) vec_mergeh ((__v16qu)__A, (__v16qu)__B);
1275}
1276
1277extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1278_mm_unpacklo_epi16 (__m128i __A, __m128i __B)
1279{
1280 return (__m128i) vec_mergeh ((__v8hi)__A, (__v8hi)__B);
1281}
1282
1283extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1284_mm_unpacklo_epi32 (__m128i __A, __m128i __B)
1285{
1286 return (__m128i) vec_mergeh ((__v4si)__A, (__v4si)__B);
1287}
1288
1289extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1290_mm_unpacklo_epi64 (__m128i __A, __m128i __B)
1291{
1292 return (__m128i) vec_mergeh ((__vector long long) __A,
1293 (__vector long long) __B);
1294}
1295
1296extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1297_mm_add_epi8 (__m128i __A, __m128i __B)
1298{
1299 return (__m128i) ((__v16qu)__A + (__v16qu)__B);
1300}
1301
1302extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1303_mm_add_epi16 (__m128i __A, __m128i __B)
1304{
1305 return (__m128i) ((__v8hu)__A + (__v8hu)__B);
1306}
1307
1308extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1309_mm_add_epi32 (__m128i __A, __m128i __B)
1310{
1311 return (__m128i) ((__v4su)__A + (__v4su)__B);
1312}
1313
1314extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1315_mm_add_epi64 (__m128i __A, __m128i __B)
1316{
1317 return (__m128i) ((__v2du)__A + (__v2du)__B);
1318}
1319
1320extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1321_mm_adds_epi8 (__m128i __A, __m128i __B)
1322{
1323 return (__m128i) vec_adds ((__v16qi)__A, (__v16qi)__B);
1324}
1325
1326extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1327_mm_adds_epi16 (__m128i __A, __m128i __B)
1328{
1329 return (__m128i) vec_adds ((__v8hi)__A, (__v8hi)__B);
1330}
1331
1332extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1333_mm_adds_epu8 (__m128i __A, __m128i __B)
1334{
1335 return (__m128i) vec_adds ((__v16qu)__A, (__v16qu)__B);
1336}
1337
1338extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1339_mm_adds_epu16 (__m128i __A, __m128i __B)
1340{
1341 return (__m128i) vec_adds ((__v8hu)__A, (__v8hu)__B);
1342}
1343
1344extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1345_mm_sub_epi8 (__m128i __A, __m128i __B)
1346{
1347 return (__m128i) ((__v16qu)__A - (__v16qu)__B);
1348}
1349
1350extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1351_mm_sub_epi16 (__m128i __A, __m128i __B)
1352{
1353 return (__m128i) ((__v8hu)__A - (__v8hu)__B);
1354}
1355
1356extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1357_mm_sub_epi32 (__m128i __A, __m128i __B)
1358{
1359 return (__m128i) ((__v4su)__A - (__v4su)__B);
1360}
1361
1362extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1363_mm_sub_epi64 (__m128i __A, __m128i __B)
1364{
1365 return (__m128i) ((__v2du)__A - (__v2du)__B);
1366}
1367
1368extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1369_mm_subs_epi8 (__m128i __A, __m128i __B)
1370{
1371 return (__m128i) vec_subs ((__v16qi)__A, (__v16qi)__B);
1372}
1373
1374extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1375_mm_subs_epi16 (__m128i __A, __m128i __B)
1376{
1377 return (__m128i) vec_subs ((__v8hi)__A, (__v8hi)__B);
1378}
1379
1380extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1381_mm_subs_epu8 (__m128i __A, __m128i __B)
1382{
1383 return (__m128i) vec_subs ((__v16qu)__A, (__v16qu)__B);
1384}
1385
1386extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1387_mm_subs_epu16 (__m128i __A, __m128i __B)
1388{
1389 return (__m128i) vec_subs ((__v8hu)__A, (__v8hu)__B);
1390}
1391
1392extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1393_mm_madd_epi16 (__m128i __A, __m128i __B)
1394{
1395 __vector signed int zero = {0, 0, 0, 0};
1396
1397 return (__m128i) vec_vmsumshm ((__v8hi)__A, (__v8hi)__B, zero);
1398}
1399
1400extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1401_mm_mulhi_epi16 (__m128i __A, __m128i __B)
1402{
1403 __vector signed int w0, w1;
1404
1405 __vector unsigned char xform1 = {
1406#ifdef __LITTLE_ENDIAN__
1407 0x02, 0x03, 0x12, 0x13, 0x06, 0x07, 0x16, 0x17,
1408 0x0A, 0x0B, 0x1A, 0x1B, 0x0E, 0x0F, 0x1E, 0x1F
1409#else
1410 0x00, 0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15,
1411 0x08, 0x09, 0x18, 0x19, 0x0C, 0x0D, 0x1C, 0x1D
1412#endif
1413 };
1414
1415 w0 = vec_vmulesh ((__v8hi)__A, (__v8hi)__B);
1416 w1 = vec_vmulosh ((__v8hi)__A, (__v8hi)__B);
1417 return (__m128i) vec_perm (w0, w1, xform1);
1418}
1419
1420extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1421_mm_mullo_epi16 (__m128i __A, __m128i __B)
1422{
1423 return (__m128i) ((__v8hi)__A * (__v8hi)__B);
1424}
1425
1426extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1427_mm_mul_su32 (__m64 __A, __m64 __B)
1428{
1429 unsigned int a = __A;
1430 unsigned int b = __B;
1431
1432 return ((__m64)a * (__m64)b);
1433}
1434
1435extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1436_mm_mul_epu32 (__m128i __A, __m128i __B)
1437{
1438#if __GNUC__ < 8
1439 __v2du result;
1440
1441#ifdef __LITTLE_ENDIAN__
1442 /* VMX Vector Multiply Odd Unsigned Word. */
1443 __asm__(
1444 "vmulouw %0,%1,%2"
1445 : "=v" (result)
1446 : "v" (__A), "v" (__B)
1447 : );
1448#else
1449 /* VMX Vector Multiply Even Unsigned Word. */
1450 __asm__(
1451 "vmuleuw %0,%1,%2"
1452 : "=v" (result)
1453 : "v" (__A), "v" (__B)
1454 : );
1455#endif
1456 return (__m128i) result;
1457#else
1458 return (__m128i) vec_mule ((__v4su)__A, (__v4su)__B);
1459#endif
1460}
1461
1462extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1463_mm_slli_epi16 (__m128i __A, int __B)
1464{
1465 __v8hu lshift;
1466 __v8hi result = { 0, 0, 0, 0, 0, 0, 0, 0 };
1467
1468 if (__B >= 0 && __B < 16)
1469 {
1470 if (__builtin_constant_p(__B))
1471 lshift = (__v8hu) vec_splat_s16(__B);
1472 else
1473 lshift = vec_splats ((unsigned short) __B);
1474
1475 result = vec_sl ((__v8hi) __A, lshift);
1476 }
1477
1478 return (__m128i) result;
1479}
1480
1481extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1482_mm_slli_epi32 (__m128i __A, int __B)
1483{
1484 __v4su lshift;
1485 __v4si result = { 0, 0, 0, 0 };
1486
1487 if (__B >= 0 && __B < 32)
1488 {
1489 if (__builtin_constant_p(__B) && __B < 16)
1490 lshift = (__v4su) vec_splat_s32(__B);
1491 else
1492 lshift = vec_splats ((unsigned int) __B);
1493
1494 result = vec_sl ((__v4si) __A, lshift);
1495 }
1496
1497 return (__m128i) result;
1498}
1499
1500#ifdef _ARCH_PWR8
1501extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1502_mm_slli_epi64 (__m128i __A, int __B)
1503{
1504 __v2du lshift;
1505 __v2di result = { 0, 0 };
1506
1507 if (__B >= 0 && __B < 64)
1508 {
1509 if (__builtin_constant_p(__B) && __B < 16)
1510 lshift = (__v2du) vec_splat_s32(__B);
1511 else
1512 lshift = (__v2du) vec_splats ((unsigned int) __B);
1513
1514 result = vec_sl ((__v2di) __A, lshift);
1515 }
1516
1517 return (__m128i) result;
1518}
1519#endif
1520
1521extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1522_mm_srai_epi16 (__m128i __A, int __B)
1523{
1524 __v8hu rshift = { 15, 15, 15, 15, 15, 15, 15, 15 };
1525 __v8hi result;
1526
1527 if (__B < 16)
1528 {
1529 if (__builtin_constant_p(__B))
1530 rshift = (__v8hu) vec_splat_s16(__B);
1531 else
1532 rshift = vec_splats ((unsigned short) __B);
1533 }
1534 result = vec_sra ((__v8hi) __A, rshift);
1535
1536 return (__m128i) result;
1537}
1538
1539extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1540_mm_srai_epi32 (__m128i __A, int __B)
1541{
1542 __v4su rshift = { 31, 31, 31, 31 };
1543 __v4si result;
1544
1545 if (__B < 32)
1546 {
1547 if (__builtin_constant_p(__B))
1548 {
1549 if (__B < 16)
1550 rshift = (__v4su) vec_splat_s32(__B);
1551 else
1552 rshift = (__v4su) vec_splats((unsigned int)__B);
1553 }
1554 else
1555 rshift = vec_splats ((unsigned int) __B);
1556 }
1557 result = vec_sra ((__v4si) __A, rshift);
1558
1559 return (__m128i) result;
1560}
1561
1562extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1563_mm_bslli_si128 (__m128i __A, const int __N)
1564{
1565 __v16qu result;
1566 const __v16qu zeros = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
1567
1568 if (__N < 16)
1569 result = vec_sld ((__v16qu) __A, zeros, __N);
1570 else
1571 result = zeros;
1572
1573 return (__m128i) result;
1574}
1575
1576extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1577_mm_bsrli_si128 (__m128i __A, const int __N)
1578{
1579 __v16qu result;
1580 const __v16qu zeros = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
1581
1582 if (__N < 16)
1583#ifdef __LITTLE_ENDIAN__
1584 if (__builtin_constant_p(__N))
1585 /* Would like to use Vector Shift Left Double by Octet
1586 Immediate here to use the immediate form and avoid
1587 load of __N * 8 value into a separate VR. */
1588 result = vec_sld (zeros, (__v16qu) __A, (16 - __N));
1589 else
1590#endif
1591 {
1592 __v16qu shift = vec_splats((unsigned char)(__N*8));
1593#ifdef __LITTLE_ENDIAN__
1594 result = vec_sro ((__v16qu)__A, shift);
1595#else
1596 result = vec_slo ((__v16qu)__A, shift);
1597#endif
1598 }
1599 else
1600 result = zeros;
1601
1602 return (__m128i) result;
1603}
1604
1605extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1606_mm_srli_si128 (__m128i __A, const int __N)
1607{
1608 return _mm_bsrli_si128 (__A, __N);
1609}
1610
1611extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1612_mm_slli_si128 (__m128i __A, const int _imm5)
1613{
1614 __v16qu result;
1615 const __v16qu zeros = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
1616
1617 if (_imm5 < 16)
1618#ifdef __LITTLE_ENDIAN__
1619 result = vec_sld ((__v16qu) __A, zeros, _imm5);
1620#else
1621 result = vec_sld (zeros, (__v16qu) __A, (16 - _imm5));
1622#endif
1623 else
1624 result = zeros;
1625
1626 return (__m128i) result;
1627}
1628
1629extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1630
1631_mm_srli_epi16 (__m128i __A, int __B)
1632{
1633 __v8hu rshift;
1634 __v8hi result = { 0, 0, 0, 0, 0, 0, 0, 0 };
1635
1636 if (__B < 16)
1637 {
1638 if (__builtin_constant_p(__B))
1639 rshift = (__v8hu) vec_splat_s16(__B);
1640 else
1641 rshift = vec_splats ((unsigned short) __B);
1642
1643 result = vec_sr ((__v8hi) __A, rshift);
1644 }
1645
1646 return (__m128i) result;
1647}
1648
1649extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1650_mm_srli_epi32 (__m128i __A, int __B)
1651{
1652 __v4su rshift;
1653 __v4si result = { 0, 0, 0, 0 };
1654
1655 if (__B < 32)
1656 {
1657 if (__builtin_constant_p(__B))
1658 {
1659 if (__B < 16)
1660 rshift = (__v4su) vec_splat_s32(__B);
1661 else
1662 rshift = (__v4su) vec_splats((unsigned int)__B);
1663 }
1664 else
1665 rshift = vec_splats ((unsigned int) __B);
1666
1667 result = vec_sr ((__v4si) __A, rshift);
1668 }
1669
1670 return (__m128i) result;
1671}
1672
1673#ifdef _ARCH_PWR8
1674extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1675_mm_srli_epi64 (__m128i __A, int __B)
1676{
1677 __v2du rshift;
1678 __v2di result = { 0, 0 };
1679
1680 if (__B < 64)
1681 {
1682 if (__builtin_constant_p(__B))
1683 {
1684 if (__B < 16)
1685 rshift = (__v2du) vec_splat_s32(__B);
1686 else
1687 rshift = (__v2du) vec_splats((unsigned long long)__B);
1688 }
1689 else
1690 rshift = (__v2du) vec_splats ((unsigned int) __B);
1691
1692 result = vec_sr ((__v2di) __A, rshift);
1693 }
1694
1695 return (__m128i) result;
1696}
1697#endif
1698
1699extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1700_mm_sll_epi16 (__m128i __A, __m128i __B)
1701{
1702 __v8hu lshift;
1703 __vector __bool short shmask;
1704 const __v8hu shmax = { 15, 15, 15, 15, 15, 15, 15, 15 };
1705 __v8hu result;
1706
1707#ifdef __LITTLE_ENDIAN__
1708 lshift = vec_splat ((__v8hu) __B, 0);
1709#else
1710 lshift = vec_splat ((__v8hu) __B, 3);
1711#endif
1712 shmask = vec_cmple (lshift, shmax);
1713 result = vec_sl ((__v8hu) __A, lshift);
1714 result = vec_sel ((__v8hu) shmask, result, shmask);
1715
1716 return (__m128i) result;
1717}
1718
1719extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1720_mm_sll_epi32 (__m128i __A, __m128i __B)
1721{
1722 __v4su lshift;
1723 __vector __bool int shmask;
1724 const __v4su shmax = { 32, 32, 32, 32 };
1725 __v4su result;
1726#ifdef __LITTLE_ENDIAN__
1727 lshift = vec_splat ((__v4su) __B, 0);
1728#else
1729 lshift = vec_splat ((__v4su) __B, 1);
1730#endif
1731 shmask = vec_cmplt (lshift, shmax);
1732 result = vec_sl ((__v4su) __A, lshift);
1733 result = vec_sel ((__v4su) shmask, result, shmask);
1734
1735 return (__m128i) result;
1736}
1737
1738#ifdef _ARCH_PWR8
1739extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1740_mm_sll_epi64 (__m128i __A, __m128i __B)
1741{
1742 __v2du lshift;
1743 __vector __bool long long shmask;
1744 const __v2du shmax = { 64, 64 };
1745 __v2du result;
1746
1747 lshift = vec_splat ((__v2du) __B, 0);
1748 shmask = vec_cmplt (lshift, shmax);
1749 result = vec_sl ((__v2du) __A, lshift);
1750 result = vec_sel ((__v2du) shmask, result, shmask);
1751
1752 return (__m128i) result;
1753}
1754#endif
1755
1756extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1757_mm_sra_epi16 (__m128i __A, __m128i __B)
1758{
1759 const __v8hu rshmax = { 15, 15, 15, 15, 15, 15, 15, 15 };
1760 __v8hu rshift;
1761 __v8hi result;
1762
1763#ifdef __LITTLE_ENDIAN__
1764 rshift = vec_splat ((__v8hu)__B, 0);
1765#else
1766 rshift = vec_splat ((__v8hu)__B, 3);
1767#endif
1768 rshift = vec_min (rshift, rshmax);
1769 result = vec_sra ((__v8hi) __A, rshift);
1770
1771 return (__m128i) result;
1772}
1773
1774extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1775_mm_sra_epi32 (__m128i __A, __m128i __B)
1776{
1777 const __v4su rshmax = { 31, 31, 31, 31 };
1778 __v4su rshift;
1779 __v4si result;
1780
1781#ifdef __LITTLE_ENDIAN__
1782 rshift = vec_splat ((__v4su)__B, 0);
1783#else
1784 rshift = vec_splat ((__v4su)__B, 1);
1785#endif
1786 rshift = vec_min (rshift, rshmax);
1787 result = vec_sra ((__v4si) __A, rshift);
1788
1789 return (__m128i) result;
1790}
1791
1792extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1793_mm_srl_epi16 (__m128i __A, __m128i __B)
1794{
1795 __v8hu rshift;
1796 __vector __bool short shmask;
1797 const __v8hu shmax = { 15, 15, 15, 15, 15, 15, 15, 15 };
1798 __v8hu result;
1799
1800#ifdef __LITTLE_ENDIAN__
1801 rshift = vec_splat ((__v8hu) __B, 0);
1802#else
1803 rshift = vec_splat ((__v8hu) __B, 3);
1804#endif
1805 shmask = vec_cmple (rshift, shmax);
1806 result = vec_sr ((__v8hu) __A, rshift);
1807 result = vec_sel ((__v8hu) shmask, result, shmask);
1808
1809 return (__m128i) result;
1810}
1811
1812extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1813_mm_srl_epi32 (__m128i __A, __m128i __B)
1814{
1815 __v4su rshift;
1816 __vector __bool int shmask;
1817 const __v4su shmax = { 32, 32, 32, 32 };
1818 __v4su result;
1819
1820#ifdef __LITTLE_ENDIAN__
1821 rshift = vec_splat ((__v4su) __B, 0);
1822#else
1823 rshift = vec_splat ((__v4su) __B, 1);
1824#endif
1825 shmask = vec_cmplt (rshift, shmax);
1826 result = vec_sr ((__v4su) __A, rshift);
1827 result = vec_sel ((__v4su) shmask, result, shmask);
1828
1829 return (__m128i) result;
1830}
1831
1832#ifdef _ARCH_PWR8
1833extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1834_mm_srl_epi64 (__m128i __A, __m128i __B)
1835{
1836 __v2du rshift;
1837 __vector __bool long long shmask;
1838 const __v2du shmax = { 64, 64 };
1839 __v2du result;
1840
1841 rshift = vec_splat ((__v2du) __B, 0);
1842 shmask = vec_cmplt (rshift, shmax);
1843 result = vec_sr ((__v2du) __A, rshift);
1844 result = vec_sel ((__v2du) shmask, result, shmask);
1845
1846 return (__m128i) result;
1847}
1848#endif
1849
1850extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1851_mm_and_pd (__m128d __A, __m128d __B)
1852{
1853 return (vec_and ((__v2df) __A, (__v2df) __B));
1854}
1855
1856extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1857_mm_andnot_pd (__m128d __A, __m128d __B)
1858{
1859 return (vec_andc ((__v2df) __B, (__v2df) __A));
1860}
1861
1862extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1863_mm_or_pd (__m128d __A, __m128d __B)
1864{
1865 return (vec_or ((__v2df) __A, (__v2df) __B));
1866}
1867
1868extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1869_mm_xor_pd (__m128d __A, __m128d __B)
1870{
1871 return (vec_xor ((__v2df) __A, (__v2df) __B));
1872}
1873
1874extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1875_mm_and_si128 (__m128i __A, __m128i __B)
1876{
1877 return (__m128i)vec_and ((__v2di) __A, (__v2di) __B);
1878}
1879
1880extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1881_mm_andnot_si128 (__m128i __A, __m128i __B)
1882{
1883 return (__m128i)vec_andc ((__v2di) __B, (__v2di) __A);
1884}
1885
1886extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1887_mm_or_si128 (__m128i __A, __m128i __B)
1888{
1889 return (__m128i)vec_or ((__v2di) __A, (__v2di) __B);
1890}
1891
1892extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1893_mm_xor_si128 (__m128i __A, __m128i __B)
1894{
1895 return (__m128i)vec_xor ((__v2di) __A, (__v2di) __B);
1896}
1897
1898extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1899_mm_cmpeq_epi8 (__m128i __A, __m128i __B)
1900{
1901 return (__m128i) vec_cmpeq ((__v16qi) __A, (__v16qi)__B);
1902}
1903
1904extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1905_mm_cmpeq_epi16 (__m128i __A, __m128i __B)
1906{
1907 return (__m128i) vec_cmpeq ((__v8hi) __A, (__v8hi)__B);
1908}
1909
1910extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1911_mm_cmpeq_epi32 (__m128i __A, __m128i __B)
1912{
1913 return (__m128i) vec_cmpeq ((__v4si) __A, (__v4si)__B);
1914}
1915
1916extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1917_mm_cmplt_epi8 (__m128i __A, __m128i __B)
1918{
1919 return (__m128i) vec_cmplt ((__v16qi) __A, (__v16qi)__B);
1920}
1921
1922extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1923_mm_cmplt_epi16 (__m128i __A, __m128i __B)
1924{
1925 return (__m128i) vec_cmplt ((__v8hi) __A, (__v8hi)__B);
1926}
1927
1928extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1929_mm_cmplt_epi32 (__m128i __A, __m128i __B)
1930{
1931 return (__m128i) vec_cmplt ((__v4si) __A, (__v4si)__B);
1932}
1933
1934extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1935_mm_cmpgt_epi8 (__m128i __A, __m128i __B)
1936{
1937 return (__m128i) vec_cmpgt ((__v16qi) __A, (__v16qi)__B);
1938}
1939
1940extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1941_mm_cmpgt_epi16 (__m128i __A, __m128i __B)
1942{
1943 return (__m128i) vec_cmpgt ((__v8hi) __A, (__v8hi)__B);
1944}
1945
1946extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1947_mm_cmpgt_epi32 (__m128i __A, __m128i __B)
1948{
1949 return (__m128i) vec_cmpgt ((__v4si) __A, (__v4si)__B);
1950}
1951
1952extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1953_mm_extract_epi16 (__m128i const __A, int const __N)
1954{
1955 return (unsigned short) ((__v8hi)__A)[__N & 7];
1956}
1957
1958extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1959_mm_insert_epi16 (__m128i const __A, int const __D, int const __N)
1960{
1961 __v8hi result = (__v8hi)__A;
1962
1963 result [(__N & 7)] = __D;
1964
1965 return (__m128i) result;
1966}
1967
1968extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1969_mm_max_epi16 (__m128i __A, __m128i __B)
1970{
1971 return (__m128i) vec_max ((__v8hi)__A, (__v8hi)__B);
1972}
1973
1974extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1975_mm_max_epu8 (__m128i __A, __m128i __B)
1976{
1977 return (__m128i) vec_max ((__v16qu) __A, (__v16qu)__B);
1978}
1979
1980extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1981_mm_min_epi16 (__m128i __A, __m128i __B)
1982{
1983 return (__m128i) vec_min ((__v8hi) __A, (__v8hi)__B);
1984}
1985
1986extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1987_mm_min_epu8 (__m128i __A, __m128i __B)
1988{
1989 return (__m128i) vec_min ((__v16qu) __A, (__v16qu)__B);
1990}
1991
1992
1993#ifdef _ARCH_PWR8
1994/* Intrinsic functions that require PowerISA 2.07 minimum. */
1995
1996/* Creates a 4-bit mask from the most significant bits of the SPFP values. */
1997extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1998_mm_movemask_epi8 (__m128i __A)
1999{
2000 __vector unsigned long long result;
2001 static const __vector unsigned char perm_mask =
2002 {
2003 0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40,
2004 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00
2005 };
2006
2007 result = ((__vector unsigned long long)
2008 vec_vbpermq ((__vector unsigned char) __A,
2009 (__vector unsigned char) perm_mask));
2010
2011#ifdef __LITTLE_ENDIAN__
2012 return result[1];
2013#else
2014 return result[0];
2015#endif
2016}
2017#endif /* _ARCH_PWR8 */
2018
2019extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2020_mm_mulhi_epu16 (__m128i __A, __m128i __B)
2021{
2022 __v4su w0, w1;
2023 __v16qu xform1 = {
2024#ifdef __LITTLE_ENDIAN__
2025 0x02, 0x03, 0x12, 0x13, 0x06, 0x07, 0x16, 0x17,
2026 0x0A, 0x0B, 0x1A, 0x1B, 0x0E, 0x0F, 0x1E, 0x1F
2027#else
2028 0x00, 0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15,
2029 0x08, 0x09, 0x18, 0x19, 0x0C, 0x0D, 0x1C, 0x1D
2030#endif
2031 };
2032
2033 w0 = vec_vmuleuh ((__v8hu)__A, (__v8hu)__B);
2034 w1 = vec_vmulouh ((__v8hu)__A, (__v8hu)__B);
2035 return (__m128i) vec_perm (w0, w1, xform1);
2036}
2037
2038extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2039_mm_shufflehi_epi16 (__m128i __A, const int __mask)
2040{
2041 unsigned long element_selector_98 = __mask & 0x03;
2042 unsigned long element_selector_BA = (__mask >> 2) & 0x03;
2043 unsigned long element_selector_DC = (__mask >> 4) & 0x03;
2044 unsigned long element_selector_FE = (__mask >> 6) & 0x03;
2045 static const unsigned short permute_selectors[4] =
2046 {
2047#ifdef __LITTLE_ENDIAN__
2048 0x0908, 0x0B0A, 0x0D0C, 0x0F0E
2049#else
2050 0x0809, 0x0A0B, 0x0C0D, 0x0E0F
2051#endif
2052 };
2053 __v2du pmask =
2054#ifdef __LITTLE_ENDIAN__
2055 { 0x1716151413121110UL, 0UL};
2056#else
2057 { 0x1011121314151617UL, 0UL};
2058#endif
2059 __m64_union t;
2060 __v2du a, r;
2061
2062 t.as_short[0] = permute_selectors[element_selector_98];
2063 t.as_short[1] = permute_selectors[element_selector_BA];
2064 t.as_short[2] = permute_selectors[element_selector_DC];
2065 t.as_short[3] = permute_selectors[element_selector_FE];
2066 pmask[1] = t.as_m64;
2067 a = (__v2du)__A;
2068 r = vec_perm (a, a, (__vector unsigned char)pmask);
2069 return (__m128i) r;
2070}
2071
2072extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2073_mm_shufflelo_epi16 (__m128i __A, const int __mask)
2074{
2075 unsigned long element_selector_10 = __mask & 0x03;
2076 unsigned long element_selector_32 = (__mask >> 2) & 0x03;
2077 unsigned long element_selector_54 = (__mask >> 4) & 0x03;
2078 unsigned long element_selector_76 = (__mask >> 6) & 0x03;
2079 static const unsigned short permute_selectors[4] =
2080 {
2081#ifdef __LITTLE_ENDIAN__
2082 0x0100, 0x0302, 0x0504, 0x0706
2083#else
2084 0x0001, 0x0203, 0x0405, 0x0607
2085#endif
2086 };
2087 __v2du pmask =
2088#ifdef __LITTLE_ENDIAN__
2089 { 0UL, 0x1f1e1d1c1b1a1918UL};
2090#else
2091 { 0UL, 0x18191a1b1c1d1e1fUL};
2092#endif
2093 __m64_union t;
2094 __v2du a, r;
2095 t.as_short[0] = permute_selectors[element_selector_10];
2096 t.as_short[1] = permute_selectors[element_selector_32];
2097 t.as_short[2] = permute_selectors[element_selector_54];
2098 t.as_short[3] = permute_selectors[element_selector_76];
2099 pmask[0] = t.as_m64;
2100 a = (__v2du)__A;
2101 r = vec_perm (a, a, (__vector unsigned char)pmask);
2102 return (__m128i) r;
2103}
2104
2105extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2106_mm_shuffle_epi32 (__m128i __A, const int __mask)
2107{
2108 unsigned long element_selector_10 = __mask & 0x03;
2109 unsigned long element_selector_32 = (__mask >> 2) & 0x03;
2110 unsigned long element_selector_54 = (__mask >> 4) & 0x03;
2111 unsigned long element_selector_76 = (__mask >> 6) & 0x03;
2112 static const unsigned int permute_selectors[4] =
2113 {
2114#ifdef __LITTLE_ENDIAN__
2115 0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
2116#else
2117 0x00010203, 0x04050607, 0x08090A0B, 0x0C0D0E0F
2118#endif
2119 };
2120 __v4su t;
2121
2122 t[0] = permute_selectors[element_selector_10];
2123 t[1] = permute_selectors[element_selector_32];
2124 t[2] = permute_selectors[element_selector_54] + 0x10101010;
2125 t[3] = permute_selectors[element_selector_76] + 0x10101010;
2126 return (__m128i)vec_perm ((__v4si) __A, (__v4si)__A, (__vector unsigned char)t);
2127}
2128
2129extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2130_mm_maskmoveu_si128 (__m128i __A, __m128i __B, char *__C)
2131{
2132 __v2du hibit = { 0x7f7f7f7f7f7f7f7fUL, 0x7f7f7f7f7f7f7f7fUL};
2133 __v16qu mask, tmp;
2134 __m128i_u *p = (__m128i_u*)__C;
2135
2136 tmp = (__v16qu)_mm_loadu_si128(p);
2137 mask = (__v16qu)vec_cmpgt ((__v16qu)__B, (__v16qu)hibit);
2138 tmp = vec_sel (tmp, (__v16qu)__A, mask);
2139 _mm_storeu_si128 (p, (__m128i)tmp);
2140}
2141
2142extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2143_mm_avg_epu8 (__m128i __A, __m128i __B)
2144{
2145 return (__m128i) vec_avg ((__v16qu)__A, (__v16qu)__B);
2146}
2147
2148extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2149_mm_avg_epu16 (__m128i __A, __m128i __B)
2150{
2151 return (__m128i) vec_avg ((__v8hu)__A, (__v8hu)__B);
2152}
2153
2154
2155extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2156_mm_sad_epu8 (__m128i __A, __m128i __B)
2157{
2158 __v16qu a, b;
2159 __v16qu vmin, vmax, vabsdiff;
2160 __v4si vsum;
2161 const __v4su zero = { 0, 0, 0, 0 };
2162 __v4si result;
2163
2164 a = (__v16qu) __A;
2165 b = (__v16qu) __B;
2166 vmin = vec_min (a, b);
2167 vmax = vec_max (a, b);
2168 vabsdiff = vec_sub (vmax, vmin);
2169 /* Sum four groups of bytes into integers. */
2170 vsum = (__vector signed int) vec_sum4s (vabsdiff, zero);
2171 /* Sum across four integers with two integer results. */
2172 result = vec_sum2s (vsum, (__vector signed int) zero);
2173 /* Rotate the sums into the correct position. */
2174#ifdef __LITTLE_ENDIAN__
2175 result = vec_sld (result, result, 4);
2176#else
2177 result = vec_sld (result, result, 6);
2178#endif
2179 /* Rotate the sums into the correct position. */
2180 return (__m128i) result;
2181}
2182
2183extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2184_mm_stream_si32 (int *__A, int __B)
2185{
2186 /* Use the data cache block touch for store transient. */
2187 __asm__ (
2188 "dcbtstt 0,%0"
2189 :
2190 : "b" (__A)
2191 : "memory"
2192 );
2193 *__A = __B;
2194}
2195
2196extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2197_mm_stream_si64 (long long int *__A, long long int __B)
2198{
2199 /* Use the data cache block touch for store transient. */
2200 __asm__ (
2201 " dcbtstt 0,%0"
2202 :
2203 : "b" (__A)
2204 : "memory"
2205 );
2206 *__A = __B;
2207}
2208
2209extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2210_mm_stream_si128 (__m128i *__A, __m128i __B)
2211{
2212 /* Use the data cache block touch for store transient. */
2213 __asm__ (
2214 "dcbtstt 0,%0"
2215 :
2216 : "b" (__A)
2217 : "memory"
2218 );
2219 *__A = __B;
2220}
2221
2222extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2223_mm_stream_pd (double *__A, __m128d __B)
2224{
2225 /* Use the data cache block touch for store transient. */
2226 __asm__ (
2227 "dcbtstt 0,%0"
2228 :
2229 : "b" (__A)
2230 : "memory"
2231 );
2232 *(__m128d*)__A = __B;
2233}
2234
2235extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2236_mm_clflush (void const *__A)
2237{
2238 /* Use the data cache block flush. */
2239 __asm__ (
2240 "dcbf 0,%0"
2241 :
2242 : "b" (__A)
2243 : "memory"
2244 );
2245}
2246
2247extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2248_mm_lfence (void)
2249{
2250 /* Use light weight sync for load to load ordering. */
2251 __atomic_thread_fence (__ATOMIC_RELEASE);
2252}
2253
2254extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2255_mm_mfence (void)
2256{
2257 /* Use heavy weight sync for any to any ordering. */
2258 __atomic_thread_fence (__ATOMIC_SEQ_CST);
2259}
2260
2261extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2262_mm_cvtsi32_si128 (int __A)
2263{
2264 return _mm_set_epi32 (0, 0, 0, __A);
2265}
2266
2267extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2268_mm_cvtsi64_si128 (long long __A)
2269{
2270 return __extension__ (__m128i)(__v2di){ __A, 0LL };
2271}
2272
2273/* Microsoft intrinsic. */
2274extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2275_mm_cvtsi64x_si128 (long long __A)
2276{
2277 return __extension__ (__m128i)(__v2di){ __A, 0LL };
2278}
2279
2280/* Casts between various SP, DP, INT vector types. Note that these do no
2281 conversion of values, they just change the type. */
2282extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2283_mm_castpd_ps(__m128d __A)
2284{
2285 return (__m128) __A;
2286}
2287
2288extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2289_mm_castpd_si128(__m128d __A)
2290{
2291 return (__m128i) __A;
2292}
2293
2294extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2295_mm_castps_pd(__m128 __A)
2296{
2297 return (__m128d) __A;
2298}
2299
2300extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2301_mm_castps_si128(__m128 __A)
2302{
2303 return (__m128i) __A;
2304}
2305
2306extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2307_mm_castsi128_ps(__m128i __A)
2308{
2309 return (__m128) __A;
2310}
2311
2312extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
2313_mm_castsi128_pd(__m128i __A)
2314{
2315 return (__m128d) __A;
2316}
2317
2318#endif /* EMMINTRIN_H_ */
lib/include/ppc_wrappers/mm_malloc.h created+44
...@@ -0,0 +1,44 @@
1/*===---- mm_malloc.h - Implementation of _mm_malloc and _mm_free ----------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10#ifndef _MM_MALLOC_H_INCLUDED
11#define _MM_MALLOC_H_INCLUDED
12
13#include <stdlib.h>
14
15/* We can't depend on <stdlib.h> since the prototype of posix_memalign
16 may not be visible. */
17#ifndef __cplusplus
18extern int posix_memalign (void **, size_t, size_t);
19#else
20extern "C" int posix_memalign (void **, size_t, size_t) throw ();
21#endif
22
23static __inline void *
24_mm_malloc (size_t size, size_t alignment)
25{
26 /* PowerPC64 ELF V2 ABI requires quadword alignment. */
27 size_t vec_align = sizeof (__vector float);
28 void *ptr;
29
30 if (alignment < vec_align)
31 alignment = vec_align;
32 if (posix_memalign (&ptr, alignment, size) == 0)
33 return ptr;
34 else
35 return NULL;
36}
37
38static __inline void
39_mm_free (void * ptr)
40{
41 free (ptr);
42}
43
44#endif /* _MM_MALLOC_H_INCLUDED */
lib/include/ppc_wrappers/mmintrin.h created+1443
...@@ -0,0 +1,1443 @@
1/*===---- mmintrin.h - Implementation of MMX intrinsics on PowerPC ---------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10/* Implemented from the specification included in the Intel C++ Compiler
11 User Guide and Reference, version 9.0. */
12
13#ifndef NO_WARN_X86_INTRINSICS
14/* This header file is to help porting code using Intel intrinsics
15 explicitly from x86_64 to powerpc64/powerpc64le.
16
17 Since PowerPC target doesn't support native 64-bit vector type, we
18 typedef __m64 to 64-bit unsigned long long in MMX intrinsics, which
19 works well for _si64 and some _pi32 operations.
20
21 For _pi16 and _pi8 operations, it's better to transfer __m64 into
22 128-bit PowerPC vector first. Power8 introduced direct register
23 move instructions which helps for more efficient implementation.
24
25 It's user's responsibility to determine if the results of such port
26 are acceptable or further changes are needed. Please note that much
27 code using Intel intrinsics CAN BE REWRITTEN in more portable and
28 efficient standard C or GNU C extensions with 64-bit scalar
29 operations, or 128-bit SSE/Altivec operations, which are more
30 recommended. */
31#error \
32 "Please read comment above. Use -DNO_WARN_X86_INTRINSICS to disable this error."
33#endif
34
35#ifndef _MMINTRIN_H_INCLUDED
36#define _MMINTRIN_H_INCLUDED
37
38#include <altivec.h>
39/* The Intel API is flexible enough that we must allow aliasing with other
40 vector types, and their scalar components. */
41typedef __attribute__((__aligned__(8))) unsigned long long __m64;
42
43typedef __attribute__((__aligned__(8))) union {
44 __m64 as_m64;
45 char as_char[8];
46 signed char as_signed_char[8];
47 short as_short[4];
48 int as_int[2];
49 long long as_long_long;
50 float as_float[2];
51 double as_double;
52} __m64_union;
53
54/* Empty the multimedia state. */
55extern __inline void
56 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
57 _mm_empty(void) {
58 /* nothing to do on PowerPC. */
59}
60
61extern __inline void
62 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
63 _m_empty(void) {
64 /* nothing to do on PowerPC. */
65}
66
67/* Convert I to a __m64 object. The integer is zero-extended to 64-bits. */
68extern __inline __m64
69 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
70 _mm_cvtsi32_si64(int __i) {
71 return (__m64)(unsigned int)__i;
72}
73
74extern __inline __m64
75 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
76 _m_from_int(int __i) {
77 return _mm_cvtsi32_si64(__i);
78}
79
80/* Convert the lower 32 bits of the __m64 object into an integer. */
81extern __inline int
82 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
83 _mm_cvtsi64_si32(__m64 __i) {
84 return ((int)__i);
85}
86
87extern __inline int
88 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
89 _m_to_int(__m64 __i) {
90 return _mm_cvtsi64_si32(__i);
91}
92
93/* Convert I to a __m64 object. */
94
95/* Intel intrinsic. */
96extern __inline __m64
97 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
98 _m_from_int64(long long __i) {
99 return (__m64)__i;
100}
101
102extern __inline __m64
103 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
104 _mm_cvtsi64_m64(long long __i) {
105 return (__m64)__i;
106}
107
108/* Microsoft intrinsic. */
109extern __inline __m64
110 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
111 _mm_cvtsi64x_si64(long long __i) {
112 return (__m64)__i;
113}
114
115extern __inline __m64
116 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
117 _mm_set_pi64x(long long __i) {
118 return (__m64)__i;
119}
120
121/* Convert the __m64 object to a 64bit integer. */
122
123/* Intel intrinsic. */
124extern __inline long long
125 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
126 _m_to_int64(__m64 __i) {
127 return (long long)__i;
128}
129
130extern __inline long long
131 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
132 _mm_cvtm64_si64(__m64 __i) {
133 return (long long)__i;
134}
135
136/* Microsoft intrinsic. */
137extern __inline long long
138 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
139 _mm_cvtsi64_si64x(__m64 __i) {
140 return (long long)__i;
141}
142
143#ifdef _ARCH_PWR8
144/* Pack the four 16-bit values from M1 into the lower four 8-bit values of
145 the result, and the four 16-bit values from M2 into the upper four 8-bit
146 values of the result, all with signed saturation. */
147extern __inline __m64
148 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
149 _mm_packs_pi16(__m64 __m1, __m64 __m2) {
150 __vector signed short vm1;
151 __vector signed char vresult;
152
153 vm1 = (__vector signed short)(__vector unsigned long long)
154#ifdef __LITTLE_ENDIAN__
155 {__m1, __m2};
156#else
157 {__m2, __m1};
158#endif
159 vresult = vec_packs(vm1, vm1);
160 return (__m64)((__vector long long)vresult)[0];
161}
162
163extern __inline __m64
164 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
165 _m_packsswb(__m64 __m1, __m64 __m2) {
166 return _mm_packs_pi16(__m1, __m2);
167}
168
169/* Pack the two 32-bit values from M1 in to the lower two 16-bit values of
170 the result, and the two 32-bit values from M2 into the upper two 16-bit
171 values of the result, all with signed saturation. */
172extern __inline __m64
173 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
174 _mm_packs_pi32(__m64 __m1, __m64 __m2) {
175 __vector signed int vm1;
176 __vector signed short vresult;
177
178 vm1 = (__vector signed int)(__vector unsigned long long)
179#ifdef __LITTLE_ENDIAN__
180 {__m1, __m2};
181#else
182 {__m2, __m1};
183#endif
184 vresult = vec_packs(vm1, vm1);
185 return (__m64)((__vector long long)vresult)[0];
186}
187
188extern __inline __m64
189 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
190 _m_packssdw(__m64 __m1, __m64 __m2) {
191 return _mm_packs_pi32(__m1, __m2);
192}
193
194/* Pack the four 16-bit values from M1 into the lower four 8-bit values of
195 the result, and the four 16-bit values from M2 into the upper four 8-bit
196 values of the result, all with unsigned saturation. */
197extern __inline __m64
198 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
199 _mm_packs_pu16(__m64 __m1, __m64 __m2) {
200 __vector unsigned char r;
201 __vector signed short vm1 = (__vector signed short)(__vector long long)
202#ifdef __LITTLE_ENDIAN__
203 {__m1, __m2};
204#else
205 {__m2, __m1};
206#endif
207 const __vector signed short __zero = {0};
208 __vector __bool short __select = vec_cmplt(vm1, __zero);
209 r = vec_packs((__vector unsigned short)vm1, (__vector unsigned short)vm1);
210 __vector __bool char packsel = vec_pack(__select, __select);
211 r = vec_sel(r, (const __vector unsigned char)__zero, packsel);
212 return (__m64)((__vector long long)r)[0];
213}
214
215extern __inline __m64
216 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
217 _m_packuswb(__m64 __m1, __m64 __m2) {
218 return _mm_packs_pu16(__m1, __m2);
219}
220#endif /* end ARCH_PWR8 */
221
222/* Interleave the four 8-bit values from the high half of M1 with the four
223 8-bit values from the high half of M2. */
224extern __inline __m64
225 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
226 _mm_unpackhi_pi8(__m64 __m1, __m64 __m2) {
227#if _ARCH_PWR8
228 __vector unsigned char a, b, c;
229
230 a = (__vector unsigned char)vec_splats(__m1);
231 b = (__vector unsigned char)vec_splats(__m2);
232 c = vec_mergel(a, b);
233 return (__m64)((__vector long long)c)[1];
234#else
235 __m64_union m1, m2, res;
236
237 m1.as_m64 = __m1;
238 m2.as_m64 = __m2;
239
240 res.as_char[0] = m1.as_char[4];
241 res.as_char[1] = m2.as_char[4];
242 res.as_char[2] = m1.as_char[5];
243 res.as_char[3] = m2.as_char[5];
244 res.as_char[4] = m1.as_char[6];
245 res.as_char[5] = m2.as_char[6];
246 res.as_char[6] = m1.as_char[7];
247 res.as_char[7] = m2.as_char[7];
248
249 return (__m64)res.as_m64;
250#endif
251}
252
253extern __inline __m64
254 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
255 _m_punpckhbw(__m64 __m1, __m64 __m2) {
256 return _mm_unpackhi_pi8(__m1, __m2);
257}
258
259/* Interleave the two 16-bit values from the high half of M1 with the two
260 16-bit values from the high half of M2. */
261extern __inline __m64
262 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
263 _mm_unpackhi_pi16(__m64 __m1, __m64 __m2) {
264 __m64_union m1, m2, res;
265
266 m1.as_m64 = __m1;
267 m2.as_m64 = __m2;
268
269 res.as_short[0] = m1.as_short[2];
270 res.as_short[1] = m2.as_short[2];
271 res.as_short[2] = m1.as_short[3];
272 res.as_short[3] = m2.as_short[3];
273
274 return (__m64)res.as_m64;
275}
276
277extern __inline __m64
278 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
279 _m_punpckhwd(__m64 __m1, __m64 __m2) {
280 return _mm_unpackhi_pi16(__m1, __m2);
281}
282/* Interleave the 32-bit value from the high half of M1 with the 32-bit
283 value from the high half of M2. */
284extern __inline __m64
285 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
286 _mm_unpackhi_pi32(__m64 __m1, __m64 __m2) {
287 __m64_union m1, m2, res;
288
289 m1.as_m64 = __m1;
290 m2.as_m64 = __m2;
291
292 res.as_int[0] = m1.as_int[1];
293 res.as_int[1] = m2.as_int[1];
294
295 return (__m64)res.as_m64;
296}
297
298extern __inline __m64
299 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
300 _m_punpckhdq(__m64 __m1, __m64 __m2) {
301 return _mm_unpackhi_pi32(__m1, __m2);
302}
303/* Interleave the four 8-bit values from the low half of M1 with the four
304 8-bit values from the low half of M2. */
305extern __inline __m64
306 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
307 _mm_unpacklo_pi8(__m64 __m1, __m64 __m2) {
308#if _ARCH_PWR8
309 __vector unsigned char a, b, c;
310
311 a = (__vector unsigned char)vec_splats(__m1);
312 b = (__vector unsigned char)vec_splats(__m2);
313 c = vec_mergel(a, b);
314 return (__m64)((__vector long long)c)[0];
315#else
316 __m64_union m1, m2, res;
317
318 m1.as_m64 = __m1;
319 m2.as_m64 = __m2;
320
321 res.as_char[0] = m1.as_char[0];
322 res.as_char[1] = m2.as_char[0];
323 res.as_char[2] = m1.as_char[1];
324 res.as_char[3] = m2.as_char[1];
325 res.as_char[4] = m1.as_char[2];
326 res.as_char[5] = m2.as_char[2];
327 res.as_char[6] = m1.as_char[3];
328 res.as_char[7] = m2.as_char[3];
329
330 return (__m64)res.as_m64;
331#endif
332}
333
334extern __inline __m64
335 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
336 _m_punpcklbw(__m64 __m1, __m64 __m2) {
337 return _mm_unpacklo_pi8(__m1, __m2);
338}
339/* Interleave the two 16-bit values from the low half of M1 with the two
340 16-bit values from the low half of M2. */
341extern __inline __m64
342 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
343 _mm_unpacklo_pi16(__m64 __m1, __m64 __m2) {
344 __m64_union m1, m2, res;
345
346 m1.as_m64 = __m1;
347 m2.as_m64 = __m2;
348
349 res.as_short[0] = m1.as_short[0];
350 res.as_short[1] = m2.as_short[0];
351 res.as_short[2] = m1.as_short[1];
352 res.as_short[3] = m2.as_short[1];
353
354 return (__m64)res.as_m64;
355}
356
357extern __inline __m64
358 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
359 _m_punpcklwd(__m64 __m1, __m64 __m2) {
360 return _mm_unpacklo_pi16(__m1, __m2);
361}
362
363/* Interleave the 32-bit value from the low half of M1 with the 32-bit
364 value from the low half of M2. */
365extern __inline __m64
366 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
367 _mm_unpacklo_pi32(__m64 __m1, __m64 __m2) {
368 __m64_union m1, m2, res;
369
370 m1.as_m64 = __m1;
371 m2.as_m64 = __m2;
372
373 res.as_int[0] = m1.as_int[0];
374 res.as_int[1] = m2.as_int[0];
375
376 return (__m64)res.as_m64;
377}
378
379extern __inline __m64
380 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
381 _m_punpckldq(__m64 __m1, __m64 __m2) {
382 return _mm_unpacklo_pi32(__m1, __m2);
383}
384
385/* Add the 8-bit values in M1 to the 8-bit values in M2. */
386extern __inline __m64
387 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
388 _mm_add_pi8(__m64 __m1, __m64 __m2) {
389#if _ARCH_PWR8
390 __vector signed char a, b, c;
391
392 a = (__vector signed char)vec_splats(__m1);
393 b = (__vector signed char)vec_splats(__m2);
394 c = vec_add(a, b);
395 return (__m64)((__vector long long)c)[0];
396#else
397 __m64_union m1, m2, res;
398
399 m1.as_m64 = __m1;
400 m2.as_m64 = __m2;
401
402 res.as_char[0] = m1.as_char[0] + m2.as_char[0];
403 res.as_char[1] = m1.as_char[1] + m2.as_char[1];
404 res.as_char[2] = m1.as_char[2] + m2.as_char[2];
405 res.as_char[3] = m1.as_char[3] + m2.as_char[3];
406 res.as_char[4] = m1.as_char[4] + m2.as_char[4];
407 res.as_char[5] = m1.as_char[5] + m2.as_char[5];
408 res.as_char[6] = m1.as_char[6] + m2.as_char[6];
409 res.as_char[7] = m1.as_char[7] + m2.as_char[7];
410
411 return (__m64)res.as_m64;
412#endif
413}
414
415extern __inline __m64
416 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
417 _m_paddb(__m64 __m1, __m64 __m2) {
418 return _mm_add_pi8(__m1, __m2);
419}
420
421/* Add the 16-bit values in M1 to the 16-bit values in M2. */
422extern __inline __m64
423 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
424 _mm_add_pi16(__m64 __m1, __m64 __m2) {
425#if _ARCH_PWR8
426 __vector signed short a, b, c;
427
428 a = (__vector signed short)vec_splats(__m1);
429 b = (__vector signed short)vec_splats(__m2);
430 c = vec_add(a, b);
431 return (__m64)((__vector long long)c)[0];
432#else
433 __m64_union m1, m2, res;
434
435 m1.as_m64 = __m1;
436 m2.as_m64 = __m2;
437
438 res.as_short[0] = m1.as_short[0] + m2.as_short[0];
439 res.as_short[1] = m1.as_short[1] + m2.as_short[1];
440 res.as_short[2] = m1.as_short[2] + m2.as_short[2];
441 res.as_short[3] = m1.as_short[3] + m2.as_short[3];
442
443 return (__m64)res.as_m64;
444#endif
445}
446
447extern __inline __m64
448 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
449 _m_paddw(__m64 __m1, __m64 __m2) {
450 return _mm_add_pi16(__m1, __m2);
451}
452
453/* Add the 32-bit values in M1 to the 32-bit values in M2. */
454extern __inline __m64
455 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
456 _mm_add_pi32(__m64 __m1, __m64 __m2) {
457#if _ARCH_PWR9
458 __vector signed int a, b, c;
459
460 a = (__vector signed int)vec_splats(__m1);
461 b = (__vector signed int)vec_splats(__m2);
462 c = vec_add(a, b);
463 return (__m64)((__vector long long)c)[0];
464#else
465 __m64_union m1, m2, res;
466
467 m1.as_m64 = __m1;
468 m2.as_m64 = __m2;
469
470 res.as_int[0] = m1.as_int[0] + m2.as_int[0];
471 res.as_int[1] = m1.as_int[1] + m2.as_int[1];
472
473 return (__m64)res.as_m64;
474#endif
475}
476
477extern __inline __m64
478 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
479 _m_paddd(__m64 __m1, __m64 __m2) {
480 return _mm_add_pi32(__m1, __m2);
481}
482
483/* Subtract the 8-bit values in M2 from the 8-bit values in M1. */
484extern __inline __m64
485 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
486 _mm_sub_pi8(__m64 __m1, __m64 __m2) {
487#if _ARCH_PWR8
488 __vector signed char a, b, c;
489
490 a = (__vector signed char)vec_splats(__m1);
491 b = (__vector signed char)vec_splats(__m2);
492 c = vec_sub(a, b);
493 return (__m64)((__vector long long)c)[0];
494#else
495 __m64_union m1, m2, res;
496
497 m1.as_m64 = __m1;
498 m2.as_m64 = __m2;
499
500 res.as_char[0] = m1.as_char[0] - m2.as_char[0];
501 res.as_char[1] = m1.as_char[1] - m2.as_char[1];
502 res.as_char[2] = m1.as_char[2] - m2.as_char[2];
503 res.as_char[3] = m1.as_char[3] - m2.as_char[3];
504 res.as_char[4] = m1.as_char[4] - m2.as_char[4];
505 res.as_char[5] = m1.as_char[5] - m2.as_char[5];
506 res.as_char[6] = m1.as_char[6] - m2.as_char[6];
507 res.as_char[7] = m1.as_char[7] - m2.as_char[7];
508
509 return (__m64)res.as_m64;
510#endif
511}
512
513extern __inline __m64
514 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
515 _m_psubb(__m64 __m1, __m64 __m2) {
516 return _mm_sub_pi8(__m1, __m2);
517}
518
519/* Subtract the 16-bit values in M2 from the 16-bit values in M1. */
520extern __inline __m64
521 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
522 _mm_sub_pi16(__m64 __m1, __m64 __m2) {
523#if _ARCH_PWR8
524 __vector signed short a, b, c;
525
526 a = (__vector signed short)vec_splats(__m1);
527 b = (__vector signed short)vec_splats(__m2);
528 c = vec_sub(a, b);
529 return (__m64)((__vector long long)c)[0];
530#else
531 __m64_union m1, m2, res;
532
533 m1.as_m64 = __m1;
534 m2.as_m64 = __m2;
535
536 res.as_short[0] = m1.as_short[0] - m2.as_short[0];
537 res.as_short[1] = m1.as_short[1] - m2.as_short[1];
538 res.as_short[2] = m1.as_short[2] - m2.as_short[2];
539 res.as_short[3] = m1.as_short[3] - m2.as_short[3];
540
541 return (__m64)res.as_m64;
542#endif
543}
544
545extern __inline __m64
546 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
547 _m_psubw(__m64 __m1, __m64 __m2) {
548 return _mm_sub_pi16(__m1, __m2);
549}
550
551/* Subtract the 32-bit values in M2 from the 32-bit values in M1. */
552extern __inline __m64
553 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
554 _mm_sub_pi32(__m64 __m1, __m64 __m2) {
555#if _ARCH_PWR9
556 __vector signed int a, b, c;
557
558 a = (__vector signed int)vec_splats(__m1);
559 b = (__vector signed int)vec_splats(__m2);
560 c = vec_sub(a, b);
561 return (__m64)((__vector long long)c)[0];
562#else
563 __m64_union m1, m2, res;
564
565 m1.as_m64 = __m1;
566 m2.as_m64 = __m2;
567
568 res.as_int[0] = m1.as_int[0] - m2.as_int[0];
569 res.as_int[1] = m1.as_int[1] - m2.as_int[1];
570
571 return (__m64)res.as_m64;
572#endif
573}
574
575extern __inline __m64
576 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
577 _m_psubd(__m64 __m1, __m64 __m2) {
578 return _mm_sub_pi32(__m1, __m2);
579}
580
581extern __inline __m64
582 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
583 _mm_add_si64(__m64 __m1, __m64 __m2) {
584 return (__m1 + __m2);
585}
586
587extern __inline __m64
588 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
589 _mm_sub_si64(__m64 __m1, __m64 __m2) {
590 return (__m1 - __m2);
591}
592
593/* Shift the 64-bit value in M left by COUNT. */
594extern __inline __m64
595 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
596 _mm_sll_si64(__m64 __m, __m64 __count) {
597 return (__m << __count);
598}
599
600extern __inline __m64
601 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
602 _m_psllq(__m64 __m, __m64 __count) {
603 return _mm_sll_si64(__m, __count);
604}
605
606extern __inline __m64
607 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
608 _mm_slli_si64(__m64 __m, const int __count) {
609 return (__m << __count);
610}
611
612extern __inline __m64
613 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
614 _m_psllqi(__m64 __m, const int __count) {
615 return _mm_slli_si64(__m, __count);
616}
617
618/* Shift the 64-bit value in M left by COUNT; shift in zeros. */
619extern __inline __m64
620 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
621 _mm_srl_si64(__m64 __m, __m64 __count) {
622 return (__m >> __count);
623}
624
625extern __inline __m64
626 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
627 _m_psrlq(__m64 __m, __m64 __count) {
628 return _mm_srl_si64(__m, __count);
629}
630
631extern __inline __m64
632 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
633 _mm_srli_si64(__m64 __m, const int __count) {
634 return (__m >> __count);
635}
636
637extern __inline __m64
638 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
639 _m_psrlqi(__m64 __m, const int __count) {
640 return _mm_srli_si64(__m, __count);
641}
642
643/* Bit-wise AND the 64-bit values in M1 and M2. */
644extern __inline __m64
645 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
646 _mm_and_si64(__m64 __m1, __m64 __m2) {
647 return (__m1 & __m2);
648}
649
650extern __inline __m64
651 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
652 _m_pand(__m64 __m1, __m64 __m2) {
653 return _mm_and_si64(__m1, __m2);
654}
655
656/* Bit-wise complement the 64-bit value in M1 and bit-wise AND it with the
657 64-bit value in M2. */
658extern __inline __m64
659 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
660 _mm_andnot_si64(__m64 __m1, __m64 __m2) {
661 return (~__m1 & __m2);
662}
663
664extern __inline __m64
665 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
666 _m_pandn(__m64 __m1, __m64 __m2) {
667 return _mm_andnot_si64(__m1, __m2);
668}
669
670/* Bit-wise inclusive OR the 64-bit values in M1 and M2. */
671extern __inline __m64
672 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
673 _mm_or_si64(__m64 __m1, __m64 __m2) {
674 return (__m1 | __m2);
675}
676
677extern __inline __m64
678 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
679 _m_por(__m64 __m1, __m64 __m2) {
680 return _mm_or_si64(__m1, __m2);
681}
682
683/* Bit-wise exclusive OR the 64-bit values in M1 and M2. */
684extern __inline __m64
685 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
686 _mm_xor_si64(__m64 __m1, __m64 __m2) {
687 return (__m1 ^ __m2);
688}
689
690extern __inline __m64
691 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
692 _m_pxor(__m64 __m1, __m64 __m2) {
693 return _mm_xor_si64(__m1, __m2);
694}
695
696/* Creates a 64-bit zero. */
697extern __inline __m64
698 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
699 _mm_setzero_si64(void) {
700 return (__m64)0;
701}
702
703/* Compare eight 8-bit values. The result of the comparison is 0xFF if the
704 test is true and zero if false. */
705extern __inline __m64
706 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
707 _mm_cmpeq_pi8(__m64 __m1, __m64 __m2) {
708#if defined(_ARCH_PWR6) && defined(__powerpc64__)
709 __m64 res;
710 __asm__("cmpb %0,%1,%2;\n" : "=r"(res) : "r"(__m1), "r"(__m2) :);
711 return (res);
712#else
713 __m64_union m1, m2, res;
714
715 m1.as_m64 = __m1;
716 m2.as_m64 = __m2;
717
718 res.as_char[0] = (m1.as_char[0] == m2.as_char[0]) ? -1 : 0;
719 res.as_char[1] = (m1.as_char[1] == m2.as_char[1]) ? -1 : 0;
720 res.as_char[2] = (m1.as_char[2] == m2.as_char[2]) ? -1 : 0;
721 res.as_char[3] = (m1.as_char[3] == m2.as_char[3]) ? -1 : 0;
722 res.as_char[4] = (m1.as_char[4] == m2.as_char[4]) ? -1 : 0;
723 res.as_char[5] = (m1.as_char[5] == m2.as_char[5]) ? -1 : 0;
724 res.as_char[6] = (m1.as_char[6] == m2.as_char[6]) ? -1 : 0;
725 res.as_char[7] = (m1.as_char[7] == m2.as_char[7]) ? -1 : 0;
726
727 return (__m64)res.as_m64;
728#endif
729}
730
731extern __inline __m64
732 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
733 _m_pcmpeqb(__m64 __m1, __m64 __m2) {
734 return _mm_cmpeq_pi8(__m1, __m2);
735}
736
737extern __inline __m64
738 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
739 _mm_cmpgt_pi8(__m64 __m1, __m64 __m2) {
740#if _ARCH_PWR8
741 __vector signed char a, b, c;
742
743 a = (__vector signed char)vec_splats(__m1);
744 b = (__vector signed char)vec_splats(__m2);
745 c = (__vector signed char)vec_cmpgt(a, b);
746 return (__m64)((__vector long long)c)[0];
747#else
748 __m64_union m1, m2, res;
749
750 m1.as_m64 = __m1;
751 m2.as_m64 = __m2;
752
753 res.as_char[0] = (m1.as_char[0] > m2.as_char[0]) ? -1 : 0;
754 res.as_char[1] = (m1.as_char[1] > m2.as_char[1]) ? -1 : 0;
755 res.as_char[2] = (m1.as_char[2] > m2.as_char[2]) ? -1 : 0;
756 res.as_char[3] = (m1.as_char[3] > m2.as_char[3]) ? -1 : 0;
757 res.as_char[4] = (m1.as_char[4] > m2.as_char[4]) ? -1 : 0;
758 res.as_char[5] = (m1.as_char[5] > m2.as_char[5]) ? -1 : 0;
759 res.as_char[6] = (m1.as_char[6] > m2.as_char[6]) ? -1 : 0;
760 res.as_char[7] = (m1.as_char[7] > m2.as_char[7]) ? -1 : 0;
761
762 return (__m64)res.as_m64;
763#endif
764}
765
766extern __inline __m64
767 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
768 _m_pcmpgtb(__m64 __m1, __m64 __m2) {
769 return _mm_cmpgt_pi8(__m1, __m2);
770}
771
772/* Compare four 16-bit values. The result of the comparison is 0xFFFF if
773 the test is true and zero if false. */
774extern __inline __m64
775 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
776 _mm_cmpeq_pi16(__m64 __m1, __m64 __m2) {
777#if _ARCH_PWR8
778 __vector signed short a, b, c;
779
780 a = (__vector signed short)vec_splats(__m1);
781 b = (__vector signed short)vec_splats(__m2);
782 c = (__vector signed short)vec_cmpeq(a, b);
783 return (__m64)((__vector long long)c)[0];
784#else
785 __m64_union m1, m2, res;
786
787 m1.as_m64 = __m1;
788 m2.as_m64 = __m2;
789
790 res.as_short[0] = (m1.as_short[0] == m2.as_short[0]) ? -1 : 0;
791 res.as_short[1] = (m1.as_short[1] == m2.as_short[1]) ? -1 : 0;
792 res.as_short[2] = (m1.as_short[2] == m2.as_short[2]) ? -1 : 0;
793 res.as_short[3] = (m1.as_short[3] == m2.as_short[3]) ? -1 : 0;
794
795 return (__m64)res.as_m64;
796#endif
797}
798
799extern __inline __m64
800 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
801 _m_pcmpeqw(__m64 __m1, __m64 __m2) {
802 return _mm_cmpeq_pi16(__m1, __m2);
803}
804
805extern __inline __m64
806 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
807 _mm_cmpgt_pi16(__m64 __m1, __m64 __m2) {
808#if _ARCH_PWR8
809 __vector signed short a, b, c;
810
811 a = (__vector signed short)vec_splats(__m1);
812 b = (__vector signed short)vec_splats(__m2);
813 c = (__vector signed short)vec_cmpgt(a, b);
814 return (__m64)((__vector long long)c)[0];
815#else
816 __m64_union m1, m2, res;
817
818 m1.as_m64 = __m1;
819 m2.as_m64 = __m2;
820
821 res.as_short[0] = (m1.as_short[0] > m2.as_short[0]) ? -1 : 0;
822 res.as_short[1] = (m1.as_short[1] > m2.as_short[1]) ? -1 : 0;
823 res.as_short[2] = (m1.as_short[2] > m2.as_short[2]) ? -1 : 0;
824 res.as_short[3] = (m1.as_short[3] > m2.as_short[3]) ? -1 : 0;
825
826 return (__m64)res.as_m64;
827#endif
828}
829
830extern __inline __m64
831 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
832 _m_pcmpgtw(__m64 __m1, __m64 __m2) {
833 return _mm_cmpgt_pi16(__m1, __m2);
834}
835
836/* Compare two 32-bit values. The result of the comparison is 0xFFFFFFFF if
837 the test is true and zero if false. */
838extern __inline __m64
839 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
840 _mm_cmpeq_pi32(__m64 __m1, __m64 __m2) {
841#if _ARCH_PWR9
842 __vector signed int a, b, c;
843
844 a = (__vector signed int)vec_splats(__m1);
845 b = (__vector signed int)vec_splats(__m2);
846 c = (__vector signed int)vec_cmpeq(a, b);
847 return (__m64)((__vector long long)c)[0];
848#else
849 __m64_union m1, m2, res;
850
851 m1.as_m64 = __m1;
852 m2.as_m64 = __m2;
853
854 res.as_int[0] = (m1.as_int[0] == m2.as_int[0]) ? -1 : 0;
855 res.as_int[1] = (m1.as_int[1] == m2.as_int[1]) ? -1 : 0;
856
857 return (__m64)res.as_m64;
858#endif
859}
860
861extern __inline __m64
862 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
863 _m_pcmpeqd(__m64 __m1, __m64 __m2) {
864 return _mm_cmpeq_pi32(__m1, __m2);
865}
866
867extern __inline __m64
868 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
869 _mm_cmpgt_pi32(__m64 __m1, __m64 __m2) {
870#if _ARCH_PWR9
871 __vector signed int a, b, c;
872
873 a = (__vector signed int)vec_splats(__m1);
874 b = (__vector signed int)vec_splats(__m2);
875 c = (__vector signed int)vec_cmpgt(a, b);
876 return (__m64)((__vector long long)c)[0];
877#else
878 __m64_union m1, m2, res;
879
880 m1.as_m64 = __m1;
881 m2.as_m64 = __m2;
882
883 res.as_int[0] = (m1.as_int[0] > m2.as_int[0]) ? -1 : 0;
884 res.as_int[1] = (m1.as_int[1] > m2.as_int[1]) ? -1 : 0;
885
886 return (__m64)res.as_m64;
887#endif
888}
889
890extern __inline __m64
891 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
892 _m_pcmpgtd(__m64 __m1, __m64 __m2) {
893 return _mm_cmpgt_pi32(__m1, __m2);
894}
895
896#if _ARCH_PWR8
897/* Add the 8-bit values in M1 to the 8-bit values in M2 using signed
898 saturated arithmetic. */
899extern __inline __m64
900 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
901 _mm_adds_pi8(__m64 __m1, __m64 __m2) {
902 __vector signed char a, b, c;
903
904 a = (__vector signed char)vec_splats(__m1);
905 b = (__vector signed char)vec_splats(__m2);
906 c = vec_adds(a, b);
907 return (__m64)((__vector long long)c)[0];
908}
909
910extern __inline __m64
911 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
912 _m_paddsb(__m64 __m1, __m64 __m2) {
913 return _mm_adds_pi8(__m1, __m2);
914}
915/* Add the 16-bit values in M1 to the 16-bit values in M2 using signed
916 saturated arithmetic. */
917extern __inline __m64
918 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
919 _mm_adds_pi16(__m64 __m1, __m64 __m2) {
920 __vector signed short a, b, c;
921
922 a = (__vector signed short)vec_splats(__m1);
923 b = (__vector signed short)vec_splats(__m2);
924 c = vec_adds(a, b);
925 return (__m64)((__vector long long)c)[0];
926}
927
928extern __inline __m64
929 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
930 _m_paddsw(__m64 __m1, __m64 __m2) {
931 return _mm_adds_pi16(__m1, __m2);
932}
933/* Add the 8-bit values in M1 to the 8-bit values in M2 using unsigned
934 saturated arithmetic. */
935extern __inline __m64
936 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
937 _mm_adds_pu8(__m64 __m1, __m64 __m2) {
938 __vector unsigned char a, b, c;
939
940 a = (__vector unsigned char)vec_splats(__m1);
941 b = (__vector unsigned char)vec_splats(__m2);
942 c = vec_adds(a, b);
943 return (__m64)((__vector long long)c)[0];
944}
945
946extern __inline __m64
947 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
948 _m_paddusb(__m64 __m1, __m64 __m2) {
949 return _mm_adds_pu8(__m1, __m2);
950}
951
952/* Add the 16-bit values in M1 to the 16-bit values in M2 using unsigned
953 saturated arithmetic. */
954extern __inline __m64
955 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
956 _mm_adds_pu16(__m64 __m1, __m64 __m2) {
957 __vector unsigned short a, b, c;
958
959 a = (__vector unsigned short)vec_splats(__m1);
960 b = (__vector unsigned short)vec_splats(__m2);
961 c = vec_adds(a, b);
962 return (__m64)((__vector long long)c)[0];
963}
964
965extern __inline __m64
966 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
967 _m_paddusw(__m64 __m1, __m64 __m2) {
968 return _mm_adds_pu16(__m1, __m2);
969}
970
971/* Subtract the 8-bit values in M2 from the 8-bit values in M1 using signed
972 saturating arithmetic. */
973extern __inline __m64
974 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
975 _mm_subs_pi8(__m64 __m1, __m64 __m2) {
976 __vector signed char a, b, c;
977
978 a = (__vector signed char)vec_splats(__m1);
979 b = (__vector signed char)vec_splats(__m2);
980 c = vec_subs(a, b);
981 return (__m64)((__vector long long)c)[0];
982}
983
984extern __inline __m64
985 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
986 _m_psubsb(__m64 __m1, __m64 __m2) {
987 return _mm_subs_pi8(__m1, __m2);
988}
989
990/* Subtract the 16-bit values in M2 from the 16-bit values in M1 using
991 signed saturating arithmetic. */
992extern __inline __m64
993 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
994 _mm_subs_pi16(__m64 __m1, __m64 __m2) {
995 __vector signed short a, b, c;
996
997 a = (__vector signed short)vec_splats(__m1);
998 b = (__vector signed short)vec_splats(__m2);
999 c = vec_subs(a, b);
1000 return (__m64)((__vector long long)c)[0];
1001}
1002
1003extern __inline __m64
1004 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1005 _m_psubsw(__m64 __m1, __m64 __m2) {
1006 return _mm_subs_pi16(__m1, __m2);
1007}
1008
1009/* Subtract the 8-bit values in M2 from the 8-bit values in M1 using
1010 unsigned saturating arithmetic. */
1011extern __inline __m64
1012 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1013 _mm_subs_pu8(__m64 __m1, __m64 __m2) {
1014 __vector unsigned char a, b, c;
1015
1016 a = (__vector unsigned char)vec_splats(__m1);
1017 b = (__vector unsigned char)vec_splats(__m2);
1018 c = vec_subs(a, b);
1019 return (__m64)((__vector long long)c)[0];
1020}
1021
1022extern __inline __m64
1023 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1024 _m_psubusb(__m64 __m1, __m64 __m2) {
1025 return _mm_subs_pu8(__m1, __m2);
1026}
1027
1028/* Subtract the 16-bit values in M2 from the 16-bit values in M1 using
1029 unsigned saturating arithmetic. */
1030extern __inline __m64
1031 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1032 _mm_subs_pu16(__m64 __m1, __m64 __m2) {
1033 __vector unsigned short a, b, c;
1034
1035 a = (__vector unsigned short)vec_splats(__m1);
1036 b = (__vector unsigned short)vec_splats(__m2);
1037 c = vec_subs(a, b);
1038 return (__m64)((__vector long long)c)[0];
1039}
1040
1041extern __inline __m64
1042 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1043 _m_psubusw(__m64 __m1, __m64 __m2) {
1044 return _mm_subs_pu16(__m1, __m2);
1045}
1046
1047/* Multiply four 16-bit values in M1 by four 16-bit values in M2 producing
1048 four 32-bit intermediate results, which are then summed by pairs to
1049 produce two 32-bit results. */
1050extern __inline __m64
1051 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1052 _mm_madd_pi16(__m64 __m1, __m64 __m2) {
1053 __vector signed short a, b;
1054 __vector signed int c;
1055 __vector signed int zero = {0, 0, 0, 0};
1056
1057 a = (__vector signed short)vec_splats(__m1);
1058 b = (__vector signed short)vec_splats(__m2);
1059 c = vec_vmsumshm(a, b, zero);
1060 return (__m64)((__vector long long)c)[0];
1061}
1062
1063extern __inline __m64
1064 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1065 _m_pmaddwd(__m64 __m1, __m64 __m2) {
1066 return _mm_madd_pi16(__m1, __m2);
1067}
1068/* Multiply four signed 16-bit values in M1 by four signed 16-bit values in
1069 M2 and produce the high 16 bits of the 32-bit results. */
1070extern __inline __m64
1071 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1072 _mm_mulhi_pi16(__m64 __m1, __m64 __m2) {
1073 __vector signed short a, b;
1074 __vector signed short c;
1075 __vector signed int w0, w1;
1076 __vector unsigned char xform1 = {
1077#ifdef __LITTLE_ENDIAN__
1078 0x02, 0x03, 0x12, 0x13, 0x06, 0x07, 0x16, 0x17, 0x0A,
1079 0x0B, 0x1A, 0x1B, 0x0E, 0x0F, 0x1E, 0x1F
1080#else
1081 0x00, 0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15, 0x00,
1082 0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15
1083#endif
1084 };
1085
1086 a = (__vector signed short)vec_splats(__m1);
1087 b = (__vector signed short)vec_splats(__m2);
1088
1089 w0 = vec_vmulesh(a, b);
1090 w1 = vec_vmulosh(a, b);
1091 c = (__vector signed short)vec_perm(w0, w1, xform1);
1092
1093 return (__m64)((__vector long long)c)[0];
1094}
1095
1096extern __inline __m64
1097 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1098 _m_pmulhw(__m64 __m1, __m64 __m2) {
1099 return _mm_mulhi_pi16(__m1, __m2);
1100}
1101
1102/* Multiply four 16-bit values in M1 by four 16-bit values in M2 and produce
1103 the low 16 bits of the results. */
1104extern __inline __m64
1105 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1106 _mm_mullo_pi16(__m64 __m1, __m64 __m2) {
1107 __vector signed short a, b, c;
1108
1109 a = (__vector signed short)vec_splats(__m1);
1110 b = (__vector signed short)vec_splats(__m2);
1111 c = a * b;
1112 return (__m64)((__vector long long)c)[0];
1113}
1114
1115extern __inline __m64
1116 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1117 _m_pmullw(__m64 __m1, __m64 __m2) {
1118 return _mm_mullo_pi16(__m1, __m2);
1119}
1120
1121/* Shift four 16-bit values in M left by COUNT. */
1122extern __inline __m64
1123 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1124 _mm_sll_pi16(__m64 __m, __m64 __count) {
1125 __vector signed short m, r;
1126 __vector unsigned short c;
1127
1128 if (__count <= 15) {
1129 m = (__vector signed short)vec_splats(__m);
1130 c = (__vector unsigned short)vec_splats((unsigned short)__count);
1131 r = vec_sl(m, (__vector unsigned short)c);
1132 return (__m64)((__vector long long)r)[0];
1133 } else
1134 return (0);
1135}
1136
1137extern __inline __m64
1138 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1139 _m_psllw(__m64 __m, __m64 __count) {
1140 return _mm_sll_pi16(__m, __count);
1141}
1142
1143extern __inline __m64
1144 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1145 _mm_slli_pi16(__m64 __m, int __count) {
1146 /* Promote int to long then invoke mm_sll_pi16. */
1147 return _mm_sll_pi16(__m, __count);
1148}
1149
1150extern __inline __m64
1151 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1152 _m_psllwi(__m64 __m, int __count) {
1153 return _mm_slli_pi16(__m, __count);
1154}
1155
1156/* Shift two 32-bit values in M left by COUNT. */
1157extern __inline __m64
1158 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1159 _mm_sll_pi32(__m64 __m, __m64 __count) {
1160 __m64_union m, res;
1161
1162 m.as_m64 = __m;
1163
1164 res.as_int[0] = m.as_int[0] << __count;
1165 res.as_int[1] = m.as_int[1] << __count;
1166 return (res.as_m64);
1167}
1168
1169extern __inline __m64
1170 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1171 _m_pslld(__m64 __m, __m64 __count) {
1172 return _mm_sll_pi32(__m, __count);
1173}
1174
1175extern __inline __m64
1176 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1177 _mm_slli_pi32(__m64 __m, int __count) {
1178 /* Promote int to long then invoke mm_sll_pi32. */
1179 return _mm_sll_pi32(__m, __count);
1180}
1181
1182extern __inline __m64
1183 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1184 _m_pslldi(__m64 __m, int __count) {
1185 return _mm_slli_pi32(__m, __count);
1186}
1187
1188/* Shift four 16-bit values in M right by COUNT; shift in the sign bit. */
1189extern __inline __m64
1190 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1191 _mm_sra_pi16(__m64 __m, __m64 __count) {
1192 __vector signed short m, r;
1193 __vector unsigned short c;
1194
1195 if (__count <= 15) {
1196 m = (__vector signed short)vec_splats(__m);
1197 c = (__vector unsigned short)vec_splats((unsigned short)__count);
1198 r = vec_sra(m, (__vector unsigned short)c);
1199 return (__m64)((__vector long long)r)[0];
1200 } else
1201 return (0);
1202}
1203
1204extern __inline __m64
1205 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1206 _m_psraw(__m64 __m, __m64 __count) {
1207 return _mm_sra_pi16(__m, __count);
1208}
1209
1210extern __inline __m64
1211 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1212 _mm_srai_pi16(__m64 __m, int __count) {
1213 /* Promote int to long then invoke mm_sra_pi32. */
1214 return _mm_sra_pi16(__m, __count);
1215}
1216
1217extern __inline __m64
1218 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1219 _m_psrawi(__m64 __m, int __count) {
1220 return _mm_srai_pi16(__m, __count);
1221}
1222
1223/* Shift two 32-bit values in M right by COUNT; shift in the sign bit. */
1224extern __inline __m64
1225 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1226 _mm_sra_pi32(__m64 __m, __m64 __count) {
1227 __m64_union m, res;
1228
1229 m.as_m64 = __m;
1230
1231 res.as_int[0] = m.as_int[0] >> __count;
1232 res.as_int[1] = m.as_int[1] >> __count;
1233 return (res.as_m64);
1234}
1235
1236extern __inline __m64
1237 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1238 _m_psrad(__m64 __m, __m64 __count) {
1239 return _mm_sra_pi32(__m, __count);
1240}
1241
1242extern __inline __m64
1243 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1244 _mm_srai_pi32(__m64 __m, int __count) {
1245 /* Promote int to long then invoke mm_sra_pi32. */
1246 return _mm_sra_pi32(__m, __count);
1247}
1248
1249extern __inline __m64
1250 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1251 _m_psradi(__m64 __m, int __count) {
1252 return _mm_srai_pi32(__m, __count);
1253}
1254
1255/* Shift four 16-bit values in M right by COUNT; shift in zeros. */
1256extern __inline __m64
1257 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1258 _mm_srl_pi16(__m64 __m, __m64 __count) {
1259 __vector unsigned short m, r;
1260 __vector unsigned short c;
1261
1262 if (__count <= 15) {
1263 m = (__vector unsigned short)vec_splats(__m);
1264 c = (__vector unsigned short)vec_splats((unsigned short)__count);
1265 r = vec_sr(m, (__vector unsigned short)c);
1266 return (__m64)((__vector long long)r)[0];
1267 } else
1268 return (0);
1269}
1270
1271extern __inline __m64
1272 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1273 _m_psrlw(__m64 __m, __m64 __count) {
1274 return _mm_srl_pi16(__m, __count);
1275}
1276
1277extern __inline __m64
1278 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1279 _mm_srli_pi16(__m64 __m, int __count) {
1280 /* Promote int to long then invoke mm_sra_pi32. */
1281 return _mm_srl_pi16(__m, __count);
1282}
1283
1284extern __inline __m64
1285 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1286 _m_psrlwi(__m64 __m, int __count) {
1287 return _mm_srli_pi16(__m, __count);
1288}
1289
1290/* Shift two 32-bit values in M right by COUNT; shift in zeros. */
1291extern __inline __m64
1292 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1293 _mm_srl_pi32(__m64 __m, __m64 __count) {
1294 __m64_union m, res;
1295
1296 m.as_m64 = __m;
1297
1298 res.as_int[0] = (unsigned int)m.as_int[0] >> __count;
1299 res.as_int[1] = (unsigned int)m.as_int[1] >> __count;
1300 return (res.as_m64);
1301}
1302
1303extern __inline __m64
1304 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1305 _m_psrld(__m64 __m, __m64 __count) {
1306 return _mm_srl_pi32(__m, __count);
1307}
1308
1309extern __inline __m64
1310 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1311 _mm_srli_pi32(__m64 __m, int __count) {
1312 /* Promote int to long then invoke mm_srl_pi32. */
1313 return _mm_srl_pi32(__m, __count);
1314}
1315
1316extern __inline __m64
1317 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1318 _m_psrldi(__m64 __m, int __count) {
1319 return _mm_srli_pi32(__m, __count);
1320}
1321#endif /* _ARCH_PWR8 */
1322
1323/* Creates a vector of two 32-bit values; I0 is least significant. */
1324extern __inline __m64
1325 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1326 _mm_set_pi32(int __i1, int __i0) {
1327 __m64_union res;
1328
1329 res.as_int[0] = __i0;
1330 res.as_int[1] = __i1;
1331 return (res.as_m64);
1332}
1333
1334/* Creates a vector of four 16-bit values; W0 is least significant. */
1335extern __inline __m64
1336 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1337 _mm_set_pi16(short __w3, short __w2, short __w1, short __w0) {
1338 __m64_union res;
1339
1340 res.as_short[0] = __w0;
1341 res.as_short[1] = __w1;
1342 res.as_short[2] = __w2;
1343 res.as_short[3] = __w3;
1344 return (res.as_m64);
1345}
1346
1347/* Creates a vector of eight 8-bit values; B0 is least significant. */
1348extern __inline __m64
1349 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1350 _mm_set_pi8(char __b7, char __b6, char __b5, char __b4, char __b3,
1351 char __b2, char __b1, char __b0) {
1352 __m64_union res;
1353
1354 res.as_char[0] = __b0;
1355 res.as_char[1] = __b1;
1356 res.as_char[2] = __b2;
1357 res.as_char[3] = __b3;
1358 res.as_char[4] = __b4;
1359 res.as_char[5] = __b5;
1360 res.as_char[6] = __b6;
1361 res.as_char[7] = __b7;
1362 return (res.as_m64);
1363}
1364
1365/* Similar, but with the arguments in reverse order. */
1366extern __inline __m64
1367 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1368 _mm_setr_pi32(int __i0, int __i1) {
1369 __m64_union res;
1370
1371 res.as_int[0] = __i0;
1372 res.as_int[1] = __i1;
1373 return (res.as_m64);
1374}
1375
1376extern __inline __m64
1377 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1378 _mm_setr_pi16(short __w0, short __w1, short __w2, short __w3) {
1379 return _mm_set_pi16(__w3, __w2, __w1, __w0);
1380}
1381
1382extern __inline __m64
1383 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1384 _mm_setr_pi8(char __b0, char __b1, char __b2, char __b3, char __b4,
1385 char __b5, char __b6, char __b7) {
1386 return _mm_set_pi8(__b7, __b6, __b5, __b4, __b3, __b2, __b1, __b0);
1387}
1388
1389/* Creates a vector of two 32-bit values, both elements containing I. */
1390extern __inline __m64
1391 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1392 _mm_set1_pi32(int __i) {
1393 __m64_union res;
1394
1395 res.as_int[0] = __i;
1396 res.as_int[1] = __i;
1397 return (res.as_m64);
1398}
1399
1400/* Creates a vector of four 16-bit values, all elements containing W. */
1401extern __inline __m64
1402 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1403 _mm_set1_pi16(short __w) {
1404#if _ARCH_PWR9
1405 __vector signed short w;
1406
1407 w = (__vector signed short)vec_splats(__w);
1408 return (__m64)((__vector long long)w)[0];
1409#else
1410 __m64_union res;
1411
1412 res.as_short[0] = __w;
1413 res.as_short[1] = __w;
1414 res.as_short[2] = __w;
1415 res.as_short[3] = __w;
1416 return (res.as_m64);
1417#endif
1418}
1419
1420/* Creates a vector of eight 8-bit values, all elements containing B. */
1421extern __inline __m64
1422 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1423 _mm_set1_pi8(signed char __b) {
1424#if _ARCH_PWR8
1425 __vector signed char b;
1426
1427 b = (__vector signed char)vec_splats(__b);
1428 return (__m64)((__vector long long)b)[0];
1429#else
1430 __m64_union res;
1431
1432 res.as_char[0] = __b;
1433 res.as_char[1] = __b;
1434 res.as_char[2] = __b;
1435 res.as_char[3] = __b;
1436 res.as_char[4] = __b;
1437 res.as_char[5] = __b;
1438 res.as_char[6] = __b;
1439 res.as_char[7] = __b;
1440 return (res.as_m64);
1441#endif
1442}
1443#endif /* _MMINTRIN_H_INCLUDED */
lib/include/ppc_wrappers/xmmintrin.h created+1838
...@@ -0,0 +1,1838 @@
1/*===---- xmmintrin.h - Implementation of SSE intrinsics on PowerPC --------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10/* Implemented from the specification included in the Intel C++ Compiler
11 User Guide and Reference, version 9.0. */
12
13#ifndef NO_WARN_X86_INTRINSICS
14/* This header file is to help porting code using Intel intrinsics
15 explicitly from x86_64 to powerpc64/powerpc64le.
16
17 Since X86 SSE intrinsics mainly handles __m128 type, PowerPC
18 VMX/VSX ISA is a good match for vector float SIMD operations.
19 However scalar float operations in vector (XMM) registers require
20 the POWER8 VSX ISA (2.07) level. There are differences for data
21 format and placement of float scalars in the vector register, which
22 require extra steps to match SSE scalar float semantics on POWER.
23
24 It should be noted that there's much difference between X86_64's
25 MXSCR and PowerISA's FPSCR/VSCR registers. It's recommended to use
26 portable <fenv.h> instead of access MXSCR directly.
27
28 Most SSE scalar float intrinsic operations can be performed more
29 efficiently as C language float scalar operations or optimized to
30 use vector SIMD operations. We recommend this for new applications. */
31#error "Please read comment above. Use -DNO_WARN_X86_INTRINSICS to disable this error."
32#endif
33
34#ifndef _XMMINTRIN_H_INCLUDED
35#define _XMMINTRIN_H_INCLUDED
36
37/* Define four value permute mask */
38#define _MM_SHUFFLE(w,x,y,z) (((w) << 6) | ((x) << 4) | ((y) << 2) | (z))
39
40#include <altivec.h>
41
42/* Avoid collisions between altivec.h and strict adherence to C++ and
43 C11 standards. This should eventually be done inside altivec.h itself,
44 but only after testing a full distro build. */
45#if defined(__STRICT_ANSI__) && (defined(__cplusplus) || \
46 (defined(__STDC_VERSION__) && \
47 __STDC_VERSION__ >= 201112L))
48#undef vector
49#undef pixel
50#undef bool
51#endif
52
53/* We need type definitions from the MMX header file. */
54#include <mmintrin.h>
55
56/* Get _mm_malloc () and _mm_free (). */
57#if __STDC_HOSTED__
58#include <mm_malloc.h>
59#endif
60
61/* The Intel API is flexible enough that we must allow aliasing with other
62 vector types, and their scalar components. */
63typedef float __m128 __attribute__ ((__vector_size__ (16), __may_alias__));
64
65/* Unaligned version of the same type. */
66typedef float __m128_u __attribute__ ((__vector_size__ (16), __may_alias__,
67 __aligned__ (1)));
68
69/* Internal data types for implementing the intrinsics. */
70typedef float __v4sf __attribute__ ((__vector_size__ (16)));
71
72/* Create an undefined vector. */
73extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
74_mm_undefined_ps (void)
75{
76 __m128 __Y = __Y;
77 return __Y;
78}
79
80/* Create a vector of zeros. */
81extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
82_mm_setzero_ps (void)
83{
84 return __extension__ (__m128){ 0.0f, 0.0f, 0.0f, 0.0f };
85}
86
87/* Load four SPFP values from P. The address must be 16-byte aligned. */
88extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
89_mm_load_ps (float const *__P)
90{
91 return ((__m128)vec_ld(0, (__v4sf*)__P));
92}
93
94/* Load four SPFP values from P. The address need not be 16-byte aligned. */
95extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
96_mm_loadu_ps (float const *__P)
97{
98 return (vec_vsx_ld(0, __P));
99}
100
101/* Load four SPFP values in reverse order. The address must be aligned. */
102extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
103_mm_loadr_ps (float const *__P)
104{
105 __v4sf __tmp;
106 __m128 result;
107 static const __vector unsigned char permute_vector =
108 { 0x1C, 0x1D, 0x1E, 0x1F, 0x18, 0x19, 0x1A, 0x1B, 0x14, 0x15, 0x16,
109 0x17, 0x10, 0x11, 0x12, 0x13 };
110
111 __tmp = vec_ld (0, (__v4sf *) __P);
112 result = (__m128) vec_perm (__tmp, __tmp, permute_vector);
113 return result;
114}
115
116/* Create a vector with all four elements equal to F. */
117extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
118_mm_set1_ps (float __F)
119{
120 return __extension__ (__m128)(__v4sf){ __F, __F, __F, __F };
121}
122
123extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
124_mm_set_ps1 (float __F)
125{
126 return _mm_set1_ps (__F);
127}
128
129/* Create the vector [Z Y X W]. */
130extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
131_mm_set_ps (const float __Z, const float __Y, const float __X, const float __W)
132{
133 return __extension__ (__m128)(__v4sf){ __W, __X, __Y, __Z };
134}
135
136/* Create the vector [W X Y Z]. */
137extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
138_mm_setr_ps (float __Z, float __Y, float __X, float __W)
139{
140 return __extension__ (__m128)(__v4sf){ __Z, __Y, __X, __W };
141}
142
143/* Store four SPFP values. The address must be 16-byte aligned. */
144extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
145_mm_store_ps (float *__P, __m128 __A)
146{
147 vec_st((__v4sf)__A, 0, (__v4sf*)__P);
148}
149
150/* Store four SPFP values. The address need not be 16-byte aligned. */
151extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
152_mm_storeu_ps (float *__P, __m128 __A)
153{
154 *(__m128_u *)__P = __A;
155}
156
157/* Store four SPFP values in reverse order. The address must be aligned. */
158extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
159_mm_storer_ps (float *__P, __m128 __A)
160{
161 __v4sf __tmp;
162 static const __vector unsigned char permute_vector =
163 { 0x1C, 0x1D, 0x1E, 0x1F, 0x18, 0x19, 0x1A, 0x1B, 0x14, 0x15, 0x16,
164 0x17, 0x10, 0x11, 0x12, 0x13 };
165
166 __tmp = (__m128) vec_perm (__A, __A, permute_vector);
167
168 _mm_store_ps (__P, __tmp);
169}
170
171/* Store the lower SPFP value across four words. */
172extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
173_mm_store1_ps (float *__P, __m128 __A)
174{
175 __v4sf __va = vec_splat((__v4sf)__A, 0);
176 _mm_store_ps (__P, __va);
177}
178
179extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
180_mm_store_ps1 (float *__P, __m128 __A)
181{
182 _mm_store1_ps (__P, __A);
183}
184
185/* Create a vector with element 0 as F and the rest zero. */
186extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
187_mm_set_ss (float __F)
188{
189 return __extension__ (__m128)(__v4sf){ __F, 0.0f, 0.0f, 0.0f };
190}
191
192/* Sets the low SPFP value of A from the low value of B. */
193extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
194_mm_move_ss (__m128 __A, __m128 __B)
195{
196 static const __vector unsigned int mask = {0xffffffff, 0, 0, 0};
197
198 return (vec_sel ((__v4sf)__A, (__v4sf)__B, mask));
199}
200
201/* Create a vector with element 0 as *P and the rest zero. */
202extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
203_mm_load_ss (float const *__P)
204{
205 return _mm_set_ss (*__P);
206}
207
208/* Stores the lower SPFP value. */
209extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
210_mm_store_ss (float *__P, __m128 __A)
211{
212 *__P = ((__v4sf)__A)[0];
213}
214
215/* Perform the respective operation on the lower SPFP (single-precision
216 floating-point) values of A and B; the upper three SPFP values are
217 passed through from A. */
218
219extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
220_mm_add_ss (__m128 __A, __m128 __B)
221{
222#ifdef _ARCH_PWR7
223 __m128 a, b, c;
224 static const __vector unsigned int mask = {0xffffffff, 0, 0, 0};
225 /* PowerISA VSX does not allow partial (for just lower double)
226 results. So to insure we don't generate spurious exceptions
227 (from the upper double values) we splat the lower double
228 before we to the operation. */
229 a = vec_splat (__A, 0);
230 b = vec_splat (__B, 0);
231 c = a + b;
232 /* Then we merge the lower float result with the original upper
233 float elements from __A. */
234 return (vec_sel (__A, c, mask));
235#else
236 __A[0] = __A[0] + __B[0];
237 return (__A);
238#endif
239}
240
241extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
242_mm_sub_ss (__m128 __A, __m128 __B)
243{
244#ifdef _ARCH_PWR7
245 __m128 a, b, c;
246 static const __vector unsigned int mask = {0xffffffff, 0, 0, 0};
247 /* PowerISA VSX does not allow partial (for just lower double)
248 results. So to insure we don't generate spurious exceptions
249 (from the upper double values) we splat the lower double
250 before we to the operation. */
251 a = vec_splat (__A, 0);
252 b = vec_splat (__B, 0);
253 c = a - b;
254 /* Then we merge the lower float result with the original upper
255 float elements from __A. */
256 return (vec_sel (__A, c, mask));
257#else
258 __A[0] = __A[0] - __B[0];
259 return (__A);
260#endif
261}
262
263extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
264_mm_mul_ss (__m128 __A, __m128 __B)
265{
266#ifdef _ARCH_PWR7
267 __m128 a, b, c;
268 static const __vector unsigned int mask = {0xffffffff, 0, 0, 0};
269 /* PowerISA VSX does not allow partial (for just lower double)
270 results. So to insure we don't generate spurious exceptions
271 (from the upper double values) we splat the lower double
272 before we to the operation. */
273 a = vec_splat (__A, 0);
274 b = vec_splat (__B, 0);
275 c = a * b;
276 /* Then we merge the lower float result with the original upper
277 float elements from __A. */
278 return (vec_sel (__A, c, mask));
279#else
280 __A[0] = __A[0] * __B[0];
281 return (__A);
282#endif
283}
284
285extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
286_mm_div_ss (__m128 __A, __m128 __B)
287{
288#ifdef _ARCH_PWR7
289 __m128 a, b, c;
290 static const __vector unsigned int mask = {0xffffffff, 0, 0, 0};
291 /* PowerISA VSX does not allow partial (for just lower double)
292 results. So to insure we don't generate spurious exceptions
293 (from the upper double values) we splat the lower double
294 before we to the operation. */
295 a = vec_splat (__A, 0);
296 b = vec_splat (__B, 0);
297 c = a / b;
298 /* Then we merge the lower float result with the original upper
299 float elements from __A. */
300 return (vec_sel (__A, c, mask));
301#else
302 __A[0] = __A[0] / __B[0];
303 return (__A);
304#endif
305}
306
307extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
308_mm_sqrt_ss (__m128 __A)
309{
310 __m128 a, c;
311 static const __vector unsigned int mask = {0xffffffff, 0, 0, 0};
312 /* PowerISA VSX does not allow partial (for just lower double)
313 * results. So to insure we don't generate spurious exceptions
314 * (from the upper double values) we splat the lower double
315 * before we to the operation. */
316 a = vec_splat (__A, 0);
317 c = vec_sqrt (a);
318 /* Then we merge the lower float result with the original upper
319 * float elements from __A. */
320 return (vec_sel (__A, c, mask));
321}
322
323/* Perform the respective operation on the four SPFP values in A and B. */
324extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
325_mm_add_ps (__m128 __A, __m128 __B)
326{
327 return (__m128) ((__v4sf)__A + (__v4sf)__B);
328}
329
330extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
331_mm_sub_ps (__m128 __A, __m128 __B)
332{
333 return (__m128) ((__v4sf)__A - (__v4sf)__B);
334}
335
336extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
337_mm_mul_ps (__m128 __A, __m128 __B)
338{
339 return (__m128) ((__v4sf)__A * (__v4sf)__B);
340}
341
342extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
343_mm_div_ps (__m128 __A, __m128 __B)
344{
345 return (__m128) ((__v4sf)__A / (__v4sf)__B);
346}
347
348extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
349_mm_sqrt_ps (__m128 __A)
350{
351 return (vec_sqrt ((__v4sf)__A));
352}
353
354extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
355_mm_rcp_ps (__m128 __A)
356{
357 return (vec_re ((__v4sf)__A));
358}
359
360extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
361_mm_rsqrt_ps (__m128 __A)
362{
363 return (vec_rsqrte (__A));
364}
365
366extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
367_mm_rcp_ss (__m128 __A)
368{
369 __m128 a, c;
370 static const __vector unsigned int mask = {0xffffffff, 0, 0, 0};
371 /* PowerISA VSX does not allow partial (for just lower double)
372 * results. So to insure we don't generate spurious exceptions
373 * (from the upper double values) we splat the lower double
374 * before we to the operation. */
375 a = vec_splat (__A, 0);
376 c = _mm_rcp_ps (a);
377 /* Then we merge the lower float result with the original upper
378 * float elements from __A. */
379 return (vec_sel (__A, c, mask));
380}
381
382extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
383_mm_rsqrt_ss (__m128 __A)
384{
385 __m128 a, c;
386 static const __vector unsigned int mask = {0xffffffff, 0, 0, 0};
387 /* PowerISA VSX does not allow partial (for just lower double)
388 * results. So to insure we don't generate spurious exceptions
389 * (from the upper double values) we splat the lower double
390 * before we to the operation. */
391 a = vec_splat (__A, 0);
392 c = vec_rsqrte (a);
393 /* Then we merge the lower float result with the original upper
394 * float elements from __A. */
395 return (vec_sel (__A, c, mask));
396}
397
398extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
399_mm_min_ss (__m128 __A, __m128 __B)
400{
401 __v4sf a, b, c;
402 static const __vector unsigned int mask = {0xffffffff, 0, 0, 0};
403 /* PowerISA VSX does not allow partial (for just lower float)
404 * results. So to insure we don't generate spurious exceptions
405 * (from the upper float values) we splat the lower float
406 * before we to the operation. */
407 a = vec_splat ((__v4sf)__A, 0);
408 b = vec_splat ((__v4sf)__B, 0);
409 c = vec_min (a, b);
410 /* Then we merge the lower float result with the original upper
411 * float elements from __A. */
412 return (vec_sel ((__v4sf)__A, c, mask));
413}
414
415extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
416_mm_max_ss (__m128 __A, __m128 __B)
417{
418 __v4sf a, b, c;
419 static const __vector unsigned int mask = {0xffffffff, 0, 0, 0};
420 /* PowerISA VSX does not allow partial (for just lower float)
421 * results. So to insure we don't generate spurious exceptions
422 * (from the upper float values) we splat the lower float
423 * before we to the operation. */
424 a = vec_splat (__A, 0);
425 b = vec_splat (__B, 0);
426 c = vec_max (a, b);
427 /* Then we merge the lower float result with the original upper
428 * float elements from __A. */
429 return (vec_sel ((__v4sf)__A, c, mask));
430}
431
432extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
433_mm_min_ps (__m128 __A, __m128 __B)
434{
435 __vector __bool int m = vec_cmpgt ((__v4sf) __B, (__v4sf) __A);
436 return vec_sel (__B, __A, m);
437}
438
439extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
440_mm_max_ps (__m128 __A, __m128 __B)
441{
442 __vector __bool int m = vec_cmpgt ((__v4sf) __A, (__v4sf) __B);
443 return vec_sel (__B, __A, m);
444}
445
446/* Perform logical bit-wise operations on 128-bit values. */
447extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
448_mm_and_ps (__m128 __A, __m128 __B)
449{
450 return ((__m128)vec_and ((__v4sf)__A, (__v4sf)__B));
451// return __builtin_ia32_andps (__A, __B);
452}
453
454extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
455_mm_andnot_ps (__m128 __A, __m128 __B)
456{
457 return ((__m128)vec_andc ((__v4sf)__B, (__v4sf)__A));
458}
459
460extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
461_mm_or_ps (__m128 __A, __m128 __B)
462{
463 return ((__m128)vec_or ((__v4sf)__A, (__v4sf)__B));
464}
465
466extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
467_mm_xor_ps (__m128 __A, __m128 __B)
468{
469 return ((__m128)vec_xor ((__v4sf)__A, (__v4sf)__B));
470}
471
472/* Perform a comparison on the four SPFP values of A and B. For each
473 element, if the comparison is true, place a mask of all ones in the
474 result, otherwise a mask of zeros. */
475extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
476_mm_cmpeq_ps (__m128 __A, __m128 __B)
477{
478 return ((__m128)vec_cmpeq ((__v4sf)__A,(__v4sf) __B));
479}
480
481extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
482_mm_cmplt_ps (__m128 __A, __m128 __B)
483{
484 return ((__m128)vec_cmplt ((__v4sf)__A, (__v4sf)__B));
485}
486
487extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
488_mm_cmple_ps (__m128 __A, __m128 __B)
489{
490 return ((__m128)vec_cmple ((__v4sf)__A, (__v4sf)__B));
491}
492
493extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
494_mm_cmpgt_ps (__m128 __A, __m128 __B)
495{
496 return ((__m128)vec_cmpgt ((__v4sf)__A, (__v4sf)__B));
497}
498
499extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
500_mm_cmpge_ps (__m128 __A, __m128 __B)
501{
502 return ((__m128)vec_cmpge ((__v4sf)__A, (__v4sf)__B));
503}
504
505extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
506_mm_cmpneq_ps (__m128 __A, __m128 __B)
507{
508 __v4sf temp = (__v4sf ) vec_cmpeq ((__v4sf) __A, (__v4sf)__B);
509 return ((__m128)vec_nor (temp, temp));
510}
511
512extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
513_mm_cmpnlt_ps (__m128 __A, __m128 __B)
514{
515 return ((__m128)vec_cmpge ((__v4sf)__A, (__v4sf)__B));
516}
517
518extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
519_mm_cmpnle_ps (__m128 __A, __m128 __B)
520{
521 return ((__m128)vec_cmpgt ((__v4sf)__A, (__v4sf)__B));
522}
523
524extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
525_mm_cmpngt_ps (__m128 __A, __m128 __B)
526{
527 return ((__m128)vec_cmple ((__v4sf)__A, (__v4sf)__B));
528}
529
530extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
531_mm_cmpnge_ps (__m128 __A, __m128 __B)
532{
533 return ((__m128)vec_cmplt ((__v4sf)__A, (__v4sf)__B));
534}
535
536extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
537_mm_cmpord_ps (__m128 __A, __m128 __B)
538{
539 __vector unsigned int a, b;
540 __vector unsigned int c, d;
541 static const __vector unsigned int float_exp_mask =
542 { 0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000 };
543
544 a = (__vector unsigned int) vec_abs ((__v4sf)__A);
545 b = (__vector unsigned int) vec_abs ((__v4sf)__B);
546 c = (__vector unsigned int) vec_cmpgt (float_exp_mask, a);
547 d = (__vector unsigned int) vec_cmpgt (float_exp_mask, b);
548 return ((__m128 ) vec_and (c, d));
549}
550
551extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
552_mm_cmpunord_ps (__m128 __A, __m128 __B)
553{
554 __vector unsigned int a, b;
555 __vector unsigned int c, d;
556 static const __vector unsigned int float_exp_mask =
557 { 0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000 };
558
559 a = (__vector unsigned int) vec_abs ((__v4sf)__A);
560 b = (__vector unsigned int) vec_abs ((__v4sf)__B);
561 c = (__vector unsigned int) vec_cmpgt (a, float_exp_mask);
562 d = (__vector unsigned int) vec_cmpgt (b, float_exp_mask);
563 return ((__m128 ) vec_or (c, d));
564}
565
566/* Perform a comparison on the lower SPFP values of A and B. If the
567 comparison is true, place a mask of all ones in the result, otherwise a
568 mask of zeros. The upper three SPFP values are passed through from A. */
569extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
570_mm_cmpeq_ss (__m128 __A, __m128 __B)
571{
572 static const __vector unsigned int mask =
573 { 0xffffffff, 0, 0, 0 };
574 __v4sf a, b, c;
575 /* PowerISA VMX does not allow partial (for just element 0)
576 * results. So to insure we don't generate spurious exceptions
577 * (from the upper elements) we splat the lower float
578 * before we to the operation. */
579 a = vec_splat ((__v4sf) __A, 0);
580 b = vec_splat ((__v4sf) __B, 0);
581 c = (__v4sf) vec_cmpeq(a, b);
582 /* Then we merge the lower float result with the original upper
583 * float elements from __A. */
584 return ((__m128)vec_sel ((__v4sf)__A, c, mask));
585}
586
587extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
588_mm_cmplt_ss (__m128 __A, __m128 __B)
589{
590 static const __vector unsigned int mask =
591 { 0xffffffff, 0, 0, 0 };
592 __v4sf a, b, c;
593 /* PowerISA VMX does not allow partial (for just element 0)
594 * results. So to insure we don't generate spurious exceptions
595 * (from the upper elements) we splat the lower float
596 * before we to the operation. */
597 a = vec_splat ((__v4sf) __A, 0);
598 b = vec_splat ((__v4sf) __B, 0);
599 c = (__v4sf) vec_cmplt(a, b);
600 /* Then we merge the lower float result with the original upper
601 * float elements from __A. */
602 return ((__m128)vec_sel ((__v4sf)__A, c, mask));
603}
604
605extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
606_mm_cmple_ss (__m128 __A, __m128 __B)
607{
608 static const __vector unsigned int mask =
609 { 0xffffffff, 0, 0, 0 };
610 __v4sf a, b, c;
611 /* PowerISA VMX does not allow partial (for just element 0)
612 * results. So to insure we don't generate spurious exceptions
613 * (from the upper elements) we splat the lower float
614 * before we to the operation. */
615 a = vec_splat ((__v4sf) __A, 0);
616 b = vec_splat ((__v4sf) __B, 0);
617 c = (__v4sf) vec_cmple(a, b);
618 /* Then we merge the lower float result with the original upper
619 * float elements from __A. */
620 return ((__m128)vec_sel ((__v4sf)__A, c, mask));
621}
622
623extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
624_mm_cmpgt_ss (__m128 __A, __m128 __B)
625{
626 static const __vector unsigned int mask =
627 { 0xffffffff, 0, 0, 0 };
628 __v4sf a, b, c;
629 /* PowerISA VMX does not allow partial (for just element 0)
630 * results. So to insure we don't generate spurious exceptions
631 * (from the upper elements) we splat the lower float
632 * before we to the operation. */
633 a = vec_splat ((__v4sf) __A, 0);
634 b = vec_splat ((__v4sf) __B, 0);
635 c = (__v4sf) vec_cmpgt(a, b);
636 /* Then we merge the lower float result with the original upper
637 * float elements from __A. */
638 return ((__m128)vec_sel ((__v4sf)__A, c, mask));
639}
640
641extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
642_mm_cmpge_ss (__m128 __A, __m128 __B)
643{
644 static const __vector unsigned int mask =
645 { 0xffffffff, 0, 0, 0 };
646 __v4sf a, b, c;
647 /* PowerISA VMX does not allow partial (for just element 0)
648 * results. So to insure we don't generate spurious exceptions
649 * (from the upper elements) we splat the lower float
650 * before we to the operation. */
651 a = vec_splat ((__v4sf) __A, 0);
652 b = vec_splat ((__v4sf) __B, 0);
653 c = (__v4sf) vec_cmpge(a, b);
654 /* Then we merge the lower float result with the original upper
655 * float elements from __A. */
656 return ((__m128)vec_sel ((__v4sf)__A, c, mask));
657}
658
659extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
660_mm_cmpneq_ss (__m128 __A, __m128 __B)
661{
662 static const __vector unsigned int mask =
663 { 0xffffffff, 0, 0, 0 };
664 __v4sf a, b, c;
665 /* PowerISA VMX does not allow partial (for just element 0)
666 * results. So to insure we don't generate spurious exceptions
667 * (from the upper elements) we splat the lower float
668 * before we to the operation. */
669 a = vec_splat ((__v4sf) __A, 0);
670 b = vec_splat ((__v4sf) __B, 0);
671 c = (__v4sf) vec_cmpeq(a, b);
672 c = vec_nor (c, c);
673 /* Then we merge the lower float result with the original upper
674 * float elements from __A. */
675 return ((__m128)vec_sel ((__v4sf)__A, c, mask));
676}
677
678extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
679_mm_cmpnlt_ss (__m128 __A, __m128 __B)
680{
681 static const __vector unsigned int mask =
682 { 0xffffffff, 0, 0, 0 };
683 __v4sf a, b, c;
684 /* PowerISA VMX does not allow partial (for just element 0)
685 * results. So to insure we don't generate spurious exceptions
686 * (from the upper elements) we splat the lower float
687 * before we to the operation. */
688 a = vec_splat ((__v4sf) __A, 0);
689 b = vec_splat ((__v4sf) __B, 0);
690 c = (__v4sf) vec_cmpge(a, b);
691 /* Then we merge the lower float result with the original upper
692 * float elements from __A. */
693 return ((__m128)vec_sel ((__v4sf)__A, c, mask));
694}
695
696extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
697_mm_cmpnle_ss (__m128 __A, __m128 __B)
698{
699 static const __vector unsigned int mask =
700 { 0xffffffff, 0, 0, 0 };
701 __v4sf a, b, c;
702 /* PowerISA VMX does not allow partial (for just element 0)
703 * results. So to insure we don't generate spurious exceptions
704 * (from the upper elements) we splat the lower float
705 * before we to the operation. */
706 a = vec_splat ((__v4sf) __A, 0);
707 b = vec_splat ((__v4sf) __B, 0);
708 c = (__v4sf) vec_cmpgt(a, b);
709 /* Then we merge the lower float result with the original upper
710 * float elements from __A. */
711 return ((__m128)vec_sel ((__v4sf)__A, c, mask));
712}
713
714extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
715_mm_cmpngt_ss (__m128 __A, __m128 __B)
716{
717 static const __vector unsigned int mask =
718 { 0xffffffff, 0, 0, 0 };
719 __v4sf a, b, c;
720 /* PowerISA VMX does not allow partial (for just element 0)
721 * results. So to insure we don't generate spurious exceptions
722 * (from the upper elements) we splat the lower float
723 * before we to the operation. */
724 a = vec_splat ((__v4sf) __A, 0);
725 b = vec_splat ((__v4sf) __B, 0);
726 c = (__v4sf) vec_cmple(a, b);
727 /* Then we merge the lower float result with the original upper
728 * float elements from __A. */
729 return ((__m128)vec_sel ((__v4sf)__A, c, mask));
730}
731
732extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
733_mm_cmpnge_ss (__m128 __A, __m128 __B)
734{
735 static const __vector unsigned int mask =
736 { 0xffffffff, 0, 0, 0 };
737 __v4sf a, b, c;
738 /* PowerISA VMX does not allow partial (for just element 0)
739 * results. So to insure we don't generate spurious exceptions
740 * (from the upper elements) we splat the lower float
741 * before we do the operation. */
742 a = vec_splat ((__v4sf) __A, 0);
743 b = vec_splat ((__v4sf) __B, 0);
744 c = (__v4sf) vec_cmplt(a, b);
745 /* Then we merge the lower float result with the original upper
746 * float elements from __A. */
747 return ((__m128)vec_sel ((__v4sf)__A, c, mask));
748}
749
750extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
751_mm_cmpord_ss (__m128 __A, __m128 __B)
752{
753 __vector unsigned int a, b;
754 __vector unsigned int c, d;
755 static const __vector unsigned int float_exp_mask =
756 { 0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000 };
757 static const __vector unsigned int mask =
758 { 0xffffffff, 0, 0, 0 };
759
760 a = (__vector unsigned int) vec_abs ((__v4sf)__A);
761 b = (__vector unsigned int) vec_abs ((__v4sf)__B);
762 c = (__vector unsigned int) vec_cmpgt (float_exp_mask, a);
763 d = (__vector unsigned int) vec_cmpgt (float_exp_mask, b);
764 c = vec_and (c, d);
765 /* Then we merge the lower float result with the original upper
766 * float elements from __A. */
767 return ((__m128)vec_sel ((__v4sf)__A, (__v4sf)c, mask));
768}
769
770extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
771_mm_cmpunord_ss (__m128 __A, __m128 __B)
772{
773 __vector unsigned int a, b;
774 __vector unsigned int c, d;
775 static const __vector unsigned int float_exp_mask =
776 { 0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000 };
777 static const __vector unsigned int mask =
778 { 0xffffffff, 0, 0, 0 };
779
780 a = (__vector unsigned int) vec_abs ((__v4sf)__A);
781 b = (__vector unsigned int) vec_abs ((__v4sf)__B);
782 c = (__vector unsigned int) vec_cmpgt (a, float_exp_mask);
783 d = (__vector unsigned int) vec_cmpgt (b, float_exp_mask);
784 c = vec_or (c, d);
785 /* Then we merge the lower float result with the original upper
786 * float elements from __A. */
787 return ((__m128)vec_sel ((__v4sf)__A, (__v4sf)c, mask));
788}
789
790/* Compare the lower SPFP values of A and B and return 1 if true
791 and 0 if false. */
792extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
793_mm_comieq_ss (__m128 __A, __m128 __B)
794{
795 return (__A[0] == __B[0]);
796}
797
798extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
799_mm_comilt_ss (__m128 __A, __m128 __B)
800{
801 return (__A[0] < __B[0]);
802}
803
804extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
805_mm_comile_ss (__m128 __A, __m128 __B)
806{
807 return (__A[0] <= __B[0]);
808}
809
810extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
811_mm_comigt_ss (__m128 __A, __m128 __B)
812{
813 return (__A[0] > __B[0]);
814}
815
816extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
817_mm_comige_ss (__m128 __A, __m128 __B)
818{
819 return (__A[0] >= __B[0]);
820}
821
822extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
823_mm_comineq_ss (__m128 __A, __m128 __B)
824{
825 return (__A[0] != __B[0]);
826}
827
828/* FIXME
829 * The __mm_ucomi??_ss implementations below are exactly the same as
830 * __mm_comi??_ss because GCC for PowerPC only generates unordered
831 * compares (scalar and vector).
832 * Technically __mm_comieq_ss et al should be using the ordered
833 * compare and signal for QNaNs.
834 * The __mm_ucomieq_sd et all should be OK, as is.
835 */
836extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
837_mm_ucomieq_ss (__m128 __A, __m128 __B)
838{
839 return (__A[0] == __B[0]);
840}
841
842extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
843_mm_ucomilt_ss (__m128 __A, __m128 __B)
844{
845 return (__A[0] < __B[0]);
846}
847
848extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
849_mm_ucomile_ss (__m128 __A, __m128 __B)
850{
851 return (__A[0] <= __B[0]);
852}
853
854extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
855_mm_ucomigt_ss (__m128 __A, __m128 __B)
856{
857 return (__A[0] > __B[0]);
858}
859
860extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
861_mm_ucomige_ss (__m128 __A, __m128 __B)
862{
863 return (__A[0] >= __B[0]);
864}
865
866extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
867_mm_ucomineq_ss (__m128 __A, __m128 __B)
868{
869 return (__A[0] != __B[0]);
870}
871
872extern __inline float __attribute__((__gnu_inline__, __always_inline__, __artificial__))
873_mm_cvtss_f32 (__m128 __A)
874{
875 return ((__v4sf)__A)[0];
876}
877
878/* Convert the lower SPFP value to a 32-bit integer according to the current
879 rounding mode. */
880extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
881_mm_cvtss_si32 (__m128 __A)
882{
883 __m64 res = 0;
884#ifdef _ARCH_PWR8
885 double dtmp;
886 __asm__(
887#ifdef __LITTLE_ENDIAN__
888 "xxsldwi %x0,%x0,%x0,3;\n"
889#endif
890 "xscvspdp %x2,%x0;\n"
891 "fctiw %2,%2;\n"
892 "mfvsrd %1,%x2;\n"
893 : "+wa" (__A),
894 "=r" (res),
895 "=f" (dtmp)
896 : );
897#else
898 res = __builtin_rint(__A[0]);
899#endif
900 return (res);
901}
902
903extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
904_mm_cvt_ss2si (__m128 __A)
905{
906 return _mm_cvtss_si32 (__A);
907}
908
909/* Convert the lower SPFP value to a 32-bit integer according to the
910 current rounding mode. */
911
912/* Intel intrinsic. */
913extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
914_mm_cvtss_si64 (__m128 __A)
915{
916 __m64 res = 0;
917#ifdef _ARCH_PWR8
918 double dtmp;
919 __asm__(
920#ifdef __LITTLE_ENDIAN__
921 "xxsldwi %x0,%x0,%x0,3;\n"
922#endif
923 "xscvspdp %x2,%x0;\n"
924 "fctid %2,%2;\n"
925 "mfvsrd %1,%x2;\n"
926 : "+wa" (__A),
927 "=r" (res),
928 "=f" (dtmp)
929 : );
930#else
931 res = __builtin_llrint(__A[0]);
932#endif
933 return (res);
934}
935
936/* Microsoft intrinsic. */
937extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
938_mm_cvtss_si64x (__m128 __A)
939{
940 return _mm_cvtss_si64 ((__v4sf) __A);
941}
942
943/* Constants for use with _mm_prefetch. */
944enum _mm_hint
945{
946 /* _MM_HINT_ET is _MM_HINT_T with set 3rd bit. */
947 _MM_HINT_ET0 = 7,
948 _MM_HINT_ET1 = 6,
949 _MM_HINT_T0 = 3,
950 _MM_HINT_T1 = 2,
951 _MM_HINT_T2 = 1,
952 _MM_HINT_NTA = 0
953};
954
955/* Loads one cache line from address P to a location "closer" to the
956 processor. The selector I specifies the type of prefetch operation. */
957extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
958_mm_prefetch (const void *__P, enum _mm_hint __I)
959{
960 /* Current PowerPC will ignores the hint parameters. */
961 __builtin_prefetch (__P);
962}
963
964/* Convert the two lower SPFP values to 32-bit integers according to the
965 current rounding mode. Return the integers in packed form. */
966extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
967_mm_cvtps_pi32 (__m128 __A)
968{
969 /* Splat two lower SPFP values to both halves. */
970 __v4sf temp, rounded;
971 __vector unsigned long long result;
972
973 /* Splat two lower SPFP values to both halves. */
974 temp = (__v4sf) vec_splat ((__vector long long)__A, 0);
975 rounded = vec_rint(temp);
976 result = (__vector unsigned long long) vec_cts (rounded, 0);
977
978 return (__m64) ((__vector long long) result)[0];
979}
980
981extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
982_mm_cvt_ps2pi (__m128 __A)
983{
984 return _mm_cvtps_pi32 (__A);
985}
986
987/* Truncate the lower SPFP value to a 32-bit integer. */
988extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
989_mm_cvttss_si32 (__m128 __A)
990{
991 /* Extract the lower float element. */
992 float temp = __A[0];
993 /* truncate to 32-bit integer and return. */
994 return temp;
995}
996
997extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
998_mm_cvtt_ss2si (__m128 __A)
999{
1000 return _mm_cvttss_si32 (__A);
1001}
1002
1003/* Intel intrinsic. */
1004extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1005_mm_cvttss_si64 (__m128 __A)
1006{
1007 /* Extract the lower float element. */
1008 float temp = __A[0];
1009 /* truncate to 32-bit integer and return. */
1010 return temp;
1011}
1012
1013/* Microsoft intrinsic. */
1014extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1015_mm_cvttss_si64x (__m128 __A)
1016{
1017 /* Extract the lower float element. */
1018 float temp = __A[0];
1019 /* truncate to 32-bit integer and return. */
1020 return temp;
1021}
1022
1023/* Truncate the two lower SPFP values to 32-bit integers. Return the
1024 integers in packed form. */
1025extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1026_mm_cvttps_pi32 (__m128 __A)
1027{
1028 __v4sf temp;
1029 __vector unsigned long long result;
1030
1031 /* Splat two lower SPFP values to both halves. */
1032 temp = (__v4sf) vec_splat ((__vector long long)__A, 0);
1033 result = (__vector unsigned long long) vec_cts (temp, 0);
1034
1035 return (__m64) ((__vector long long) result)[0];
1036}
1037
1038extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1039_mm_cvtt_ps2pi (__m128 __A)
1040{
1041 return _mm_cvttps_pi32 (__A);
1042}
1043
1044/* Convert B to a SPFP value and insert it as element zero in A. */
1045extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1046_mm_cvtsi32_ss (__m128 __A, int __B)
1047{
1048 float temp = __B;
1049 __A[0] = temp;
1050
1051 return __A;
1052}
1053
1054extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1055_mm_cvt_si2ss (__m128 __A, int __B)
1056{
1057 return _mm_cvtsi32_ss (__A, __B);
1058}
1059
1060/* Convert B to a SPFP value and insert it as element zero in A. */
1061/* Intel intrinsic. */
1062extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1063_mm_cvtsi64_ss (__m128 __A, long long __B)
1064{
1065 float temp = __B;
1066 __A[0] = temp;
1067
1068 return __A;
1069}
1070
1071/* Microsoft intrinsic. */
1072extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1073_mm_cvtsi64x_ss (__m128 __A, long long __B)
1074{
1075 return _mm_cvtsi64_ss (__A, __B);
1076}
1077
1078/* Convert the two 32-bit values in B to SPFP form and insert them
1079 as the two lower elements in A. */
1080extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1081_mm_cvtpi32_ps (__m128 __A, __m64 __B)
1082{
1083 __vector signed int vm1;
1084 __vector float vf1;
1085
1086 vm1 = (__vector signed int) (__vector unsigned long long) {__B, __B};
1087 vf1 = (__vector float) vec_ctf (vm1, 0);
1088
1089 return ((__m128) (__vector unsigned long long)
1090 { ((__vector unsigned long long)vf1) [0],
1091 ((__vector unsigned long long)__A) [1]});
1092}
1093
1094extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1095_mm_cvt_pi2ps (__m128 __A, __m64 __B)
1096{
1097 return _mm_cvtpi32_ps (__A, __B);
1098}
1099
1100/* Convert the four signed 16-bit values in A to SPFP form. */
1101extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1102_mm_cvtpi16_ps (__m64 __A)
1103{
1104 __vector signed short vs8;
1105 __vector signed int vi4;
1106 __vector float vf1;
1107
1108 vs8 = (__vector signed short) (__vector unsigned long long) { __A, __A };
1109 vi4 = vec_vupklsh (vs8);
1110 vf1 = (__vector float) vec_ctf (vi4, 0);
1111
1112 return (__m128) vf1;
1113}
1114
1115/* Convert the four unsigned 16-bit values in A to SPFP form. */
1116extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1117_mm_cvtpu16_ps (__m64 __A)
1118{
1119 const __vector unsigned short zero =
1120 { 0, 0, 0, 0, 0, 0, 0, 0 };
1121 __vector unsigned short vs8;
1122 __vector unsigned int vi4;
1123 __vector float vf1;
1124
1125 vs8 = (__vector unsigned short) (__vector unsigned long long) { __A, __A };
1126 vi4 = (__vector unsigned int) vec_mergel
1127#ifdef __LITTLE_ENDIAN__
1128 (vs8, zero);
1129#else
1130 (zero, vs8);
1131#endif
1132 vf1 = (__vector float) vec_ctf (vi4, 0);
1133
1134 return (__m128) vf1;
1135}
1136
1137/* Convert the low four signed 8-bit values in A to SPFP form. */
1138extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1139_mm_cvtpi8_ps (__m64 __A)
1140{
1141 __vector signed char vc16;
1142 __vector signed short vs8;
1143 __vector signed int vi4;
1144 __vector float vf1;
1145
1146 vc16 = (__vector signed char) (__vector unsigned long long) { __A, __A };
1147 vs8 = vec_vupkhsb (vc16);
1148 vi4 = vec_vupkhsh (vs8);
1149 vf1 = (__vector float) vec_ctf (vi4, 0);
1150
1151 return (__m128) vf1;
1152}
1153
1154/* Convert the low four unsigned 8-bit values in A to SPFP form. */
1155extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1156
1157_mm_cvtpu8_ps (__m64 __A)
1158{
1159 const __vector unsigned char zero =
1160 { 0, 0, 0, 0, 0, 0, 0, 0 };
1161 __vector unsigned char vc16;
1162 __vector unsigned short vs8;
1163 __vector unsigned int vi4;
1164 __vector float vf1;
1165
1166 vc16 = (__vector unsigned char) (__vector unsigned long long) { __A, __A };
1167#ifdef __LITTLE_ENDIAN__
1168 vs8 = (__vector unsigned short) vec_mergel (vc16, zero);
1169 vi4 = (__vector unsigned int) vec_mergeh (vs8,
1170 (__vector unsigned short) zero);
1171#else
1172 vs8 = (__vector unsigned short) vec_mergel (zero, vc16);
1173 vi4 = (__vector unsigned int) vec_mergeh ((__vector unsigned short) zero,
1174 vs8);
1175#endif
1176 vf1 = (__vector float) vec_ctf (vi4, 0);
1177
1178 return (__m128) vf1;
1179}
1180
1181/* Convert the four signed 32-bit values in A and B to SPFP form. */
1182extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1183_mm_cvtpi32x2_ps (__m64 __A, __m64 __B)
1184{
1185 __vector signed int vi4;
1186 __vector float vf4;
1187
1188 vi4 = (__vector signed int) (__vector unsigned long long) { __A, __B };
1189 vf4 = (__vector float) vec_ctf (vi4, 0);
1190 return (__m128) vf4;
1191}
1192
1193/* Convert the four SPFP values in A to four signed 16-bit integers. */
1194extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1195_mm_cvtps_pi16 (__m128 __A)
1196{
1197 __v4sf rounded;
1198 __vector signed int temp;
1199 __vector unsigned long long result;
1200
1201 rounded = vec_rint(__A);
1202 temp = vec_cts (rounded, 0);
1203 result = (__vector unsigned long long) vec_pack (temp, temp);
1204
1205 return (__m64) ((__vector long long) result)[0];
1206}
1207
1208/* Convert the four SPFP values in A to four signed 8-bit integers. */
1209extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1210_mm_cvtps_pi8 (__m128 __A)
1211{
1212 __v4sf rounded;
1213 __vector signed int tmp_i;
1214 static const __vector signed int zero = {0, 0, 0, 0};
1215 __vector signed short tmp_s;
1216 __vector signed char res_v;
1217
1218 rounded = vec_rint(__A);
1219 tmp_i = vec_cts (rounded, 0);
1220 tmp_s = vec_pack (tmp_i, zero);
1221 res_v = vec_pack (tmp_s, tmp_s);
1222 return (__m64) ((__vector long long) res_v)[0];
1223}
1224
1225/* Selects four specific SPFP values from A and B based on MASK. */
1226extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1227
1228_mm_shuffle_ps (__m128 __A, __m128 __B, int const __mask)
1229{
1230 unsigned long element_selector_10 = __mask & 0x03;
1231 unsigned long element_selector_32 = (__mask >> 2) & 0x03;
1232 unsigned long element_selector_54 = (__mask >> 4) & 0x03;
1233 unsigned long element_selector_76 = (__mask >> 6) & 0x03;
1234 static const unsigned int permute_selectors[4] =
1235 {
1236#ifdef __LITTLE_ENDIAN__
1237 0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
1238#else
1239 0x00010203, 0x04050607, 0x08090A0B, 0x0C0D0E0F
1240#endif
1241 };
1242 __vector unsigned int t;
1243
1244 t[0] = permute_selectors[element_selector_10];
1245 t[1] = permute_selectors[element_selector_32];
1246 t[2] = permute_selectors[element_selector_54] + 0x10101010;
1247 t[3] = permute_selectors[element_selector_76] + 0x10101010;
1248 return vec_perm ((__v4sf) __A, (__v4sf)__B, (__vector unsigned char)t);
1249}
1250
1251/* Selects and interleaves the upper two SPFP values from A and B. */
1252extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1253_mm_unpackhi_ps (__m128 __A, __m128 __B)
1254{
1255 return (__m128) vec_vmrglw ((__v4sf) __A, (__v4sf)__B);
1256}
1257
1258/* Selects and interleaves the lower two SPFP values from A and B. */
1259extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1260_mm_unpacklo_ps (__m128 __A, __m128 __B)
1261{
1262 return (__m128) vec_vmrghw ((__v4sf) __A, (__v4sf)__B);
1263}
1264
1265/* Sets the upper two SPFP values with 64-bits of data loaded from P;
1266 the lower two values are passed through from A. */
1267extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1268_mm_loadh_pi (__m128 __A, __m64 const *__P)
1269{
1270 __vector unsigned long long __a = (__vector unsigned long long)__A;
1271 __vector unsigned long long __p = vec_splats(*__P);
1272 __a [1] = __p [1];
1273
1274 return (__m128)__a;
1275}
1276
1277/* Stores the upper two SPFP values of A into P. */
1278extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1279_mm_storeh_pi (__m64 *__P, __m128 __A)
1280{
1281 __vector unsigned long long __a = (__vector unsigned long long) __A;
1282
1283 *__P = __a[1];
1284}
1285
1286/* Moves the upper two values of B into the lower two values of A. */
1287extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1288_mm_movehl_ps (__m128 __A, __m128 __B)
1289{
1290 return (__m128) vec_mergel ((__vector unsigned long long)__B,
1291 (__vector unsigned long long)__A);
1292}
1293
1294/* Moves the lower two values of B into the upper two values of A. */
1295extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1296_mm_movelh_ps (__m128 __A, __m128 __B)
1297{
1298 return (__m128) vec_mergeh ((__vector unsigned long long)__A,
1299 (__vector unsigned long long)__B);
1300}
1301
1302/* Sets the lower two SPFP values with 64-bits of data loaded from P;
1303 the upper two values are passed through from A. */
1304extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1305_mm_loadl_pi (__m128 __A, __m64 const *__P)
1306{
1307 __vector unsigned long long __a = (__vector unsigned long long)__A;
1308 __vector unsigned long long __p = vec_splats(*__P);
1309 __a [0] = __p [0];
1310
1311 return (__m128)__a;
1312}
1313
1314/* Stores the lower two SPFP values of A into P. */
1315extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1316_mm_storel_pi (__m64 *__P, __m128 __A)
1317{
1318 __vector unsigned long long __a = (__vector unsigned long long) __A;
1319
1320 *__P = __a[0];
1321}
1322
1323#ifdef _ARCH_PWR8
1324/* Intrinsic functions that require PowerISA 2.07 minimum. */
1325
1326/* Creates a 4-bit mask from the most significant bits of the SPFP values. */
1327extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1328_mm_movemask_ps (__m128 __A)
1329{
1330 __vector unsigned long long result;
1331 static const __vector unsigned int perm_mask =
1332 {
1333#ifdef __LITTLE_ENDIAN__
1334 0x00204060, 0x80808080, 0x80808080, 0x80808080
1335#else
1336 0x80808080, 0x80808080, 0x80808080, 0x00204060
1337#endif
1338 };
1339
1340 result = ((__vector unsigned long long)
1341 vec_vbpermq ((__vector unsigned char) __A,
1342 (__vector unsigned char) perm_mask));
1343
1344#ifdef __LITTLE_ENDIAN__
1345 return result[1];
1346#else
1347 return result[0];
1348#endif
1349}
1350#endif /* _ARCH_PWR8 */
1351
1352/* Create a vector with all four elements equal to *P. */
1353extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1354_mm_load1_ps (float const *__P)
1355{
1356 return _mm_set1_ps (*__P);
1357}
1358
1359extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1360_mm_load_ps1 (float const *__P)
1361{
1362 return _mm_load1_ps (__P);
1363}
1364
1365/* Extracts one of the four words of A. The selector N must be immediate. */
1366extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1367_mm_extract_pi16 (__m64 const __A, int const __N)
1368{
1369 unsigned int shiftr = __N & 3;
1370#ifdef __BIG_ENDIAN__
1371 shiftr = 3 - shiftr;
1372#endif
1373
1374 return ((__A >> (shiftr * 16)) & 0xffff);
1375}
1376
1377extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1378_m_pextrw (__m64 const __A, int const __N)
1379{
1380 return _mm_extract_pi16 (__A, __N);
1381}
1382
1383/* Inserts word D into one of four words of A. The selector N must be
1384 immediate. */
1385extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1386_mm_insert_pi16 (__m64 const __A, int const __D, int const __N)
1387{
1388 const int shiftl = (__N & 3) * 16;
1389 const __m64 shiftD = (const __m64) __D << shiftl;
1390 const __m64 mask = 0xffffUL << shiftl;
1391 __m64 result = (__A & (~mask)) | (shiftD & mask);
1392
1393 return (result);
1394}
1395
1396extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1397_m_pinsrw (__m64 const __A, int const __D, int const __N)
1398{
1399 return _mm_insert_pi16 (__A, __D, __N);
1400}
1401
1402/* Compute the element-wise maximum of signed 16-bit values. */
1403extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1404
1405_mm_max_pi16 (__m64 __A, __m64 __B)
1406{
1407#if _ARCH_PWR8
1408 __vector signed short a, b, r;
1409 __vector __bool short c;
1410
1411 a = (__vector signed short)vec_splats (__A);
1412 b = (__vector signed short)vec_splats (__B);
1413 c = (__vector __bool short)vec_cmpgt (a, b);
1414 r = vec_sel (b, a, c);
1415 return (__m64) ((__vector long long) r)[0];
1416#else
1417 __m64_union m1, m2, res;
1418
1419 m1.as_m64 = __A;
1420 m2.as_m64 = __B;
1421
1422 res.as_short[0] =
1423 (m1.as_short[0] > m2.as_short[0]) ? m1.as_short[0] : m2.as_short[0];
1424 res.as_short[1] =
1425 (m1.as_short[1] > m2.as_short[1]) ? m1.as_short[1] : m2.as_short[1];
1426 res.as_short[2] =
1427 (m1.as_short[2] > m2.as_short[2]) ? m1.as_short[2] : m2.as_short[2];
1428 res.as_short[3] =
1429 (m1.as_short[3] > m2.as_short[3]) ? m1.as_short[3] : m2.as_short[3];
1430
1431 return (__m64) res.as_m64;
1432#endif
1433}
1434
1435extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1436_m_pmaxsw (__m64 __A, __m64 __B)
1437{
1438 return _mm_max_pi16 (__A, __B);
1439}
1440
1441/* Compute the element-wise maximum of unsigned 8-bit values. */
1442extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1443_mm_max_pu8 (__m64 __A, __m64 __B)
1444{
1445#if _ARCH_PWR8
1446 __vector unsigned char a, b, r;
1447 __vector __bool char c;
1448
1449 a = (__vector unsigned char)vec_splats (__A);
1450 b = (__vector unsigned char)vec_splats (__B);
1451 c = (__vector __bool char)vec_cmpgt (a, b);
1452 r = vec_sel (b, a, c);
1453 return (__m64) ((__vector long long) r)[0];
1454#else
1455 __m64_union m1, m2, res;
1456 long i;
1457
1458 m1.as_m64 = __A;
1459 m2.as_m64 = __B;
1460
1461
1462 for (i = 0; i < 8; i++)
1463 res.as_char[i] =
1464 ((unsigned char) m1.as_char[i] > (unsigned char) m2.as_char[i]) ?
1465 m1.as_char[i] : m2.as_char[i];
1466
1467 return (__m64) res.as_m64;
1468#endif
1469}
1470
1471extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1472_m_pmaxub (__m64 __A, __m64 __B)
1473{
1474 return _mm_max_pu8 (__A, __B);
1475}
1476
1477/* Compute the element-wise minimum of signed 16-bit values. */
1478extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1479_mm_min_pi16 (__m64 __A, __m64 __B)
1480{
1481#if _ARCH_PWR8
1482 __vector signed short a, b, r;
1483 __vector __bool short c;
1484
1485 a = (__vector signed short)vec_splats (__A);
1486 b = (__vector signed short)vec_splats (__B);
1487 c = (__vector __bool short)vec_cmplt (a, b);
1488 r = vec_sel (b, a, c);
1489 return (__m64) ((__vector long long) r)[0];
1490#else
1491 __m64_union m1, m2, res;
1492
1493 m1.as_m64 = __A;
1494 m2.as_m64 = __B;
1495
1496 res.as_short[0] =
1497 (m1.as_short[0] < m2.as_short[0]) ? m1.as_short[0] : m2.as_short[0];
1498 res.as_short[1] =
1499 (m1.as_short[1] < m2.as_short[1]) ? m1.as_short[1] : m2.as_short[1];
1500 res.as_short[2] =
1501 (m1.as_short[2] < m2.as_short[2]) ? m1.as_short[2] : m2.as_short[2];
1502 res.as_short[3] =
1503 (m1.as_short[3] < m2.as_short[3]) ? m1.as_short[3] : m2.as_short[3];
1504
1505 return (__m64) res.as_m64;
1506#endif
1507}
1508
1509extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1510_m_pminsw (__m64 __A, __m64 __B)
1511{
1512 return _mm_min_pi16 (__A, __B);
1513}
1514
1515/* Compute the element-wise minimum of unsigned 8-bit values. */
1516extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1517_mm_min_pu8 (__m64 __A, __m64 __B)
1518{
1519#if _ARCH_PWR8
1520 __vector unsigned char a, b, r;
1521 __vector __bool char c;
1522
1523 a = (__vector unsigned char)vec_splats (__A);
1524 b = (__vector unsigned char)vec_splats (__B);
1525 c = (__vector __bool char)vec_cmplt (a, b);
1526 r = vec_sel (b, a, c);
1527 return (__m64) ((__vector long long) r)[0];
1528#else
1529 __m64_union m1, m2, res;
1530 long i;
1531
1532 m1.as_m64 = __A;
1533 m2.as_m64 = __B;
1534
1535
1536 for (i = 0; i < 8; i++)
1537 res.as_char[i] =
1538 ((unsigned char) m1.as_char[i] < (unsigned char) m2.as_char[i]) ?
1539 m1.as_char[i] : m2.as_char[i];
1540
1541 return (__m64) res.as_m64;
1542#endif
1543}
1544
1545extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1546_m_pminub (__m64 __A, __m64 __B)
1547{
1548 return _mm_min_pu8 (__A, __B);
1549}
1550
1551/* Create an 8-bit mask of the signs of 8-bit values. */
1552extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1553_mm_movemask_pi8 (__m64 __A)
1554{
1555 unsigned long long p =
1556#ifdef __LITTLE_ENDIAN__
1557 0x0008101820283038UL; // permute control for sign bits
1558#else
1559 0x3830282018100800UL; // permute control for sign bits
1560#endif
1561 return __builtin_bpermd (p, __A);
1562}
1563
1564extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1565_m_pmovmskb (__m64 __A)
1566{
1567 return _mm_movemask_pi8 (__A);
1568}
1569
1570/* Multiply four unsigned 16-bit values in A by four unsigned 16-bit values
1571 in B and produce the high 16 bits of the 32-bit results. */
1572extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1573_mm_mulhi_pu16 (__m64 __A, __m64 __B)
1574{
1575 __vector unsigned short a, b;
1576 __vector unsigned short c;
1577 __vector unsigned int w0, w1;
1578 __vector unsigned char xform1 = {
1579#ifdef __LITTLE_ENDIAN__
1580 0x02, 0x03, 0x12, 0x13, 0x06, 0x07, 0x16, 0x17,
1581 0x0A, 0x0B, 0x1A, 0x1B, 0x0E, 0x0F, 0x1E, 0x1F
1582#else
1583 0x00, 0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15,
1584 0x00, 0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15
1585#endif
1586 };
1587
1588 a = (__vector unsigned short)vec_splats (__A);
1589 b = (__vector unsigned short)vec_splats (__B);
1590
1591 w0 = vec_vmuleuh (a, b);
1592 w1 = vec_vmulouh (a, b);
1593 c = (__vector unsigned short)vec_perm (w0, w1, xform1);
1594
1595 return (__m64) ((__vector long long) c)[0];
1596}
1597
1598extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1599_m_pmulhuw (__m64 __A, __m64 __B)
1600{
1601 return _mm_mulhi_pu16 (__A, __B);
1602}
1603
1604/* Return a combination of the four 16-bit values in A. The selector
1605 must be an immediate. */
1606extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1607_mm_shuffle_pi16 (__m64 __A, int const __N)
1608{
1609 unsigned long element_selector_10 = __N & 0x03;
1610 unsigned long element_selector_32 = (__N >> 2) & 0x03;
1611 unsigned long element_selector_54 = (__N >> 4) & 0x03;
1612 unsigned long element_selector_76 = (__N >> 6) & 0x03;
1613 static const unsigned short permute_selectors[4] =
1614 {
1615#ifdef __LITTLE_ENDIAN__
1616 0x0908, 0x0B0A, 0x0D0C, 0x0F0E
1617#else
1618 0x0607, 0x0405, 0x0203, 0x0001
1619#endif
1620 };
1621 __m64_union t;
1622 __vector unsigned long long a, p, r;
1623
1624#ifdef __LITTLE_ENDIAN__
1625 t.as_short[0] = permute_selectors[element_selector_10];
1626 t.as_short[1] = permute_selectors[element_selector_32];
1627 t.as_short[2] = permute_selectors[element_selector_54];
1628 t.as_short[3] = permute_selectors[element_selector_76];
1629#else
1630 t.as_short[3] = permute_selectors[element_selector_10];
1631 t.as_short[2] = permute_selectors[element_selector_32];
1632 t.as_short[1] = permute_selectors[element_selector_54];
1633 t.as_short[0] = permute_selectors[element_selector_76];
1634#endif
1635 p = vec_splats (t.as_m64);
1636 a = vec_splats (__A);
1637 r = vec_perm (a, a, (__vector unsigned char)p);
1638 return (__m64) ((__vector long long) r)[0];
1639}
1640
1641extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1642_m_pshufw (__m64 __A, int const __N)
1643{
1644 return _mm_shuffle_pi16 (__A, __N);
1645}
1646
1647/* Conditionally store byte elements of A into P. The high bit of each
1648 byte in the selector N determines whether the corresponding byte from
1649 A is stored. */
1650extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1651_mm_maskmove_si64 (__m64 __A, __m64 __N, char *__P)
1652{
1653 __m64 hibit = 0x8080808080808080UL;
1654 __m64 mask, tmp;
1655 __m64 *p = (__m64*)__P;
1656
1657 tmp = *p;
1658 mask = _mm_cmpeq_pi8 ((__N & hibit), hibit);
1659 tmp = (tmp & (~mask)) | (__A & mask);
1660 *p = tmp;
1661}
1662
1663extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1664_m_maskmovq (__m64 __A, __m64 __N, char *__P)
1665{
1666 _mm_maskmove_si64 (__A, __N, __P);
1667}
1668
1669/* Compute the rounded averages of the unsigned 8-bit values in A and B. */
1670extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1671_mm_avg_pu8 (__m64 __A, __m64 __B)
1672{
1673 __vector unsigned char a, b, c;
1674
1675 a = (__vector unsigned char)vec_splats (__A);
1676 b = (__vector unsigned char)vec_splats (__B);
1677 c = vec_avg (a, b);
1678 return (__m64) ((__vector long long) c)[0];
1679}
1680
1681extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1682_m_pavgb (__m64 __A, __m64 __B)
1683{
1684 return _mm_avg_pu8 (__A, __B);
1685}
1686
1687/* Compute the rounded averages of the unsigned 16-bit values in A and B. */
1688extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1689_mm_avg_pu16 (__m64 __A, __m64 __B)
1690{
1691 __vector unsigned short a, b, c;
1692
1693 a = (__vector unsigned short)vec_splats (__A);
1694 b = (__vector unsigned short)vec_splats (__B);
1695 c = vec_avg (a, b);
1696 return (__m64) ((__vector long long) c)[0];
1697}
1698
1699extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1700_m_pavgw (__m64 __A, __m64 __B)
1701{
1702 return _mm_avg_pu16 (__A, __B);
1703}
1704
1705/* Compute the sum of the absolute differences of the unsigned 8-bit
1706 values in A and B. Return the value in the lower 16-bit word; the
1707 upper words are cleared. */
1708extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1709_mm_sad_pu8 (__m64 __A, __m64 __B)
1710{
1711 __vector unsigned char a, b;
1712 __vector unsigned char vmin, vmax, vabsdiff;
1713 __vector signed int vsum;
1714 const __vector unsigned int zero =
1715 { 0, 0, 0, 0 };
1716 __m64_union result = {0};
1717
1718 a = (__vector unsigned char) (__vector unsigned long long) { 0UL, __A };
1719 b = (__vector unsigned char) (__vector unsigned long long) { 0UL, __B };
1720 vmin = vec_min (a, b);
1721 vmax = vec_max (a, b);
1722 vabsdiff = vec_sub (vmax, vmin);
1723 /* Sum four groups of bytes into integers. */
1724 vsum = (__vector signed int) vec_sum4s (vabsdiff, zero);
1725 /* Sum across four integers with integer result. */
1726 vsum = vec_sums (vsum, (__vector signed int) zero);
1727 /* The sum is in the right most 32-bits of the vector result.
1728 Transfer to a GPR and truncate to 16 bits. */
1729 result.as_short[0] = vsum[3];
1730 return result.as_m64;
1731}
1732
1733extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1734_m_psadbw (__m64 __A, __m64 __B)
1735{
1736 return _mm_sad_pu8 (__A, __B);
1737}
1738
1739/* Stores the data in A to the address P without polluting the caches. */
1740extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1741_mm_stream_pi (__m64 *__P, __m64 __A)
1742{
1743 /* Use the data cache block touch for store transient. */
1744 __asm__ (
1745 " dcbtstt 0,%0"
1746 :
1747 : "b" (__P)
1748 : "memory"
1749 );
1750 *__P = __A;
1751}
1752
1753/* Likewise. The address must be 16-byte aligned. */
1754extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1755_mm_stream_ps (float *__P, __m128 __A)
1756{
1757 /* Use the data cache block touch for store transient. */
1758 __asm__ (
1759 " dcbtstt 0,%0"
1760 :
1761 : "b" (__P)
1762 : "memory"
1763 );
1764 _mm_store_ps (__P, __A);
1765}
1766
1767/* Guarantees that every preceding store is globally visible before
1768 any subsequent store. */
1769extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1770_mm_sfence (void)
1771{
1772 /* Generate a light weight sync. */
1773 __atomic_thread_fence (__ATOMIC_RELEASE);
1774}
1775
1776/* The execution of the next instruction is delayed by an implementation
1777 specific amount of time. The instruction does not modify the
1778 architectural state. This is after the pop_options pragma because
1779 it does not require SSE support in the processor--the encoding is a
1780 nop on processors that do not support it. */
1781extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1782_mm_pause (void)
1783{
1784 /* There is no exact match with this construct, but the following is
1785 close to the desired effect. */
1786#if _ARCH_PWR8
1787 /* On power8 and later processors we can depend on Program Priority
1788 (PRI) and associated "very low" PPI setting. Since we don't know
1789 what PPI this thread is running at we: 1) save the current PRI
1790 from the PPR SPR into a local GRP, 2) set the PRI to "very low*
1791 via the special or 31,31,31 encoding. 3) issue an "isync" to
1792 insure the PRI change takes effect before we execute any more
1793 instructions.
1794 Now we can execute a lwsync (release barrier) while we execute
1795 this thread at "very low" PRI. Finally we restore the original
1796 PRI and continue execution. */
1797 unsigned long __PPR;
1798
1799 __asm__ volatile (
1800 " mfppr %0;"
1801 " or 31,31,31;"
1802 " isync;"
1803 " lwsync;"
1804 " isync;"
1805 " mtppr %0;"
1806 : "=r" (__PPR)
1807 :
1808 : "memory"
1809 );
1810#else
1811 /* For older processor where we may not even have Program Priority
1812 controls we can only depend on Heavy Weight Sync. */
1813 __atomic_thread_fence (__ATOMIC_SEQ_CST);
1814#endif
1815}
1816
1817/* Transpose the 4x4 matrix composed of row[0-3]. */
1818#define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \
1819do { \
1820 __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3); \
1821 __v4sf __t0 = vec_vmrghw (__r0, __r1); \
1822 __v4sf __t1 = vec_vmrghw (__r2, __r3); \
1823 __v4sf __t2 = vec_vmrglw (__r0, __r1); \
1824 __v4sf __t3 = vec_vmrglw (__r2, __r3); \
1825 (row0) = (__v4sf)vec_mergeh ((__vector long long)__t0, \
1826 (__vector long long)__t1); \
1827 (row1) = (__v4sf)vec_mergel ((__vector long long)__t0, \
1828 (__vector long long)__t1); \
1829 (row2) = (__v4sf)vec_mergeh ((__vector long long)__t2, \
1830 (__vector long long)__t3); \
1831 (row3) = (__v4sf)vec_mergel ((__vector long long)__t2, \
1832 (__vector long long)__t3); \
1833} while (0)
1834
1835/* For backward source compatibility. */
1836//# include <emmintrin.h>
1837
1838#endif /* _XMMINTRIN_H_INCLUDED */
lib/include/prfchwintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- prfchwintrin.h - PREFETCHW intrinsic -----------------------------===1/*===---- prfchwintrin.h - PREFETCHW intrinsic -----------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/ptwriteintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===------------ ptwriteintrin.h - PTWRITE intrinsic --------------------===1/*===------------ ptwriteintrin.h - PTWRITE intrinsic --------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/rdseedintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- rdseedintrin.h - RDSEED intrinsics -------------------------------===1/*===---- rdseedintrin.h - RDSEED intrinsics -------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/rtmintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- rtmintrin.h - RTM intrinsics -------------------------------------===1/*===---- rtmintrin.h - RTM intrinsics -------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/s390intrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- s390intrin.h - SystemZ intrinsics --------------------------------===1/*===---- s390intrin.h - SystemZ intrinsics --------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/sgxintrin.h+7-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- sgxintrin.h - X86 SGX intrinsics configuration -------------------===1/*===---- sgxintrin.h - X86 SGX intrinsics configuration -------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -28,6 +14,8 @@...@@ -28,6 +14,8 @@
28#ifndef __SGXINTRIN_H14#ifndef __SGXINTRIN_H
29#define __SGXINTRIN_H15#define __SGXINTRIN_H
3016
17#if __has_extension(gnu_asm)
18
31/* Define the default attributes for the functions in this file. */19/* Define the default attributes for the functions in this file. */
32#define __DEFAULT_FN_ATTRS \20#define __DEFAULT_FN_ATTRS \
33 __attribute__((__always_inline__, __nodebug__, __target__("sgx")))21 __attribute__((__always_inline__, __nodebug__, __target__("sgx")))
...@@ -67,4 +55,6 @@ _enclv_u32(unsigned int __leaf, __SIZE_TYPE__ __d[])...@@ -67,4 +55,6 @@ _enclv_u32(unsigned int __leaf, __SIZE_TYPE__ __d[])
6755
68#undef __DEFAULT_FN_ATTRS56#undef __DEFAULT_FN_ATTRS
6957
58#endif /* __has_extension(gnu_asm) */
59
70#endif60#endif
lib/include/shaintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- shaintrin.h - SHA intrinsics -------------------------------------===1/*===---- shaintrin.h - SHA intrinsics -------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/smmintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- smmintrin.h - SSE4 intrinsics ------------------------------------===1/*===---- smmintrin.h - SSE4 intrinsics ------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/stdalign.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- stdalign.h - Standard header for alignment ------------------------===1/*===---- stdalign.h - Standard header for alignment ------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/stdarg.h+3-19
...@@ -1,24 +1,8 @@...@@ -1,24 +1,8 @@
1/*===---- stdarg.h - Variable argument handling ----------------------------===1/*===---- stdarg.h - Variable argument handling ----------------------------===
2 *2 *
3 * Copyright (c) 2008 Eli Friedman3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 *4 * See https://llvm.org/LICENSE.txt for license information.
5 * Permission is hereby granted, free of charge, to any person obtaining a copy5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * of this software and associated documentation files (the "Software"), to deal
7 * in the Software without restriction, including without limitation the rights
8 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 * copies of the Software, and to permit persons to whom the Software is
10 * furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 * THE SOFTWARE.
22 *6 *
23 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
24 */8 */
lib/include/stdatomic.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- stdatomic.h - Standard header for atomic types and operations -----===1/*===---- stdatomic.h - Standard header for atomic types and operations -----===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/stdbool.h+3-19
...@@ -1,24 +1,8 @@...@@ -1,24 +1,8 @@
1/*===---- stdbool.h - Standard header for booleans -------------------------===1/*===---- stdbool.h - Standard header for booleans -------------------------===
2 *2 *
3 * Copyright (c) 2008 Eli Friedman3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 *4 * See https://llvm.org/LICENSE.txt for license information.
5 * Permission is hereby granted, free of charge, to any person obtaining a copy5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * of this software and associated documentation files (the "Software"), to deal
7 * in the Software without restriction, including without limitation the rights
8 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 * copies of the Software, and to permit persons to whom the Software is
10 * furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 * THE SOFTWARE.
22 *6 *
23 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
24 */8 */
lib/include/stddef.h+3-19
...@@ -1,24 +1,8 @@...@@ -1,24 +1,8 @@
1/*===---- stddef.h - Basic type definitions --------------------------------===1/*===---- stddef.h - Basic type definitions --------------------------------===
2 *2 *
3 * Copyright (c) 2008 Eli Friedman3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 *4 * See https://llvm.org/LICENSE.txt for license information.
5 * Permission is hereby granted, free of charge, to any person obtaining a copy5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * of this software and associated documentation files (the "Software"), to deal
7 * in the Software without restriction, including without limitation the rights
8 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 * copies of the Software, and to permit persons to whom the Software is
10 * furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 * THE SOFTWARE.
22 *6 *
23 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
24 */8 */
lib/include/stdint.h+8-19
...@@ -1,29 +1,18 @@...@@ -1,29 +1,18 @@
1/*===---- stdint.h - Standard header for sized integer types --------------===*\1/*===---- stdint.h - Standard header for sized integer types --------------===*\
2 *2 *
3 * Copyright (c) 2009 Chris Lattner3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 *4 * See https://llvm.org/LICENSE.txt for license information.
5 * Permission is hereby granted, free of charge, to any person obtaining a copy5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * of this software and associated documentation files (the "Software"), to deal
7 * in the Software without restriction, including without limitation the rights
8 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 * copies of the Software, and to permit persons to whom the Software is
10 * furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 * THE SOFTWARE.
22 *6 *
23\*===----------------------------------------------------------------------===*/7\*===----------------------------------------------------------------------===*/
248
25#ifndef __CLANG_STDINT_H9#ifndef __CLANG_STDINT_H
10// AIX system headers need stdint.h to be re-enterable while _STD_TYPES_T
11// is defined until an inclusion of it without _STD_TYPES_T occurs, in which
12// case the header guard macro is defined.
13#if !defined(_AIX) || !defined(_STD_TYPES_T) || !defined(__STDC_HOSTED__)
26#define __CLANG_STDINT_H14#define __CLANG_STDINT_H
15#endif
2716
28/* If we're hosted, fall back to the system's stdint.h, which might have17/* If we're hosted, fall back to the system's stdint.h, which might have
29 * additional definitions.18 * additional definitions.
lib/include/stdnoreturn.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- stdnoreturn.h - Standard header for noreturn macro ---------------===1/*===---- stdnoreturn.h - Standard header for noreturn macro ---------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/tbmintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- tbmintrin.h - TBM intrinsics -------------------------------------===1/*===---- tbmintrin.h - TBM intrinsics -------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/tgmath.h+3-19
...@@ -1,24 +1,8 @@...@@ -1,24 +1,8 @@
1/*===---- tgmath.h - Standard header for type generic math ----------------===*\1/*===---- tgmath.h - Standard header for type generic math ----------------===*\
2 *2 *
3 * Copyright (c) 2009 Howard Hinnant3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 *4 * See https://llvm.org/LICENSE.txt for license information.
5 * Permission is hereby granted, free of charge, to any person obtaining a copy5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * of this software and associated documentation files (the "Software"), to deal
7 * in the Software without restriction, including without limitation the rights
8 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 * copies of the Software, and to permit persons to whom the Software is
10 * furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 * THE SOFTWARE.
22 *6 *
23\*===----------------------------------------------------------------------===*/7\*===----------------------------------------------------------------------===*/
248
lib/include/tmmintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- tmmintrin.h - SSSE3 intrinsics -----------------------------------===1/*===---- tmmintrin.h - SSSE3 intrinsics -----------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/unwind.h+5-19
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- unwind.h - Stack unwinding ----------------------------------------===1/*===---- unwind.h - Stack unwinding ----------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -66,8 +52,8 @@ extern "C" {...@@ -66,8 +52,8 @@ extern "C" {
66#pragma GCC visibility push(default)52#pragma GCC visibility push(default)
67#endif53#endif
6854
69typedef uintptr_t _Unwind_Word;55typedef uintptr_t _Unwind_Word __attribute__((__mode__(__unwind_word__)));
70typedef intptr_t _Unwind_Sword;56typedef intptr_t _Unwind_Sword __attribute__((__mode__(__unwind_word__)));
71typedef uintptr_t _Unwind_Ptr;57typedef uintptr_t _Unwind_Ptr;
72typedef uintptr_t _Unwind_Internal_Ptr;58typedef uintptr_t _Unwind_Internal_Ptr;
73typedef uint64_t _Unwind_Exception_Class;59typedef uint64_t _Unwind_Exception_Class;
lib/include/vadefs.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/* ===-------- vadefs.h ---------------------------------------------------===1/* ===-------- vadefs.h ---------------------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/vaesintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===------------------ vaesintrin.h - VAES intrinsics ---------------------===1/*===------------------ vaesintrin.h - VAES intrinsics ---------------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/varargs.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- varargs.h - Variable argument handling -------------------------------------===1/*===---- varargs.h - Variable argument handling -------------------------------------===
2*2*
3* Permission is hereby granted, free of charge, to any person obtaining a copy3* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4* of this software and associated documentation files (the "Software"), to deal4* See https://llvm.org/LICENSE.txt for license information.
5* in the Software without restriction, including without limitation the rights5* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7* copies of the Software, and to permit persons to whom the Software is
8* furnished to do so, subject to the following conditions:
9*
10* The above copyright notice and this permission notice shall be included in
11* all copies or substantial portions of the Software.
12*
13* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19* THE SOFTWARE.
20*6*
21*===-----------------------------------------------------------------------===7*===-----------------------------------------------------------------------===
22*/8*/
lib/include/vecintrin.h+409-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- vecintrin.h - Vector intrinsics ----------------------------------===1/*===---- vecintrin.h - Vector intrinsics ----------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -531,6 +517,141 @@ vec_bperm_u128(vector unsigned char __a, vector unsigned char __b) {...@@ -531,6 +517,141 @@ vec_bperm_u128(vector unsigned char __a, vector unsigned char __b) {
531}517}
532#endif518#endif
533519
520/*-- vec_revb ---------------------------------------------------------------*/
521
522static inline __ATTRS_o_ai vector signed short
523vec_revb(vector signed short __vec) {
524 return (vector signed short)
525 __builtin_s390_vlbrh((vector unsigned short)__vec);
526}
527
528static inline __ATTRS_o_ai vector unsigned short
529vec_revb(vector unsigned short __vec) {
530 return __builtin_s390_vlbrh(__vec);
531}
532
533static inline __ATTRS_o_ai vector signed int
534vec_revb(vector signed int __vec) {
535 return (vector signed int)
536 __builtin_s390_vlbrf((vector unsigned int)__vec);
537}
538
539static inline __ATTRS_o_ai vector unsigned int
540vec_revb(vector unsigned int __vec) {
541 return __builtin_s390_vlbrf(__vec);
542}
543
544static inline __ATTRS_o_ai vector signed long long
545vec_revb(vector signed long long __vec) {
546 return (vector signed long long)
547 __builtin_s390_vlbrg((vector unsigned long long)__vec);
548}
549
550static inline __ATTRS_o_ai vector unsigned long long
551vec_revb(vector unsigned long long __vec) {
552 return __builtin_s390_vlbrg(__vec);
553}
554
555#if __ARCH__ >= 12
556static inline __ATTRS_o_ai vector float
557vec_revb(vector float __vec) {
558 return (vector float)
559 __builtin_s390_vlbrf((vector unsigned int)__vec);
560}
561#endif
562
563static inline __ATTRS_o_ai vector double
564vec_revb(vector double __vec) {
565 return (vector double)
566 __builtin_s390_vlbrg((vector unsigned long long)__vec);
567}
568
569/*-- vec_reve ---------------------------------------------------------------*/
570
571static inline __ATTRS_o_ai vector signed char
572vec_reve(vector signed char __vec) {
573 return (vector signed char) { __vec[15], __vec[14], __vec[13], __vec[12],
574 __vec[11], __vec[10], __vec[9], __vec[8],
575 __vec[7], __vec[6], __vec[5], __vec[4],
576 __vec[3], __vec[2], __vec[1], __vec[0] };
577}
578
579static inline __ATTRS_o_ai vector unsigned char
580vec_reve(vector unsigned char __vec) {
581 return (vector unsigned char) { __vec[15], __vec[14], __vec[13], __vec[12],
582 __vec[11], __vec[10], __vec[9], __vec[8],
583 __vec[7], __vec[6], __vec[5], __vec[4],
584 __vec[3], __vec[2], __vec[1], __vec[0] };
585}
586
587static inline __ATTRS_o_ai vector bool char
588vec_reve(vector bool char __vec) {
589 return (vector bool char) { __vec[15], __vec[14], __vec[13], __vec[12],
590 __vec[11], __vec[10], __vec[9], __vec[8],
591 __vec[7], __vec[6], __vec[5], __vec[4],
592 __vec[3], __vec[2], __vec[1], __vec[0] };
593}
594
595static inline __ATTRS_o_ai vector signed short
596vec_reve(vector signed short __vec) {
597 return (vector signed short) { __vec[7], __vec[6], __vec[5], __vec[4],
598 __vec[3], __vec[2], __vec[1], __vec[0] };
599}
600
601static inline __ATTRS_o_ai vector unsigned short
602vec_reve(vector unsigned short __vec) {
603 return (vector unsigned short) { __vec[7], __vec[6], __vec[5], __vec[4],
604 __vec[3], __vec[2], __vec[1], __vec[0] };
605}
606
607static inline __ATTRS_o_ai vector bool short
608vec_reve(vector bool short __vec) {
609 return (vector bool short) { __vec[7], __vec[6], __vec[5], __vec[4],
610 __vec[3], __vec[2], __vec[1], __vec[0] };
611}
612
613static inline __ATTRS_o_ai vector signed int
614vec_reve(vector signed int __vec) {
615 return (vector signed int) { __vec[3], __vec[2], __vec[1], __vec[0] };
616}
617
618static inline __ATTRS_o_ai vector unsigned int
619vec_reve(vector unsigned int __vec) {
620 return (vector unsigned int) { __vec[3], __vec[2], __vec[1], __vec[0] };
621}
622
623static inline __ATTRS_o_ai vector bool int
624vec_reve(vector bool int __vec) {
625 return (vector bool int) { __vec[3], __vec[2], __vec[1], __vec[0] };
626}
627
628static inline __ATTRS_o_ai vector signed long long
629vec_reve(vector signed long long __vec) {
630 return (vector signed long long) { __vec[1], __vec[0] };
631}
632
633static inline __ATTRS_o_ai vector unsigned long long
634vec_reve(vector unsigned long long __vec) {
635 return (vector unsigned long long) { __vec[1], __vec[0] };
636}
637
638static inline __ATTRS_o_ai vector bool long long
639vec_reve(vector bool long long __vec) {
640 return (vector bool long long) { __vec[1], __vec[0] };
641}
642
643#if __ARCH__ >= 12
644static inline __ATTRS_o_ai vector float
645vec_reve(vector float __vec) {
646 return (vector float) { __vec[3], __vec[2], __vec[1], __vec[0] };
647}
648#endif
649
650static inline __ATTRS_o_ai vector double
651vec_reve(vector double __vec) {
652 return (vector double) { __vec[1], __vec[0] };
653}
654
534/*-- vec_sel ----------------------------------------------------------------*/655/*-- vec_sel ----------------------------------------------------------------*/
535656
536static inline __ATTRS_o_ai vector signed char657static inline __ATTRS_o_ai vector signed char
...@@ -6849,6 +6970,56 @@ vec_sldw(vector double __a, vector double __b, int __c)...@@ -6849,6 +6970,56 @@ vec_sldw(vector double __a, vector double __b, int __c)
6849 __builtin_s390_vsldb((vector unsigned char)(X), \6970 __builtin_s390_vsldb((vector unsigned char)(X), \
6850 (vector unsigned char)(Y), (Z) * 4))6971 (vector unsigned char)(Y), (Z) * 4))
68516972
6973/*-- vec_sldb ---------------------------------------------------------------*/
6974
6975#if __ARCH__ >= 13
6976
6977extern __ATTRS_o vector signed char
6978vec_sldb(vector signed char __a, vector signed char __b, int __c)
6979 __constant_range(__c, 0, 7);
6980
6981extern __ATTRS_o vector unsigned char
6982vec_sldb(vector unsigned char __a, vector unsigned char __b, int __c)
6983 __constant_range(__c, 0, 7);
6984
6985extern __ATTRS_o vector signed short
6986vec_sldb(vector signed short __a, vector signed short __b, int __c)
6987 __constant_range(__c, 0, 7);
6988
6989extern __ATTRS_o vector unsigned short
6990vec_sldb(vector unsigned short __a, vector unsigned short __b, int __c)
6991 __constant_range(__c, 0, 7);
6992
6993extern __ATTRS_o vector signed int
6994vec_sldb(vector signed int __a, vector signed int __b, int __c)
6995 __constant_range(__c, 0, 7);
6996
6997extern __ATTRS_o vector unsigned int
6998vec_sldb(vector unsigned int __a, vector unsigned int __b, int __c)
6999 __constant_range(__c, 0, 7);
7000
7001extern __ATTRS_o vector signed long long
7002vec_sldb(vector signed long long __a, vector signed long long __b, int __c)
7003 __constant_range(__c, 0, 7);
7004
7005extern __ATTRS_o vector unsigned long long
7006vec_sldb(vector unsigned long long __a, vector unsigned long long __b, int __c)
7007 __constant_range(__c, 0, 7);
7008
7009extern __ATTRS_o vector float
7010vec_sldb(vector float __a, vector float __b, int __c)
7011 __constant_range(__c, 0, 7);
7012
7013extern __ATTRS_o vector double
7014vec_sldb(vector double __a, vector double __b, int __c)
7015 __constant_range(__c, 0, 7);
7016
7017#define vec_sldb(X, Y, Z) ((__typeof__((vec_sldb)((X), (Y), (Z)))) \
7018 __builtin_s390_vsld((vector unsigned char)(X), \
7019 (vector unsigned char)(Y), (Z)))
7020
7021#endif
7022
6852/*-- vec_sral ---------------------------------------------------------------*/7023/*-- vec_sral ---------------------------------------------------------------*/
68537024
6854static inline __ATTRS_o_ai vector signed char7025static inline __ATTRS_o_ai vector signed char
...@@ -7579,6 +7750,56 @@ vec_srb(vector double __a, vector unsigned long long __b) {...@@ -7579,6 +7750,56 @@ vec_srb(vector double __a, vector unsigned long long __b) {
7579 (vector unsigned char)__a, (vector unsigned char)__b);7750 (vector unsigned char)__a, (vector unsigned char)__b);
7580}7751}
75817752
7753/*-- vec_srdb ---------------------------------------------------------------*/
7754
7755#if __ARCH__ >= 13
7756
7757extern __ATTRS_o vector signed char
7758vec_srdb(vector signed char __a, vector signed char __b, int __c)
7759 __constant_range(__c, 0, 7);
7760
7761extern __ATTRS_o vector unsigned char
7762vec_srdb(vector unsigned char __a, vector unsigned char __b, int __c)
7763 __constant_range(__c, 0, 7);
7764
7765extern __ATTRS_o vector signed short
7766vec_srdb(vector signed short __a, vector signed short __b, int __c)
7767 __constant_range(__c, 0, 7);
7768
7769extern __ATTRS_o vector unsigned short
7770vec_srdb(vector unsigned short __a, vector unsigned short __b, int __c)
7771 __constant_range(__c, 0, 7);
7772
7773extern __ATTRS_o vector signed int
7774vec_srdb(vector signed int __a, vector signed int __b, int __c)
7775 __constant_range(__c, 0, 7);
7776
7777extern __ATTRS_o vector unsigned int
7778vec_srdb(vector unsigned int __a, vector unsigned int __b, int __c)
7779 __constant_range(__c, 0, 7);
7780
7781extern __ATTRS_o vector signed long long
7782vec_srdb(vector signed long long __a, vector signed long long __b, int __c)
7783 __constant_range(__c, 0, 7);
7784
7785extern __ATTRS_o vector unsigned long long
7786vec_srdb(vector unsigned long long __a, vector unsigned long long __b, int __c)
7787 __constant_range(__c, 0, 7);
7788
7789extern __ATTRS_o vector float
7790vec_srdb(vector float __a, vector float __b, int __c)
7791 __constant_range(__c, 0, 7);
7792
7793extern __ATTRS_o vector double
7794vec_srdb(vector double __a, vector double __b, int __c)
7795 __constant_range(__c, 0, 7);
7796
7797#define vec_srdb(X, Y, Z) ((__typeof__((vec_srdb)((X), (Y), (Z)))) \
7798 __builtin_s390_vsrd((vector unsigned char)(X), \
7799 (vector unsigned char)(Y), (Z)))
7800
7801#endif
7802
7582/*-- vec_abs ----------------------------------------------------------------*/7803/*-- vec_abs ----------------------------------------------------------------*/
75837804
7584static inline __ATTRS_o_ai vector signed char7805static inline __ATTRS_o_ai vector signed char
...@@ -8725,6 +8946,22 @@ vec_double(vector unsigned long long __a) {...@@ -8725,6 +8946,22 @@ vec_double(vector unsigned long long __a) {
8725 return __builtin_convertvector(__a, vector double);8946 return __builtin_convertvector(__a, vector double);
8726}8947}
87278948
8949/*-- vec_float --------------------------------------------------------------*/
8950
8951#if __ARCH__ >= 13
8952
8953static inline __ATTRS_o_ai vector float
8954vec_float(vector signed int __a) {
8955 return __builtin_convertvector(__a, vector float);
8956}
8957
8958static inline __ATTRS_o_ai vector float
8959vec_float(vector unsigned int __a) {
8960 return __builtin_convertvector(__a, vector float);
8961}
8962
8963#endif
8964
8728/*-- vec_signed -------------------------------------------------------------*/8965/*-- vec_signed -------------------------------------------------------------*/
87298966
8730static inline __ATTRS_o_ai vector signed long long8967static inline __ATTRS_o_ai vector signed long long
...@@ -8732,6 +8969,13 @@ vec_signed(vector double __a) {...@@ -8732,6 +8969,13 @@ vec_signed(vector double __a) {
8732 return __builtin_convertvector(__a, vector signed long long);8969 return __builtin_convertvector(__a, vector signed long long);
8733}8970}
87348971
8972#if __ARCH__ >= 13
8973static inline __ATTRS_o_ai vector signed int
8974vec_signed(vector float __a) {
8975 return __builtin_convertvector(__a, vector signed int);
8976}
8977#endif
8978
8735/*-- vec_unsigned -----------------------------------------------------------*/8979/*-- vec_unsigned -----------------------------------------------------------*/
87368980
8737static inline __ATTRS_o_ai vector unsigned long long8981static inline __ATTRS_o_ai vector unsigned long long
...@@ -8739,6 +8983,13 @@ vec_unsigned(vector double __a) {...@@ -8739,6 +8983,13 @@ vec_unsigned(vector double __a) {
8739 return __builtin_convertvector(__a, vector unsigned long long);8983 return __builtin_convertvector(__a, vector unsigned long long);
8740}8984}
87418985
8986#if __ARCH__ >= 13
8987static inline __ATTRS_o_ai vector unsigned int
8988vec_unsigned(vector float __a) {
8989 return __builtin_convertvector(__a, vector unsigned int);
8990}
8991#endif
8992
8742/*-- vec_roundp -------------------------------------------------------------*/8993/*-- vec_roundp -------------------------------------------------------------*/
87438994
8744#if __ARCH__ >= 128995#if __ARCH__ >= 12
...@@ -10456,6 +10707,147 @@ vec_find_any_ne_or_0_idx_cc(vector unsigned int __a, vector unsigned int __b,...@@ -10456,6 +10707,147 @@ vec_find_any_ne_or_0_idx_cc(vector unsigned int __a, vector unsigned int __b,
10456 return __builtin_s390_vfaezfs(__a, __b, 8, __cc);10707 return __builtin_s390_vfaezfs(__a, __b, 8, __cc);
10457}10708}
1045810709
10710/*-- vec_search_string_cc ---------------------------------------------------*/
10711
10712#if __ARCH__ >= 13
10713
10714static inline __ATTRS_o_ai vector unsigned char
10715vec_search_string_cc(vector signed char __a, vector signed char __b,
10716 vector unsigned char __c, int *__cc) {
10717 return __builtin_s390_vstrsb((vector unsigned char)__a,
10718 (vector unsigned char)__b, __c, __cc);
10719}
10720
10721static inline __ATTRS_o_ai vector unsigned char
10722vec_search_string_cc(vector bool char __a, vector bool char __b,
10723 vector unsigned char __c, int *__cc) {
10724 return __builtin_s390_vstrsb((vector unsigned char)__a,
10725 (vector unsigned char)__b, __c, __cc);
10726}
10727
10728static inline __ATTRS_o_ai vector unsigned char
10729vec_search_string_cc(vector unsigned char __a, vector unsigned char __b,
10730 vector unsigned char __c, int *__cc) {
10731 return __builtin_s390_vstrsb(__a, __b, __c, __cc);
10732}
10733
10734static inline __ATTRS_o_ai vector unsigned char
10735vec_search_string_cc(vector signed short __a, vector signed short __b,
10736 vector unsigned char __c, int *__cc) {
10737 return __builtin_s390_vstrsh((vector unsigned short)__a,
10738 (vector unsigned short)__b, __c, __cc);
10739}
10740
10741static inline __ATTRS_o_ai vector unsigned char
10742vec_search_string_cc(vector bool short __a, vector bool short __b,
10743 vector unsigned char __c, int *__cc) {
10744 return __builtin_s390_vstrsh((vector unsigned short)__a,
10745 (vector unsigned short)__b, __c, __cc);
10746}
10747
10748static inline __ATTRS_o_ai vector unsigned char
10749vec_search_string_cc(vector unsigned short __a, vector unsigned short __b,
10750 vector unsigned char __c, int *__cc) {
10751 return __builtin_s390_vstrsh(__a, __b, __c, __cc);
10752}
10753
10754static inline __ATTRS_o_ai vector unsigned char
10755vec_search_string_cc(vector signed int __a, vector signed int __b,
10756 vector unsigned char __c, int *__cc) {
10757 return __builtin_s390_vstrsf((vector unsigned int)__a,
10758 (vector unsigned int)__b, __c, __cc);
10759}
10760
10761static inline __ATTRS_o_ai vector unsigned char
10762vec_search_string_cc(vector bool int __a, vector bool int __b,
10763 vector unsigned char __c, int *__cc) {
10764 return __builtin_s390_vstrsf((vector unsigned int)__a,
10765 (vector unsigned int)__b, __c, __cc);
10766}
10767
10768static inline __ATTRS_o_ai vector unsigned char
10769vec_search_string_cc(vector unsigned int __a, vector unsigned int __b,
10770 vector unsigned char __c, int *__cc) {
10771 return __builtin_s390_vstrsf(__a, __b, __c, __cc);
10772}
10773
10774#endif
10775
10776/*-- vec_search_string_until_zero_cc ----------------------------------------*/
10777
10778#if __ARCH__ >= 13
10779
10780static inline __ATTRS_o_ai vector unsigned char
10781vec_search_string_until_zero_cc(vector signed char __a,
10782 vector signed char __b,
10783 vector unsigned char __c, int *__cc) {
10784 return __builtin_s390_vstrszb((vector unsigned char)__a,
10785 (vector unsigned char)__b, __c, __cc);
10786}
10787
10788static inline __ATTRS_o_ai vector unsigned char
10789vec_search_string_until_zero_cc(vector bool char __a,
10790 vector bool char __b,
10791 vector unsigned char __c, int *__cc) {
10792 return __builtin_s390_vstrszb((vector unsigned char)__a,
10793 (vector unsigned char)__b, __c, __cc);
10794}
10795
10796static inline __ATTRS_o_ai vector unsigned char
10797vec_search_string_until_zero_cc(vector unsigned char __a,
10798 vector unsigned char __b,
10799 vector unsigned char __c, int *__cc) {
10800 return __builtin_s390_vstrszb(__a, __b, __c, __cc);
10801}
10802
10803static inline __ATTRS_o_ai vector unsigned char
10804vec_search_string_until_zero_cc(vector signed short __a,
10805 vector signed short __b,
10806 vector unsigned char __c, int *__cc) {
10807 return __builtin_s390_vstrszh((vector unsigned short)__a,
10808 (vector unsigned short)__b, __c, __cc);
10809}
10810
10811static inline __ATTRS_o_ai vector unsigned char
10812vec_search_string_until_zero_cc(vector bool short __a,
10813 vector bool short __b,
10814 vector unsigned char __c, int *__cc) {
10815 return __builtin_s390_vstrszh((vector unsigned short)__a,
10816 (vector unsigned short)__b, __c, __cc);
10817}
10818
10819static inline __ATTRS_o_ai vector unsigned char
10820vec_search_string_until_zero_cc(vector unsigned short __a,
10821 vector unsigned short __b,
10822 vector unsigned char __c, int *__cc) {
10823 return __builtin_s390_vstrszh(__a, __b, __c, __cc);
10824}
10825
10826static inline __ATTRS_o_ai vector unsigned char
10827vec_search_string_until_zero_cc(vector signed int __a,
10828 vector signed int __b,
10829 vector unsigned char __c, int *__cc) {
10830 return __builtin_s390_vstrszf((vector unsigned int)__a,
10831 (vector unsigned int)__b, __c, __cc);
10832}
10833
10834static inline __ATTRS_o_ai vector unsigned char
10835vec_search_string_until_zero_cc(vector bool int __a,
10836 vector bool int __b,
10837 vector unsigned char __c, int *__cc) {
10838 return __builtin_s390_vstrszf((vector unsigned int)__a,
10839 (vector unsigned int)__b, __c, __cc);
10840}
10841
10842static inline __ATTRS_o_ai vector unsigned char
10843vec_search_string_until_zero_cc(vector unsigned int __a,
10844 vector unsigned int __b,
10845 vector unsigned char __c, int *__cc) {
10846 return __builtin_s390_vstrszf(__a, __b, __c, __cc);
10847}
10848
10849#endif
10850
10459#undef __constant_pow2_range10851#undef __constant_pow2_range
10460#undef __constant_range10852#undef __constant_range
10461#undef __constant10853#undef __constant
lib/include/vpclmulqdqintrin.h+3-17
...@@ -1,23 +1,9 @@...@@ -1,23 +1,9 @@
1/*===------------ vpclmulqdqintrin.h - VPCLMULQDQ intrinsics ---------------===1/*===------------ vpclmulqdqintrin.h - VPCLMULQDQ intrinsics ---------------===
2 *2 *
3 *3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * of this software and associated documentation files (the "Software"), to deal5 * See https://llvm.org/LICENSE.txt for license information.
6 * in the Software without restriction, including without limitation the rights6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 *7 *
22 *===-----------------------------------------------------------------------===8 *===-----------------------------------------------------------------------===
23 */9 */
lib/include/waitpkgintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===----------------------- waitpkgintrin.h - WAITPKG --------------------===1/*===----------------------- waitpkgintrin.h - WAITPKG --------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/wbnoinvdintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===-------------- wbnoinvdintrin.h - wbnoinvd intrinsic-------------------===1/*===-------------- wbnoinvdintrin.h - wbnoinvd intrinsic-------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/wmmintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- wmmintrin.h - AES intrinsics ------------------------------------===1/*===---- wmmintrin.h - AES intrinsics ------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/x86intrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- x86intrin.h - X86 intrinsics -------------------------------------===1/*===---- x86intrin.h - X86 intrinsics -------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/xmmintrin.h+18-22
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- xmmintrin.h - SSE intrinsics -------------------------------------===1/*===---- xmmintrin.h - SSE intrinsics -------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -28,7 +14,9 @@...@@ -28,7 +14,9 @@
2814
29typedef int __v4si __attribute__((__vector_size__(16)));15typedef int __v4si __attribute__((__vector_size__(16)));
30typedef float __v4sf __attribute__((__vector_size__(16)));16typedef float __v4sf __attribute__((__vector_size__(16)));
31typedef float __m128 __attribute__((__vector_size__(16)));17typedef float __m128 __attribute__((__vector_size__(16), __aligned__(16)));
18
19typedef float __m128_u __attribute__((__vector_size__(16), __aligned__(1)));
3220
33/* Unsigned types */21/* Unsigned types */
34typedef unsigned int __v4su __attribute__((__vector_size__(16)));22typedef unsigned int __v4su __attribute__((__vector_size__(16)));
...@@ -1752,7 +1740,7 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS...@@ -1752,7 +1740,7 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS
1752_mm_loadu_ps(const float *__p)1740_mm_loadu_ps(const float *__p)
1753{1741{
1754 struct __loadu_ps {1742 struct __loadu_ps {
1755 __m128 __v;1743 __m128_u __v;
1756 } __attribute__((__packed__, __may_alias__));1744 } __attribute__((__packed__, __may_alias__));
1757 return ((struct __loadu_ps*)__p)->__v;1745 return ((struct __loadu_ps*)__p)->__v;
1758}1746}
...@@ -1931,7 +1919,11 @@ _mm_setzero_ps(void)...@@ -1931,7 +1919,11 @@ _mm_setzero_ps(void)
1931static __inline__ void __DEFAULT_FN_ATTRS1919static __inline__ void __DEFAULT_FN_ATTRS
1932_mm_storeh_pi(__m64 *__p, __m128 __a)1920_mm_storeh_pi(__m64 *__p, __m128 __a)
1933{1921{
1934 __builtin_ia32_storehps((__v2si *)__p, (__v4sf)__a);1922 typedef float __mm_storeh_pi_v2f32 __attribute__((__vector_size__(8)));
1923 struct __mm_storeh_pi_struct {
1924 __mm_storeh_pi_v2f32 __u;
1925 } __attribute__((__packed__, __may_alias__));
1926 ((struct __mm_storeh_pi_struct*)__p)->__u = __builtin_shufflevector(__a, __a, 2, 3);
1935}1927}
19361928
1937/// Stores the lower 64 bits of a 128-bit vector of [4 x float] to a1929/// Stores the lower 64 bits of a 128-bit vector of [4 x float] to a
...@@ -1948,7 +1940,11 @@ _mm_storeh_pi(__m64 *__p, __m128 __a)...@@ -1948,7 +1940,11 @@ _mm_storeh_pi(__m64 *__p, __m128 __a)
1948static __inline__ void __DEFAULT_FN_ATTRS1940static __inline__ void __DEFAULT_FN_ATTRS
1949_mm_storel_pi(__m64 *__p, __m128 __a)1941_mm_storel_pi(__m64 *__p, __m128 __a)
1950{1942{
1951 __builtin_ia32_storelps((__v2si *)__p, (__v4sf)__a);1943 typedef float __mm_storeh_pi_v2f32 __attribute__((__vector_size__(8)));
1944 struct __mm_storeh_pi_struct {
1945 __mm_storeh_pi_v2f32 __u;
1946 } __attribute__((__packed__, __may_alias__));
1947 ((struct __mm_storeh_pi_struct*)__p)->__u = __builtin_shufflevector(__a, __a, 0, 1);
1952}1948}
19531949
1954/// Stores the lower 32 bits of a 128-bit vector of [4 x float] to a1950/// Stores the lower 32 bits of a 128-bit vector of [4 x float] to a
...@@ -1987,7 +1983,7 @@ static __inline__ void __DEFAULT_FN_ATTRS...@@ -1987,7 +1983,7 @@ static __inline__ void __DEFAULT_FN_ATTRS
1987_mm_storeu_ps(float *__p, __m128 __a)1983_mm_storeu_ps(float *__p, __m128 __a)
1988{1984{
1989 struct __storeu_ps {1985 struct __storeu_ps {
1990 __m128 __v;1986 __m128_u __v;
1991 } __attribute__((__packed__, __may_alias__));1987 } __attribute__((__packed__, __may_alias__));
1992 ((struct __storeu_ps*)__p)->__v = __a;1988 ((struct __storeu_ps*)__p)->__v = __a;
1993}1989}
lib/include/xopintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- xopintrin.h - XOP intrinsics -------------------------------------===1/*===---- xopintrin.h - XOP intrinsics -------------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/xsavecintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- xsavecintrin.h - XSAVEC intrinsic --------------------------------===1/*===---- xsavecintrin.h - XSAVEC intrinsic --------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/xsaveintrin.h+22-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- xsaveintrin.h - XSAVE intrinsic ----------------------------------===1/*===---- xsaveintrin.h - XSAVE intrinsic ----------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
...@@ -28,6 +14,10 @@...@@ -28,6 +14,10 @@
28#ifndef __XSAVEINTRIN_H14#ifndef __XSAVEINTRIN_H
29#define __XSAVEINTRIN_H15#define __XSAVEINTRIN_H
3016
17#ifdef _MSC_VER
18#define _XCR_XFEATURE_ENABLED_MASK 0
19#endif
20
31/* Define the default attributes for the functions in this file. */21/* Define the default attributes for the functions in this file. */
32#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("xsave")))22#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("xsave")))
3323
...@@ -41,6 +31,20 @@ _xrstor(void *__p, unsigned long long __m) {...@@ -41,6 +31,20 @@ _xrstor(void *__p, unsigned long long __m) {
41 __builtin_ia32_xrstor(__p, __m);31 __builtin_ia32_xrstor(__p, __m);
42}32}
4333
34#ifndef _MSC_VER
35#define _xgetbv(A) __builtin_ia32_xgetbv((long long)(A))
36#define _xsetbv(A, B) __builtin_ia32_xsetbv((unsigned int)(A), (unsigned long long)(B))
37#else
38#ifdef __cplusplus
39extern "C" {
40#endif
41unsigned __int64 __cdecl _xgetbv(unsigned int);
42void __cdecl _xsetbv(unsigned int, unsigned __int64);
43#ifdef __cplusplus
44}
45#endif
46#endif /* _MSC_VER */
47
44#ifdef __x86_64__48#ifdef __x86_64__
45static __inline__ void __DEFAULT_FN_ATTRS49static __inline__ void __DEFAULT_FN_ATTRS
46_xsave64(void *__p, unsigned long long __m) {50_xsave64(void *__p, unsigned long long __m) {
...@@ -51,6 +55,7 @@ static __inline__ void __DEFAULT_FN_ATTRS...@@ -51,6 +55,7 @@ static __inline__ void __DEFAULT_FN_ATTRS
51_xrstor64(void *__p, unsigned long long __m) {55_xrstor64(void *__p, unsigned long long __m) {
52 __builtin_ia32_xrstor64(__p, __m);56 __builtin_ia32_xrstor64(__p, __m);
53}57}
58
54#endif59#endif
5560
56#undef __DEFAULT_FN_ATTRS61#undef __DEFAULT_FN_ATTRS
lib/include/xsaveoptintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- xsaveoptintrin.h - XSAVEOPT intrinsic ----------------------------===1/*===---- xsaveoptintrin.h - XSAVEOPT intrinsic ----------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/xsavesintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- xsavesintrin.h - XSAVES intrinsic --------------------------------===1/*===---- xsavesintrin.h - XSAVES intrinsic --------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */
lib/include/xtestintrin.h+3-17
...@@ -1,22 +1,8 @@...@@ -1,22 +1,8 @@
1/*===---- xtestintrin.h - XTEST intrinsic ----------------------------------===1/*===---- xtestintrin.h - XTEST intrinsic ----------------------------------===
2 *2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * of this software and associated documentation files (the "Software"), to deal4 * See https://llvm.org/LICENSE.txt for license information.
5 * in the Software without restriction, including without limitation the rights5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *6 *
21 *===-----------------------------------------------------------------------===7 *===-----------------------------------------------------------------------===
22 */8 */