From 9f88510f6f3531461dad8a660ba77b944510d0fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20R=C3=B8nne=20Petersen?= Date: Mon, 3 Aug 2026 12:13:28 +0200 Subject: [PATCH] zig cc: update intrinsic headers to LLVM 23 --- .../__clang_spirv_libdevice_declares.h | 170 + lib/include/__clang_spirv_math.h | 719 ++ lib/include/altivec.h | 88 +- lib/include/amdhsa_abi.h | 83 + lib/include/amo.h | 89 + lib/include/amxtf32intrin.h | 108 - lib/include/andes_vector.h | 5 + lib/include/arm64_neon.h | 21 + lib/include/arm64intr.h | 10 + lib/include/arm_acle.h | 1 + lib/include/arm_neon.h | 7262 +++++++++-------- lib/include/arm_sme.h | 12 + lib/include/arm_sve.h | 420 + lib/include/avx2intrin.h | 9 +- lib/include/avx512bmmintrin.h | 174 + lib/include/avx512bmmvlintrin.h | 245 + lib/include/avx512bwintrin.h | 48 +- lib/include/avx512fintrin.h | 151 +- lib/include/avx512fp16intrin.h | 30 +- lib/include/avx512vbmi2intrin.h | 28 +- lib/include/avx512vlbwintrin.h | 60 +- lib/include/avx512vlintrin.h | 176 +- lib/include/avx512vlvbmi2intrin.h | 56 +- lib/include/avx512vlvnniintrin.h | 63 +- lib/include/avx512vnniintrin.h | 67 +- lib/include/avx512vp2intersectintrin.h | 2 +- lib/include/avxvnniintrin.h | 41 +- lib/include/crc32intrin.h | 18 +- lib/include/emmintrin.h | 16 +- lib/include/endian.h | 91 + lib/include/hvx_hexagon_protos.h | 182 +- lib/include/immintrin.h | 6 +- lib/include/intrin.h | 30 +- lib/include/larchintrin.h | 2 + lib/include/llvm_libc_wrappers/ctype.h | 5 +- lib/include/llvm_libc_wrappers/string.h | 5 +- lib/include/module.modulemap | 21 +- .../__clang_openmp_device_functions.h | 14 + lib/include/openmp_wrappers/complex | 8 +- lib/include/openmp_wrappers/complex.h | 6 + lib/include/openmp_wrappers/math.h | 10 + lib/include/ptrauth.h | 102 +- lib/include/riscv_bitmanip.h | 7 + lib/include/riscv_corev_alu.h | 2 + lib/include/riscv_crypto.h | 8 + lib/include/riscv_mips.h | 2 + lib/include/riscv_nds.h | 3 + lib/include/riscv_ntlh.h | 2 + lib/include/riscv_packed_simd.h | 644 ++ lib/include/riscv_vector.h | 43 + lib/include/sifive_vector.h | 18 + lib/include/spirvintrin.h | 31 +- lib/include/stdint.h | 19 +- lib/include/vecintrin.h | 2 +- lib/include/wasm_simd128.h | 17 + lib/include/xmmintrin.h | 15 +- 56 files changed, 7215 insertions(+), 4252 deletions(-) create mode 100644 lib/include/__clang_spirv_libdevice_declares.h create mode 100644 lib/include/__clang_spirv_math.h create mode 100644 lib/include/amdhsa_abi.h delete mode 100644 lib/include/amxtf32intrin.h create mode 100644 lib/include/arm64_neon.h create mode 100644 lib/include/avx512bmmintrin.h create mode 100644 lib/include/avx512bmmvlintrin.h create mode 100644 lib/include/endian.h create mode 100644 lib/include/riscv_packed_simd.h diff --git a/lib/include/__clang_spirv_libdevice_declares.h b/lib/include/__clang_spirv_libdevice_declares.h new file mode 100644 index 0000000000000000000000000000000000000000..3530c92305c4984a83cf2b7f9637a5dcb2c53b60 --- /dev/null +++ b/lib/include/__clang_spirv_libdevice_declares.h @@ -0,0 +1,170 @@ +/*===-- __clang_spirv_libdevice_declares.h - decls for libdevice functions --=== + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===-----------------------------------------------------------------------=== + */ + +#ifndef __CLANG_SPIRV_LIBDEVICE_DECLARES_H__ +#define __CLANG_SPIRV_LIBDEVICE_DECLARES_H__ + +#if defined(__cplusplus) +extern "C" { +#else +_Pragma("push_macro(\"bool\")"); +#define bool _Bool +#endif + +#define _CLC_OVERLOAD [[clang::overloadable]] +#define _CLC_CONSTFN [[gnu::const]] +// TODO: Add vector versions of the API in case it is required. +_CLC_OVERLOAD _CLC_CONSTFN unsigned int __spirv_ocl_s_abs(int); +_CLC_OVERLOAD _CLC_CONSTFN unsigned long __spirv_ocl_s_abs(long); +_CLC_OVERLOAD _CLC_CONSTFN unsigned long long __spirv_ocl_s_abs(long long); +_CLC_OVERLOAD _CLC_CONSTFN int __spirv_ocl_s_max(int, int); +_CLC_OVERLOAD _CLC_CONSTFN long __spirv_ocl_s_max(long, long); +_CLC_OVERLOAD _CLC_CONSTFN long long __spirv_ocl_s_max(long long, long long); +_CLC_OVERLOAD _CLC_CONSTFN int __spirv_ocl_s_min(int, int); +_CLC_OVERLOAD _CLC_CONSTFN long __spirv_ocl_s_min(long, long); +_CLC_OVERLOAD _CLC_CONSTFN long long __spirv_ocl_s_min(long long, long long); +_CLC_OVERLOAD _CLC_CONSTFN unsigned int __spirv_ocl_u_max(unsigned int, + unsigned int); +_CLC_OVERLOAD _CLC_CONSTFN unsigned long __spirv_ocl_u_max(unsigned long, + unsigned long); +_CLC_OVERLOAD _CLC_CONSTFN unsigned long long +__spirv_ocl_u_max(unsigned long long, unsigned long long); +_CLC_OVERLOAD _CLC_CONSTFN unsigned int __spirv_ocl_u_min(unsigned int, + unsigned int); +_CLC_OVERLOAD _CLC_CONSTFN unsigned long __spirv_ocl_u_min(unsigned long, + unsigned long); +_CLC_OVERLOAD _CLC_CONSTFN unsigned long long +__spirv_ocl_u_min(unsigned long long, unsigned long long); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_acos(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_acos(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_acosh(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_acosh(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_asin(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_asin(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_asinh(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_asinh(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_atan(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_atan(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_atan2(float, float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_atan2(double, double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_atanh(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_atanh(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_cbrt(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_cbrt(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_ceil(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_ceil(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_cos(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_cos(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_cosh(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_cosh(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_cospi(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_cospi(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_erf(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_erf(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_erfc(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_erfc(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_exp(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_exp(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_exp2(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_exp2(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_exp10(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_exp10(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_expm1(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_expm1(double); +_CLC_OVERLOAD _CLC_CONSTFN bool __spirv_IsNan(float); +_CLC_OVERLOAD _CLC_CONSTFN bool __spirv_IsNan(double); +_CLC_OVERLOAD _CLC_CONSTFN bool __spirv_IsInf(float); +_CLC_OVERLOAD _CLC_CONSTFN bool __spirv_IsInf(double); +_CLC_OVERLOAD _CLC_CONSTFN bool __spirv_IsFinite(float); +_CLC_OVERLOAD _CLC_CONSTFN bool __spirv_IsFinite(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_copysign(float, float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_copysign(double, double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_ldexp(float, int); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_ldexp(double, int); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_fabs(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_fabs(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_logb(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_logb(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_fmax(float, float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_fmax(double, double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_fmin(float, float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_fmin(double, double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_fdim(float, float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_fdim(double, double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_floor(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_floor(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_fma(float, float, float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_fma(double, double, double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_fmod(float, float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_fmod(double, double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_frexp(float, int *); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_frexp(double, int *); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_hypot(float, float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_hypot(double, double); +_CLC_OVERLOAD _CLC_CONSTFN int __spirv_ocl_ilogb(float); +_CLC_OVERLOAD _CLC_CONSTFN int __spirv_ocl_ilogb(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_lgamma(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_lgamma(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_round(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_round(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_log(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_log(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_log10(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_log10(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_log1p(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_log1p(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_log2(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_log2(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_modf(float, float *); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_modf(double, double *); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_nan(int); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_nan(unsigned int); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_nan(long); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_nan(unsigned long); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_nextafter(float, float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_nextafter(double, double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_sqrt(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_sqrt(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_rsqrt(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_rsqrt(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_pow(float, float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_pow(double, double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_pown(float, int); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_pown(double, int); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_remainder(float, float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_remainder(double, double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_remquo(float, float, int *); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_remquo(double, double, int *); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_sin(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_sin(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_sincos(float, float *); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_sincos(double, double *); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_sinh(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_sinh(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_sinpi(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_sinpi(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_tan(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_tan(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_tanh(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_tanh(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_tgamma(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_tgamma(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_trunc(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_trunc(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_rint(float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_rint(double); +_CLC_OVERLOAD _CLC_CONSTFN float __spirv_ocl_fclamp(float, float, float); +_CLC_OVERLOAD _CLC_CONSTFN double __spirv_ocl_fclamp(double, double, double); + +#if defined(__cplusplus) +} // extern "C" +#else +_Pragma("pop_macro(\"bool\")"); +#endif +#endif // __CLANG_SPIRV_LIBDEVICE_DECLARES_H__ diff --git a/lib/include/__clang_spirv_math.h b/lib/include/__clang_spirv_math.h new file mode 100644 index 0000000000000000000000000000000000000000..761083c10fda1327c1df0b4e1dd36e2cdb0c7036 --- /dev/null +++ b/lib/include/__clang_spirv_math.h @@ -0,0 +1,719 @@ +/*===---- __clang_spirv_math.h - Device-side SPIRV math support ------------=== + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===-----------------------------------------------------------------------=== + */ +#ifndef __CLANG_SPIRV_MATH_H__ +#define __CLANG_SPIRV_MATH_H__ + +#if !defined(__SPIRV__) && !defined(__OPENMP_SPIRV__) +#error "This file is for SPIRV and OpenMP SPIRV device compilation only." +#endif + +// The __CLANG_GPU_DISABLE_MATH_WRAPPERS macro provides a way to let standard +// libcalls reach the link step instead of being eagerly replaced. +#ifndef __CLANG_GPU_DISABLE_MATH_WRAPPERS + +// __DEVICE__ is a helper macro with common set of attributes for the wrappers +// we implement in this file. We need static in order to avoid emitting unused +// functions and __forceinline__ helps inlining these wrappers at -O1. +#pragma push_macro("__DEVICE__") +#ifdef __OPENMP_SPIRV__ +#if defined(__cplusplus) +#define __DEVICE__ static constexpr __attribute__((always_inline, nothrow)) +#else +#define __DEVICE__ static __attribute__((always_inline, nothrow)) +#endif +#else +#define __DEVICE__ static __device__ __forceinline__ +#endif + +__DEVICE__ +float __cosf(float __x) { return __spirv_ocl_cos(__x); } +__DEVICE__ +float __exp10f(float __x) { return __spirv_ocl_exp10(__x); } +__DEVICE__ +float __expf(float __x) { return __spirv_ocl_exp(__x); } + +__DEVICE__ +float __fadd_rd(float __x, float __y) { + float sum = __x + __y; + float rounded = __spirv_ocl_floor(sum); + if (rounded > sum) + rounded -= 1.0f; + return rounded; +} + +__DEVICE__ +float __fadd_rn(float __x, float __y) { return __spirv_ocl_rint(__x + __y); } +__DEVICE__ +float __fadd_ru(float __x, float __y) { return __spirv_ocl_ceil(__x + __y); } +__DEVICE__ +float __fadd_rz(float __x, float __y) { return __spirv_ocl_trunc(__x + __y); } + +__DEVICE__ +float __fdiv_rd(float __x, float __y) { + float res = __x / __y; + float rounded = __spirv_ocl_floor(res); + if (rounded > res) + rounded -= 1.0f; + return rounded; +} +__DEVICE__ +float __fdiv_rn(float __x, float __y) { return __spirv_ocl_rint(__x / __y); } +__DEVICE__ +float __fdiv_ru(float __x, float __y) { return __spirv_ocl_ceil(__x / __y); } +__DEVICE__ +float __fdiv_rz(float __x, float __y) { return __spirv_ocl_trunc(__x / __y); } +__DEVICE__ +float __fdividef(float __x, float __y) { return __x / __y; } + +__DEVICE__ +float __fmaf_rd(float __x, float __y, float __z) { + float res = __x * __y + __z; + float rounded = __spirv_ocl_floor(res); + if (rounded > res) + rounded -= 1.0f; + return rounded; +} +__DEVICE__ +float __fmaf_rn(float __x, float __y, float __z) { + return __spirv_ocl_rint(__x * __y + __z); +} +__DEVICE__ +float __fmaf_ru(float __x, float __y, float __z) { + return __spirv_ocl_ceil(__x * __y + __z); +} +__DEVICE__ +float __fmaf_rz(float __x, float __y, float __z) { + return __spirv_ocl_trunc(__x * __y + __z); +} + +__DEVICE__ +float __fmul_rd(float __x, float __y) { + float res = __x * __y; + float rounded = __spirv_ocl_floor(res); + if (rounded > res) + rounded -= 1.0f; + return rounded; +} +__DEVICE__ +float __fmul_rn(float __x, float __y) { return __spirv_ocl_rint(__x * __y); } +__DEVICE__ +float __fmul_ru(float __x, float __y) { return __spirv_ocl_ceil(__x * __y); } +__DEVICE__ +float __fmul_rz(float __x, float __y) { return __spirv_ocl_trunc(__x * __y); } + +__DEVICE__ +float __frcp_rd(float __x) { return __fdiv_rd(1.0f, __x); } +__DEVICE__ +float __frcp_rn(float __x) { return __fdiv_rn(1.0f, __x); } +__DEVICE__ +float __frcp_ru(float __x) { return __fdiv_ru(1.0f, __x); } +__DEVICE__ +float __frcp_rz(float __x) { return __fdiv_rz(1.0f, __x); } +__DEVICE__ + +float __frsqrt_rn(float __x) { + return __spirv_ocl_rint(__spirv_ocl_rsqrt(__x)); +} + +__DEVICE__ +float __fsqrt_rd(float __x) { + float res = __spirv_ocl_sqrt(__x); + float rounded = __spirv_ocl_floor(res); + if (rounded > res) + rounded -= 1.0f; + return rounded; +} +__DEVICE__ +float __fsqrt_rn(float __x) { return __spirv_ocl_rint(__spirv_ocl_sqrt(__x)); } +__DEVICE__ +float __fsqrt_ru(float __x) { return __spirv_ocl_ceil(__spirv_ocl_sqrt(__x)); } +__DEVICE__ +float __fsqrt_rz(float __x) { return __spirv_ocl_trunc(__spirv_ocl_sqrt(__x)); } + +__DEVICE__ +float __fsub_rd(float __x, float __y) { + float res = __x - __y; + float rounded = __spirv_ocl_floor(res); + if (rounded > res) + rounded -= 1.0f; + return rounded; +} +__DEVICE__ +float __fsub_rn(float __x, float __y) { return __spirv_ocl_rint(__x - __y); } +__DEVICE__ +float __fsub_ru(float __x, float __y) { return __spirv_ocl_ceil(__x - __y); } +__DEVICE__ +float __fsub_rz(float __x, float __y) { return __spirv_ocl_trunc(__x - __y); } +__DEVICE__ +float __log10f(float __x) { return __spirv_ocl_log10(__x); } +__DEVICE__ +float __log2f(float __x) { return __spirv_ocl_log2(__x); } +__DEVICE__ +float __logf(float __x) { return __spirv_ocl_log(__x); } +__DEVICE__ +float __powf(float __x, float __y) { return __spirv_ocl_pow(__x, __y); } + +__DEVICE__ +float __saturatef(float __x) { return __spirv_ocl_fclamp(__x, 0.0f, 1.0f); } + +__DEVICE__ +void __sincosf(float __x, float *__sinptr, float *__cosptr) { + *__sinptr = __spirv_ocl_sincos(__x, __cosptr); +} + +__DEVICE__ +float __sinf(float __x) { return __spirv_ocl_sin(__x); } + +__DEVICE__ +float __tanf(float __x) { return __spirv_ocl_tan(__x); } + +__DEVICE__ +int __finitef(float __x) { return !__spirv_IsInf(__x) && !__spirv_IsNan(__x); } +__DEVICE__ +int __isinff(float __x) { return __spirv_IsInf(__x); } +__DEVICE__ +int __isnanf(float __x) { return __spirv_IsNan(__x); } +__DEVICE__ +int __signbitf(float __x) { return __builtin_signbitf(__x); } + +__DEVICE__ +int __finite(double __x) { return !__spirv_IsInf(__x) && !__spirv_IsNan(__x); } + +__DEVICE__ +int __isinf(double __x) { return __spirv_IsInf(__x); } + +__DEVICE__ +int __isnan(double __x) { return __spirv_IsNan(__x); } +__DEVICE__ +int __signbit(double __x) { return __builtin_signbit(__x); } + +__DEVICE__ +double __dadd_rd(double __x, double __y) { + double sum = __x + __y; + double rounded = __spirv_ocl_floor(sum); + if (rounded > sum) + rounded -= 1.0; + return rounded; +} +__DEVICE__ +double __dadd_rn(double __x, double __y) { return __spirv_ocl_rint(__x + __y); } +__DEVICE__ +double __dadd_ru(double __x, double __y) { return __spirv_ocl_ceil(__x + __y); } +__DEVICE__ +double __dadd_rz(double __x, double __y) { + return __spirv_ocl_trunc(__x + __y); +} +__DEVICE__ +double __ddiv_rd(double __x, double __y) { + double res = __x / __y; + double rounded = __spirv_ocl_floor(res); + if (rounded > res) + rounded -= 1.0; + return rounded; +} +__DEVICE__ +double __ddiv_rn(double __x, double __y) { return __spirv_ocl_rint(__x / __y); } +__DEVICE__ +double __ddiv_ru(double __x, double __y) { return __spirv_ocl_ceil(__x / __y); } +__DEVICE__ +double __ddiv_rz(double __x, double __y) { + return __spirv_ocl_trunc(__x / __y); +} + +__DEVICE__ +double __dmul_rd(double __x, double __y) { + double res = __x * __y; + double rounded = __spirv_ocl_floor(res); + if (rounded > res) + rounded -= 1.0; + return rounded; +} +__DEVICE__ +double __dmul_rn(double __x, double __y) { return __spirv_ocl_rint(__x * __y); } +__DEVICE__ +double __dmul_ru(double __x, double __y) { return __spirv_ocl_ceil(__x * __y); } +__DEVICE__ +double __dmul_rz(double __x, double __y) { + return __spirv_ocl_trunc(__x * __y); +} + +__DEVICE__ +double __drcp_rd(double __x) { return __ddiv_rd(1.0, __x); } +__DEVICE__ +double __drcp_rn(double __x) { return __ddiv_rn(1.0, __x); } +__DEVICE__ +double __drcp_ru(double __x) { return __ddiv_ru(1.0, __x); } +__DEVICE__ +double __drcp_rz(double __x) { return __ddiv_rz(1.0, __x); } + +__DEVICE__ +double __dsqrt_rd(double __x) { + double res = __spirv_ocl_sqrt(__x); + double rounded = __spirv_ocl_floor(res); + if (rounded > res) + rounded -= 1.0; + return rounded; +} +__DEVICE__ +double __dsqrt_rn(double __x) { + return __spirv_ocl_rint(__spirv_ocl_sqrt(__x)); +} +__DEVICE__ +double __dsqrt_ru(double __x) { + return __spirv_ocl_ceil(__spirv_ocl_sqrt(__x)); +} +__DEVICE__ +double __dsqrt_rz(double __x) { + return __spirv_ocl_trunc(__spirv_ocl_sqrt(__x)); +} + +__DEVICE__ +double __dsub_rd(double __x, double __y) { + double res = __x - __y; + double rounded = __spirv_ocl_floor(res); + if (rounded > res) + rounded -= 1.0; + return rounded; +} +__DEVICE__ +double __dsub_rn(double __x, double __y) { return __spirv_ocl_rint(__x - __y); } +__DEVICE__ +double __dsub_ru(double __x, double __y) { return __spirv_ocl_ceil(__x - __y); } +__DEVICE__ +double __dsub_rz(double __x, double __y) { + return __spirv_ocl_trunc(__x - __y); +} + +__DEVICE__ +double __fma_rd(double __x, double __y, double __z) { + double res = __x * __y + __z; + double rounded = __spirv_ocl_floor(res); + if (rounded > res) + rounded -= 1.0; + return rounded; +} +__DEVICE__ +double __fma_rn(double __x, double __y, double __z) { + return __spirv_ocl_rint(__x * __y + __z); +} +__DEVICE__ +double __fma_ru(double __x, double __y, double __z) { + return __spirv_ocl_ceil(__x * __y + __z); +} +__DEVICE__ +double __fma_rz(double __x, double __y, double __z) { + return __spirv_ocl_trunc(__x * __y + __z); +} + +__DEVICE__ int abs(int __a) { return __spirv_ocl_s_abs(__a); } +__DEVICE__ double fabs(double __a) { return __spirv_ocl_fabs(__a); } +__DEVICE__ double acos(double __a) { return __spirv_ocl_acos(__a); } +__DEVICE__ float acosf(float __a) { return __spirv_ocl_acos(__a); } +__DEVICE__ double acosh(double __a) { return __spirv_ocl_acosh(__a); } +__DEVICE__ float acoshf(float __a) { return __spirv_ocl_acosh(__a); } +__DEVICE__ double asin(double __a) { return __spirv_ocl_asin(__a); } +__DEVICE__ float asinf(float __a) { return __spirv_ocl_asin(__a); } +__DEVICE__ double asinh(double __a) { return __spirv_ocl_asinh(__a); } +__DEVICE__ float asinhf(float __a) { return __spirv_ocl_asinh(__a); } +__DEVICE__ double atan(double __a) { return __spirv_ocl_atan(__a); } +__DEVICE__ double atan2(double __a, double __b) { + return __spirv_ocl_atan2(__a, __b); +} +__DEVICE__ float atan2f(float __a, float __b) { + return __spirv_ocl_atan2(__a, __b); +} +__DEVICE__ float atanf(float __a) { return __spirv_ocl_atan(__a); } +__DEVICE__ double atanh(double __a) { return __spirv_ocl_atanh(__a); } +__DEVICE__ float atanhf(float __a) { return __spirv_ocl_atanh(__a); } +__DEVICE__ double cbrt(double __a) { return __spirv_ocl_cbrt(__a); } +__DEVICE__ float cbrtf(float __a) { return __spirv_ocl_cbrt(__a); } +__DEVICE__ double ceil(double __a) { return __spirv_ocl_ceil(__a); } +__DEVICE__ float ceilf(float __a) { return __spirv_ocl_ceil(__a); } +__DEVICE__ double copysign(double __a, double __b) { + return __spirv_ocl_copysign(__a, __b); +} +__DEVICE__ float copysignf(float __a, float __b) { + return __spirv_ocl_copysign(__a, __b); +} +__DEVICE__ double cos(double __a) { return __spirv_ocl_cos(__a); } +__DEVICE__ float cosf(float __a) { return __spirv_ocl_cos(__a); } +__DEVICE__ double cosh(double __a) { return __spirv_ocl_cosh(__a); } +__DEVICE__ float coshf(float __a) { return __spirv_ocl_cosh(__a); } +__DEVICE__ double cospi(double __a) { return __spirv_ocl_cospi(__a); } +__DEVICE__ float cospif(float __a) { return __spirv_ocl_cospi(__a); } +__DEVICE__ double erf(double __a) { return __spirv_ocl_erf(__a); } +__DEVICE__ double erfc(double __a) { return __spirv_ocl_erfc(__a); } +__DEVICE__ float erfcf(float __a) { return __spirv_ocl_erfc(__a); } +__DEVICE__ double erfcx(double __a) { + return __spirv_ocl_exp(__a * __a) * __spirv_ocl_erfc(__a); +} +__DEVICE__ float erfcxf(float __a) { + return __spirv_ocl_exp(__a * __a) * __spirv_ocl_erfc(__a); +} +__DEVICE__ float erff(float __a) { return __spirv_ocl_erf(__a); } +__DEVICE__ double exp(double __a) { return __spirv_ocl_exp(__a); } +__DEVICE__ double exp10(double __a) { return __spirv_ocl_exp10(__a); } +__DEVICE__ float exp10f(float __a) { return __spirv_ocl_exp10(__a); } +__DEVICE__ double exp2(double __a) { return __spirv_ocl_exp2(__a); } +__DEVICE__ float exp2f(float __a) { return __spirv_ocl_exp2(__a); } +__DEVICE__ float expf(float __a) { return __spirv_ocl_exp(__a); } +__DEVICE__ double expm1(double __a) { return __spirv_ocl_expm1(__a); } +__DEVICE__ float expm1f(float __a) { return __spirv_ocl_expm1(__a); } +__DEVICE__ float fabsf(float __a) { return __spirv_ocl_fabs(__a); } +__DEVICE__ double fdim(double __a, double __b) { + return __spirv_ocl_fdim(__a, __b); +} +__DEVICE__ float fdimf(float __a, float __b) { + return __spirv_ocl_fdim(__a, __b); +} +__DEVICE__ double fdivide(double __a, double __b) { return __a / __b; } +__DEVICE__ float fdividef(float __a, float __b) { return __a / __b; } +__DEVICE__ double floor(double __f) { return __spirv_ocl_floor(__f); } +__DEVICE__ float floorf(float __f) { return __spirv_ocl_floor(__f); } +__DEVICE__ double fma(double __a, double __b, double __c) { + return __spirv_ocl_fma(__a, __b, __c); +} +__DEVICE__ float fmaf(float __a, float __b, float __c) { + return __spirv_ocl_fma(__a, __b, __c); +} +__DEVICE__ double fmax(double __a, double __b) { + return __spirv_ocl_fmax(__a, __b); +} +__DEVICE__ float fmaxf(float __a, float __b) { + return __spirv_ocl_fmax(__a, __b); +} +__DEVICE__ double fmin(double __a, double __b) { + return __spirv_ocl_fmin(__a, __b); +} +__DEVICE__ float fminf(float __a, float __b) { + return __spirv_ocl_fmin(__a, __b); +} +__DEVICE__ double fmod(double __a, double __b) { + return __spirv_ocl_fmod(__a, __b); +} +__DEVICE__ float fmodf(float __a, float __b) { + return __spirv_ocl_fmod(__a, __b); +} +__DEVICE__ double frexp(double __a, int *__b) { + return __spirv_ocl_frexp(__a, __b); +} +__DEVICE__ float frexpf(float __a, int *__b) { + return __spirv_ocl_frexp(__a, __b); +} +__DEVICE__ double hypot(double __a, double __b) { + return __spirv_ocl_hypot(__a, __b); +} +__DEVICE__ float hypotf(float __a, float __b) { + return __spirv_ocl_hypot(__a, __b); +} +__DEVICE__ int ilogb(double __a) { return __spirv_ocl_ilogb(__a); } +__DEVICE__ int ilogbf(float __a) { return __spirv_ocl_ilogb(__a); } +__DEVICE__ long labs(long __a) { return __spirv_ocl_s_abs(__a); }; +__DEVICE__ double ldexp(double __a, int __b) { + return __spirv_ocl_ldexp(__a, __b); +} +__DEVICE__ float ldexpf(float __a, int __b) { + return __spirv_ocl_ldexp(__a, __b); +} +__DEVICE__ double lgamma(double __a) { return __spirv_ocl_lgamma(__a); } +__DEVICE__ float lgammaf(float __a) { return __spirv_ocl_lgamma(__a); } +__DEVICE__ long long llabs(long long __a) { return __spirv_ocl_s_abs(__a); } +__DEVICE__ long long llmax(long long __a, long long __b) { + return __spirv_ocl_s_max(__a, __b); +} +__DEVICE__ long long llmin(long long __a, long long __b) { + return __spirv_ocl_s_min(__a, __b); +} +__DEVICE__ long long llrint(double __a) { return __builtin_rint(__a); } +__DEVICE__ long long llrintf(float __a) { return __builtin_rintf(__a); } +__DEVICE__ long long llround(double __a) { return __builtin_round(__a); } +__DEVICE__ long long llroundf(float __a) { return __builtin_roundf(__a); } +__DEVICE__ double round(double __a) { return __spirv_ocl_round(__a); } +__DEVICE__ float roundf(float __a) { return __spirv_ocl_round(__a); } +__DEVICE__ double log(double __a) { return __spirv_ocl_log(__a); } +__DEVICE__ double log10(double __a) { return __spirv_ocl_log10(__a); } +__DEVICE__ float log10f(float __a) { return __spirv_ocl_log10(__a); } +__DEVICE__ double log1p(double __a) { return __spirv_ocl_log1p(__a); } +__DEVICE__ float log1pf(float __a) { return __spirv_ocl_log1p(__a); } +__DEVICE__ double log2(double __a) { return __spirv_ocl_log2(__a); } +__DEVICE__ float log2f(float __a) { return __spirv_ocl_log2(__a); } +__DEVICE__ double logb(double __a) { return __spirv_ocl_logb(__a); } +__DEVICE__ float logbf(float __a) { return __spirv_ocl_logb(__a); } +__DEVICE__ float logf(float __a) { return __spirv_ocl_log(__a); } +__DEVICE__ long lrint(double __a) { return __builtin_rint(__a); } +__DEVICE__ long lrintf(float __a) { return __builtin_rintf(__a); } +__DEVICE__ long lround(double __a) { return __builtin_round(__a); } +__DEVICE__ long lroundf(float __a) { return __builtin_roundf(__a); } +__DEVICE__ int max(int __a, int __b) { return __spirv_ocl_s_max(__a, __b); } +__DEVICE__ int min(int __a, int __b) { return __spirv_ocl_s_min(__a, __b); } +__DEVICE__ double modf(double __a, double *__b) { + return __spirv_ocl_modf(__a, __b); +} +__DEVICE__ float modff(float __a, float *__b) { + return __spirv_ocl_modf(__a, __b); +} +__DEVICE__ double nearbyint(double __a) { return __spirv_ocl_rint(__a); } +__DEVICE__ float nearbyintf(float __a) { return __spirv_ocl_rint(__a); } +__DEVICE__ double nextafter(double __a, double __b) { + return __spirv_ocl_nextafter(__a, __b); +} +__DEVICE__ float nextafterf(float __a, float __b) { + return __spirv_ocl_nextafter(__a, __b); +} + +__DEVICE__ double norm(int __dim, const double *__a) { + double __r = 0; + while (__dim--) { + __r += __a[0] * __a[0]; + ++__a; + } + + return __spirv_ocl_sqrt(__r); +} +__DEVICE__ double norm3d(double __a, double __b, double __c) { + return __spirv_ocl_sqrt(__a * __a + __b * __b + __c * __c); +} +__DEVICE__ float norm3df(float __a, float __b, float __c) { + return __spirv_ocl_sqrt(__a * __a + __b * __b + __c * __c); +} +__DEVICE__ double norm4d(double __a, double __b, double __c, double __d) { + return __spirv_ocl_sqrt(__a * __a + __b * __b + __c * __c + __d * __d); +} +__DEVICE__ float norm4df(float __a, float __b, float __c, float __d) { + return __spirv_ocl_sqrt(__a * __a + __b * __b + __c * __c + __d * __d); +} +__DEVICE__ double normcdf(double __a) { + return 0.5 * (1.0 + __spirv_ocl_erf(__a * __spirv_ocl_rsqrt(2.0))); +} +__DEVICE__ float normcdff(float __a) { + return 0.5f * (1.0f + __spirv_ocl_erf(__a * __spirv_ocl_rsqrt(2.0f))); +} +__DEVICE__ float normf(int __dim, const float *__a) { + float __r = 0; + while (__dim--) { + __r += __a[0] * __a[0]; + ++__a; + } + return __spirv_ocl_sqrt(__r); +} +__DEVICE__ double pow(double __a, double __b) { + return __spirv_ocl_pow(__a, __b); +} +__DEVICE__ float powf(float __a, float __b) { + return __spirv_ocl_pow(__a, __b); +} +__DEVICE__ double powi(double __a, int __b) { return pow(__a, (double)__b); } +__DEVICE__ float powif(float __a, int __b) { return pow(__a, (float)__b); } +__DEVICE__ double rcbrt(double __a) { return 1.0 / __spirv_ocl_cbrt(__a); } +__DEVICE__ float rcbrtf(float __a) { return 1.0f / __spirv_ocl_cbrt(__a); } +__DEVICE__ double remainder(double __a, double __b) { + return __spirv_ocl_remainder(__a, __b); +} +__DEVICE__ float remainderf(float __a, float __b) { + return __spirv_ocl_remainder(__a, __b); +} +__DEVICE__ double remquo(double __a, double __b, int *__c) { + return __spirv_ocl_remquo(__a, __b, __c); +} +__DEVICE__ float remquof(float __a, float __b, int *__c) { + return __spirv_ocl_remquo(__a, __b, __c); +} +__DEVICE__ double rhypot(double __a, double __b) { + return __spirv_ocl_hypot(__a, __b); +} +__DEVICE__ float rhypotf(float __a, float __b) { + return __spirv_ocl_hypot(__a, __b); +} +__DEVICE__ double rint(double __a) { return __spirv_ocl_rint(__a); } +__DEVICE__ float rintf(float __a) { return __spirv_ocl_rint(__a); } +__DEVICE__ double rnorm(int __dim, const double *__a) { + double __r = 0; + while (__dim--) { + __r += __a[0] * __a[0]; + ++__a; + } + + return __spirv_ocl_rsqrt(__r); +} +__DEVICE__ double rnorm3d(double __a, double __b, double __c) { + return __spirv_ocl_rsqrt(__a * __a + __b * __b + __c * __c); +} +__DEVICE__ float rnorm3df(float __a, float __b, float __c) { + return __spirv_ocl_rsqrt(__a * __a + __b * __b + __c * __c); +} +__DEVICE__ double rnorm4d(double __a, double __b, double __c, double __d) { + return __spirv_ocl_rsqrt(__a * __a + __b * __b + __c * __c + __d * __d); +} +__DEVICE__ float rnorm4df(float __a, float __b, float __c, float __d) { + return __spirv_ocl_rsqrt(__a * __a + __b * __b + __c * __c + __d * __d); +} +__DEVICE__ float rnormf(int __dim, const float *__a) { + float __r = 0; + while (__dim--) { + __r += __a[0] * __a[0]; + ++__a; + } + + return __spirv_ocl_rsqrt(__r); +} +__DEVICE__ double rsqrt(double __a) { return __spirv_ocl_rsqrt(__a); } +__DEVICE__ float rsqrtf(float __a) { return __spirv_ocl_rsqrt(__a); } +__DEVICE__ double scalbn(double __a, int __b) { + return __spirv_ocl_ldexp(__a, __b); +} +__DEVICE__ float scalbnf(float __a, int __b) { + return __spirv_ocl_ldexp(__a, __b); +} +__DEVICE__ double scalbln(double __a, long __b) { + if (__b > INT_MAX) + return __a > 0 ? HUGE_VAL : -HUGE_VAL; + if (__b < INT_MIN) + return __a > 0 ? 0.0 : -0.0; + return scalbn(__a, (int)__b); +} +__DEVICE__ float scalblnf(float __a, long __b) { + if (__b > INT_MAX) + return __a > 0 ? HUGE_VALF : -HUGE_VALF; + if (__b < INT_MIN) + return __a > 0 ? 0.f : -0.f; + return scalbnf(__a, (int)__b); +} +__DEVICE__ double sin(double __a) { return __spirv_ocl_sin(__a); } +__DEVICE__ void sincos(double __a, double *__s, double *__c) { + *__s = __spirv_ocl_sincos(__a, __c); +} +__DEVICE__ void sincosf(float __a, float *__s, float *__c) { + *__s = __spirv_ocl_sincos(__a, __c); +} +__DEVICE__ void sincospi(double __a, double *__s, double *__c) { + *__s = __spirv_ocl_sinpi(__a); + *__c = __spirv_ocl_cospi(__a); +} +__DEVICE__ void sincospif(float __a, float *__s, float *__c) { + *__s = __spirv_ocl_sinpi(__a); + *__c = __spirv_ocl_cospi(__a); +} +__DEVICE__ float sinf(float __a) { return __spirv_ocl_sin(__a); } +__DEVICE__ double sinh(double __a) { return __spirv_ocl_sinh(__a); } +__DEVICE__ float sinhf(float __a) { return __spirv_ocl_sinh(__a); } +__DEVICE__ double sinpi(double __a) { return __spirv_ocl_sinpi(__a); } +__DEVICE__ float sinpif(float __a) { return __spirv_ocl_sinpi(__a); } +__DEVICE__ double sqrt(double __a) { return __spirv_ocl_sqrt(__a); } +__DEVICE__ float sqrtf(float __a) { return __spirv_ocl_sqrt(__a); } +__DEVICE__ double tan(double __a) { return __spirv_ocl_tan(__a); } +__DEVICE__ float tanf(float __a) { return __spirv_ocl_tan(__a); } +__DEVICE__ double tanh(double __a) { return __spirv_ocl_tanh(__a); } +__DEVICE__ float tanhf(float __a) { return __spirv_ocl_tanh(__a); } +__DEVICE__ double tgamma(double __a) { return __spirv_ocl_tgamma(__a); } +__DEVICE__ float tgammaf(float __a) { return __spirv_ocl_tgamma(__a); } +__DEVICE__ double trunc(double __a) { return __spirv_ocl_trunc(__a); } +__DEVICE__ float truncf(float __a) { return __spirv_ocl_trunc(__a); } +__DEVICE__ unsigned long long ullmax(unsigned long long __a, + unsigned long long __b) { + return __spirv_ocl_u_max(__a, __b); +} +__DEVICE__ unsigned long long ullmin(unsigned long long __a, + unsigned long long __b) { + return __spirv_ocl_u_min(__a, __b); +} +__DEVICE__ unsigned int umax(unsigned int __a, unsigned int __b) { + return __spirv_ocl_u_max(__a, __b); +} +__DEVICE__ unsigned int umin(unsigned int __a, unsigned int __b) { + return __spirv_ocl_u_min(__a, __b); +} + +#if !defined(__cplusplus) && __STDC_VERSION__ >= 201112L +#define isfinite(__x) _Generic((__x), float: __finitef, double: __finite)(__x) +#define isinf(__x) _Generic((__x), float: __isinff, double: __isinf)(__x) +#define isnan(__x) _Generic((__x), float: __isnanf, double: __isnan)(__x) +#define signbit(__x) _Generic((__x), float: __signbitf, double: __signbit)(__x) +#endif // !defined(__cplusplus) && __STDC_VERSION__ >= 201112L + +__DEVICE__ +unsigned long __make_mantissa_base8(const char *__tagp) { + unsigned long __r = 0; + while (*__tagp != '\0') { + char __tmp = *__tagp; + + if (__tmp >= '0' && __tmp <= '7') + __r = (__r * 8u) + __tmp - '0'; + else + return 0; + + ++__tagp; + } + + return __r; +} + +__DEVICE__ +unsigned long __make_mantissa_base10(const char *__tagp) { + unsigned long __r = 0; + while (*__tagp != '\0') { + char __tmp = *__tagp; + + if (__tmp >= '0' && __tmp <= '9') + __r = (__r * 10u) + __tmp - '0'; + else + return 0; + + ++__tagp; + } + + return __r; +} + +__DEVICE__ +unsigned long __make_mantissa_base16(const char *__tagp) { + unsigned long __r = 0; + while (*__tagp != '\0') { + char __tmp = *__tagp; + + if (__tmp >= '0' && __tmp <= '9') + __r = (__r * 16u) + __tmp - '0'; + else if (__tmp >= 'a' && __tmp <= 'f') + __r = (__r * 16u) + __tmp - 'a' + 10; + else if (__tmp >= 'A' && __tmp <= 'F') + __r = (__r * 16u) + __tmp - 'A' + 10; + else + return 0; + + ++__tagp; + } + + return __r; +} + +__DEVICE__ +unsigned long __make_mantissa(const char *__tagp) { + if (!__tagp) + return 0; + if (*__tagp == '0') { + ++__tagp; + + if (*__tagp == 'x' || *__tagp == 'X') + return __make_mantissa_base16(++__tagp); + else + return __make_mantissa_base8(__tagp); + } + + return __make_mantissa_base10(__tagp); +} + +float nanf(const char *__tagp) { + return __spirv_ocl_nan((unsigned int)__make_mantissa(__tagp)); +} +double nan(const char *__tagp) { + return __spirv_ocl_nan(__make_mantissa(__tagp)); +} + +#pragma pop_macro("__DEVICE__") +#endif // __CLANG_GPU_DISABLE_MATH_WRAPPERS +#endif // __CLANG_SPIRV_MATH_H__ diff --git a/lib/include/altivec.h b/lib/include/altivec.h index 71d8d3c0c0771422a501c958c52d4e523a4b3175..2df3f46823732de59daa99d7b767e7aedc851846 100644 --- a/lib/include/altivec.h +++ b/lib/include/altivec.h @@ -2117,7 +2117,7 @@ vec_cmpne(vector unsigned long long __a, vector unsigned long long __b) { } #endif -#ifdef __VSX__ +#ifdef __POWER8_VECTOR__ static __inline__ vector bool long long __ATTRS_o_ai vec_cmpne(vector double __a, vector double __b) { return (vector bool long long) @@ -6404,6 +6404,18 @@ vec_mulh(vector unsigned long long __a, vector unsigned long long __b) { } #endif +#ifdef __FUTURE_VECTOR__ +static __inline__ vector signed short + __ATTRS_o_ai vec_mulh(vector signed short __a, vector signed short __b) { + return __builtin_altivec_vmulhsh(__a, __b); +} + +static __inline__ vector unsigned short __ATTRS_o_ai +vec_mulh(vector unsigned short __a, vector unsigned short __b) { + return __builtin_altivec_vmulhuh(__a, __b); +} +#endif + /* vec_mulo */ static __inline__ vector short __ATTRS_o_ai vec_mulo(vector signed char __a, @@ -17809,6 +17821,7 @@ vec_xl(ptrdiff_t __offset, const unsigned __int128 *__ptr) { /* vec_xl_be */ #ifdef __LITTLE_ENDIAN__ +#ifdef __VSX__ static __inline__ vector signed char __ATTRS_o_ai vec_xl_be(ptrdiff_t __offset, const signed char *__ptr) { vector signed char __vec = (vector signed char)__builtin_vsx_lxvd2x_be(__offset, __ptr); @@ -17850,7 +17863,6 @@ vec_xl_be(signed long long __offset, const float *__ptr) { return (vector float)__builtin_vsx_lxvw4x_be(__offset, __ptr); } -#ifdef __VSX__ static __inline__ vector signed long long __ATTRS_o_ai vec_xl_be(signed long long __offset, const signed long long *__ptr) { return (vector signed long long)__builtin_vsx_lxvd2x_be(__offset, __ptr); @@ -17865,7 +17877,6 @@ static __inline__ vector double __ATTRS_o_ai vec_xl_be(signed long long __offset, const double *__ptr) { return (vector double)__builtin_vsx_lxvd2x_be(__offset, __ptr); } -#endif #if defined(__POWER8_VECTOR__) && defined(__powerpc64__) && \ defined(__SIZEOF_INT128__) @@ -17879,8 +17890,9 @@ vec_xl_be(signed long long __offset, const unsigned __int128 *__ptr) { return vec_xl(__offset, __ptr); } #endif -#else - #define vec_xl_be vec_xl +#endif // __VSX__ +#else // ! __LITTLE_ENDIAN__ +#define vec_xl_be vec_xl #endif #if defined(__POWER10_VECTOR__) && defined(__VSX__) && \ @@ -18130,6 +18142,7 @@ vec_xst_trunc(vector unsigned __int128 __vec, ptrdiff_t __offset, /* vec_xst_be */ #ifdef __LITTLE_ENDIAN__ +#ifdef __VSX__ static __inline__ void __ATTRS_o_ai vec_xst_be(vector signed char __vec, signed long long __offset, signed char *__ptr) { @@ -18186,7 +18199,6 @@ static __inline__ void __ATTRS_o_ai vec_xst_be(vector float __vec, __builtin_vsx_stxvw4x_be((vector int)__vec, __offset, __ptr); } -#ifdef __VSX__ static __inline__ void __ATTRS_o_ai vec_xst_be(vector signed long long __vec, signed long long __offset, signed long long *__ptr) { @@ -18204,7 +18216,6 @@ static __inline__ void __ATTRS_o_ai vec_xst_be(vector double __vec, double *__ptr) { __builtin_vsx_stxvd2x_be((vector double)__vec, __offset, __ptr); } -#endif #if defined(__POWER8_VECTOR__) && defined(__powerpc64__) && \ defined(__SIZEOF_INT128__) @@ -18220,8 +18231,9 @@ static __inline__ void __ATTRS_o_ai vec_xst_be(vector unsigned __int128 __vec, vec_xst(__vec, __offset, __ptr); } #endif -#else - #define vec_xst_be vec_xst +#endif // VSX +#else // ! __LITTLE_ENDIAN__ +#define vec_xst_be vec_xst #endif #ifdef __POWER9_VECTOR__ @@ -19314,6 +19326,64 @@ vec_sra(vector signed __int128 __a, vector unsigned __int128 __b) { #endif /* __SIZEOF_INT128__ */ #endif /* __POWER10_VECTOR__ */ +#ifdef __FUTURE_VECTOR__ + +/* vec_uncompress* - Deeply Compressed Weights builtins */ + +static __inline__ vector unsigned char __ATTRS_o_ai +vec_uncompresshn(vector unsigned char __a, vector unsigned char __b) { + return __builtin_altivec_vucmprhn(__a, __b); +} + +static __inline__ vector unsigned char __ATTRS_o_ai +vec_uncompressln(vector unsigned char __a, vector unsigned char __b) { + return __builtin_altivec_vucmprln(__a, __b); +} + +static __inline__ vector unsigned char __ATTRS_o_ai +vec_uncompresshb(vector unsigned char __a, vector unsigned char __b) { + return __builtin_altivec_vucmprhb(__a, __b); +} + +static __inline__ vector unsigned char __ATTRS_o_ai +vec_uncompresslb(vector unsigned char __a, vector unsigned char __b) { + return __builtin_altivec_vucmprlb(__a, __b); +} + +static __inline__ vector unsigned char __ATTRS_o_ai +vec_uncompresshh(vector unsigned char __a, vector unsigned char __b) { + return __builtin_altivec_vucmprhh(__a, __b); +} + +static __inline__ vector unsigned char __ATTRS_o_ai +vec_uncompresslh(vector unsigned char __a, vector unsigned char __b) { + return __builtin_altivec_vucmprlh(__a, __b); +} + +static __inline__ vector unsigned char __ATTRS_o_ai +vec_unpack_hsn_to_byte(vector unsigned char __a) { + return __builtin_altivec_vupkhsntob(__a); +} + +static __inline__ vector unsigned char __ATTRS_o_ai +vec_unpack_lsn_to_byte(vector unsigned char __a) { + return __builtin_altivec_vupklsntob(__a); +} + +#define vec_unpack_int4_to_bf16(__a, __imm) \ + __builtin_altivec_vupkint4tobf16((__a), (__imm)) + +#define vec_unpack_int8_to_bf16(__a, __imm) \ + __builtin_altivec_vupkint8tobf16((__a), (__imm)) + +#define vec_unpack_int4_to_fp32(__a, __imm) \ + __builtin_altivec_vupkint4tofp32((__a), (__imm)) + +#define vec_unpack_int8_to_fp32(__a, __imm) \ + __builtin_altivec_vupkint8tofp32((__a), (__imm)) + +#endif /* __FUTURE_VECTOR__ */ + #ifdef __POWER8_VECTOR__ #define __bcdadd(__a, __b, __ps) __builtin_ppc_bcdadd((__a), (__b), (__ps)) #define __bcdsub(__a, __b, __ps) __builtin_ppc_bcdsub((__a), (__b), (__ps)) diff --git a/lib/include/amdhsa_abi.h b/lib/include/amdhsa_abi.h new file mode 100644 index 0000000000000000000000000000000000000000..f76fe55a6cc21de99016b9056491dc45e552d2ed --- /dev/null +++ b/lib/include/amdhsa_abi.h @@ -0,0 +1,83 @@ +//===-- amdhsa_abi.h - AMDHSA ABI definition utilities --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __AMDHSA_ABI_H +#define __AMDHSA_ABI_H + +#include +#include + +typedef struct __attribute__((aligned(8))) amdhsa_implicit_kernarg_v5 { + uint32_t block_count[3]; + uint16_t group_size[3]; + uint16_t remainder[3]; + char reserved0[16]; + uint64_t global_offset[3]; + uint16_t grid_dims; + char reserved1[6]; + __attribute__((opencl_global)) void *printf_buffer; + __attribute__((opencl_global)) void *hostcall_buffer; + __attribute__((opencl_global)) void *multigrid_sync_arg; + __attribute__((opencl_global)) void *heap_v1; + __attribute__((opencl_global)) void *default_queue; + __attribute__((opencl_global)) void *completion_action; + char reserved2[72]; + uint32_t private_base; // Unused on gfx9+ + uint32_t shared_base; // Unused on gfx9+ + __attribute__((opencl_global)) void *queue_ptr; + char reserved3[48]; +} amdhsa_implicit_kernarg_v5; + +_Static_assert(sizeof(amdhsa_implicit_kernarg_v5) == 256, "wrong struct size"); + +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, block_count[0]) == 0, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, block_count[1]) == 4, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, block_count[2]) == 8, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, group_size[0]) == 12, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, group_size[1]) == 14, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, group_size[2]) == 16, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, remainder[0]) == 18, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, remainder[1]) == 20, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, remainder[2]) == 22, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, global_offset[0]) == 40, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, global_offset[1]) == 48, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, global_offset[2]) == 56, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, grid_dims) == 64, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, printf_buffer) == 72, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, hostcall_buffer) == 80, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, multigrid_sync_arg) == 88, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, heap_v1) == 96, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, default_queue) == 104, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, completion_action) == 112, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, private_base) == 192, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, shared_base) == 196, + "wrong offset"); +_Static_assert(offsetof(amdhsa_implicit_kernarg_v5, queue_ptr) == 200, + "wrong offset"); + +#endif // __AMDHSA_ABI_H diff --git a/lib/include/amo.h b/lib/include/amo.h index 97eff35e9c5dc52b3e1f55e707eaaa53657475ce..c7563f203d1f88cd1948145db34a8b7d71b3174a 100644 --- a/lib/include/amo.h +++ b/lib/include/amo.h @@ -124,6 +124,95 @@ static inline int64_t amo_ldat_sswap(int64_t *ptr, int64_t val) { return __builtin_amo_ldat_s(ptr, val, _AMO_LD_SWAP); } +/* AMO Store Operation Codes (FC values) */ +enum _AMO_ST { + _AMO_ST_ADD = 0x00, /* Store Add */ + _AMO_ST_XOR = 0x01, /* Store Xor */ + _AMO_ST_IOR = 0x02, /* Store Ior */ + _AMO_ST_AND = 0x03, /* Store And */ + _AMO_ST_UMAX = 0x04, /* Store Unsigned Maximum */ + _AMO_ST_SMAX = 0x05, /* Store Signed Maximum */ + _AMO_ST_UMIN = 0x06, /* Store Unsigned Minimum */ + _AMO_ST_SMIN = 0x07, /* Store Signed Minimum */ + _AMO_ST_TWIN = 0x18 /* Store Twin */ +}; + +/* 32-bit unsigned AMO store operations */ +static inline void amo_stwat_add(uint32_t *ptr, uint32_t val) { + __builtin_amo_stwat(ptr, val, _AMO_ST_ADD); +} + +static inline void amo_stwat_xor(uint32_t *ptr, uint32_t val) { + __builtin_amo_stwat(ptr, val, _AMO_ST_XOR); +} + +static inline void amo_stwat_ior(uint32_t *ptr, uint32_t val) { + __builtin_amo_stwat(ptr, val, _AMO_ST_IOR); +} + +static inline void amo_stwat_and(uint32_t *ptr, uint32_t val) { + __builtin_amo_stwat(ptr, val, _AMO_ST_AND); +} + +static inline void amo_stwat_umax(uint32_t *ptr, uint32_t val) { + __builtin_amo_stwat(ptr, val, _AMO_ST_UMAX); +} + +static inline void amo_stwat_umin(uint32_t *ptr, uint32_t val) { + __builtin_amo_stwat(ptr, val, _AMO_ST_UMIN); +} + +/* 32-bit signed AMO store operations */ +static inline void amo_stwat_sadd(int32_t *ptr, int32_t val) { + __builtin_amo_stwat_s(ptr, val, _AMO_ST_ADD); +} + +static inline void amo_stwat_smax(int32_t *ptr, int32_t val) { + __builtin_amo_stwat_s(ptr, val, _AMO_ST_SMAX); +} + +static inline void amo_stwat_smin(int32_t *ptr, int32_t val) { + __builtin_amo_stwat_s(ptr, val, _AMO_ST_SMIN); +} + +/* 64-bit unsigned AMO store operations */ +static inline void amo_stdat_add(uint64_t *ptr, uint64_t val) { + __builtin_amo_stdat(ptr, val, _AMO_ST_ADD); +} + +static inline void amo_stdat_xor(uint64_t *ptr, uint64_t val) { + __builtin_amo_stdat(ptr, val, _AMO_ST_XOR); +} + +static inline void amo_stdat_ior(uint64_t *ptr, uint64_t val) { + __builtin_amo_stdat(ptr, val, _AMO_ST_IOR); +} + +static inline void amo_stdat_and(uint64_t *ptr, uint64_t val) { + __builtin_amo_stdat(ptr, val, _AMO_ST_AND); +} + +static inline void amo_stdat_umax(uint64_t *ptr, uint64_t val) { + __builtin_amo_stdat(ptr, val, _AMO_ST_UMAX); +} + +static inline void amo_stdat_umin(uint64_t *ptr, uint64_t val) { + __builtin_amo_stdat(ptr, val, _AMO_ST_UMIN); +} + +/* 64-bit signed AMO store operations */ +static inline void amo_stdat_sadd(int64_t *ptr, int64_t val) { + __builtin_amo_stdat_s(ptr, val, _AMO_ST_ADD); +} + +static inline void amo_stdat_smax(int64_t *ptr, int64_t val) { + __builtin_amo_stdat_s(ptr, val, _AMO_ST_SMAX); +} + +static inline void amo_stdat_smin(int64_t *ptr, int64_t val) { + __builtin_amo_stdat_s(ptr, val, _AMO_ST_SMIN); +} + #ifdef __cplusplus } #endif diff --git a/lib/include/amxtf32intrin.h b/lib/include/amxtf32intrin.h deleted file mode 100644 index 44d002c6600d6fbd230b01ff7923d86871b3ab4f..0000000000000000000000000000000000000000 --- a/lib/include/amxtf32intrin.h +++ /dev/null @@ -1,108 +0,0 @@ -/*===------------- amxtf32intrin.h - AMX_TF32 intrinsics -*- C++ -*---------=== - * - * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. - * See https://llvm.org/LICENSE.txt for license information. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - * - *===------------------------------------------------------------------------=== - */ - -#ifndef __IMMINTRIN_H -#error "Never use directly; include instead." -#endif // __IMMINTRIN_H - -#ifndef __AMX_TF32INTRIN_H -#define __AMX_TF32INTRIN_H -#ifdef __x86_64__ - -#define __DEFAULT_FN_ATTRS_TF32 \ - __attribute__((__always_inline__, __nodebug__, __target__("amx-tf32"))) - -/// Do Matrix Multiplication of \a a and \a b, and then do Matrix Plus -/// with \a srcdst. -/// All the calculation is base on float32 but with the lower 13-bit set to 0. -/// -/// \headerfile -/// -/// \code -/// void _tile_mmultf32ps(constexpr int srcdst, constexpr int a, \ -/// constexpr int b); -/// \endcode -/// -/// This intrinsic corresponds to the TMMULTF32PS instruction. -/// -/// \param srcdst -/// The destination tile. Max size is 1024 Bytes. -/// \param a -/// The 1st source tile. Max size is 1024 Bytes. -/// \param b -/// The 2nd source tile. Max size is 1024 Bytes. -/// -/// \code{.operation} -/// DEFINE zero_lower_mantissa_bits_fp32(x[31:0]) { -/// dword[12:0] := 0 -/// dword[31:13] := x[31:13] -/// return dword -/// } -/// -/// DEFINE silence_snan_fp32(x[31:0]) { -/// IF (x.exponent == 255 and x.fraction != 0 and x.fraction[22] == 0) -/// x.fraction[22] := 1 -/// return x -/// } -/// -/// elements_a := a.colsb / 4 -/// elements_dest := srcdst.colsb / 4 -/// -/// FOR m = 0 TO (srcdst.rows-1) -/// tmp[511:0] := 0 -/// FOR k = 0 TO (elements_a-1) -/// FOR n = 0 TO (elements_dest-1) -/// af := silence_snan_fp32(a.row[m].fp32[k]) -/// bf := silence_snan_fp32(b.row[k].fp32[n]) -/// tmp.fp32[n] += zero_lower_mantissa_bits_fp32(af) -/// * zero_lower_mantissa_bits_fp32(bf) -/// ENDFOR -/// ENDFOR -/// -/// FOR n = 0 TO (elements_dest-1) -/// tmp.fp32[n] += srcdst.row[m].fp32[n] -/// ENDFOR -/// write_row_and_zero(srcdst, m, tmp, srcdst.colsb) -/// -/// ENDFOR -/// -/// zero_upper_rows(srcdst, srcdst.rows) -/// zero_tileconfig_start() -/// \endcode -#define _tile_mmultf32ps(srcdst, a, b) \ - __builtin_ia32_tmmultf32ps((srcdst), (a), (b)) - -static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TF32 -_tile_mmultf32ps_internal(unsigned short m, unsigned short n, unsigned short k, - _tile1024i dst, _tile1024i src1, _tile1024i src2) { - return __builtin_ia32_tmmultf32ps_internal(m, n, k, dst, src1, src2); -} - -/// Do Matrix Multiplication of src0 and src1, and then do Matrix Plus with dst. -/// All the calculation is base on float32 but with the lower 13-bit set to 0. -/// -/// \headerfile -/// -/// This intrinsic corresponds to the TMMULTF32PS instruction. -/// -/// \param dst -/// The destination tile. Max size is 1024 Bytes. -/// \param src0 -/// The 1st source tile. Max size is 1024 Bytes. -/// \param src1 -/// The 2nd source tile. Max size is 1024 Bytes. -__DEFAULT_FN_ATTRS_TF32 -static void __tile_mmultf32ps(__tile1024i *dst, __tile1024i src0, - __tile1024i src1) { - dst->tile = _tile_mmultf32ps_internal(src0.row, src1.col, src0.col, dst->tile, - src0.tile, src1.tile); -} - -#endif // __x86_64__ -#endif // __AMX_TF32INTRIN_H diff --git a/lib/include/andes_vector.h b/lib/include/andes_vector.h index dc717e6d805b94683df9d583118208f00f645088..968a922b6a21c43248b022086df00fe1853197ac 100644 --- a/lib/include/andes_vector.h +++ b/lib/include/andes_vector.h @@ -13,4 +13,9 @@ #pragma clang riscv intrinsic andes_vector +#define __riscv_intrinsic_xandesvbfhcvt 1 +#define __riscv_intrinsic_xandesvdot 1 +#define __riscv_intrinsic_xandesvpackfph 1 +#define __riscv_intrinsic_xandesvsintload 1 + #endif //_ANDES_VECTOR_H_ diff --git a/lib/include/arm64_neon.h b/lib/include/arm64_neon.h new file mode 100644 index 0000000000000000000000000000000000000000..29803a8e455ae57646687856b9a379a1330a7a17 --- /dev/null +++ b/lib/include/arm64_neon.h @@ -0,0 +1,21 @@ +/*===---- arm64_neon.h - ARM64 NEON intrinsics -----------------------------=== + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===-----------------------------------------------------------------------=== + */ + +/* Only include this if we're compiling for the windows platform. */ +#ifndef _MSC_VER +#include_next +#else + +#ifndef __ARM64_NEON_H +#define __ARM64_NEON_H + +#include + +#endif /* __ARM64_NEON_H */ +#endif /* _MSC_VER */ diff --git a/lib/include/arm64intr.h b/lib/include/arm64intr.h index 4943b2db69d0273ff3da21560d312c4a04bc7e43..53a3d57a6e9d1b4433c4e4b5b6e48ffd18bffa98 100644 --- a/lib/include/arm64intr.h +++ b/lib/include/arm64intr.h @@ -15,6 +15,16 @@ #ifndef __ARM64INTR_H #define __ARM64INTR_H +/* Encode an AArch64 system register for use with + _ReadStatusReg/_WriteStatusReg. op0 must be 2 or 3; only the low bit is + stored. */ +#define ARM64_SYSREG(op0, op1, CRn, CRm, op2) \ + ((((op0) & 0x1) << 14) | (((op1) & 0x7) << 11) | (((CRn) & 0xF) << 7) | \ + (((CRm) & 0xF) << 3) | ((op2) & 0x7)) + +#define ARM64_FPCR ARM64_SYSREG(3, 3, 4, 4, 0) +#define ARM64_FPSR ARM64_SYSREG(3, 3, 4, 4, 1) + typedef enum { _ARM64_BARRIER_SY = 0xF, diff --git a/lib/include/arm_acle.h b/lib/include/arm_acle.h index 622e8f3d6aa7b6f4abade19311ad8c6894ce6e5d..9a6b6a837fa5a2c508037e172aa17d61f615c90b 100644 --- a/lib/include/arm_acle.h +++ b/lib/include/arm_acle.h @@ -115,6 +115,7 @@ __swp(uint32_t __x, volatile uint32_t *__p) { #else #define __plix(cache_level, retention_policy, addr) \ __builtin_arm_prefetch(addr, 0, cache_level, retention_policy, 0) +#define __pldir(addr) __builtin_arm_prefetch_ir(addr) #endif /* 7.7 NOP */ diff --git a/lib/include/arm_neon.h b/lib/include/arm_neon.h index 392184c61b8c4cab685122a2a439bcdac97d2229..920a4738ad34038eee2751cd4fa7f1481e32f0b0 100644 --- a/lib/include/arm_neon.h +++ b/lib/include/arm_neon.h @@ -144,102 +144,6 @@ typedef struct poly64x2x4_t { #define __lane_reverse_128_8 7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8 #endif #endif -#ifdef __LITTLE_ENDIAN__ -#define splatq_lane_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_splatq_lane_bf16(__builtin_bit_cast(int8x8_t, __s0), __p1, 11)); \ - __ret; \ -}) -#else -#define splatq_lane_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_splatq_lane_bf16(__builtin_bit_cast(int8x8_t, __rev0), __p1, 11)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ - __ret; \ -}) -#define __noswap_splatq_lane_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_splatq_lane_bf16(__builtin_bit_cast(int8x8_t, __s0), __p1, 11)); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define splat_lane_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x4_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_splat_lane_bf16(__builtin_bit_cast(int8x8_t, __s0), __p1, 11)); \ - __ret; \ -}) -#else -#define splat_lane_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x4_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_splat_lane_bf16(__builtin_bit_cast(int8x8_t, __rev0), __p1, 11)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \ - __ret; \ -}) -#define __noswap_splat_lane_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x4_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_splat_lane_bf16(__builtin_bit_cast(int8x8_t, __s0), __p1, 11)); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define splatq_laneq_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_splatq_laneq_bf16(__builtin_bit_cast(int8x16_t, __s0), __p1, 43)); \ - __ret; \ -}) -#else -#define splatq_laneq_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_splatq_laneq_bf16(__builtin_bit_cast(int8x16_t, __rev0), __p1, 43)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ - __ret; \ -}) -#define __noswap_splatq_laneq_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_splatq_laneq_bf16(__builtin_bit_cast(int8x16_t, __s0), __p1, 43)); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define splat_laneq_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x4_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_splat_laneq_bf16(__builtin_bit_cast(int8x16_t, __s0), __p1, 43)); \ - __ret; \ -}) -#else -#define splat_laneq_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x4_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_splat_laneq_bf16(__builtin_bit_cast(int8x16_t, __rev0), __p1, 43)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \ - __ret; \ -}) -#define __noswap_splat_laneq_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x4_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_splat_laneq_bf16(__builtin_bit_cast(int8x16_t, __s0), __p1, 43)); \ - __ret; \ -}) -#endif - #ifdef __LITTLE_ENDIAN__ __ai __attribute__((target("bf16,neon"))) float32x4_t vbfdotq_f32(float32x4_t __p0, bfloat16x8_t __p1, bfloat16x8_t __p2) { float32x4_t __ret; @@ -350,34 +254,6 @@ __ai __attribute__((target("bf16,neon"))) float32x4_t vbfmmlaq_f32(float32x4_t _ } #endif -#ifdef __LITTLE_ENDIAN__ -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcombine_bf16(bfloat16x4_t __p0, bfloat16x4_t __p1) { - bfloat16x8_t __ret; - __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); - return __ret; -} -#else -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcombine_bf16(bfloat16x4_t __p0, bfloat16x4_t __p1) { - bfloat16x8_t __ret; - bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_64_16); - bfloat16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, __lane_reverse_64_16); - __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7); - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t __noswap_vcombine_bf16(bfloat16x4_t __p0, bfloat16x4_t __p1) { - bfloat16x8_t __ret; - __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); - return __ret; -} -#endif - -#define vcreate_bf16(__p0) __extension__ ({ \ - bfloat16x4_t __ret; \ - uint64_t __promote = __p0; \ - __ret = __builtin_bit_cast(bfloat16x4_t, __promote); \ - __ret; \ -}) __ai __attribute__((target("bf16,neon"))) float32_t vcvtah_f32_bf16(bfloat16_t __p0) { float32_t __ret; __ret = __builtin_bit_cast(float32_t, (uint32_t)(__builtin_bit_cast(uint16_t, __p0)) << 16); @@ -388,1188 +264,6 @@ __ai __attribute__((target("bf16,neon"))) bfloat16_t vcvth_bf16_f32(float32_t __ __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vcvth_bf16_f32(__p0)); return __ret; } -#ifdef __LITTLE_ENDIAN__ -#define vduph_lane_bf16(__p0, __p1) __extension__ ({ \ - bfloat16_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vduph_lane_bf16(__s0, __p1)); \ - __ret; \ -}) -#else -#define vduph_lane_bf16(__p0, __p1) __extension__ ({ \ - bfloat16_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ - __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vduph_lane_bf16(__rev0, __p1)); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vdupq_lane_bf16(__p0_0, __p1_0) __extension__ ({ \ - bfloat16x8_t __ret_0; \ - bfloat16x4_t __s0_0 = __p0_0; \ - __ret_0 = splatq_lane_bf16(__s0_0, __p1_0); \ - __ret_0; \ -}) -#else -#define vdupq_lane_bf16(__p0_1, __p1_1) __extension__ ({ \ - bfloat16x8_t __ret_1; \ - bfloat16x4_t __s0_1 = __p0_1; \ - bfloat16x4_t __rev0_1; __rev0_1 = __builtin_shufflevector(__s0_1, __s0_1, __lane_reverse_64_16); \ - __ret_1 = __noswap_splatq_lane_bf16(__rev0_1, __p1_1); \ - __ret_1 = __builtin_shufflevector(__ret_1, __ret_1, __lane_reverse_128_16); \ - __ret_1; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vdup_lane_bf16(__p0_2, __p1_2) __extension__ ({ \ - bfloat16x4_t __ret_2; \ - bfloat16x4_t __s0_2 = __p0_2; \ - __ret_2 = splat_lane_bf16(__s0_2, __p1_2); \ - __ret_2; \ -}) -#else -#define vdup_lane_bf16(__p0_3, __p1_3) __extension__ ({ \ - bfloat16x4_t __ret_3; \ - bfloat16x4_t __s0_3 = __p0_3; \ - bfloat16x4_t __rev0_3; __rev0_3 = __builtin_shufflevector(__s0_3, __s0_3, __lane_reverse_64_16); \ - __ret_3 = __noswap_splat_lane_bf16(__rev0_3, __p1_3); \ - __ret_3 = __builtin_shufflevector(__ret_3, __ret_3, __lane_reverse_64_16); \ - __ret_3; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vduph_laneq_bf16(__p0, __p1) __extension__ ({ \ - bfloat16_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vduph_laneq_bf16(__s0, __p1)); \ - __ret; \ -}) -#else -#define vduph_laneq_bf16(__p0, __p1) __extension__ ({ \ - bfloat16_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ - __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vduph_laneq_bf16(__rev0, __p1)); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vdupq_laneq_bf16(__p0_4, __p1_4) __extension__ ({ \ - bfloat16x8_t __ret_4; \ - bfloat16x8_t __s0_4 = __p0_4; \ - __ret_4 = splatq_laneq_bf16(__s0_4, __p1_4); \ - __ret_4; \ -}) -#else -#define vdupq_laneq_bf16(__p0_5, __p1_5) __extension__ ({ \ - bfloat16x8_t __ret_5; \ - bfloat16x8_t __s0_5 = __p0_5; \ - bfloat16x8_t __rev0_5; __rev0_5 = __builtin_shufflevector(__s0_5, __s0_5, __lane_reverse_128_16); \ - __ret_5 = __noswap_splatq_laneq_bf16(__rev0_5, __p1_5); \ - __ret_5 = __builtin_shufflevector(__ret_5, __ret_5, __lane_reverse_128_16); \ - __ret_5; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vdup_laneq_bf16(__p0_6, __p1_6) __extension__ ({ \ - bfloat16x4_t __ret_6; \ - bfloat16x8_t __s0_6 = __p0_6; \ - __ret_6 = splat_laneq_bf16(__s0_6, __p1_6); \ - __ret_6; \ -}) -#else -#define vdup_laneq_bf16(__p0_7, __p1_7) __extension__ ({ \ - bfloat16x4_t __ret_7; \ - bfloat16x8_t __s0_7 = __p0_7; \ - bfloat16x8_t __rev0_7; __rev0_7 = __builtin_shufflevector(__s0_7, __s0_7, __lane_reverse_128_16); \ - __ret_7 = __noswap_splat_laneq_bf16(__rev0_7, __p1_7); \ - __ret_7 = __builtin_shufflevector(__ret_7, __ret_7, __lane_reverse_64_16); \ - __ret_7; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vdupq_n_bf16(bfloat16_t __p0) { - bfloat16x8_t __ret; - __ret = (bfloat16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; - return __ret; -} -#else -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vdupq_n_bf16(bfloat16_t __p0) { - bfloat16x8_t __ret; - __ret = (bfloat16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); - return __ret; -} -#endif - -#ifdef __LITTLE_ENDIAN__ -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vdup_n_bf16(bfloat16_t __p0) { - bfloat16x4_t __ret; - __ret = (bfloat16x4_t) {__p0, __p0, __p0, __p0}; - return __ret; -} -#else -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vdup_n_bf16(bfloat16_t __p0) { - bfloat16x4_t __ret; - __ret = (bfloat16x4_t) {__p0, __p0, __p0, __p0}; - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); - return __ret; -} -#endif - -#ifdef __LITTLE_ENDIAN__ -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vget_high_bf16(bfloat16x8_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); - return __ret; -} -#else -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vget_high_bf16(bfloat16x8_t __p0) { - bfloat16x4_t __ret; - bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_16); - __ret = __builtin_shufflevector(__rev0, __rev0, 4, 5, 6, 7); - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t __noswap_vget_high_bf16(bfloat16x8_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); - return __ret; -} -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vgetq_lane_bf16(__p0, __p1) __extension__ ({ \ - bfloat16_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vgetq_lane_bf16(__s0, __p1)); \ - __ret; \ -}) -#else -#define vgetq_lane_bf16(__p0, __p1) __extension__ ({ \ - bfloat16_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ - __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vgetq_lane_bf16(__rev0, __p1)); \ - __ret; \ -}) -#define __noswap_vgetq_lane_bf16(__p0, __p1) __extension__ ({ \ - bfloat16_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vgetq_lane_bf16(__s0, __p1)); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vget_lane_bf16(__p0, __p1) __extension__ ({ \ - bfloat16_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vget_lane_bf16(__s0, __p1)); \ - __ret; \ -}) -#else -#define vget_lane_bf16(__p0, __p1) __extension__ ({ \ - bfloat16_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ - __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vget_lane_bf16(__rev0, __p1)); \ - __ret; \ -}) -#define __noswap_vget_lane_bf16(__p0, __p1) __extension__ ({ \ - bfloat16_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vget_lane_bf16(__s0, __p1)); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vget_low_bf16(bfloat16x8_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3); - return __ret; -} -#else -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vget_low_bf16(bfloat16x8_t __p0) { - bfloat16x4_t __ret; - bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_16); - __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3); - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t __noswap_vget_low_bf16(bfloat16x8_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3); - return __ret; -} -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld1q_bf16(__p0) __extension__ ({ \ - bfloat16x8_t __ret; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vld1q_bf16(__p0, 43)); \ - __ret; \ -}) -#else -#define vld1q_bf16(__p0) __extension__ ({ \ - bfloat16x8_t __ret; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vld1q_bf16(__p0, 43)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld1_bf16(__p0) __extension__ ({ \ - bfloat16x4_t __ret; \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vld1_bf16(__p0, 11)); \ - __ret; \ -}) -#else -#define vld1_bf16(__p0) __extension__ ({ \ - bfloat16x4_t __ret; \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vld1_bf16(__p0, 11)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld1q_dup_bf16(__p0) __extension__ ({ \ - bfloat16x8_t __ret; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vld1q_dup_bf16(__p0, 43)); \ - __ret; \ -}) -#else -#define vld1q_dup_bf16(__p0) __extension__ ({ \ - bfloat16x8_t __ret; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vld1q_dup_bf16(__p0, 43)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld1_dup_bf16(__p0) __extension__ ({ \ - bfloat16x4_t __ret; \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vld1_dup_bf16(__p0, 11)); \ - __ret; \ -}) -#else -#define vld1_dup_bf16(__p0) __extension__ ({ \ - bfloat16x4_t __ret; \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vld1_dup_bf16(__p0, 11)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld1q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x8_t __s1 = __p1; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vld1q_lane_bf16(__p0, __builtin_bit_cast(int8x16_t, __s1), __p2, 43)); \ - __ret; \ -}) -#else -#define vld1q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x8_t __s1 = __p1; \ - bfloat16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_16); \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vld1q_lane_bf16(__p0, __builtin_bit_cast(int8x16_t, __rev1), __p2, 43)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld1_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4_t __ret; \ - bfloat16x4_t __s1 = __p1; \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vld1_lane_bf16(__p0, __builtin_bit_cast(int8x8_t, __s1), __p2, 11)); \ - __ret; \ -}) -#else -#define vld1_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4_t __ret; \ - bfloat16x4_t __s1 = __p1; \ - bfloat16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vld1_lane_bf16(__p0, __builtin_bit_cast(int8x8_t, __rev1), __p2, 11)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld1q_bf16_x2(__p0) __extension__ ({ \ - bfloat16x8x2_t __ret; \ - __builtin_neon_vld1q_bf16_x2(&__ret, __p0, 43); \ - __ret; \ -}) -#else -#define vld1q_bf16_x2(__p0) __extension__ ({ \ - bfloat16x8x2_t __ret; \ - __builtin_neon_vld1q_bf16_x2(&__ret, __p0, 43); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld1_bf16_x2(__p0) __extension__ ({ \ - bfloat16x4x2_t __ret; \ - __builtin_neon_vld1_bf16_x2(&__ret, __p0, 11); \ - __ret; \ -}) -#else -#define vld1_bf16_x2(__p0) __extension__ ({ \ - bfloat16x4x2_t __ret; \ - __builtin_neon_vld1_bf16_x2(&__ret, __p0, 11); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld1q_bf16_x3(__p0) __extension__ ({ \ - bfloat16x8x3_t __ret; \ - __builtin_neon_vld1q_bf16_x3(&__ret, __p0, 43); \ - __ret; \ -}) -#else -#define vld1q_bf16_x3(__p0) __extension__ ({ \ - bfloat16x8x3_t __ret; \ - __builtin_neon_vld1q_bf16_x3(&__ret, __p0, 43); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld1_bf16_x3(__p0) __extension__ ({ \ - bfloat16x4x3_t __ret; \ - __builtin_neon_vld1_bf16_x3(&__ret, __p0, 11); \ - __ret; \ -}) -#else -#define vld1_bf16_x3(__p0) __extension__ ({ \ - bfloat16x4x3_t __ret; \ - __builtin_neon_vld1_bf16_x3(&__ret, __p0, 11); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld1q_bf16_x4(__p0) __extension__ ({ \ - bfloat16x8x4_t __ret; \ - __builtin_neon_vld1q_bf16_x4(&__ret, __p0, 43); \ - __ret; \ -}) -#else -#define vld1q_bf16_x4(__p0) __extension__ ({ \ - bfloat16x8x4_t __ret; \ - __builtin_neon_vld1q_bf16_x4(&__ret, __p0, 43); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ - __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld1_bf16_x4(__p0) __extension__ ({ \ - bfloat16x4x4_t __ret; \ - __builtin_neon_vld1_bf16_x4(&__ret, __p0, 11); \ - __ret; \ -}) -#else -#define vld1_bf16_x4(__p0) __extension__ ({ \ - bfloat16x4x4_t __ret; \ - __builtin_neon_vld1_bf16_x4(&__ret, __p0, 11); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ - __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld2q_bf16(__p0) __extension__ ({ \ - bfloat16x8x2_t __ret; \ - __builtin_neon_vld2q_bf16(&__ret, __p0, 43); \ - __ret; \ -}) -#else -#define vld2q_bf16(__p0) __extension__ ({ \ - bfloat16x8x2_t __ret; \ - __builtin_neon_vld2q_bf16(&__ret, __p0, 43); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld2_bf16(__p0) __extension__ ({ \ - bfloat16x4x2_t __ret; \ - __builtin_neon_vld2_bf16(&__ret, __p0, 11); \ - __ret; \ -}) -#else -#define vld2_bf16(__p0) __extension__ ({ \ - bfloat16x4x2_t __ret; \ - __builtin_neon_vld2_bf16(&__ret, __p0, 11); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld2q_dup_bf16(__p0) __extension__ ({ \ - bfloat16x8x2_t __ret; \ - __builtin_neon_vld2q_dup_bf16(&__ret, __p0, 43); \ - __ret; \ -}) -#else -#define vld2q_dup_bf16(__p0) __extension__ ({ \ - bfloat16x8x2_t __ret; \ - __builtin_neon_vld2q_dup_bf16(&__ret, __p0, 43); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld2_dup_bf16(__p0) __extension__ ({ \ - bfloat16x4x2_t __ret; \ - __builtin_neon_vld2_dup_bf16(&__ret, __p0, 11); \ - __ret; \ -}) -#else -#define vld2_dup_bf16(__p0) __extension__ ({ \ - bfloat16x4x2_t __ret; \ - __builtin_neon_vld2_dup_bf16(&__ret, __p0, 11); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8x2_t __ret; \ - bfloat16x8x2_t __s1 = __p1; \ - __builtin_neon_vld2q_lane_bf16(&__ret, __p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __p2, 43); \ - __ret; \ -}) -#else -#define vld2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8x2_t __ret; \ - bfloat16x8x2_t __s1 = __p1; \ - bfloat16x8x2_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ - __builtin_neon_vld2q_lane_bf16(&__ret, __p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __p2, 43); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4x2_t __ret; \ - bfloat16x4x2_t __s1 = __p1; \ - __builtin_neon_vld2_lane_bf16(&__ret, __p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __p2, 11); \ - __ret; \ -}) -#else -#define vld2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4x2_t __ret; \ - bfloat16x4x2_t __s1 = __p1; \ - bfloat16x4x2_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ - __builtin_neon_vld2_lane_bf16(&__ret, __p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __p2, 11); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld3q_bf16(__p0) __extension__ ({ \ - bfloat16x8x3_t __ret; \ - __builtin_neon_vld3q_bf16(&__ret, __p0, 43); \ - __ret; \ -}) -#else -#define vld3q_bf16(__p0) __extension__ ({ \ - bfloat16x8x3_t __ret; \ - __builtin_neon_vld3q_bf16(&__ret, __p0, 43); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld3_bf16(__p0) __extension__ ({ \ - bfloat16x4x3_t __ret; \ - __builtin_neon_vld3_bf16(&__ret, __p0, 11); \ - __ret; \ -}) -#else -#define vld3_bf16(__p0) __extension__ ({ \ - bfloat16x4x3_t __ret; \ - __builtin_neon_vld3_bf16(&__ret, __p0, 11); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld3q_dup_bf16(__p0) __extension__ ({ \ - bfloat16x8x3_t __ret; \ - __builtin_neon_vld3q_dup_bf16(&__ret, __p0, 43); \ - __ret; \ -}) -#else -#define vld3q_dup_bf16(__p0) __extension__ ({ \ - bfloat16x8x3_t __ret; \ - __builtin_neon_vld3q_dup_bf16(&__ret, __p0, 43); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld3_dup_bf16(__p0) __extension__ ({ \ - bfloat16x4x3_t __ret; \ - __builtin_neon_vld3_dup_bf16(&__ret, __p0, 11); \ - __ret; \ -}) -#else -#define vld3_dup_bf16(__p0) __extension__ ({ \ - bfloat16x4x3_t __ret; \ - __builtin_neon_vld3_dup_bf16(&__ret, __p0, 11); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld3q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8x3_t __ret; \ - bfloat16x8x3_t __s1 = __p1; \ - __builtin_neon_vld3q_lane_bf16(&__ret, __p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), __p2, 43); \ - __ret; \ -}) -#else -#define vld3q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8x3_t __ret; \ - bfloat16x8x3_t __s1 = __p1; \ - bfloat16x8x3_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ - __builtin_neon_vld3q_lane_bf16(&__ret, __p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), __p2, 43); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld3_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4x3_t __ret; \ - bfloat16x4x3_t __s1 = __p1; \ - __builtin_neon_vld3_lane_bf16(&__ret, __p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), __p2, 11); \ - __ret; \ -}) -#else -#define vld3_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4x3_t __ret; \ - bfloat16x4x3_t __s1 = __p1; \ - bfloat16x4x3_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ - __builtin_neon_vld3_lane_bf16(&__ret, __p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), __p2, 11); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld4q_bf16(__p0) __extension__ ({ \ - bfloat16x8x4_t __ret; \ - __builtin_neon_vld4q_bf16(&__ret, __p0, 43); \ - __ret; \ -}) -#else -#define vld4q_bf16(__p0) __extension__ ({ \ - bfloat16x8x4_t __ret; \ - __builtin_neon_vld4q_bf16(&__ret, __p0, 43); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ - __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld4_bf16(__p0) __extension__ ({ \ - bfloat16x4x4_t __ret; \ - __builtin_neon_vld4_bf16(&__ret, __p0, 11); \ - __ret; \ -}) -#else -#define vld4_bf16(__p0) __extension__ ({ \ - bfloat16x4x4_t __ret; \ - __builtin_neon_vld4_bf16(&__ret, __p0, 11); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ - __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld4q_dup_bf16(__p0) __extension__ ({ \ - bfloat16x8x4_t __ret; \ - __builtin_neon_vld4q_dup_bf16(&__ret, __p0, 43); \ - __ret; \ -}) -#else -#define vld4q_dup_bf16(__p0) __extension__ ({ \ - bfloat16x8x4_t __ret; \ - __builtin_neon_vld4q_dup_bf16(&__ret, __p0, 43); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ - __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld4_dup_bf16(__p0) __extension__ ({ \ - bfloat16x4x4_t __ret; \ - __builtin_neon_vld4_dup_bf16(&__ret, __p0, 11); \ - __ret; \ -}) -#else -#define vld4_dup_bf16(__p0) __extension__ ({ \ - bfloat16x4x4_t __ret; \ - __builtin_neon_vld4_dup_bf16(&__ret, __p0, 11); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ - __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld4q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8x4_t __ret; \ - bfloat16x8x4_t __s1 = __p1; \ - __builtin_neon_vld4q_lane_bf16(&__ret, __p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), __builtin_bit_cast(int8x16_t, __s1.val[3]), __p2, 43); \ - __ret; \ -}) -#else -#define vld4q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8x4_t __ret; \ - bfloat16x8x4_t __s1 = __p1; \ - bfloat16x8x4_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ - __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_128_16); \ - __builtin_neon_vld4q_lane_bf16(&__ret, __p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), __builtin_bit_cast(int8x16_t, __rev1.val[3]), __p2, 43); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ - __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vld4_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4x4_t __ret; \ - bfloat16x4x4_t __s1 = __p1; \ - __builtin_neon_vld4_lane_bf16(&__ret, __p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), __builtin_bit_cast(int8x8_t, __s1.val[3]), __p2, 11); \ - __ret; \ -}) -#else -#define vld4_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4x4_t __ret; \ - bfloat16x4x4_t __s1 = __p1; \ - bfloat16x4x4_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ - __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_64_16); \ - __builtin_neon_vld4_lane_bf16(&__ret, __p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), __builtin_bit_cast(int8x8_t, __rev1.val[3]), __p2, 11); \ - \ - __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ - __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ - __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ - __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vsetq_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16_t __s0 = __p0; \ - bfloat16x8_t __s1 = __p1; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vsetq_lane_bf16(__s0, __s1, __p2)); \ - __ret; \ -}) -#else -#define vsetq_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16_t __s0 = __p0; \ - bfloat16x8_t __s1 = __p1; \ - bfloat16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_16); \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vsetq_lane_bf16(__s0, __rev1, __p2)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ - __ret; \ -}) -#define __noswap_vsetq_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16_t __s0 = __p0; \ - bfloat16x8_t __s1 = __p1; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vsetq_lane_bf16(__s0, __s1, __p2)); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vset_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4_t __ret; \ - bfloat16_t __s0 = __p0; \ - bfloat16x4_t __s1 = __p1; \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vset_lane_bf16(__s0, __s1, __p2)); \ - __ret; \ -}) -#else -#define vset_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4_t __ret; \ - bfloat16_t __s0 = __p0; \ - bfloat16x4_t __s1 = __p1; \ - bfloat16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vset_lane_bf16(__s0, __rev1, __p2)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \ - __ret; \ -}) -#define __noswap_vset_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4_t __ret; \ - bfloat16_t __s0 = __p0; \ - bfloat16x4_t __s1 = __p1; \ - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vset_lane_bf16(__s0, __s1, __p2)); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst1q_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x8_t __s1 = __p1; \ - __builtin_neon_vst1q_bf16(__p0, __builtin_bit_cast(int8x16_t, __s1), 43); \ -}) -#else -#define vst1q_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x8_t __s1 = __p1; \ - bfloat16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_16); \ - __builtin_neon_vst1q_bf16(__p0, __builtin_bit_cast(int8x16_t, __rev1), 43); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst1_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x4_t __s1 = __p1; \ - __builtin_neon_vst1_bf16(__p0, __builtin_bit_cast(int8x8_t, __s1), 11); \ -}) -#else -#define vst1_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x4_t __s1 = __p1; \ - bfloat16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \ - __builtin_neon_vst1_bf16(__p0, __builtin_bit_cast(int8x8_t, __rev1), 11); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst1q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __s1 = __p1; \ - __builtin_neon_vst1q_lane_bf16(__p0, __builtin_bit_cast(int8x16_t, __s1), __p2, 43); \ -}) -#else -#define vst1q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __s1 = __p1; \ - bfloat16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_16); \ - __builtin_neon_vst1q_lane_bf16(__p0, __builtin_bit_cast(int8x16_t, __rev1), __p2, 43); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst1_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4_t __s1 = __p1; \ - __builtin_neon_vst1_lane_bf16(__p0, __builtin_bit_cast(int8x8_t, __s1), __p2, 11); \ -}) -#else -#define vst1_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4_t __s1 = __p1; \ - bfloat16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \ - __builtin_neon_vst1_lane_bf16(__p0, __builtin_bit_cast(int8x8_t, __rev1), __p2, 11); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst1q_bf16_x2(__p0, __p1) __extension__ ({ \ - bfloat16x8x2_t __s1 = __p1; \ - __builtin_neon_vst1q_bf16_x2(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), 43); \ -}) -#else -#define vst1q_bf16_x2(__p0, __p1) __extension__ ({ \ - bfloat16x8x2_t __s1 = __p1; \ - bfloat16x8x2_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ - __builtin_neon_vst1q_bf16_x2(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), 43); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst1_bf16_x2(__p0, __p1) __extension__ ({ \ - bfloat16x4x2_t __s1 = __p1; \ - __builtin_neon_vst1_bf16_x2(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), 11); \ -}) -#else -#define vst1_bf16_x2(__p0, __p1) __extension__ ({ \ - bfloat16x4x2_t __s1 = __p1; \ - bfloat16x4x2_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ - __builtin_neon_vst1_bf16_x2(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), 11); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst1q_bf16_x3(__p0, __p1) __extension__ ({ \ - bfloat16x8x3_t __s1 = __p1; \ - __builtin_neon_vst1q_bf16_x3(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), 43); \ -}) -#else -#define vst1q_bf16_x3(__p0, __p1) __extension__ ({ \ - bfloat16x8x3_t __s1 = __p1; \ - bfloat16x8x3_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ - __builtin_neon_vst1q_bf16_x3(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), 43); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst1_bf16_x3(__p0, __p1) __extension__ ({ \ - bfloat16x4x3_t __s1 = __p1; \ - __builtin_neon_vst1_bf16_x3(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), 11); \ -}) -#else -#define vst1_bf16_x3(__p0, __p1) __extension__ ({ \ - bfloat16x4x3_t __s1 = __p1; \ - bfloat16x4x3_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ - __builtin_neon_vst1_bf16_x3(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), 11); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst1q_bf16_x4(__p0, __p1) __extension__ ({ \ - bfloat16x8x4_t __s1 = __p1; \ - __builtin_neon_vst1q_bf16_x4(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), __builtin_bit_cast(int8x16_t, __s1.val[3]), 43); \ -}) -#else -#define vst1q_bf16_x4(__p0, __p1) __extension__ ({ \ - bfloat16x8x4_t __s1 = __p1; \ - bfloat16x8x4_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ - __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_128_16); \ - __builtin_neon_vst1q_bf16_x4(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), __builtin_bit_cast(int8x16_t, __rev1.val[3]), 43); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst1_bf16_x4(__p0, __p1) __extension__ ({ \ - bfloat16x4x4_t __s1 = __p1; \ - __builtin_neon_vst1_bf16_x4(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), __builtin_bit_cast(int8x8_t, __s1.val[3]), 11); \ -}) -#else -#define vst1_bf16_x4(__p0, __p1) __extension__ ({ \ - bfloat16x4x4_t __s1 = __p1; \ - bfloat16x4x4_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ - __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_64_16); \ - __builtin_neon_vst1_bf16_x4(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), __builtin_bit_cast(int8x8_t, __rev1.val[3]), 11); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst2q_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x8x2_t __s1 = __p1; \ - __builtin_neon_vst2q_bf16(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), 43); \ -}) -#else -#define vst2q_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x8x2_t __s1 = __p1; \ - bfloat16x8x2_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ - __builtin_neon_vst2q_bf16(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), 43); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst2_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x4x2_t __s1 = __p1; \ - __builtin_neon_vst2_bf16(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), 11); \ -}) -#else -#define vst2_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x4x2_t __s1 = __p1; \ - bfloat16x4x2_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ - __builtin_neon_vst2_bf16(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), 11); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8x2_t __s1 = __p1; \ - __builtin_neon_vst2q_lane_bf16(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __p2, 43); \ -}) -#else -#define vst2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8x2_t __s1 = __p1; \ - bfloat16x8x2_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ - __builtin_neon_vst2q_lane_bf16(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __p2, 43); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4x2_t __s1 = __p1; \ - __builtin_neon_vst2_lane_bf16(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __p2, 11); \ -}) -#else -#define vst2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4x2_t __s1 = __p1; \ - bfloat16x4x2_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ - __builtin_neon_vst2_lane_bf16(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __p2, 11); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst3q_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x8x3_t __s1 = __p1; \ - __builtin_neon_vst3q_bf16(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), 43); \ -}) -#else -#define vst3q_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x8x3_t __s1 = __p1; \ - bfloat16x8x3_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ - __builtin_neon_vst3q_bf16(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), 43); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst3_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x4x3_t __s1 = __p1; \ - __builtin_neon_vst3_bf16(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), 11); \ -}) -#else -#define vst3_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x4x3_t __s1 = __p1; \ - bfloat16x4x3_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ - __builtin_neon_vst3_bf16(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), 11); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst3q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8x3_t __s1 = __p1; \ - __builtin_neon_vst3q_lane_bf16(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), __p2, 43); \ -}) -#else -#define vst3q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8x3_t __s1 = __p1; \ - bfloat16x8x3_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ - __builtin_neon_vst3q_lane_bf16(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), __p2, 43); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst3_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4x3_t __s1 = __p1; \ - __builtin_neon_vst3_lane_bf16(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), __p2, 11); \ -}) -#else -#define vst3_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4x3_t __s1 = __p1; \ - bfloat16x4x3_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ - __builtin_neon_vst3_lane_bf16(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), __p2, 11); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst4q_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x8x4_t __s1 = __p1; \ - __builtin_neon_vst4q_bf16(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), __builtin_bit_cast(int8x16_t, __s1.val[3]), 43); \ -}) -#else -#define vst4q_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x8x4_t __s1 = __p1; \ - bfloat16x8x4_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ - __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_128_16); \ - __builtin_neon_vst4q_bf16(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), __builtin_bit_cast(int8x16_t, __rev1.val[3]), 43); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst4_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x4x4_t __s1 = __p1; \ - __builtin_neon_vst4_bf16(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), __builtin_bit_cast(int8x8_t, __s1.val[3]), 11); \ -}) -#else -#define vst4_bf16(__p0, __p1) __extension__ ({ \ - bfloat16x4x4_t __s1 = __p1; \ - bfloat16x4x4_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ - __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_64_16); \ - __builtin_neon_vst4_bf16(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), __builtin_bit_cast(int8x8_t, __rev1.val[3]), 11); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst4q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8x4_t __s1 = __p1; \ - __builtin_neon_vst4q_lane_bf16(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), __builtin_bit_cast(int8x16_t, __s1.val[3]), __p2, 43); \ -}) -#else -#define vst4q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8x4_t __s1 = __p1; \ - bfloat16x8x4_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ - __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_128_16); \ - __builtin_neon_vst4q_lane_bf16(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), __builtin_bit_cast(int8x16_t, __rev1.val[3]), __p2, 43); \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vst4_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4x4_t __s1 = __p1; \ - __builtin_neon_vst4_lane_bf16(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), __builtin_bit_cast(int8x8_t, __s1.val[3]), __p2, 11); \ -}) -#else -#define vst4_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x4x4_t __s1 = __p1; \ - bfloat16x4x4_t __rev1; \ - __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ - __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ - __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ - __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_64_16); \ - __builtin_neon_vst4_lane_bf16(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), __builtin_bit_cast(int8x8_t, __rev1.val[3]), __p2, 11); \ -}) -#endif - #ifdef __LITTLE_ENDIAN__ __ai __attribute__((target("dotprod,neon"))) uint32x4_t vdotq_u32(uint32x4_t __p0, uint8x16_t __p1, uint8x16_t __p2) { uint32x4_t __ret; @@ -3925,6 +2619,54 @@ __ai __attribute__((target("i8mm,neon"))) int32x4_t vusmmlaq_s32(int32x4_t __p0, }) #endif +#ifdef __LITTLE_ENDIAN__ +#define splatq_lane_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_splatq_lane_v(__builtin_bit_cast(int8x8_t, __s0), __p1, 11)); \ + __ret; \ +}) +#else +#define splatq_lane_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_splatq_lane_v(__builtin_bit_cast(int8x8_t, __rev0), __p1, 11)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ + __ret; \ +}) +#define __noswap_splatq_lane_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_splatq_lane_v(__builtin_bit_cast(int8x8_t, __s0), __p1, 11)); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define splat_lane_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x4_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_splat_lane_v(__builtin_bit_cast(int8x8_t, __s0), __p1, 11)); \ + __ret; \ +}) +#else +#define splat_lane_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x4_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_splat_lane_v(__builtin_bit_cast(int8x8_t, __rev0), __p1, 11)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \ + __ret; \ +}) +#define __noswap_splat_lane_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x4_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_splat_lane_v(__builtin_bit_cast(int8x8_t, __s0), __p1, 11)); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define splat_laneq_p8(__p0, __p1) __extension__ ({ \ poly8x8_t __ret; \ @@ -4593,6 +3335,54 @@ __ai __attribute__((target("i8mm,neon"))) int32x4_t vusmmlaq_s32(int32x4_t __p0, }) #endif +#ifdef __LITTLE_ENDIAN__ +#define splatq_laneq_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_splatq_laneq_v(__builtin_bit_cast(int8x16_t, __s0), __p1, 43)); \ + __ret; \ +}) +#else +#define splatq_laneq_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_splatq_laneq_v(__builtin_bit_cast(int8x16_t, __rev0), __p1, 43)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ + __ret; \ +}) +#define __noswap_splatq_laneq_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_splatq_laneq_v(__builtin_bit_cast(int8x16_t, __s0), __p1, 43)); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define splat_laneq_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x4_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_splat_laneq_v(__builtin_bit_cast(int8x16_t, __s0), __p1, 43)); \ + __ret; \ +}) +#else +#define splat_laneq_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x4_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_splat_laneq_v(__builtin_bit_cast(int8x16_t, __rev0), __p1, 43)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \ + __ret; \ +}) +#define __noswap_splat_laneq_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x4_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_splat_laneq_v(__builtin_bit_cast(int8x16_t, __s0), __p1, 43)); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ __ai __attribute__((target("neon"))) uint8x16_t vabdq_u8(uint8x16_t __p0, uint8x16_t __p1) { uint8x16_t __ret; @@ -8505,6 +7295,28 @@ __ai __attribute__((target("neon"))) int16x8_t __noswap_vcombine_s16(int16x4_t _ } #endif +#ifdef __LITTLE_ENDIAN__ +__ai __attribute__((target("neon"))) bfloat16x8_t vcombine_bf16(bfloat16x4_t __p0, bfloat16x4_t __p1) { + bfloat16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai __attribute__((target("neon"))) bfloat16x8_t vcombine_bf16(bfloat16x4_t __p0, bfloat16x4_t __p1) { + bfloat16x8_t __ret; + bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_64_16); + bfloat16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, __lane_reverse_64_16); + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t __noswap_vcombine_bf16(bfloat16x4_t __p0, bfloat16x4_t __p1) { + bfloat16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#endif + #define vcreate_p8(__p0) __extension__ ({ \ poly8x8_t __ret; \ uint64_t __promote = __p0; \ @@ -8577,6 +7389,12 @@ __ai __attribute__((target("neon"))) int16x8_t __noswap_vcombine_s16(int16x4_t _ __ret = __builtin_bit_cast(int16x4_t, __promote); \ __ret; \ }) +#define vcreate_bf16(__p0) __extension__ ({ \ + bfloat16x4_t __ret; \ + uint64_t __promote = __p0; \ + __ret = __builtin_bit_cast(bfloat16x4_t, __promote); \ + __ret; \ +}) #ifdef __LITTLE_ENDIAN__ __ai __attribute__((target("neon"))) float32x4_t vcvtq_f32_u32(uint32x4_t __p0) { float32x4_t __ret; @@ -8850,406 +7668,512 @@ __ai __attribute__((target("neon"))) uint32x2_t vcvt_u32_f32(float32x2_t __p0) { #endif #ifdef __LITTLE_ENDIAN__ -#define vdup_lane_p8(__p0_8, __p1_8) __extension__ ({ \ - poly8x8_t __ret_8; \ - poly8x8_t __s0_8 = __p0_8; \ - __ret_8 = splat_lane_p8(__s0_8, __p1_8); \ +#define vduph_lane_bf16(__p0, __p1) __extension__ ({ \ + bfloat16_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vduph_lane_bf16(__s0, __p1)); \ + __ret; \ +}) +#else +#define vduph_lane_bf16(__p0, __p1) __extension__ ({ \ + bfloat16_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ + __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vduph_lane_bf16(__rev0, __p1)); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_p8(__p0_0, __p1_0) __extension__ ({ \ + poly8x8_t __ret_0; \ + poly8x8_t __s0_0 = __p0_0; \ + __ret_0 = splat_lane_p8(__s0_0, __p1_0); \ + __ret_0; \ +}) +#else +#define vdup_lane_p8(__p0_1, __p1_1) __extension__ ({ \ + poly8x8_t __ret_1; \ + poly8x8_t __s0_1 = __p0_1; \ + poly8x8_t __rev0_1; __rev0_1 = __builtin_shufflevector(__s0_1, __s0_1, __lane_reverse_64_8); \ + __ret_1 = __noswap_splat_lane_p8(__rev0_1, __p1_1); \ + __ret_1 = __builtin_shufflevector(__ret_1, __ret_1, __lane_reverse_64_8); \ + __ret_1; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_p16(__p0_2, __p1_2) __extension__ ({ \ + poly16x4_t __ret_2; \ + poly16x4_t __s0_2 = __p0_2; \ + __ret_2 = splat_lane_p16(__s0_2, __p1_2); \ + __ret_2; \ +}) +#else +#define vdup_lane_p16(__p0_3, __p1_3) __extension__ ({ \ + poly16x4_t __ret_3; \ + poly16x4_t __s0_3 = __p0_3; \ + poly16x4_t __rev0_3; __rev0_3 = __builtin_shufflevector(__s0_3, __s0_3, __lane_reverse_64_16); \ + __ret_3 = __noswap_splat_lane_p16(__rev0_3, __p1_3); \ + __ret_3 = __builtin_shufflevector(__ret_3, __ret_3, __lane_reverse_64_16); \ + __ret_3; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_p8(__p0_4, __p1_4) __extension__ ({ \ + poly8x16_t __ret_4; \ + poly8x8_t __s0_4 = __p0_4; \ + __ret_4 = splatq_lane_p8(__s0_4, __p1_4); \ + __ret_4; \ +}) +#else +#define vdupq_lane_p8(__p0_5, __p1_5) __extension__ ({ \ + poly8x16_t __ret_5; \ + poly8x8_t __s0_5 = __p0_5; \ + poly8x8_t __rev0_5; __rev0_5 = __builtin_shufflevector(__s0_5, __s0_5, __lane_reverse_64_8); \ + __ret_5 = __noswap_splatq_lane_p8(__rev0_5, __p1_5); \ + __ret_5 = __builtin_shufflevector(__ret_5, __ret_5, __lane_reverse_128_8); \ + __ret_5; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_p16(__p0_6, __p1_6) __extension__ ({ \ + poly16x8_t __ret_6; \ + poly16x4_t __s0_6 = __p0_6; \ + __ret_6 = splatq_lane_p16(__s0_6, __p1_6); \ + __ret_6; \ +}) +#else +#define vdupq_lane_p16(__p0_7, __p1_7) __extension__ ({ \ + poly16x8_t __ret_7; \ + poly16x4_t __s0_7 = __p0_7; \ + poly16x4_t __rev0_7; __rev0_7 = __builtin_shufflevector(__s0_7, __s0_7, __lane_reverse_64_16); \ + __ret_7 = __noswap_splatq_lane_p16(__rev0_7, __p1_7); \ + __ret_7 = __builtin_shufflevector(__ret_7, __ret_7, __lane_reverse_128_16); \ + __ret_7; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_u8(__p0_8, __p1_8) __extension__ ({ \ + uint8x16_t __ret_8; \ + uint8x8_t __s0_8 = __p0_8; \ + __ret_8 = splatq_lane_u8(__s0_8, __p1_8); \ __ret_8; \ }) #else -#define vdup_lane_p8(__p0_9, __p1_9) __extension__ ({ \ - poly8x8_t __ret_9; \ - poly8x8_t __s0_9 = __p0_9; \ - poly8x8_t __rev0_9; __rev0_9 = __builtin_shufflevector(__s0_9, __s0_9, __lane_reverse_64_8); \ - __ret_9 = __noswap_splat_lane_p8(__rev0_9, __p1_9); \ - __ret_9 = __builtin_shufflevector(__ret_9, __ret_9, __lane_reverse_64_8); \ +#define vdupq_lane_u8(__p0_9, __p1_9) __extension__ ({ \ + uint8x16_t __ret_9; \ + uint8x8_t __s0_9 = __p0_9; \ + uint8x8_t __rev0_9; __rev0_9 = __builtin_shufflevector(__s0_9, __s0_9, __lane_reverse_64_8); \ + __ret_9 = __noswap_splatq_lane_u8(__rev0_9, __p1_9); \ + __ret_9 = __builtin_shufflevector(__ret_9, __ret_9, __lane_reverse_128_8); \ __ret_9; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vdup_lane_p16(__p0_10, __p1_10) __extension__ ({ \ - poly16x4_t __ret_10; \ - poly16x4_t __s0_10 = __p0_10; \ - __ret_10 = splat_lane_p16(__s0_10, __p1_10); \ +#define vdupq_lane_u32(__p0_10, __p1_10) __extension__ ({ \ + uint32x4_t __ret_10; \ + uint32x2_t __s0_10 = __p0_10; \ + __ret_10 = splatq_lane_u32(__s0_10, __p1_10); \ __ret_10; \ }) #else -#define vdup_lane_p16(__p0_11, __p1_11) __extension__ ({ \ - poly16x4_t __ret_11; \ - poly16x4_t __s0_11 = __p0_11; \ - poly16x4_t __rev0_11; __rev0_11 = __builtin_shufflevector(__s0_11, __s0_11, __lane_reverse_64_16); \ - __ret_11 = __noswap_splat_lane_p16(__rev0_11, __p1_11); \ - __ret_11 = __builtin_shufflevector(__ret_11, __ret_11, __lane_reverse_64_16); \ +#define vdupq_lane_u32(__p0_11, __p1_11) __extension__ ({ \ + uint32x4_t __ret_11; \ + uint32x2_t __s0_11 = __p0_11; \ + uint32x2_t __rev0_11; __rev0_11 = __builtin_shufflevector(__s0_11, __s0_11, __lane_reverse_64_32); \ + __ret_11 = __noswap_splatq_lane_u32(__rev0_11, __p1_11); \ + __ret_11 = __builtin_shufflevector(__ret_11, __ret_11, __lane_reverse_128_32); \ __ret_11; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vdupq_lane_p8(__p0_12, __p1_12) __extension__ ({ \ - poly8x16_t __ret_12; \ - poly8x8_t __s0_12 = __p0_12; \ - __ret_12 = splatq_lane_p8(__s0_12, __p1_12); \ +#define vdupq_lane_u64(__p0_12, __p1_12) __extension__ ({ \ + uint64x2_t __ret_12; \ + uint64x1_t __s0_12 = __p0_12; \ + __ret_12 = splatq_lane_u64(__s0_12, __p1_12); \ __ret_12; \ }) #else -#define vdupq_lane_p8(__p0_13, __p1_13) __extension__ ({ \ - poly8x16_t __ret_13; \ - poly8x8_t __s0_13 = __p0_13; \ - poly8x8_t __rev0_13; __rev0_13 = __builtin_shufflevector(__s0_13, __s0_13, __lane_reverse_64_8); \ - __ret_13 = __noswap_splatq_lane_p8(__rev0_13, __p1_13); \ - __ret_13 = __builtin_shufflevector(__ret_13, __ret_13, __lane_reverse_128_8); \ +#define vdupq_lane_u64(__p0_13, __p1_13) __extension__ ({ \ + uint64x2_t __ret_13; \ + uint64x1_t __s0_13 = __p0_13; \ + __ret_13 = __noswap_splatq_lane_u64(__s0_13, __p1_13); \ + __ret_13 = __builtin_shufflevector(__ret_13, __ret_13, __lane_reverse_128_64); \ __ret_13; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vdupq_lane_p16(__p0_14, __p1_14) __extension__ ({ \ - poly16x8_t __ret_14; \ - poly16x4_t __s0_14 = __p0_14; \ - __ret_14 = splatq_lane_p16(__s0_14, __p1_14); \ +#define vdupq_lane_u16(__p0_14, __p1_14) __extension__ ({ \ + uint16x8_t __ret_14; \ + uint16x4_t __s0_14 = __p0_14; \ + __ret_14 = splatq_lane_u16(__s0_14, __p1_14); \ __ret_14; \ }) #else -#define vdupq_lane_p16(__p0_15, __p1_15) __extension__ ({ \ - poly16x8_t __ret_15; \ - poly16x4_t __s0_15 = __p0_15; \ - poly16x4_t __rev0_15; __rev0_15 = __builtin_shufflevector(__s0_15, __s0_15, __lane_reverse_64_16); \ - __ret_15 = __noswap_splatq_lane_p16(__rev0_15, __p1_15); \ +#define vdupq_lane_u16(__p0_15, __p1_15) __extension__ ({ \ + uint16x8_t __ret_15; \ + uint16x4_t __s0_15 = __p0_15; \ + uint16x4_t __rev0_15; __rev0_15 = __builtin_shufflevector(__s0_15, __s0_15, __lane_reverse_64_16); \ + __ret_15 = __noswap_splatq_lane_u16(__rev0_15, __p1_15); \ __ret_15 = __builtin_shufflevector(__ret_15, __ret_15, __lane_reverse_128_16); \ __ret_15; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vdupq_lane_u8(__p0_16, __p1_16) __extension__ ({ \ - uint8x16_t __ret_16; \ - uint8x8_t __s0_16 = __p0_16; \ - __ret_16 = splatq_lane_u8(__s0_16, __p1_16); \ +#define vdupq_lane_s8(__p0_16, __p1_16) __extension__ ({ \ + int8x16_t __ret_16; \ + int8x8_t __s0_16 = __p0_16; \ + __ret_16 = splatq_lane_s8(__s0_16, __p1_16); \ __ret_16; \ }) #else -#define vdupq_lane_u8(__p0_17, __p1_17) __extension__ ({ \ - uint8x16_t __ret_17; \ - uint8x8_t __s0_17 = __p0_17; \ - uint8x8_t __rev0_17; __rev0_17 = __builtin_shufflevector(__s0_17, __s0_17, __lane_reverse_64_8); \ - __ret_17 = __noswap_splatq_lane_u8(__rev0_17, __p1_17); \ +#define vdupq_lane_s8(__p0_17, __p1_17) __extension__ ({ \ + int8x16_t __ret_17; \ + int8x8_t __s0_17 = __p0_17; \ + int8x8_t __rev0_17; __rev0_17 = __builtin_shufflevector(__s0_17, __s0_17, __lane_reverse_64_8); \ + __ret_17 = __noswap_splatq_lane_s8(__rev0_17, __p1_17); \ __ret_17 = __builtin_shufflevector(__ret_17, __ret_17, __lane_reverse_128_8); \ __ret_17; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vdupq_lane_u32(__p0_18, __p1_18) __extension__ ({ \ - uint32x4_t __ret_18; \ - uint32x2_t __s0_18 = __p0_18; \ - __ret_18 = splatq_lane_u32(__s0_18, __p1_18); \ +#define vdupq_lane_f32(__p0_18, __p1_18) __extension__ ({ \ + float32x4_t __ret_18; \ + float32x2_t __s0_18 = __p0_18; \ + __ret_18 = splatq_lane_f32(__s0_18, __p1_18); \ __ret_18; \ }) #else -#define vdupq_lane_u32(__p0_19, __p1_19) __extension__ ({ \ - uint32x4_t __ret_19; \ - uint32x2_t __s0_19 = __p0_19; \ - uint32x2_t __rev0_19; __rev0_19 = __builtin_shufflevector(__s0_19, __s0_19, __lane_reverse_64_32); \ - __ret_19 = __noswap_splatq_lane_u32(__rev0_19, __p1_19); \ +#define vdupq_lane_f32(__p0_19, __p1_19) __extension__ ({ \ + float32x4_t __ret_19; \ + float32x2_t __s0_19 = __p0_19; \ + float32x2_t __rev0_19; __rev0_19 = __builtin_shufflevector(__s0_19, __s0_19, __lane_reverse_64_32); \ + __ret_19 = __noswap_splatq_lane_f32(__rev0_19, __p1_19); \ __ret_19 = __builtin_shufflevector(__ret_19, __ret_19, __lane_reverse_128_32); \ __ret_19; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vdupq_lane_u64(__p0_20, __p1_20) __extension__ ({ \ - uint64x2_t __ret_20; \ - uint64x1_t __s0_20 = __p0_20; \ - __ret_20 = splatq_lane_u64(__s0_20, __p1_20); \ +#define vdupq_lane_f16(__p0_20, __p1_20) __extension__ ({ \ + float16x8_t __ret_20; \ + float16x4_t __s0_20 = __p0_20; \ + __ret_20 = splatq_lane_f16(__s0_20, __p1_20); \ __ret_20; \ }) #else -#define vdupq_lane_u64(__p0_21, __p1_21) __extension__ ({ \ - uint64x2_t __ret_21; \ - uint64x1_t __s0_21 = __p0_21; \ - __ret_21 = __noswap_splatq_lane_u64(__s0_21, __p1_21); \ - __ret_21 = __builtin_shufflevector(__ret_21, __ret_21, __lane_reverse_128_64); \ +#define vdupq_lane_f16(__p0_21, __p1_21) __extension__ ({ \ + float16x8_t __ret_21; \ + float16x4_t __s0_21 = __p0_21; \ + float16x4_t __rev0_21; __rev0_21 = __builtin_shufflevector(__s0_21, __s0_21, __lane_reverse_64_16); \ + __ret_21 = __noswap_splatq_lane_f16(__rev0_21, __p1_21); \ + __ret_21 = __builtin_shufflevector(__ret_21, __ret_21, __lane_reverse_128_16); \ __ret_21; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vdupq_lane_u16(__p0_22, __p1_22) __extension__ ({ \ - uint16x8_t __ret_22; \ - uint16x4_t __s0_22 = __p0_22; \ - __ret_22 = splatq_lane_u16(__s0_22, __p1_22); \ +#define vdupq_lane_s32(__p0_22, __p1_22) __extension__ ({ \ + int32x4_t __ret_22; \ + int32x2_t __s0_22 = __p0_22; \ + __ret_22 = splatq_lane_s32(__s0_22, __p1_22); \ __ret_22; \ }) #else -#define vdupq_lane_u16(__p0_23, __p1_23) __extension__ ({ \ - uint16x8_t __ret_23; \ - uint16x4_t __s0_23 = __p0_23; \ - uint16x4_t __rev0_23; __rev0_23 = __builtin_shufflevector(__s0_23, __s0_23, __lane_reverse_64_16); \ - __ret_23 = __noswap_splatq_lane_u16(__rev0_23, __p1_23); \ - __ret_23 = __builtin_shufflevector(__ret_23, __ret_23, __lane_reverse_128_16); \ +#define vdupq_lane_s32(__p0_23, __p1_23) __extension__ ({ \ + int32x4_t __ret_23; \ + int32x2_t __s0_23 = __p0_23; \ + int32x2_t __rev0_23; __rev0_23 = __builtin_shufflevector(__s0_23, __s0_23, __lane_reverse_64_32); \ + __ret_23 = __noswap_splatq_lane_s32(__rev0_23, __p1_23); \ + __ret_23 = __builtin_shufflevector(__ret_23, __ret_23, __lane_reverse_128_32); \ __ret_23; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vdupq_lane_s8(__p0_24, __p1_24) __extension__ ({ \ - int8x16_t __ret_24; \ - int8x8_t __s0_24 = __p0_24; \ - __ret_24 = splatq_lane_s8(__s0_24, __p1_24); \ +#define vdupq_lane_s64(__p0_24, __p1_24) __extension__ ({ \ + int64x2_t __ret_24; \ + int64x1_t __s0_24 = __p0_24; \ + __ret_24 = splatq_lane_s64(__s0_24, __p1_24); \ __ret_24; \ }) #else -#define vdupq_lane_s8(__p0_25, __p1_25) __extension__ ({ \ - int8x16_t __ret_25; \ - int8x8_t __s0_25 = __p0_25; \ - int8x8_t __rev0_25; __rev0_25 = __builtin_shufflevector(__s0_25, __s0_25, __lane_reverse_64_8); \ - __ret_25 = __noswap_splatq_lane_s8(__rev0_25, __p1_25); \ - __ret_25 = __builtin_shufflevector(__ret_25, __ret_25, __lane_reverse_128_8); \ +#define vdupq_lane_s64(__p0_25, __p1_25) __extension__ ({ \ + int64x2_t __ret_25; \ + int64x1_t __s0_25 = __p0_25; \ + __ret_25 = __noswap_splatq_lane_s64(__s0_25, __p1_25); \ + __ret_25 = __builtin_shufflevector(__ret_25, __ret_25, __lane_reverse_128_64); \ __ret_25; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vdupq_lane_f32(__p0_26, __p1_26) __extension__ ({ \ - float32x4_t __ret_26; \ - float32x2_t __s0_26 = __p0_26; \ - __ret_26 = splatq_lane_f32(__s0_26, __p1_26); \ +#define vdupq_lane_s16(__p0_26, __p1_26) __extension__ ({ \ + int16x8_t __ret_26; \ + int16x4_t __s0_26 = __p0_26; \ + __ret_26 = splatq_lane_s16(__s0_26, __p1_26); \ __ret_26; \ }) #else -#define vdupq_lane_f32(__p0_27, __p1_27) __extension__ ({ \ - float32x4_t __ret_27; \ - float32x2_t __s0_27 = __p0_27; \ - float32x2_t __rev0_27; __rev0_27 = __builtin_shufflevector(__s0_27, __s0_27, __lane_reverse_64_32); \ - __ret_27 = __noswap_splatq_lane_f32(__rev0_27, __p1_27); \ - __ret_27 = __builtin_shufflevector(__ret_27, __ret_27, __lane_reverse_128_32); \ +#define vdupq_lane_s16(__p0_27, __p1_27) __extension__ ({ \ + int16x8_t __ret_27; \ + int16x4_t __s0_27 = __p0_27; \ + int16x4_t __rev0_27; __rev0_27 = __builtin_shufflevector(__s0_27, __s0_27, __lane_reverse_64_16); \ + __ret_27 = __noswap_splatq_lane_s16(__rev0_27, __p1_27); \ + __ret_27 = __builtin_shufflevector(__ret_27, __ret_27, __lane_reverse_128_16); \ __ret_27; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vdupq_lane_f16(__p0_28, __p1_28) __extension__ ({ \ - float16x8_t __ret_28; \ - float16x4_t __s0_28 = __p0_28; \ - __ret_28 = splatq_lane_f16(__s0_28, __p1_28); \ +#define vdup_lane_u8(__p0_28, __p1_28) __extension__ ({ \ + uint8x8_t __ret_28; \ + uint8x8_t __s0_28 = __p0_28; \ + __ret_28 = splat_lane_u8(__s0_28, __p1_28); \ __ret_28; \ }) #else -#define vdupq_lane_f16(__p0_29, __p1_29) __extension__ ({ \ - float16x8_t __ret_29; \ - float16x4_t __s0_29 = __p0_29; \ - float16x4_t __rev0_29; __rev0_29 = __builtin_shufflevector(__s0_29, __s0_29, __lane_reverse_64_16); \ - __ret_29 = __noswap_splatq_lane_f16(__rev0_29, __p1_29); \ - __ret_29 = __builtin_shufflevector(__ret_29, __ret_29, __lane_reverse_128_16); \ +#define vdup_lane_u8(__p0_29, __p1_29) __extension__ ({ \ + uint8x8_t __ret_29; \ + uint8x8_t __s0_29 = __p0_29; \ + uint8x8_t __rev0_29; __rev0_29 = __builtin_shufflevector(__s0_29, __s0_29, __lane_reverse_64_8); \ + __ret_29 = __noswap_splat_lane_u8(__rev0_29, __p1_29); \ + __ret_29 = __builtin_shufflevector(__ret_29, __ret_29, __lane_reverse_64_8); \ __ret_29; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vdupq_lane_s32(__p0_30, __p1_30) __extension__ ({ \ - int32x4_t __ret_30; \ - int32x2_t __s0_30 = __p0_30; \ - __ret_30 = splatq_lane_s32(__s0_30, __p1_30); \ +#define vdup_lane_u32(__p0_30, __p1_30) __extension__ ({ \ + uint32x2_t __ret_30; \ + uint32x2_t __s0_30 = __p0_30; \ + __ret_30 = splat_lane_u32(__s0_30, __p1_30); \ __ret_30; \ }) #else -#define vdupq_lane_s32(__p0_31, __p1_31) __extension__ ({ \ - int32x4_t __ret_31; \ - int32x2_t __s0_31 = __p0_31; \ - int32x2_t __rev0_31; __rev0_31 = __builtin_shufflevector(__s0_31, __s0_31, __lane_reverse_64_32); \ - __ret_31 = __noswap_splatq_lane_s32(__rev0_31, __p1_31); \ - __ret_31 = __builtin_shufflevector(__ret_31, __ret_31, __lane_reverse_128_32); \ +#define vdup_lane_u32(__p0_31, __p1_31) __extension__ ({ \ + uint32x2_t __ret_31; \ + uint32x2_t __s0_31 = __p0_31; \ + uint32x2_t __rev0_31; __rev0_31 = __builtin_shufflevector(__s0_31, __s0_31, __lane_reverse_64_32); \ + __ret_31 = __noswap_splat_lane_u32(__rev0_31, __p1_31); \ + __ret_31 = __builtin_shufflevector(__ret_31, __ret_31, __lane_reverse_64_32); \ __ret_31; \ }) #endif -#ifdef __LITTLE_ENDIAN__ -#define vdupq_lane_s64(__p0_32, __p1_32) __extension__ ({ \ - int64x2_t __ret_32; \ - int64x1_t __s0_32 = __p0_32; \ - __ret_32 = splatq_lane_s64(__s0_32, __p1_32); \ +#define vdup_lane_u64(__p0_32, __p1_32) __extension__ ({ \ + uint64x1_t __ret_32; \ + uint64x1_t __s0_32 = __p0_32; \ + __ret_32 = splat_lane_u64(__s0_32, __p1_32); \ __ret_32; \ }) -#else -#define vdupq_lane_s64(__p0_33, __p1_33) __extension__ ({ \ - int64x2_t __ret_33; \ - int64x1_t __s0_33 = __p0_33; \ - __ret_33 = __noswap_splatq_lane_s64(__s0_33, __p1_33); \ - __ret_33 = __builtin_shufflevector(__ret_33, __ret_33, __lane_reverse_128_64); \ +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_u16(__p0_33, __p1_33) __extension__ ({ \ + uint16x4_t __ret_33; \ + uint16x4_t __s0_33 = __p0_33; \ + __ret_33 = splat_lane_u16(__s0_33, __p1_33); \ __ret_33; \ }) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vdupq_lane_s16(__p0_34, __p1_34) __extension__ ({ \ - int16x8_t __ret_34; \ - int16x4_t __s0_34 = __p0_34; \ - __ret_34 = splatq_lane_s16(__s0_34, __p1_34); \ +#else +#define vdup_lane_u16(__p0_34, __p1_34) __extension__ ({ \ + uint16x4_t __ret_34; \ + uint16x4_t __s0_34 = __p0_34; \ + uint16x4_t __rev0_34; __rev0_34 = __builtin_shufflevector(__s0_34, __s0_34, __lane_reverse_64_16); \ + __ret_34 = __noswap_splat_lane_u16(__rev0_34, __p1_34); \ + __ret_34 = __builtin_shufflevector(__ret_34, __ret_34, __lane_reverse_64_16); \ __ret_34; \ }) -#else -#define vdupq_lane_s16(__p0_35, __p1_35) __extension__ ({ \ - int16x8_t __ret_35; \ - int16x4_t __s0_35 = __p0_35; \ - int16x4_t __rev0_35; __rev0_35 = __builtin_shufflevector(__s0_35, __s0_35, __lane_reverse_64_16); \ - __ret_35 = __noswap_splatq_lane_s16(__rev0_35, __p1_35); \ - __ret_35 = __builtin_shufflevector(__ret_35, __ret_35, __lane_reverse_128_16); \ +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_s8(__p0_35, __p1_35) __extension__ ({ \ + int8x8_t __ret_35; \ + int8x8_t __s0_35 = __p0_35; \ + __ret_35 = splat_lane_s8(__s0_35, __p1_35); \ __ret_35; \ }) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vdup_lane_u8(__p0_36, __p1_36) __extension__ ({ \ - uint8x8_t __ret_36; \ - uint8x8_t __s0_36 = __p0_36; \ - __ret_36 = splat_lane_u8(__s0_36, __p1_36); \ +#else +#define vdup_lane_s8(__p0_36, __p1_36) __extension__ ({ \ + int8x8_t __ret_36; \ + int8x8_t __s0_36 = __p0_36; \ + int8x8_t __rev0_36; __rev0_36 = __builtin_shufflevector(__s0_36, __s0_36, __lane_reverse_64_8); \ + __ret_36 = __noswap_splat_lane_s8(__rev0_36, __p1_36); \ + __ret_36 = __builtin_shufflevector(__ret_36, __ret_36, __lane_reverse_64_8); \ __ret_36; \ }) -#else -#define vdup_lane_u8(__p0_37, __p1_37) __extension__ ({ \ - uint8x8_t __ret_37; \ - uint8x8_t __s0_37 = __p0_37; \ - uint8x8_t __rev0_37; __rev0_37 = __builtin_shufflevector(__s0_37, __s0_37, __lane_reverse_64_8); \ - __ret_37 = __noswap_splat_lane_u8(__rev0_37, __p1_37); \ - __ret_37 = __builtin_shufflevector(__ret_37, __ret_37, __lane_reverse_64_8); \ +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_f32(__p0_37, __p1_37) __extension__ ({ \ + float32x2_t __ret_37; \ + float32x2_t __s0_37 = __p0_37; \ + __ret_37 = splat_lane_f32(__s0_37, __p1_37); \ __ret_37; \ }) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vdup_lane_u32(__p0_38, __p1_38) __extension__ ({ \ - uint32x2_t __ret_38; \ - uint32x2_t __s0_38 = __p0_38; \ - __ret_38 = splat_lane_u32(__s0_38, __p1_38); \ +#else +#define vdup_lane_f32(__p0_38, __p1_38) __extension__ ({ \ + float32x2_t __ret_38; \ + float32x2_t __s0_38 = __p0_38; \ + float32x2_t __rev0_38; __rev0_38 = __builtin_shufflevector(__s0_38, __s0_38, __lane_reverse_64_32); \ + __ret_38 = __noswap_splat_lane_f32(__rev0_38, __p1_38); \ + __ret_38 = __builtin_shufflevector(__ret_38, __ret_38, __lane_reverse_64_32); \ __ret_38; \ }) -#else -#define vdup_lane_u32(__p0_39, __p1_39) __extension__ ({ \ - uint32x2_t __ret_39; \ - uint32x2_t __s0_39 = __p0_39; \ - uint32x2_t __rev0_39; __rev0_39 = __builtin_shufflevector(__s0_39, __s0_39, __lane_reverse_64_32); \ - __ret_39 = __noswap_splat_lane_u32(__rev0_39, __p1_39); \ - __ret_39 = __builtin_shufflevector(__ret_39, __ret_39, __lane_reverse_64_32); \ +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_f16(__p0_39, __p1_39) __extension__ ({ \ + float16x4_t __ret_39; \ + float16x4_t __s0_39 = __p0_39; \ + __ret_39 = splat_lane_f16(__s0_39, __p1_39); \ __ret_39; \ }) -#endif - -#define vdup_lane_u64(__p0_40, __p1_40) __extension__ ({ \ - uint64x1_t __ret_40; \ - uint64x1_t __s0_40 = __p0_40; \ - __ret_40 = splat_lane_u64(__s0_40, __p1_40); \ +#else +#define vdup_lane_f16(__p0_40, __p1_40) __extension__ ({ \ + float16x4_t __ret_40; \ + float16x4_t __s0_40 = __p0_40; \ + float16x4_t __rev0_40; __rev0_40 = __builtin_shufflevector(__s0_40, __s0_40, __lane_reverse_64_16); \ + __ret_40 = __noswap_splat_lane_f16(__rev0_40, __p1_40); \ + __ret_40 = __builtin_shufflevector(__ret_40, __ret_40, __lane_reverse_64_16); \ __ret_40; \ }) +#endif + #ifdef __LITTLE_ENDIAN__ -#define vdup_lane_u16(__p0_41, __p1_41) __extension__ ({ \ - uint16x4_t __ret_41; \ - uint16x4_t __s0_41 = __p0_41; \ - __ret_41 = splat_lane_u16(__s0_41, __p1_41); \ +#define vdup_lane_s32(__p0_41, __p1_41) __extension__ ({ \ + int32x2_t __ret_41; \ + int32x2_t __s0_41 = __p0_41; \ + __ret_41 = splat_lane_s32(__s0_41, __p1_41); \ __ret_41; \ }) #else -#define vdup_lane_u16(__p0_42, __p1_42) __extension__ ({ \ - uint16x4_t __ret_42; \ - uint16x4_t __s0_42 = __p0_42; \ - uint16x4_t __rev0_42; __rev0_42 = __builtin_shufflevector(__s0_42, __s0_42, __lane_reverse_64_16); \ - __ret_42 = __noswap_splat_lane_u16(__rev0_42, __p1_42); \ - __ret_42 = __builtin_shufflevector(__ret_42, __ret_42, __lane_reverse_64_16); \ +#define vdup_lane_s32(__p0_42, __p1_42) __extension__ ({ \ + int32x2_t __ret_42; \ + int32x2_t __s0_42 = __p0_42; \ + int32x2_t __rev0_42; __rev0_42 = __builtin_shufflevector(__s0_42, __s0_42, __lane_reverse_64_32); \ + __ret_42 = __noswap_splat_lane_s32(__rev0_42, __p1_42); \ + __ret_42 = __builtin_shufflevector(__ret_42, __ret_42, __lane_reverse_64_32); \ __ret_42; \ }) #endif -#ifdef __LITTLE_ENDIAN__ -#define vdup_lane_s8(__p0_43, __p1_43) __extension__ ({ \ - int8x8_t __ret_43; \ - int8x8_t __s0_43 = __p0_43; \ - __ret_43 = splat_lane_s8(__s0_43, __p1_43); \ +#define vdup_lane_s64(__p0_43, __p1_43) __extension__ ({ \ + int64x1_t __ret_43; \ + int64x1_t __s0_43 = __p0_43; \ + __ret_43 = splat_lane_s64(__s0_43, __p1_43); \ __ret_43; \ }) -#else -#define vdup_lane_s8(__p0_44, __p1_44) __extension__ ({ \ - int8x8_t __ret_44; \ - int8x8_t __s0_44 = __p0_44; \ - int8x8_t __rev0_44; __rev0_44 = __builtin_shufflevector(__s0_44, __s0_44, __lane_reverse_64_8); \ - __ret_44 = __noswap_splat_lane_s8(__rev0_44, __p1_44); \ - __ret_44 = __builtin_shufflevector(__ret_44, __ret_44, __lane_reverse_64_8); \ +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_s16(__p0_44, __p1_44) __extension__ ({ \ + int16x4_t __ret_44; \ + int16x4_t __s0_44 = __p0_44; \ + __ret_44 = splat_lane_s16(__s0_44, __p1_44); \ __ret_44; \ }) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vdup_lane_f32(__p0_45, __p1_45) __extension__ ({ \ - float32x2_t __ret_45; \ - float32x2_t __s0_45 = __p0_45; \ - __ret_45 = splat_lane_f32(__s0_45, __p1_45); \ +#else +#define vdup_lane_s16(__p0_45, __p1_45) __extension__ ({ \ + int16x4_t __ret_45; \ + int16x4_t __s0_45 = __p0_45; \ + int16x4_t __rev0_45; __rev0_45 = __builtin_shufflevector(__s0_45, __s0_45, __lane_reverse_64_16); \ + __ret_45 = __noswap_splat_lane_s16(__rev0_45, __p1_45); \ + __ret_45 = __builtin_shufflevector(__ret_45, __ret_45, __lane_reverse_64_16); \ __ret_45; \ }) -#else -#define vdup_lane_f32(__p0_46, __p1_46) __extension__ ({ \ - float32x2_t __ret_46; \ - float32x2_t __s0_46 = __p0_46; \ - float32x2_t __rev0_46; __rev0_46 = __builtin_shufflevector(__s0_46, __s0_46, __lane_reverse_64_32); \ - __ret_46 = __noswap_splat_lane_f32(__rev0_46, __p1_46); \ - __ret_46 = __builtin_shufflevector(__ret_46, __ret_46, __lane_reverse_64_32); \ +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_bf16(__p0_46, __p1_46) __extension__ ({ \ + bfloat16x8_t __ret_46; \ + bfloat16x4_t __s0_46 = __p0_46; \ + __ret_46 = splatq_lane_bf16(__s0_46, __p1_46); \ __ret_46; \ }) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vdup_lane_f16(__p0_47, __p1_47) __extension__ ({ \ - float16x4_t __ret_47; \ - float16x4_t __s0_47 = __p0_47; \ - __ret_47 = splat_lane_f16(__s0_47, __p1_47); \ +#else +#define vdupq_lane_bf16(__p0_47, __p1_47) __extension__ ({ \ + bfloat16x8_t __ret_47; \ + bfloat16x4_t __s0_47 = __p0_47; \ + bfloat16x4_t __rev0_47; __rev0_47 = __builtin_shufflevector(__s0_47, __s0_47, __lane_reverse_64_16); \ + __ret_47 = __noswap_splatq_lane_bf16(__rev0_47, __p1_47); \ + __ret_47 = __builtin_shufflevector(__ret_47, __ret_47, __lane_reverse_128_16); \ __ret_47; \ }) -#else -#define vdup_lane_f16(__p0_48, __p1_48) __extension__ ({ \ - float16x4_t __ret_48; \ - float16x4_t __s0_48 = __p0_48; \ - float16x4_t __rev0_48; __rev0_48 = __builtin_shufflevector(__s0_48, __s0_48, __lane_reverse_64_16); \ - __ret_48 = __noswap_splat_lane_f16(__rev0_48, __p1_48); \ - __ret_48 = __builtin_shufflevector(__ret_48, __ret_48, __lane_reverse_64_16); \ +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_bf16(__p0_48, __p1_48) __extension__ ({ \ + bfloat16x4_t __ret_48; \ + bfloat16x4_t __s0_48 = __p0_48; \ + __ret_48 = splat_lane_bf16(__s0_48, __p1_48); \ __ret_48; \ }) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vdup_lane_s32(__p0_49, __p1_49) __extension__ ({ \ - int32x2_t __ret_49; \ - int32x2_t __s0_49 = __p0_49; \ - __ret_49 = splat_lane_s32(__s0_49, __p1_49); \ +#else +#define vdup_lane_bf16(__p0_49, __p1_49) __extension__ ({ \ + bfloat16x4_t __ret_49; \ + bfloat16x4_t __s0_49 = __p0_49; \ + bfloat16x4_t __rev0_49; __rev0_49 = __builtin_shufflevector(__s0_49, __s0_49, __lane_reverse_64_16); \ + __ret_49 = __noswap_splat_lane_bf16(__rev0_49, __p1_49); \ + __ret_49 = __builtin_shufflevector(__ret_49, __ret_49, __lane_reverse_64_16); \ __ret_49; \ }) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_laneq_bf16(__p0, __p1) __extension__ ({ \ + bfloat16_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vduph_laneq_bf16(__s0, __p1)); \ + __ret; \ +}) #else -#define vdup_lane_s32(__p0_50, __p1_50) __extension__ ({ \ - int32x2_t __ret_50; \ - int32x2_t __s0_50 = __p0_50; \ - int32x2_t __rev0_50; __rev0_50 = __builtin_shufflevector(__s0_50, __s0_50, __lane_reverse_64_32); \ - __ret_50 = __noswap_splat_lane_s32(__rev0_50, __p1_50); \ - __ret_50 = __builtin_shufflevector(__ret_50, __ret_50, __lane_reverse_64_32); \ +#define vduph_laneq_bf16(__p0, __p1) __extension__ ({ \ + bfloat16_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ + __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vduph_laneq_bf16(__rev0, __p1)); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_bf16(__p0_50, __p1_50) __extension__ ({ \ + bfloat16x8_t __ret_50; \ + bfloat16x8_t __s0_50 = __p0_50; \ + __ret_50 = splatq_laneq_bf16(__s0_50, __p1_50); \ __ret_50; \ }) -#endif - -#define vdup_lane_s64(__p0_51, __p1_51) __extension__ ({ \ - int64x1_t __ret_51; \ - int64x1_t __s0_51 = __p0_51; \ - __ret_51 = splat_lane_s64(__s0_51, __p1_51); \ +#else +#define vdupq_laneq_bf16(__p0_51, __p1_51) __extension__ ({ \ + bfloat16x8_t __ret_51; \ + bfloat16x8_t __s0_51 = __p0_51; \ + bfloat16x8_t __rev0_51; __rev0_51 = __builtin_shufflevector(__s0_51, __s0_51, __lane_reverse_128_16); \ + __ret_51 = __noswap_splatq_laneq_bf16(__rev0_51, __p1_51); \ + __ret_51 = __builtin_shufflevector(__ret_51, __ret_51, __lane_reverse_128_16); \ __ret_51; \ }) +#endif + #ifdef __LITTLE_ENDIAN__ -#define vdup_lane_s16(__p0_52, __p1_52) __extension__ ({ \ - int16x4_t __ret_52; \ - int16x4_t __s0_52 = __p0_52; \ - __ret_52 = splat_lane_s16(__s0_52, __p1_52); \ +#define vdup_laneq_bf16(__p0_52, __p1_52) __extension__ ({ \ + bfloat16x4_t __ret_52; \ + bfloat16x8_t __s0_52 = __p0_52; \ + __ret_52 = splat_laneq_bf16(__s0_52, __p1_52); \ __ret_52; \ }) #else -#define vdup_lane_s16(__p0_53, __p1_53) __extension__ ({ \ - int16x4_t __ret_53; \ - int16x4_t __s0_53 = __p0_53; \ - int16x4_t __rev0_53; __rev0_53 = __builtin_shufflevector(__s0_53, __s0_53, __lane_reverse_64_16); \ - __ret_53 = __noswap_splat_lane_s16(__rev0_53, __p1_53); \ +#define vdup_laneq_bf16(__p0_53, __p1_53) __extension__ ({ \ + bfloat16x4_t __ret_53; \ + bfloat16x8_t __s0_53 = __p0_53; \ + bfloat16x8_t __rev0_53; __rev0_53 = __builtin_shufflevector(__s0_53, __s0_53, __lane_reverse_128_16); \ + __ret_53 = __noswap_splat_laneq_bf16(__rev0_53, __p1_53); \ __ret_53 = __builtin_shufflevector(__ret_53, __ret_53, __lane_reverse_64_16); \ __ret_53; \ }) @@ -9599,6 +8523,36 @@ __ai __attribute__((target("neon"))) int16x4_t vdup_n_s16(int16_t __p0) { } #endif +#ifdef __LITTLE_ENDIAN__ +__ai __attribute__((target("neon"))) bfloat16x8_t vdupq_n_bf16(bfloat16_t __p0) { + bfloat16x8_t __ret; + __ret = (bfloat16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai __attribute__((target("neon"))) bfloat16x8_t vdupq_n_bf16(bfloat16_t __p0) { + bfloat16x8_t __ret; + __ret = (bfloat16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai __attribute__((target("neon"))) bfloat16x4_t vdup_n_bf16(bfloat16_t __p0) { + bfloat16x4_t __ret; + __ret = (bfloat16x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai __attribute__((target("neon"))) bfloat16x4_t vdup_n_bf16(bfloat16_t __p0) { + bfloat16x4_t __ret; + __ret = (bfloat16x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); + return __ret; +} +#endif + #ifdef __LITTLE_ENDIAN__ __ai __attribute__((target("neon"))) uint8x16_t veorq_u8(uint8x16_t __p0, uint8x16_t __p1) { uint8x16_t __ret; @@ -10558,6 +9512,27 @@ __ai __attribute__((target("neon"))) int16x4_t __noswap_vget_high_s16(int16x8_t } #endif +#ifdef __LITTLE_ENDIAN__ +__ai __attribute__((target("neon"))) bfloat16x4_t vget_high_bf16(bfloat16x8_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#else +__ai __attribute__((target("neon"))) bfloat16x4_t vget_high_bf16(bfloat16x8_t __p0) { + bfloat16x4_t __ret; + bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_16); + __ret = __builtin_shufflevector(__rev0, __rev0, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t __noswap_vget_high_bf16(bfloat16x8_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#endif + #ifdef __LITTLE_ENDIAN__ #define vget_lane_p8(__p0, __p1) __extension__ ({ \ poly8_t __ret; \ @@ -11030,6 +10005,52 @@ __ai __attribute__((target("neon"))) int16x4_t __noswap_vget_high_s16(int16x8_t }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_bf16(__p0, __p1) __extension__ ({ \ + bfloat16_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vgetq_lane_bf16(__s0, __p1)); \ + __ret; \ +}) +#else +#define vgetq_lane_bf16(__p0, __p1) __extension__ ({ \ + bfloat16_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ + __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vgetq_lane_bf16(__rev0, __p1)); \ + __ret; \ +}) +#define __noswap_vgetq_lane_bf16(__p0, __p1) __extension__ ({ \ + bfloat16_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vgetq_lane_bf16(__s0, __p1)); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_bf16(__p0, __p1) __extension__ ({ \ + bfloat16_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vget_lane_bf16(__s0, __p1)); \ + __ret; \ +}) +#else +#define vget_lane_bf16(__p0, __p1) __extension__ ({ \ + bfloat16_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ + __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vget_lane_bf16(__rev0, __p1)); \ + __ret; \ +}) +#define __noswap_vget_lane_bf16(__p0, __p1) __extension__ ({ \ + bfloat16_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + __ret = __builtin_bit_cast(bfloat16_t, __builtin_neon_vget_lane_bf16(__s0, __p1)); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ __ai __attribute__((target("neon"))) poly8x8_t vget_low_p8(poly8x16_t __p0) { poly8x8_t __ret; @@ -11220,6 +10241,27 @@ __ai __attribute__((target("neon"))) int16x4_t vget_low_s16(int16x8_t __p0) { } #endif +#ifdef __LITTLE_ENDIAN__ +__ai __attribute__((target("neon"))) bfloat16x4_t vget_low_bf16(bfloat16x8_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3); + return __ret; +} +#else +__ai __attribute__((target("neon"))) bfloat16x4_t vget_low_bf16(bfloat16x8_t __p0) { + bfloat16x4_t __ret; + bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_16); + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t __noswap_vget_low_bf16(bfloat16x8_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3); + return __ret; +} +#endif + #ifdef __LITTLE_ENDIAN__ __ai __attribute__((target("neon"))) uint8x16_t vhaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { uint8x16_t __ret; @@ -11938,6 +10980,36 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld1q_bf16(__p0) __extension__ ({ \ + bfloat16x8_t __ret; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vld1q_v(__p0, 43)); \ + __ret; \ +}) +#else +#define vld1q_bf16(__p0) __extension__ ({ \ + bfloat16x8_t __ret; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vld1q_v(__p0, 43)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_bf16(__p0) __extension__ ({ \ + bfloat16x4_t __ret; \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vld1_v(__p0, 11)); \ + __ret; \ +}) +#else +#define vld1_bf16(__p0) __extension__ ({ \ + bfloat16x4_t __ret; \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vld1_v(__p0, 11)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vld1_dup_p8(__p0) __extension__ ({ \ poly8x8_t __ret; \ @@ -12248,6 +11320,36 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_bf16(__p0) __extension__ ({ \ + bfloat16x8_t __ret; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vld1q_dup_v(__p0, 43)); \ + __ret; \ +}) +#else +#define vld1q_dup_bf16(__p0) __extension__ ({ \ + bfloat16x8_t __ret; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vld1q_dup_v(__p0, 43)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_bf16(__p0) __extension__ ({ \ + bfloat16x4_t __ret; \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vld1_dup_v(__p0, 11)); \ + __ret; \ +}) +#else +#define vld1_dup_bf16(__p0) __extension__ ({ \ + bfloat16x4_t __ret; \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vld1_dup_v(__p0, 11)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vld1_lane_p8(__p0, __p1, __p2) __extension__ ({ \ poly8x8_t __ret; \ @@ -12620,6 +11722,42 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x8_t __s1 = __p1; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vld1q_lane_v(__p0, __builtin_bit_cast(int8x16_t, __s1), __p2, 43)); \ + __ret; \ +}) +#else +#define vld1q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x8_t __s1 = __p1; \ + bfloat16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_16); \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vld1q_lane_v(__p0, __builtin_bit_cast(int8x16_t, __rev1), __p2, 43)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4_t __ret; \ + bfloat16x4_t __s1 = __p1; \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vld1_lane_v(__p0, __builtin_bit_cast(int8x8_t, __s1), __p2, 11)); \ + __ret; \ +}) +#else +#define vld1_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4_t __ret; \ + bfloat16x4_t __s1 = __p1; \ + bfloat16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vld1_lane_v(__p0, __builtin_bit_cast(int8x8_t, __rev1), __p2, 11)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vld1_p8_x2(__p0) __extension__ ({ \ poly8x8x2_t __ret; \ @@ -12970,6 +12108,40 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld1q_bf16_x2(__p0) __extension__ ({ \ + bfloat16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 43); \ + __ret; \ +}) +#else +#define vld1q_bf16_x2(__p0) __extension__ ({ \ + bfloat16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 43); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_bf16_x2(__p0) __extension__ ({ \ + bfloat16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 11); \ + __ret; \ +}) +#else +#define vld1_bf16_x2(__p0) __extension__ ({ \ + bfloat16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 11); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vld1_p8_x3(__p0) __extension__ ({ \ poly8x8x3_t __ret; \ @@ -13340,6 +12512,42 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld1q_bf16_x3(__p0) __extension__ ({ \ + bfloat16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 43); \ + __ret; \ +}) +#else +#define vld1q_bf16_x3(__p0) __extension__ ({ \ + bfloat16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 43); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_bf16_x3(__p0) __extension__ ({ \ + bfloat16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 11); \ + __ret; \ +}) +#else +#define vld1_bf16_x3(__p0) __extension__ ({ \ + bfloat16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 11); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vld1_p8_x4(__p0) __extension__ ({ \ poly8x8x4_t __ret; \ @@ -13730,6 +12938,44 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld1q_bf16_x4(__p0) __extension__ ({ \ + bfloat16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 43); \ + __ret; \ +}) +#else +#define vld1q_bf16_x4(__p0) __extension__ ({ \ + bfloat16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 43); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_bf16_x4(__p0) __extension__ ({ \ + bfloat16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 11); \ + __ret; \ +}) +#else +#define vld1_bf16_x4(__p0) __extension__ ({ \ + bfloat16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 11); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vld2_p8(__p0) __extension__ ({ \ poly8x8x2_t __ret; \ @@ -14046,6 +13292,40 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld2q_bf16(__p0) __extension__ ({ \ + bfloat16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 43); \ + __ret; \ +}) +#else +#define vld2q_bf16(__p0) __extension__ ({ \ + bfloat16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 43); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_bf16(__p0) __extension__ ({ \ + bfloat16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 11); \ + __ret; \ +}) +#else +#define vld2_bf16(__p0) __extension__ ({ \ + bfloat16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 11); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vld2_dup_p8(__p0) __extension__ ({ \ poly8x8x2_t __ret; \ @@ -14396,6 +13676,40 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_bf16(__p0) __extension__ ({ \ + bfloat16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 43); \ + __ret; \ +}) +#else +#define vld2q_dup_bf16(__p0) __extension__ ({ \ + bfloat16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 43); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_bf16(__p0) __extension__ ({ \ + bfloat16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 11); \ + __ret; \ +}) +#else +#define vld2_dup_bf16(__p0) __extension__ ({ \ + bfloat16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 11); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vld2_lane_p8(__p0, __p1, __p2) __extension__ ({ \ poly8x8x2_t __ret; \ @@ -14726,6 +14040,50 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8x2_t __ret; \ + bfloat16x8x2_t __s1 = __p1; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __p2, 43); \ + __ret; \ +}) +#else +#define vld2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8x2_t __ret; \ + bfloat16x8x2_t __s1 = __p1; \ + bfloat16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __p2, 43); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4x2_t __ret; \ + bfloat16x4x2_t __s1 = __p1; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __p2, 11); \ + __ret; \ +}) +#else +#define vld2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4x2_t __ret; \ + bfloat16x4x2_t __s1 = __p1; \ + bfloat16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __p2, 11); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vld3_p8(__p0) __extension__ ({ \ poly8x8x3_t __ret; \ @@ -15060,6 +14418,42 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld3q_bf16(__p0) __extension__ ({ \ + bfloat16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 43); \ + __ret; \ +}) +#else +#define vld3q_bf16(__p0) __extension__ ({ \ + bfloat16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 43); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_bf16(__p0) __extension__ ({ \ + bfloat16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 11); \ + __ret; \ +}) +#else +#define vld3_bf16(__p0) __extension__ ({ \ + bfloat16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 11); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vld3_dup_p8(__p0) __extension__ ({ \ poly8x8x3_t __ret; \ @@ -15430,6 +14824,42 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_bf16(__p0) __extension__ ({ \ + bfloat16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 43); \ + __ret; \ +}) +#else +#define vld3q_dup_bf16(__p0) __extension__ ({ \ + bfloat16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 43); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_bf16(__p0) __extension__ ({ \ + bfloat16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 11); \ + __ret; \ +}) +#else +#define vld3_dup_bf16(__p0) __extension__ ({ \ + bfloat16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 11); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vld3_lane_p8(__p0, __p1, __p2) __extension__ ({ \ poly8x8x3_t __ret; \ @@ -15790,6 +15220,54 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8x3_t __ret; \ + bfloat16x8x3_t __s1 = __p1; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), __p2, 43); \ + __ret; \ +}) +#else +#define vld3q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8x3_t __ret; \ + bfloat16x8x3_t __s1 = __p1; \ + bfloat16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), __p2, 43); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4x3_t __ret; \ + bfloat16x4x3_t __s1 = __p1; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), __p2, 11); \ + __ret; \ +}) +#else +#define vld3_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4x3_t __ret; \ + bfloat16x4x3_t __s1 = __p1; \ + bfloat16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), __p2, 11); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vld4_p8(__p0) __extension__ ({ \ poly8x8x4_t __ret; \ @@ -16142,6 +15620,44 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld4q_bf16(__p0) __extension__ ({ \ + bfloat16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 43); \ + __ret; \ +}) +#else +#define vld4q_bf16(__p0) __extension__ ({ \ + bfloat16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 43); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_bf16(__p0) __extension__ ({ \ + bfloat16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 11); \ + __ret; \ +}) +#else +#define vld4_bf16(__p0) __extension__ ({ \ + bfloat16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 11); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vld4_dup_p8(__p0) __extension__ ({ \ poly8x8x4_t __ret; \ @@ -16532,6 +16048,44 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_bf16(__p0) __extension__ ({ \ + bfloat16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 43); \ + __ret; \ +}) +#else +#define vld4q_dup_bf16(__p0) __extension__ ({ \ + bfloat16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 43); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_bf16(__p0) __extension__ ({ \ + bfloat16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 11); \ + __ret; \ +}) +#else +#define vld4_dup_bf16(__p0) __extension__ ({ \ + bfloat16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 11); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vld4_lane_p8(__p0, __p1, __p2) __extension__ ({ \ poly8x8x4_t __ret; \ @@ -16922,6 +16476,58 @@ __ai __attribute__((target("neon"))) int16x4_t vhsub_s16(int16x4_t __p0, int16x4 }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8x4_t __ret; \ + bfloat16x8x4_t __s1 = __p1; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), __builtin_bit_cast(int8x16_t, __s1.val[3]), __p2, 43); \ + __ret; \ +}) +#else +#define vld4q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8x4_t __ret; \ + bfloat16x8x4_t __s1 = __p1; \ + bfloat16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_128_16); \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), __builtin_bit_cast(int8x16_t, __rev1.val[3]), __p2, 43); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_128_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_128_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_128_16); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4x4_t __ret; \ + bfloat16x4x4_t __s1 = __p1; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), __builtin_bit_cast(int8x8_t, __s1.val[3]), __p2, 11); \ + __ret; \ +}) +#else +#define vld4_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4x4_t __ret; \ + bfloat16x4x4_t __s1 = __p1; \ + bfloat16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_64_16); \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), __builtin_bit_cast(int8x8_t, __rev1.val[3]), __p2, 11); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], __lane_reverse_64_16); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], __lane_reverse_64_16); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], __lane_reverse_64_16); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], __lane_reverse_64_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ __ai __attribute__((target("neon"))) uint8x16_t vmaxq_u8(uint8x16_t __p0, uint8x16_t __p1) { uint8x16_t __ret; @@ -27576,6 +27182,60 @@ __ai __attribute__((target("neon"))) int8x8_t __noswap_vrsubhn_s16(int16x8_t __p }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16_t __s0 = __p0; \ + bfloat16x8_t __s1 = __p1; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vsetq_lane_bf16(__s0, __s1, __p2)); \ + __ret; \ +}) +#else +#define vsetq_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16_t __s0 = __p0; \ + bfloat16x8_t __s1 = __p1; \ + bfloat16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_16); \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vsetq_lane_bf16(__s0, __rev1, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ + __ret; \ +}) +#define __noswap_vsetq_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16_t __s0 = __p0; \ + bfloat16x8_t __s1 = __p1; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vsetq_lane_bf16(__s0, __s1, __p2)); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4_t __ret; \ + bfloat16_t __s0 = __p0; \ + bfloat16x4_t __s1 = __p1; \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vset_lane_bf16(__s0, __s1, __p2)); \ + __ret; \ +}) +#else +#define vset_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4_t __ret; \ + bfloat16_t __s0 = __p0; \ + bfloat16x4_t __s1 = __p1; \ + bfloat16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vset_lane_bf16(__s0, __rev1, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \ + __ret; \ +}) +#define __noswap_vset_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4_t __ret; \ + bfloat16_t __s0 = __p0; \ + bfloat16x4_t __s1 = __p1; \ + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vset_lane_bf16(__s0, __s1, __p2)); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ __ai __attribute__((target("neon"))) uint8x16_t vshlq_u8(uint8x16_t __p0, int8x16_t __p1) { uint8x16_t __ret; @@ -30000,6 +29660,32 @@ __ai __attribute__((target("neon"))) int16x4_t vshl_s16(int16x4_t __p0, int16x4_ }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vst1q_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, __builtin_bit_cast(int8x16_t, __s1), 43); \ +}) +#else +#define vst1q_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x8_t __s1 = __p1; \ + bfloat16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_16); \ + __builtin_neon_vst1q_v(__p0, __builtin_bit_cast(int8x16_t, __rev1), 43); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x4_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, __builtin_bit_cast(int8x8_t, __s1), 11); \ +}) +#else +#define vst1_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x4_t __s1 = __p1; \ + bfloat16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \ + __builtin_neon_vst1_v(__p0, __builtin_bit_cast(int8x8_t, __rev1), 11); \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vst1_lane_p8(__p0, __p1, __p2) __extension__ ({ \ poly8x8_t __s1 = __p1; \ @@ -30268,6 +29954,32 @@ __ai __attribute__((target("neon"))) int16x4_t vshl_s16(int16x4_t __p0, int16x4_ }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, __builtin_bit_cast(int8x16_t, __s1), __p2, 43); \ +}) +#else +#define vst1q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __s1 = __p1; \ + bfloat16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_16); \ + __builtin_neon_vst1q_lane_v(__p0, __builtin_bit_cast(int8x16_t, __rev1), __p2, 43); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, __builtin_bit_cast(int8x8_t, __s1), __p2, 11); \ +}) +#else +#define vst1_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4_t __s1 = __p1; \ + bfloat16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \ + __builtin_neon_vst1_lane_v(__p0, __builtin_bit_cast(int8x8_t, __rev1), __p2, 11); \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vst1_p8_x2(__p0, __p1) __extension__ ({ \ poly8x8x2_t __s1 = __p1; \ @@ -30576,6 +30288,36 @@ __ai __attribute__((target("neon"))) int16x4_t vshl_s16(int16x4_t __p0, int16x4_ }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vst1q_bf16_x2(__p0, __p1) __extension__ ({ \ + bfloat16x8x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), 43); \ +}) +#else +#define vst1q_bf16_x2(__p0, __p1) __extension__ ({ \ + bfloat16x8x2_t __s1 = __p1; \ + bfloat16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ + __builtin_neon_vst1q_x2_v(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), 43); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_bf16_x2(__p0, __p1) __extension__ ({ \ + bfloat16x4x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), 11); \ +}) +#else +#define vst1_bf16_x2(__p0, __p1) __extension__ ({ \ + bfloat16x4x2_t __s1 = __p1; \ + bfloat16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ + __builtin_neon_vst1_x2_v(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), 11); \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vst1_p8_x3(__p0, __p1) __extension__ ({ \ poly8x8x3_t __s1 = __p1; \ @@ -30904,6 +30646,38 @@ __ai __attribute__((target("neon"))) int16x4_t vshl_s16(int16x4_t __p0, int16x4_ }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vst1q_bf16_x3(__p0, __p1) __extension__ ({ \ + bfloat16x8x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), 43); \ +}) +#else +#define vst1q_bf16_x3(__p0, __p1) __extension__ ({ \ + bfloat16x8x3_t __s1 = __p1; \ + bfloat16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ + __builtin_neon_vst1q_x3_v(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), 43); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_bf16_x3(__p0, __p1) __extension__ ({ \ + bfloat16x4x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), 11); \ +}) +#else +#define vst1_bf16_x3(__p0, __p1) __extension__ ({ \ + bfloat16x4x3_t __s1 = __p1; \ + bfloat16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ + __builtin_neon_vst1_x3_v(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), 11); \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vst1_p8_x4(__p0, __p1) __extension__ ({ \ poly8x8x4_t __s1 = __p1; \ @@ -31252,6 +31026,40 @@ __ai __attribute__((target("neon"))) int16x4_t vshl_s16(int16x4_t __p0, int16x4_ }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vst1q_bf16_x4(__p0, __p1) __extension__ ({ \ + bfloat16x8x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), __builtin_bit_cast(int8x16_t, __s1.val[3]), 43); \ +}) +#else +#define vst1q_bf16_x4(__p0, __p1) __extension__ ({ \ + bfloat16x8x4_t __s1 = __p1; \ + bfloat16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_128_16); \ + __builtin_neon_vst1q_x4_v(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), __builtin_bit_cast(int8x16_t, __rev1.val[3]), 43); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_bf16_x4(__p0, __p1) __extension__ ({ \ + bfloat16x4x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), __builtin_bit_cast(int8x8_t, __s1.val[3]), 11); \ +}) +#else +#define vst1_bf16_x4(__p0, __p1) __extension__ ({ \ + bfloat16x4x4_t __s1 = __p1; \ + bfloat16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_64_16); \ + __builtin_neon_vst1_x4_v(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), __builtin_bit_cast(int8x8_t, __rev1.val[3]), 11); \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vst2_p8(__p0, __p1) __extension__ ({ \ poly8x8x2_t __s1 = __p1; \ @@ -31530,6 +31338,36 @@ __ai __attribute__((target("neon"))) int16x4_t vshl_s16(int16x4_t __p0, int16x4_ }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vst2q_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), 43); \ +}) +#else +#define vst2q_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x8x2_t __s1 = __p1; \ + bfloat16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ + __builtin_neon_vst2q_v(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), 43); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), 11); \ +}) +#else +#define vst2_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x4x2_t __s1 = __p1; \ + bfloat16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ + __builtin_neon_vst2_v(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), 11); \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vst2_lane_p8(__p0, __p1, __p2) __extension__ ({ \ poly8x8x2_t __s1 = __p1; \ @@ -31755,6 +31593,36 @@ __ai __attribute__((target("neon"))) int16x4_t vshl_s16(int16x4_t __p0, int16x4_ }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __p2, 43); \ +}) +#else +#define vst2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8x2_t __s1 = __p1; \ + bfloat16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ + __builtin_neon_vst2q_lane_v(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __p2, 43); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __p2, 11); \ +}) +#else +#define vst2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4x2_t __s1 = __p1; \ + bfloat16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ + __builtin_neon_vst2_lane_v(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __p2, 11); \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vst3_p8(__p0, __p1) __extension__ ({ \ poly8x8x3_t __s1 = __p1; \ @@ -32051,6 +31919,38 @@ __ai __attribute__((target("neon"))) int16x4_t vshl_s16(int16x4_t __p0, int16x4_ }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vst3q_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), 43); \ +}) +#else +#define vst3q_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x8x3_t __s1 = __p1; \ + bfloat16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ + __builtin_neon_vst3q_v(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), 43); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), 11); \ +}) +#else +#define vst3_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x4x3_t __s1 = __p1; \ + bfloat16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ + __builtin_neon_vst3_v(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), 11); \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vst3_lane_p8(__p0, __p1, __p2) __extension__ ({ \ poly8x8x3_t __s1 = __p1; \ @@ -32291,6 +32191,38 @@ __ai __attribute__((target("neon"))) int16x4_t vshl_s16(int16x4_t __p0, int16x4_ }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), __p2, 43); \ +}) +#else +#define vst3q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8x3_t __s1 = __p1; \ + bfloat16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ + __builtin_neon_vst3q_lane_v(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), __p2, 43); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), __p2, 11); \ +}) +#else +#define vst3_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4x3_t __s1 = __p1; \ + bfloat16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ + __builtin_neon_vst3_lane_v(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), __p2, 11); \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vst4_p8(__p0, __p1) __extension__ ({ \ poly8x8x4_t __s1 = __p1; \ @@ -32605,6 +32537,40 @@ __ai __attribute__((target("neon"))) int16x4_t vshl_s16(int16x4_t __p0, int16x4_ }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vst4q_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), __builtin_bit_cast(int8x16_t, __s1.val[3]), 43); \ +}) +#else +#define vst4q_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x8x4_t __s1 = __p1; \ + bfloat16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_128_16); \ + __builtin_neon_vst4q_v(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), __builtin_bit_cast(int8x16_t, __rev1.val[3]), 43); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), __builtin_bit_cast(int8x8_t, __s1.val[3]), 11); \ +}) +#else +#define vst4_bf16(__p0, __p1) __extension__ ({ \ + bfloat16x4x4_t __s1 = __p1; \ + bfloat16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_64_16); \ + __builtin_neon_vst4_v(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), __builtin_bit_cast(int8x8_t, __rev1.val[3]), 11); \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vst4_lane_p8(__p0, __p1, __p2) __extension__ ({ \ poly8x8x4_t __s1 = __p1; \ @@ -32860,6 +32826,40 @@ __ai __attribute__((target("neon"))) int16x4_t vshl_s16(int16x4_t __p0, int16x4_ }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, __builtin_bit_cast(int8x16_t, __s1.val[0]), __builtin_bit_cast(int8x16_t, __s1.val[1]), __builtin_bit_cast(int8x16_t, __s1.val[2]), __builtin_bit_cast(int8x16_t, __s1.val[3]), __p2, 43); \ +}) +#else +#define vst4q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8x4_t __s1 = __p1; \ + bfloat16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_128_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_128_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_128_16); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_128_16); \ + __builtin_neon_vst4q_lane_v(__p0, __builtin_bit_cast(int8x16_t, __rev1.val[0]), __builtin_bit_cast(int8x16_t, __rev1.val[1]), __builtin_bit_cast(int8x16_t, __rev1.val[2]), __builtin_bit_cast(int8x16_t, __rev1.val[3]), __p2, 43); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __builtin_bit_cast(int8x8_t, __s1.val[0]), __builtin_bit_cast(int8x8_t, __s1.val[1]), __builtin_bit_cast(int8x8_t, __s1.val[2]), __builtin_bit_cast(int8x8_t, __s1.val[3]), __p2, 11); \ +}) +#else +#define vst4_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x4x4_t __s1 = __p1; \ + bfloat16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], __lane_reverse_64_16); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], __lane_reverse_64_16); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], __lane_reverse_64_16); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], __lane_reverse_64_16); \ + __builtin_neon_vst4_lane_v(__p0, __builtin_bit_cast(int8x8_t, __rev1.val[0]), __builtin_bit_cast(int8x8_t, __rev1.val[1]), __builtin_bit_cast(int8x8_t, __rev1.val[2]), __builtin_bit_cast(int8x8_t, __rev1.val[3]), __p2, 11); \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ __ai __attribute__((target("neon"))) uint8x16_t vsubq_u8(uint8x16_t __p0, uint8x16_t __p1) { uint8x16_t __ret; @@ -37083,266 +37083,6 @@ __ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_low_bf16_f32(float3 } #endif -__ai __attribute__((target("bf16,neon"))) poly8x8_t vreinterpret_p8_bf16(bfloat16x4_t __p0) { - poly8x8_t __ret; - __ret = __builtin_bit_cast(poly8x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) poly64x1_t vreinterpret_p64_bf16(bfloat16x4_t __p0) { - poly64x1_t __ret; - __ret = __builtin_bit_cast(poly64x1_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) poly16x4_t vreinterpret_p16_bf16(bfloat16x4_t __p0) { - poly16x4_t __ret; - __ret = __builtin_bit_cast(poly16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) poly8x16_t vreinterpretq_p8_bf16(bfloat16x8_t __p0) { - poly8x16_t __ret; - __ret = __builtin_bit_cast(poly8x16_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) poly64x2_t vreinterpretq_p64_bf16(bfloat16x8_t __p0) { - poly64x2_t __ret; - __ret = __builtin_bit_cast(poly64x2_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) poly16x8_t vreinterpretq_p16_bf16(bfloat16x8_t __p0) { - poly16x8_t __ret; - __ret = __builtin_bit_cast(poly16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint8x16_t vreinterpretq_u8_bf16(bfloat16x8_t __p0) { - uint8x16_t __ret; - __ret = __builtin_bit_cast(uint8x16_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint32x4_t vreinterpretq_u32_bf16(bfloat16x8_t __p0) { - uint32x4_t __ret; - __ret = __builtin_bit_cast(uint32x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint64x2_t vreinterpretq_u64_bf16(bfloat16x8_t __p0) { - uint64x2_t __ret; - __ret = __builtin_bit_cast(uint64x2_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint16x8_t vreinterpretq_u16_bf16(bfloat16x8_t __p0) { - uint16x8_t __ret; - __ret = __builtin_bit_cast(uint16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int8x16_t vreinterpretq_s8_bf16(bfloat16x8_t __p0) { - int8x16_t __ret; - __ret = __builtin_bit_cast(int8x16_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) float32x4_t vreinterpretq_f32_bf16(bfloat16x8_t __p0) { - float32x4_t __ret; - __ret = __builtin_bit_cast(float32x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) float16x8_t vreinterpretq_f16_bf16(bfloat16x8_t __p0) { - float16x8_t __ret; - __ret = __builtin_bit_cast(float16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int32x4_t vreinterpretq_s32_bf16(bfloat16x8_t __p0) { - int32x4_t __ret; - __ret = __builtin_bit_cast(int32x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int64x2_t vreinterpretq_s64_bf16(bfloat16x8_t __p0) { - int64x2_t __ret; - __ret = __builtin_bit_cast(int64x2_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int16x8_t vreinterpretq_s16_bf16(bfloat16x8_t __p0) { - int16x8_t __ret; - __ret = __builtin_bit_cast(int16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint8x8_t vreinterpret_u8_bf16(bfloat16x4_t __p0) { - uint8x8_t __ret; - __ret = __builtin_bit_cast(uint8x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint32x2_t vreinterpret_u32_bf16(bfloat16x4_t __p0) { - uint32x2_t __ret; - __ret = __builtin_bit_cast(uint32x2_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint64x1_t vreinterpret_u64_bf16(bfloat16x4_t __p0) { - uint64x1_t __ret; - __ret = __builtin_bit_cast(uint64x1_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint16x4_t vreinterpret_u16_bf16(bfloat16x4_t __p0) { - uint16x4_t __ret; - __ret = __builtin_bit_cast(uint16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int8x8_t vreinterpret_s8_bf16(bfloat16x4_t __p0) { - int8x8_t __ret; - __ret = __builtin_bit_cast(int8x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) float32x2_t vreinterpret_f32_bf16(bfloat16x4_t __p0) { - float32x2_t __ret; - __ret = __builtin_bit_cast(float32x2_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) float16x4_t vreinterpret_f16_bf16(bfloat16x4_t __p0) { - float16x4_t __ret; - __ret = __builtin_bit_cast(float16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int32x2_t vreinterpret_s32_bf16(bfloat16x4_t __p0) { - int32x2_t __ret; - __ret = __builtin_bit_cast(int32x2_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int64x1_t vreinterpret_s64_bf16(bfloat16x4_t __p0) { - int64x1_t __ret; - __ret = __builtin_bit_cast(int64x1_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int16x4_t vreinterpret_s16_bf16(bfloat16x4_t __p0) { - int16x4_t __ret; - __ret = __builtin_bit_cast(int16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_p8(poly8x16_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_p64(poly64x2_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_p16(poly16x8_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_u8(uint8x16_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_u32(uint32x4_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_u64(uint64x2_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_u16(uint16x8_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_s8(int8x16_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_f32(float32x4_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_f16(float16x8_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_s32(int32x4_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_s64(int64x2_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_s16(int16x8_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_p8(poly8x8_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_p64(poly64x1_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_p16(poly16x4_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_u8(uint8x8_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_u32(uint32x2_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_u64(uint64x1_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_u16(uint16x4_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_s8(int8x8_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_f32(float32x2_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_f16(float16x4_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_s32(int32x2_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_s64(int64x1_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_s16(int16x4_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} #ifdef __LITTLE_ENDIAN__ #define vqdmulhq_lane_s32(__p0_214, __p1_214, __p2_214) __extension__ ({ \ int32x4_t __ret_214; \ @@ -38831,6 +38571,266 @@ __ai __attribute__((target("neon"))) int16x4_t vreinterpret_s16_s64(int64x1_t __ __ret = __builtin_bit_cast(int16x4_t, __p0); return __ret; } +__ai __attribute__((target("neon"))) poly8x8_t vreinterpret_p8_bf16(bfloat16x4_t __p0) { + poly8x8_t __ret; + __ret = __builtin_bit_cast(poly8x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) poly64x1_t vreinterpret_p64_bf16(bfloat16x4_t __p0) { + poly64x1_t __ret; + __ret = __builtin_bit_cast(poly64x1_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) poly16x4_t vreinterpret_p16_bf16(bfloat16x4_t __p0) { + poly16x4_t __ret; + __ret = __builtin_bit_cast(poly16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) poly8x16_t vreinterpretq_p8_bf16(bfloat16x8_t __p0) { + poly8x16_t __ret; + __ret = __builtin_bit_cast(poly8x16_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) poly64x2_t vreinterpretq_p64_bf16(bfloat16x8_t __p0) { + poly64x2_t __ret; + __ret = __builtin_bit_cast(poly64x2_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) poly16x8_t vreinterpretq_p16_bf16(bfloat16x8_t __p0) { + poly16x8_t __ret; + __ret = __builtin_bit_cast(poly16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint8x16_t vreinterpretq_u8_bf16(bfloat16x8_t __p0) { + uint8x16_t __ret; + __ret = __builtin_bit_cast(uint8x16_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint32x4_t vreinterpretq_u32_bf16(bfloat16x8_t __p0) { + uint32x4_t __ret; + __ret = __builtin_bit_cast(uint32x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint64x2_t vreinterpretq_u64_bf16(bfloat16x8_t __p0) { + uint64x2_t __ret; + __ret = __builtin_bit_cast(uint64x2_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint16x8_t vreinterpretq_u16_bf16(bfloat16x8_t __p0) { + uint16x8_t __ret; + __ret = __builtin_bit_cast(uint16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int8x16_t vreinterpretq_s8_bf16(bfloat16x8_t __p0) { + int8x16_t __ret; + __ret = __builtin_bit_cast(int8x16_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) float32x4_t vreinterpretq_f32_bf16(bfloat16x8_t __p0) { + float32x4_t __ret; + __ret = __builtin_bit_cast(float32x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) float16x8_t vreinterpretq_f16_bf16(bfloat16x8_t __p0) { + float16x8_t __ret; + __ret = __builtin_bit_cast(float16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int32x4_t vreinterpretq_s32_bf16(bfloat16x8_t __p0) { + int32x4_t __ret; + __ret = __builtin_bit_cast(int32x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int64x2_t vreinterpretq_s64_bf16(bfloat16x8_t __p0) { + int64x2_t __ret; + __ret = __builtin_bit_cast(int64x2_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int16x8_t vreinterpretq_s16_bf16(bfloat16x8_t __p0) { + int16x8_t __ret; + __ret = __builtin_bit_cast(int16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint8x8_t vreinterpret_u8_bf16(bfloat16x4_t __p0) { + uint8x8_t __ret; + __ret = __builtin_bit_cast(uint8x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint32x2_t vreinterpret_u32_bf16(bfloat16x4_t __p0) { + uint32x2_t __ret; + __ret = __builtin_bit_cast(uint32x2_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint64x1_t vreinterpret_u64_bf16(bfloat16x4_t __p0) { + uint64x1_t __ret; + __ret = __builtin_bit_cast(uint64x1_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint16x4_t vreinterpret_u16_bf16(bfloat16x4_t __p0) { + uint16x4_t __ret; + __ret = __builtin_bit_cast(uint16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int8x8_t vreinterpret_s8_bf16(bfloat16x4_t __p0) { + int8x8_t __ret; + __ret = __builtin_bit_cast(int8x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) float32x2_t vreinterpret_f32_bf16(bfloat16x4_t __p0) { + float32x2_t __ret; + __ret = __builtin_bit_cast(float32x2_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) float16x4_t vreinterpret_f16_bf16(bfloat16x4_t __p0) { + float16x4_t __ret; + __ret = __builtin_bit_cast(float16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int32x2_t vreinterpret_s32_bf16(bfloat16x4_t __p0) { + int32x2_t __ret; + __ret = __builtin_bit_cast(int32x2_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int64x1_t vreinterpret_s64_bf16(bfloat16x4_t __p0) { + int64x1_t __ret; + __ret = __builtin_bit_cast(int64x1_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int16x4_t vreinterpret_s16_bf16(bfloat16x4_t __p0) { + int16x4_t __ret; + __ret = __builtin_bit_cast(int16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_p8(poly8x16_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_p64(poly64x2_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_p16(poly16x8_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_u8(uint8x16_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_u32(uint32x4_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_u64(uint64x2_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_u16(uint16x8_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_s8(int8x16_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_f32(float32x4_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_f16(float16x8_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_s32(int32x4_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_s64(int64x2_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_s16(int16x8_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_p8(poly8x8_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_p64(poly64x1_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_p16(poly16x4_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_u8(uint8x8_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_u32(uint32x2_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_u64(uint64x1_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_u16(uint16x4_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_s8(int8x8_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_f32(float32x2_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_f16(float16x4_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_s32(int32x2_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_s64(int64x1_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_s16(int16x4_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} #endif #if (__ARM_FP & 2) #ifdef __LITTLE_ENDIAN__ @@ -41141,6 +41141,174 @@ __ai __attribute__((target("neon"))) float32x2_t vfms_f32(float32x2_t __p0, floa #endif #if defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai __attribute__((target("f16f32dot,neon"))) float32x4_t vdotq_f32_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { + float32x4_t __ret; + __ret = __builtin_bit_cast(float32x4_t, __builtin_neon_vdotq_f32_f16(__builtin_bit_cast(int8x16_t, __p0), __builtin_bit_cast(int8x16_t, __p1), __builtin_bit_cast(int8x16_t, __p2), 41)); + return __ret; +} +#else +__ai __attribute__((target("f16f32dot,neon"))) float32x4_t vdotq_f32_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { + float32x4_t __ret; + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_32); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, __lane_reverse_128_16); + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, __lane_reverse_128_16); + __ret = __builtin_bit_cast(float32x4_t, __builtin_neon_vdotq_f32_f16(__builtin_bit_cast(int8x16_t, __rev0), __builtin_bit_cast(int8x16_t, __rev1), __builtin_bit_cast(int8x16_t, __rev2), 41)); + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_32); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai __attribute__((target("f16f32dot,neon"))) float32x2_t vdot_f32_f16(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) { + float32x2_t __ret; + __ret = __builtin_bit_cast(float32x2_t, __builtin_neon_vdot_f32_f16(__builtin_bit_cast(int8x8_t, __p0), __builtin_bit_cast(int8x8_t, __p1), __builtin_bit_cast(int8x8_t, __p2), 9)); + return __ret; +} +#else +__ai __attribute__((target("f16f32dot,neon"))) float32x2_t vdot_f32_f16(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) { + float32x2_t __ret; + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_64_32); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, __lane_reverse_64_16); + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, __lane_reverse_64_16); + __ret = __builtin_bit_cast(float32x2_t, __builtin_neon_vdot_f32_f16(__builtin_bit_cast(int8x8_t, __rev0), __builtin_bit_cast(int8x8_t, __rev1), __builtin_bit_cast(int8x8_t, __rev2), 9)); + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_32); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdotq_lane_f32_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __ret; \ + float32x4_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + __ret = __builtin_bit_cast(float32x4_t, __builtin_neon_vdotq_lane_f32_f16(__builtin_bit_cast(int8x16_t, __s0), __builtin_bit_cast(int8x16_t, __s1), __builtin_bit_cast(int8x8_t, __s2), __p3, 41)); \ + __ret; \ +}) +#else +#define vdotq_lane_f32_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __ret; \ + float32x4_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_32); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_16); \ + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, __lane_reverse_64_16); \ + __ret = __builtin_bit_cast(float32x4_t, __builtin_neon_vdotq_lane_f32_f16(__builtin_bit_cast(int8x16_t, __rev0), __builtin_bit_cast(int8x16_t, __rev1), __builtin_bit_cast(int8x8_t, __rev2), __p3, 41)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_32); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdot_lane_f32_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __ret; \ + float32x2_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + __ret = __builtin_bit_cast(float32x2_t, __builtin_neon_vdot_lane_f32_f16(__builtin_bit_cast(int8x8_t, __s0), __builtin_bit_cast(int8x8_t, __s1), __builtin_bit_cast(int8x8_t, __s2), __p3, 9)); \ + __ret; \ +}) +#else +#define vdot_lane_f32_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __ret; \ + float32x2_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_32); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \ + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, __lane_reverse_64_16); \ + __ret = __builtin_bit_cast(float32x2_t, __builtin_neon_vdot_lane_f32_f16(__builtin_bit_cast(int8x8_t, __rev0), __builtin_bit_cast(int8x8_t, __rev1), __builtin_bit_cast(int8x8_t, __rev2), __p3, 9)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_32); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdotq_laneq_f32_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __ret; \ + float32x4_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + __ret = __builtin_bit_cast(float32x4_t, __builtin_neon_vdotq_laneq_f32_f16(__builtin_bit_cast(int8x16_t, __s0), __builtin_bit_cast(int8x16_t, __s1), __builtin_bit_cast(int8x16_t, __s2), __p3, 41)); \ + __ret; \ +}) +#else +#define vdotq_laneq_f32_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __ret; \ + float32x4_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_32); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_16); \ + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, __lane_reverse_128_16); \ + __ret = __builtin_bit_cast(float32x4_t, __builtin_neon_vdotq_laneq_f32_f16(__builtin_bit_cast(int8x16_t, __rev0), __builtin_bit_cast(int8x16_t, __rev1), __builtin_bit_cast(int8x16_t, __rev2), __p3, 41)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_32); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdot_laneq_f32_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __ret; \ + float32x2_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + __ret = __builtin_bit_cast(float32x2_t, __builtin_neon_vdot_laneq_f32_f16(__builtin_bit_cast(int8x8_t, __s0), __builtin_bit_cast(int8x8_t, __s1), __builtin_bit_cast(int8x16_t, __s2), __p3, 9)); \ + __ret; \ +}) +#else +#define vdot_laneq_f32_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __ret; \ + float32x2_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_32); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \ + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, __lane_reverse_128_16); \ + __ret = __builtin_bit_cast(float32x2_t, __builtin_neon_vdot_laneq_f32_f16(__builtin_bit_cast(int8x8_t, __rev0), __builtin_bit_cast(int8x8_t, __rev1), __builtin_bit_cast(int8x16_t, __rev2), __p3, 9)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_32); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai __attribute__((target("f16f32mm,neon"))) float32x4_t vmmlaq_f32_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { + float32x4_t __ret; + __ret = __builtin_bit_cast(float32x4_t, __builtin_neon_vmmlaq_f32_f16(__builtin_bit_cast(int8x16_t, __p0), __builtin_bit_cast(int8x16_t, __p1), __builtin_bit_cast(int8x16_t, __p2), 41)); + return __ret; +} +#else +__ai __attribute__((target("f16f32mm,neon"))) float32x4_t vmmlaq_f32_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { + float32x4_t __ret; + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_32); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, __lane_reverse_128_16); + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, __lane_reverse_128_16); + __ret = __builtin_bit_cast(float32x4_t, __builtin_neon_vmmlaq_f32_f16(__builtin_bit_cast(int8x16_t, __rev0), __builtin_bit_cast(int8x16_t, __rev1), __builtin_bit_cast(int8x16_t, __rev2), 41)); + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_32); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai __attribute__((target("f16mm,neon"))) float16x8_t vmmlaq_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + float16x8_t __ret; + __ret = __builtin_bit_cast(float16x8_t, __builtin_neon_vmmlaq_f16(__builtin_bit_cast(int8x16_t, __p0), __builtin_bit_cast(int8x16_t, __p1), __builtin_bit_cast(int8x16_t, __p2), 40)); + return __ret; +} +#else +__ai __attribute__((target("f16mm,neon"))) float16x8_t vmmlaq_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + float16x8_t __ret; + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_16); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, __lane_reverse_128_16); + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, __lane_reverse_128_16); + __ret = __builtin_bit_cast(float16x8_t, __builtin_neon_vmmlaq_f16(__builtin_bit_cast(int8x16_t, __rev0), __builtin_bit_cast(int8x16_t, __rev1), __builtin_bit_cast(int8x16_t, __rev2), 40)); + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); + return __ret; +} +#endif + #ifdef __LITTLE_ENDIAN__ __ai __attribute__((target("f8f16mm,neon"))) float16x8_t vmmlaq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) { float16x8_t __ret; @@ -42391,6 +42559,48 @@ __ai __attribute__((target("fp8fma,neon"))) float16x8_t vmlaltq_f16_mf8_fpm(floa }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vluti2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2q_lane_bf16(__builtin_bit_cast(int8x16_t, __s0), __builtin_bit_cast(int8x8_t, __s1), __p2, 43)); \ + __ret; \ +}) +#else +#define vluti2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_8); \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2q_lane_bf16(__builtin_bit_cast(int8x16_t, __rev0), __builtin_bit_cast(int8x8_t, __rev1), __p2, 43)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vluti2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2_lane_bf16(__builtin_bit_cast(int8x8_t, __s0), __builtin_bit_cast(int8x8_t, __s1), __p2, 43)); \ + __ret; \ +}) +#else +#define vluti2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_8); \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2_lane_bf16(__builtin_bit_cast(int8x8_t, __rev0), __builtin_bit_cast(int8x8_t, __rev1), __p2, 43)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vluti2_lane_p16(__p0, __p1, __p2) __extension__ ({ \ poly16x8_t __ret; \ @@ -42559,6 +42769,48 @@ __ai __attribute__((target("fp8fma,neon"))) float16x8_t vmlaltq_f16_mf8_fpm(floa }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vluti2q_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2q_laneq_bf16(__builtin_bit_cast(int8x16_t, __s0), __builtin_bit_cast(int8x16_t, __s1), __p2, 43)); \ + __ret; \ +}) +#else +#define vluti2q_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x8_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_8); \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2q_laneq_bf16(__builtin_bit_cast(int8x16_t, __rev0), __builtin_bit_cast(int8x16_t, __rev1), __p2, 43)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vluti2_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2_laneq_bf16(__builtin_bit_cast(int8x8_t, __s0), __builtin_bit_cast(int8x16_t, __s1), __p2, 43)); \ + __ret; \ +}) +#else +#define vluti2_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x4_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_8); \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2_laneq_bf16(__builtin_bit_cast(int8x8_t, __rev0), __builtin_bit_cast(int8x16_t, __rev1), __p2, 43)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vluti2_laneq_p8(__p0, __p1, __p2) __extension__ ({ \ poly8x16_t __ret; \ @@ -42979,6 +43231,29 @@ __ai __attribute__((target("fp8fma,neon"))) float16x8_t vmlaltq_f16_mf8_fpm(floa }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vluti4q_lane_bf16_x2(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x8x2_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti4q_lane_bf16_x2(__builtin_bit_cast(int8x16_t, __s0.val[0]), __builtin_bit_cast(int8x16_t, __s0.val[1]), __builtin_bit_cast(int8x8_t, __s1), __p2, 43)); \ + __ret; \ +}) +#else +#define vluti4q_lane_bf16_x2(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x8x2_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + bfloat16x8x2_t __rev0; \ + __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], __lane_reverse_128_16); \ + __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], __lane_reverse_128_16); \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_8); \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti4q_lane_bf16_x2(__builtin_bit_cast(int8x16_t, __rev0.val[0]), __builtin_bit_cast(int8x16_t, __rev0.val[1]), __builtin_bit_cast(int8x8_t, __rev1), __p2, 43)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vluti4q_lane_p16_x2(__p0, __p1, __p2) __extension__ ({ \ poly16x8_t __ret; \ @@ -43155,6 +43430,29 @@ __ai __attribute__((target("fp8fma,neon"))) float16x8_t vmlaltq_f16_mf8_fpm(floa }) #endif +#ifdef __LITTLE_ENDIAN__ +#define vluti4q_laneq_bf16_x2(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x8x2_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti4q_laneq_bf16_x2(__builtin_bit_cast(int8x16_t, __s0.val[0]), __builtin_bit_cast(int8x16_t, __s0.val[1]), __builtin_bit_cast(int8x16_t, __s1), __p2, 43)); \ + __ret; \ +}) +#else +#define vluti4q_laneq_bf16_x2(__p0, __p1, __p2) __extension__ ({ \ + bfloat16x8_t __ret; \ + bfloat16x8x2_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + bfloat16x8x2_t __rev0; \ + __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], __lane_reverse_128_16); \ + __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], __lane_reverse_128_16); \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_8); \ + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti4q_laneq_bf16_x2(__builtin_bit_cast(int8x16_t, __rev0.val[0]), __builtin_bit_cast(int8x16_t, __rev0.val[1]), __builtin_bit_cast(int8x16_t, __rev1), __p2, 43)); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ + __ret; \ +}) +#endif + #ifdef __LITTLE_ENDIAN__ #define vluti4q_laneq_p16_x2(__p0, __p1, __p2) __extension__ ({ \ poly16x8_t __ret; \ @@ -43247,136 +43545,6 @@ __ai __attribute__((target("fp8fma,neon"))) float16x8_t vmlaltq_f16_mf8_fpm(floa }) #endif -#ifdef __LITTLE_ENDIAN__ -#define vluti2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - uint8x8_t __s1 = __p1; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2q_lane_bf16(__builtin_bit_cast(int8x16_t, __s0), __builtin_bit_cast(int8x8_t, __s1), __p2, 43)); \ - __ret; \ -}) -#else -#define vluti2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - uint8x8_t __s1 = __p1; \ - bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ - uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_8); \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2q_lane_bf16(__builtin_bit_cast(int8x16_t, __rev0), __builtin_bit_cast(int8x8_t, __rev1), __p2, 43)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vluti2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - uint8x8_t __s1 = __p1; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2_lane_bf16(__builtin_bit_cast(int8x8_t, __s0), __builtin_bit_cast(int8x8_t, __s1), __p2, 43)); \ - __ret; \ -}) -#else -#define vluti2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - uint8x8_t __s1 = __p1; \ - bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ - uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_8); \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2_lane_bf16(__builtin_bit_cast(int8x8_t, __rev0), __builtin_bit_cast(int8x8_t, __rev1), __p2, 43)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vluti2q_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - uint8x16_t __s1 = __p1; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2q_laneq_bf16(__builtin_bit_cast(int8x16_t, __s0), __builtin_bit_cast(int8x16_t, __s1), __p2, 43)); \ - __ret; \ -}) -#else -#define vluti2q_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x8_t __s0 = __p0; \ - uint8x16_t __s1 = __p1; \ - bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ - uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_8); \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2q_laneq_bf16(__builtin_bit_cast(int8x16_t, __rev0), __builtin_bit_cast(int8x16_t, __rev1), __p2, 43)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vluti2_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - uint8x16_t __s1 = __p1; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2_laneq_bf16(__builtin_bit_cast(int8x8_t, __s0), __builtin_bit_cast(int8x16_t, __s1), __p2, 43)); \ - __ret; \ -}) -#else -#define vluti2_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x4_t __s0 = __p0; \ - uint8x16_t __s1 = __p1; \ - bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ - uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_8); \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti2_laneq_bf16(__builtin_bit_cast(int8x8_t, __rev0), __builtin_bit_cast(int8x16_t, __rev1), __p2, 43)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vluti4q_lane_bf16_x2(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x8x2_t __s0 = __p0; \ - uint8x8_t __s1 = __p1; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti4q_lane_bf16_x2(__builtin_bit_cast(int8x16_t, __s0.val[0]), __builtin_bit_cast(int8x16_t, __s0.val[1]), __builtin_bit_cast(int8x8_t, __s1), __p2, 43)); \ - __ret; \ -}) -#else -#define vluti4q_lane_bf16_x2(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x8x2_t __s0 = __p0; \ - uint8x8_t __s1 = __p1; \ - bfloat16x8x2_t __rev0; \ - __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], __lane_reverse_128_16); \ - __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], __lane_reverse_128_16); \ - uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_8); \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti4q_lane_bf16_x2(__builtin_bit_cast(int8x16_t, __rev0.val[0]), __builtin_bit_cast(int8x16_t, __rev0.val[1]), __builtin_bit_cast(int8x8_t, __rev1), __p2, 43)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vluti4q_laneq_bf16_x2(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x8x2_t __s0 = __p0; \ - uint8x16_t __s1 = __p1; \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti4q_laneq_bf16_x2(__builtin_bit_cast(int8x16_t, __s0.val[0]), __builtin_bit_cast(int8x16_t, __s0.val[1]), __builtin_bit_cast(int8x16_t, __s1), __p2, 43)); \ - __ret; \ -}) -#else -#define vluti4q_laneq_bf16_x2(__p0, __p1, __p2) __extension__ ({ \ - bfloat16x8_t __ret; \ - bfloat16x8x2_t __s0 = __p0; \ - uint8x16_t __s1 = __p1; \ - bfloat16x8x2_t __rev0; \ - __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], __lane_reverse_128_16); \ - __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], __lane_reverse_128_16); \ - uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_8); \ - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vluti4q_laneq_bf16_x2(__builtin_bit_cast(int8x16_t, __rev0.val[0]), __builtin_bit_cast(int8x16_t, __rev0.val[1]), __builtin_bit_cast(int8x16_t, __rev1), __p2, 43)); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ - __ret; \ -}) -#endif - #ifdef __LITTLE_ENDIAN__ #define splatq_lane_mf8(__p0, __p1) __extension__ ({ \ mfloat8x16_t __ret; \ @@ -44348,524 +44516,150 @@ __ai __attribute__((target("aes,neon"))) poly128_t vmull_p64(poly64_t __p0, poly return __ret; } #ifdef __LITTLE_ENDIAN__ -#define vcopyq_lane_bf16(__p0_234, __p1_234, __p2_234, __p3_234) __extension__ ({ \ - bfloat16x8_t __ret_234; \ - bfloat16x8_t __s0_234 = __p0_234; \ - bfloat16x4_t __s2_234 = __p2_234; \ - __ret_234 = vsetq_lane_bf16(vget_lane_bf16(__s2_234, __p3_234), __s0_234, __p1_234); \ +__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vcvt_bf16_f32(float32x4_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vcvt_bf16_f32(__builtin_bit_cast(int8x16_t, __p0), 11)); + return __ret; +} +#else +__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vcvt_bf16_f32(float32x4_t __p0) { + bfloat16x4_t __ret; + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_32); + __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vcvt_bf16_f32(__builtin_bit_cast(int8x16_t, __rev0), 11)); + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_high_bf16_f32(bfloat16x8_t __p0, float32x4_t __p1) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vcvtq_high_bf16_f32(__builtin_bit_cast(int8x16_t, __p0), __builtin_bit_cast(int8x16_t, __p1), 43)); + return __ret; +} +#else +__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_high_bf16_f32(bfloat16x8_t __p0, float32x4_t __p1) { + bfloat16x8_t __ret; + bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_16); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, __lane_reverse_128_32); + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vcvtq_high_bf16_f32(__builtin_bit_cast(int8x16_t, __rev0), __builtin_bit_cast(int8x16_t, __rev1), 43)); + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_low_bf16_f32(float32x4_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vcvtq_low_bf16_f32(__builtin_bit_cast(int8x16_t, __p0), 43)); + return __ret; +} +#else +__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_low_bf16_f32(float32x4_t __p0) { + bfloat16x8_t __ret; + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_32); + __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vcvtq_low_bf16_f32(__builtin_bit_cast(int8x16_t, __rev0), 43)); + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdotq_laneq_u32(__p0_234, __p1_234, __p2_234, __p3_234) __extension__ ({ \ + uint32x4_t __ret_234; \ + uint32x4_t __s0_234 = __p0_234; \ + uint8x16_t __s1_234 = __p1_234; \ + uint8x16_t __s2_234 = __p2_234; \ + __ret_234 = vdotq_u32(__s0_234, __s1_234, __builtin_bit_cast(uint8x16_t, splatq_laneq_u32(__builtin_bit_cast(uint32x4_t, __s2_234), __p3_234))); \ __ret_234; \ }) #else -#define vcopyq_lane_bf16(__p0_235, __p1_235, __p2_235, __p3_235) __extension__ ({ \ - bfloat16x8_t __ret_235; \ - bfloat16x8_t __s0_235 = __p0_235; \ - bfloat16x4_t __s2_235 = __p2_235; \ - bfloat16x8_t __rev0_235; __rev0_235 = __builtin_shufflevector(__s0_235, __s0_235, __lane_reverse_128_16); \ - bfloat16x4_t __rev2_235; __rev2_235 = __builtin_shufflevector(__s2_235, __s2_235, __lane_reverse_64_16); \ - __ret_235 = __noswap_vsetq_lane_bf16(__noswap_vget_lane_bf16(__rev2_235, __p3_235), __rev0_235, __p1_235); \ - __ret_235 = __builtin_shufflevector(__ret_235, __ret_235, __lane_reverse_128_16); \ +#define vdotq_laneq_u32(__p0_235, __p1_235, __p2_235, __p3_235) __extension__ ({ \ + uint32x4_t __ret_235; \ + uint32x4_t __s0_235 = __p0_235; \ + uint8x16_t __s1_235 = __p1_235; \ + uint8x16_t __s2_235 = __p2_235; \ + uint32x4_t __rev0_235; __rev0_235 = __builtin_shufflevector(__s0_235, __s0_235, __lane_reverse_128_32); \ + uint8x16_t __rev1_235; __rev1_235 = __builtin_shufflevector(__s1_235, __s1_235, __lane_reverse_128_8); \ + uint8x16_t __rev2_235; __rev2_235 = __builtin_shufflevector(__s2_235, __s2_235, __lane_reverse_128_8); \ + __ret_235 = __noswap_vdotq_u32(__rev0_235, __rev1_235, __builtin_bit_cast(uint8x16_t, __noswap_splatq_laneq_u32(__builtin_bit_cast(uint32x4_t, __rev2_235), __p3_235))); \ + __ret_235 = __builtin_shufflevector(__ret_235, __ret_235, __lane_reverse_128_32); \ __ret_235; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_lane_bf16(__p0_236, __p1_236, __p2_236, __p3_236) __extension__ ({ \ - bfloat16x4_t __ret_236; \ - bfloat16x4_t __s0_236 = __p0_236; \ - bfloat16x4_t __s2_236 = __p2_236; \ - __ret_236 = vset_lane_bf16(vget_lane_bf16(__s2_236, __p3_236), __s0_236, __p1_236); \ +#define vdotq_laneq_s32(__p0_236, __p1_236, __p2_236, __p3_236) __extension__ ({ \ + int32x4_t __ret_236; \ + int32x4_t __s0_236 = __p0_236; \ + int8x16_t __s1_236 = __p1_236; \ + int8x16_t __s2_236 = __p2_236; \ + __ret_236 = vdotq_s32(__s0_236, __s1_236, __builtin_bit_cast(int8x16_t, splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_236), __p3_236))); \ __ret_236; \ }) #else -#define vcopy_lane_bf16(__p0_237, __p1_237, __p2_237, __p3_237) __extension__ ({ \ - bfloat16x4_t __ret_237; \ - bfloat16x4_t __s0_237 = __p0_237; \ - bfloat16x4_t __s2_237 = __p2_237; \ - bfloat16x4_t __rev0_237; __rev0_237 = __builtin_shufflevector(__s0_237, __s0_237, __lane_reverse_64_16); \ - bfloat16x4_t __rev2_237; __rev2_237 = __builtin_shufflevector(__s2_237, __s2_237, __lane_reverse_64_16); \ - __ret_237 = __noswap_vset_lane_bf16(__noswap_vget_lane_bf16(__rev2_237, __p3_237), __rev0_237, __p1_237); \ - __ret_237 = __builtin_shufflevector(__ret_237, __ret_237, __lane_reverse_64_16); \ +#define vdotq_laneq_s32(__p0_237, __p1_237, __p2_237, __p3_237) __extension__ ({ \ + int32x4_t __ret_237; \ + int32x4_t __s0_237 = __p0_237; \ + int8x16_t __s1_237 = __p1_237; \ + int8x16_t __s2_237 = __p2_237; \ + int32x4_t __rev0_237; __rev0_237 = __builtin_shufflevector(__s0_237, __s0_237, __lane_reverse_128_32); \ + int8x16_t __rev1_237; __rev1_237 = __builtin_shufflevector(__s1_237, __s1_237, __lane_reverse_128_8); \ + int8x16_t __rev2_237; __rev2_237 = __builtin_shufflevector(__s2_237, __s2_237, __lane_reverse_128_8); \ + __ret_237 = __noswap_vdotq_s32(__rev0_237, __rev1_237, __builtin_bit_cast(int8x16_t, __noswap_splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_237), __p3_237))); \ + __ret_237 = __builtin_shufflevector(__ret_237, __ret_237, __lane_reverse_128_32); \ __ret_237; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_laneq_bf16(__p0_238, __p1_238, __p2_238, __p3_238) __extension__ ({ \ - bfloat16x8_t __ret_238; \ - bfloat16x8_t __s0_238 = __p0_238; \ - bfloat16x8_t __s2_238 = __p2_238; \ - __ret_238 = vsetq_lane_bf16(vgetq_lane_bf16(__s2_238, __p3_238), __s0_238, __p1_238); \ +#define vdot_laneq_u32(__p0_238, __p1_238, __p2_238, __p3_238) __extension__ ({ \ + uint32x2_t __ret_238; \ + uint32x2_t __s0_238 = __p0_238; \ + uint8x8_t __s1_238 = __p1_238; \ + uint8x16_t __s2_238 = __p2_238; \ + __ret_238 = vdot_u32(__s0_238, __s1_238, __builtin_bit_cast(uint8x8_t, splat_laneq_u32(__builtin_bit_cast(uint32x4_t, __s2_238), __p3_238))); \ __ret_238; \ }) #else -#define vcopyq_laneq_bf16(__p0_239, __p1_239, __p2_239, __p3_239) __extension__ ({ \ - bfloat16x8_t __ret_239; \ - bfloat16x8_t __s0_239 = __p0_239; \ - bfloat16x8_t __s2_239 = __p2_239; \ - bfloat16x8_t __rev0_239; __rev0_239 = __builtin_shufflevector(__s0_239, __s0_239, __lane_reverse_128_16); \ - bfloat16x8_t __rev2_239; __rev2_239 = __builtin_shufflevector(__s2_239, __s2_239, __lane_reverse_128_16); \ - __ret_239 = __noswap_vsetq_lane_bf16(__noswap_vgetq_lane_bf16(__rev2_239, __p3_239), __rev0_239, __p1_239); \ - __ret_239 = __builtin_shufflevector(__ret_239, __ret_239, __lane_reverse_128_16); \ +#define vdot_laneq_u32(__p0_239, __p1_239, __p2_239, __p3_239) __extension__ ({ \ + uint32x2_t __ret_239; \ + uint32x2_t __s0_239 = __p0_239; \ + uint8x8_t __s1_239 = __p1_239; \ + uint8x16_t __s2_239 = __p2_239; \ + uint32x2_t __rev0_239; __rev0_239 = __builtin_shufflevector(__s0_239, __s0_239, __lane_reverse_64_32); \ + uint8x8_t __rev1_239; __rev1_239 = __builtin_shufflevector(__s1_239, __s1_239, __lane_reverse_64_8); \ + uint8x16_t __rev2_239; __rev2_239 = __builtin_shufflevector(__s2_239, __s2_239, __lane_reverse_128_8); \ + __ret_239 = __noswap_vdot_u32(__rev0_239, __rev1_239, __builtin_bit_cast(uint8x8_t, __noswap_splat_laneq_u32(__builtin_bit_cast(uint32x4_t, __rev2_239), __p3_239))); \ + __ret_239 = __builtin_shufflevector(__ret_239, __ret_239, __lane_reverse_64_32); \ __ret_239; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_laneq_bf16(__p0_240, __p1_240, __p2_240, __p3_240) __extension__ ({ \ - bfloat16x4_t __ret_240; \ - bfloat16x4_t __s0_240 = __p0_240; \ - bfloat16x8_t __s2_240 = __p2_240; \ - __ret_240 = vset_lane_bf16(vgetq_lane_bf16(__s2_240, __p3_240), __s0_240, __p1_240); \ +#define vdot_laneq_s32(__p0_240, __p1_240, __p2_240, __p3_240) __extension__ ({ \ + int32x2_t __ret_240; \ + int32x2_t __s0_240 = __p0_240; \ + int8x8_t __s1_240 = __p1_240; \ + int8x16_t __s2_240 = __p2_240; \ + __ret_240 = vdot_s32(__s0_240, __s1_240, __builtin_bit_cast(int8x8_t, splat_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_240), __p3_240))); \ __ret_240; \ }) #else -#define vcopy_laneq_bf16(__p0_241, __p1_241, __p2_241, __p3_241) __extension__ ({ \ - bfloat16x4_t __ret_241; \ - bfloat16x4_t __s0_241 = __p0_241; \ - bfloat16x8_t __s2_241 = __p2_241; \ - bfloat16x4_t __rev0_241; __rev0_241 = __builtin_shufflevector(__s0_241, __s0_241, __lane_reverse_64_16); \ - bfloat16x8_t __rev2_241; __rev2_241 = __builtin_shufflevector(__s2_241, __s2_241, __lane_reverse_128_16); \ - __ret_241 = __noswap_vset_lane_bf16(__noswap_vgetq_lane_bf16(__rev2_241, __p3_241), __rev0_241, __p1_241); \ - __ret_241 = __builtin_shufflevector(__ret_241, __ret_241, __lane_reverse_64_16); \ +#define vdot_laneq_s32(__p0_241, __p1_241, __p2_241, __p3_241) __extension__ ({ \ + int32x2_t __ret_241; \ + int32x2_t __s0_241 = __p0_241; \ + int8x8_t __s1_241 = __p1_241; \ + int8x16_t __s2_241 = __p2_241; \ + int32x2_t __rev0_241; __rev0_241 = __builtin_shufflevector(__s0_241, __s0_241, __lane_reverse_64_32); \ + int8x8_t __rev1_241; __rev1_241 = __builtin_shufflevector(__s1_241, __s1_241, __lane_reverse_64_8); \ + int8x16_t __rev2_241; __rev2_241 = __builtin_shufflevector(__s2_241, __s2_241, __lane_reverse_128_8); \ + __ret_241 = __noswap_vdot_s32(__rev0_241, __rev1_241, __builtin_bit_cast(int8x8_t, __noswap_splat_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_241), __p3_241))); \ + __ret_241 = __builtin_shufflevector(__ret_241, __ret_241, __lane_reverse_64_32); \ __ret_241; \ }) #endif -#ifdef __LITTLE_ENDIAN__ -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vcvt_bf16_f32(float32x4_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vcvt_bf16_f32(__builtin_bit_cast(int8x16_t, __p0), 11)); - return __ret; -} -#else -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vcvt_bf16_f32(float32x4_t __p0) { - bfloat16x4_t __ret; - float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_32); - __ret = __builtin_bit_cast(bfloat16x4_t, __builtin_neon_vcvt_bf16_f32(__builtin_bit_cast(int8x16_t, __rev0), 11)); - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); - return __ret; -} -#endif - -#ifdef __LITTLE_ENDIAN__ -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_high_bf16_f32(bfloat16x8_t __p0, float32x4_t __p1) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vcvtq_high_bf16_f32(__builtin_bit_cast(int8x16_t, __p0), __builtin_bit_cast(int8x16_t, __p1), 43)); - return __ret; -} -#else -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_high_bf16_f32(bfloat16x8_t __p0, float32x4_t __p1) { - bfloat16x8_t __ret; - bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_16); - float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, __lane_reverse_128_32); - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vcvtq_high_bf16_f32(__builtin_bit_cast(int8x16_t, __rev0), __builtin_bit_cast(int8x16_t, __rev1), 43)); - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); - return __ret; -} -#endif - -#ifdef __LITTLE_ENDIAN__ -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_low_bf16_f32(float32x4_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vcvtq_low_bf16_f32(__builtin_bit_cast(int8x16_t, __p0), 43)); - return __ret; -} -#else -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_low_bf16_f32(float32x4_t __p0) { - bfloat16x8_t __ret; - float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_32); - __ret = __builtin_bit_cast(bfloat16x8_t, __builtin_neon_vcvtq_low_bf16_f32(__builtin_bit_cast(int8x16_t, __rev0), 43)); - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); - return __ret; -} -#endif - -__ai __attribute__((target("bf16,neon"))) poly8x8_t vreinterpret_p8_bf16(bfloat16x4_t __p0) { - poly8x8_t __ret; - __ret = __builtin_bit_cast(poly8x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) poly64x1_t vreinterpret_p64_bf16(bfloat16x4_t __p0) { - poly64x1_t __ret; - __ret = __builtin_bit_cast(poly64x1_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) poly16x4_t vreinterpret_p16_bf16(bfloat16x4_t __p0) { - poly16x4_t __ret; - __ret = __builtin_bit_cast(poly16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) poly8x16_t vreinterpretq_p8_bf16(bfloat16x8_t __p0) { - poly8x16_t __ret; - __ret = __builtin_bit_cast(poly8x16_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) poly128_t vreinterpretq_p128_bf16(bfloat16x8_t __p0) { - poly128_t __ret; - __ret = __builtin_bit_cast(poly128_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) poly64x2_t vreinterpretq_p64_bf16(bfloat16x8_t __p0) { - poly64x2_t __ret; - __ret = __builtin_bit_cast(poly64x2_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) poly16x8_t vreinterpretq_p16_bf16(bfloat16x8_t __p0) { - poly16x8_t __ret; - __ret = __builtin_bit_cast(poly16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint8x16_t vreinterpretq_u8_bf16(bfloat16x8_t __p0) { - uint8x16_t __ret; - __ret = __builtin_bit_cast(uint8x16_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint32x4_t vreinterpretq_u32_bf16(bfloat16x8_t __p0) { - uint32x4_t __ret; - __ret = __builtin_bit_cast(uint32x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint64x2_t vreinterpretq_u64_bf16(bfloat16x8_t __p0) { - uint64x2_t __ret; - __ret = __builtin_bit_cast(uint64x2_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint16x8_t vreinterpretq_u16_bf16(bfloat16x8_t __p0) { - uint16x8_t __ret; - __ret = __builtin_bit_cast(uint16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int8x16_t vreinterpretq_s8_bf16(bfloat16x8_t __p0) { - int8x16_t __ret; - __ret = __builtin_bit_cast(int8x16_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) float64x2_t vreinterpretq_f64_bf16(bfloat16x8_t __p0) { - float64x2_t __ret; - __ret = __builtin_bit_cast(float64x2_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) float32x4_t vreinterpretq_f32_bf16(bfloat16x8_t __p0) { - float32x4_t __ret; - __ret = __builtin_bit_cast(float32x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) float16x8_t vreinterpretq_f16_bf16(bfloat16x8_t __p0) { - float16x8_t __ret; - __ret = __builtin_bit_cast(float16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int32x4_t vreinterpretq_s32_bf16(bfloat16x8_t __p0) { - int32x4_t __ret; - __ret = __builtin_bit_cast(int32x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int64x2_t vreinterpretq_s64_bf16(bfloat16x8_t __p0) { - int64x2_t __ret; - __ret = __builtin_bit_cast(int64x2_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int16x8_t vreinterpretq_s16_bf16(bfloat16x8_t __p0) { - int16x8_t __ret; - __ret = __builtin_bit_cast(int16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint8x8_t vreinterpret_u8_bf16(bfloat16x4_t __p0) { - uint8x8_t __ret; - __ret = __builtin_bit_cast(uint8x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint32x2_t vreinterpret_u32_bf16(bfloat16x4_t __p0) { - uint32x2_t __ret; - __ret = __builtin_bit_cast(uint32x2_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint64x1_t vreinterpret_u64_bf16(bfloat16x4_t __p0) { - uint64x1_t __ret; - __ret = __builtin_bit_cast(uint64x1_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) uint16x4_t vreinterpret_u16_bf16(bfloat16x4_t __p0) { - uint16x4_t __ret; - __ret = __builtin_bit_cast(uint16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int8x8_t vreinterpret_s8_bf16(bfloat16x4_t __p0) { - int8x8_t __ret; - __ret = __builtin_bit_cast(int8x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) float64x1_t vreinterpret_f64_bf16(bfloat16x4_t __p0) { - float64x1_t __ret; - __ret = __builtin_bit_cast(float64x1_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) float32x2_t vreinterpret_f32_bf16(bfloat16x4_t __p0) { - float32x2_t __ret; - __ret = __builtin_bit_cast(float32x2_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) float16x4_t vreinterpret_f16_bf16(bfloat16x4_t __p0) { - float16x4_t __ret; - __ret = __builtin_bit_cast(float16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int32x2_t vreinterpret_s32_bf16(bfloat16x4_t __p0) { - int32x2_t __ret; - __ret = __builtin_bit_cast(int32x2_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int64x1_t vreinterpret_s64_bf16(bfloat16x4_t __p0) { - int64x1_t __ret; - __ret = __builtin_bit_cast(int64x1_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) int16x4_t vreinterpret_s16_bf16(bfloat16x4_t __p0) { - int16x4_t __ret; - __ret = __builtin_bit_cast(int16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_p8(poly8x16_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_p128(poly128_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_p64(poly64x2_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_p16(poly16x8_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_u8(uint8x16_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_u32(uint32x4_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_u64(uint64x2_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_u16(uint16x8_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_s8(int8x16_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_f64(float64x2_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_f32(float32x4_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_f16(float16x8_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_s32(int32x4_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_s64(int64x2_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vreinterpretq_bf16_s16(int16x8_t __p0) { - bfloat16x8_t __ret; - __ret = __builtin_bit_cast(bfloat16x8_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_p8(poly8x8_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_p64(poly64x1_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_p16(poly16x4_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_u8(uint8x8_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_u32(uint32x2_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_u64(uint64x1_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_u16(uint16x4_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_s8(int8x8_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_f64(float64x1_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_f32(float32x2_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_f16(float16x4_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_s32(int32x2_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_s64(int64x1_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_s16(int16x4_t __p0) { - bfloat16x4_t __ret; - __ret = __builtin_bit_cast(bfloat16x4_t, __p0); - return __ret; -} -#ifdef __LITTLE_ENDIAN__ -#define vdotq_laneq_u32(__p0_242, __p1_242, __p2_242, __p3_242) __extension__ ({ \ - uint32x4_t __ret_242; \ - uint32x4_t __s0_242 = __p0_242; \ - uint8x16_t __s1_242 = __p1_242; \ - uint8x16_t __s2_242 = __p2_242; \ - __ret_242 = vdotq_u32(__s0_242, __s1_242, __builtin_bit_cast(uint8x16_t, splatq_laneq_u32(__builtin_bit_cast(uint32x4_t, __s2_242), __p3_242))); \ - __ret_242; \ -}) -#else -#define vdotq_laneq_u32(__p0_243, __p1_243, __p2_243, __p3_243) __extension__ ({ \ - uint32x4_t __ret_243; \ - uint32x4_t __s0_243 = __p0_243; \ - uint8x16_t __s1_243 = __p1_243; \ - uint8x16_t __s2_243 = __p2_243; \ - uint32x4_t __rev0_243; __rev0_243 = __builtin_shufflevector(__s0_243, __s0_243, __lane_reverse_128_32); \ - uint8x16_t __rev1_243; __rev1_243 = __builtin_shufflevector(__s1_243, __s1_243, __lane_reverse_128_8); \ - uint8x16_t __rev2_243; __rev2_243 = __builtin_shufflevector(__s2_243, __s2_243, __lane_reverse_128_8); \ - __ret_243 = __noswap_vdotq_u32(__rev0_243, __rev1_243, __builtin_bit_cast(uint8x16_t, __noswap_splatq_laneq_u32(__builtin_bit_cast(uint32x4_t, __rev2_243), __p3_243))); \ - __ret_243 = __builtin_shufflevector(__ret_243, __ret_243, __lane_reverse_128_32); \ - __ret_243; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vdotq_laneq_s32(__p0_244, __p1_244, __p2_244, __p3_244) __extension__ ({ \ - int32x4_t __ret_244; \ - int32x4_t __s0_244 = __p0_244; \ - int8x16_t __s1_244 = __p1_244; \ - int8x16_t __s2_244 = __p2_244; \ - __ret_244 = vdotq_s32(__s0_244, __s1_244, __builtin_bit_cast(int8x16_t, splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_244), __p3_244))); \ - __ret_244; \ -}) -#else -#define vdotq_laneq_s32(__p0_245, __p1_245, __p2_245, __p3_245) __extension__ ({ \ - int32x4_t __ret_245; \ - int32x4_t __s0_245 = __p0_245; \ - int8x16_t __s1_245 = __p1_245; \ - int8x16_t __s2_245 = __p2_245; \ - int32x4_t __rev0_245; __rev0_245 = __builtin_shufflevector(__s0_245, __s0_245, __lane_reverse_128_32); \ - int8x16_t __rev1_245; __rev1_245 = __builtin_shufflevector(__s1_245, __s1_245, __lane_reverse_128_8); \ - int8x16_t __rev2_245; __rev2_245 = __builtin_shufflevector(__s2_245, __s2_245, __lane_reverse_128_8); \ - __ret_245 = __noswap_vdotq_s32(__rev0_245, __rev1_245, __builtin_bit_cast(int8x16_t, __noswap_splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_245), __p3_245))); \ - __ret_245 = __builtin_shufflevector(__ret_245, __ret_245, __lane_reverse_128_32); \ - __ret_245; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vdot_laneq_u32(__p0_246, __p1_246, __p2_246, __p3_246) __extension__ ({ \ - uint32x2_t __ret_246; \ - uint32x2_t __s0_246 = __p0_246; \ - uint8x8_t __s1_246 = __p1_246; \ - uint8x16_t __s2_246 = __p2_246; \ - __ret_246 = vdot_u32(__s0_246, __s1_246, __builtin_bit_cast(uint8x8_t, splat_laneq_u32(__builtin_bit_cast(uint32x4_t, __s2_246), __p3_246))); \ - __ret_246; \ -}) -#else -#define vdot_laneq_u32(__p0_247, __p1_247, __p2_247, __p3_247) __extension__ ({ \ - uint32x2_t __ret_247; \ - uint32x2_t __s0_247 = __p0_247; \ - uint8x8_t __s1_247 = __p1_247; \ - uint8x16_t __s2_247 = __p2_247; \ - uint32x2_t __rev0_247; __rev0_247 = __builtin_shufflevector(__s0_247, __s0_247, __lane_reverse_64_32); \ - uint8x8_t __rev1_247; __rev1_247 = __builtin_shufflevector(__s1_247, __s1_247, __lane_reverse_64_8); \ - uint8x16_t __rev2_247; __rev2_247 = __builtin_shufflevector(__s2_247, __s2_247, __lane_reverse_128_8); \ - __ret_247 = __noswap_vdot_u32(__rev0_247, __rev1_247, __builtin_bit_cast(uint8x8_t, __noswap_splat_laneq_u32(__builtin_bit_cast(uint32x4_t, __rev2_247), __p3_247))); \ - __ret_247 = __builtin_shufflevector(__ret_247, __ret_247, __lane_reverse_64_32); \ - __ret_247; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vdot_laneq_s32(__p0_248, __p1_248, __p2_248, __p3_248) __extension__ ({ \ - int32x2_t __ret_248; \ - int32x2_t __s0_248 = __p0_248; \ - int8x8_t __s1_248 = __p1_248; \ - int8x16_t __s2_248 = __p2_248; \ - __ret_248 = vdot_s32(__s0_248, __s1_248, __builtin_bit_cast(int8x8_t, splat_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_248), __p3_248))); \ - __ret_248; \ -}) -#else -#define vdot_laneq_s32(__p0_249, __p1_249, __p2_249, __p3_249) __extension__ ({ \ - int32x2_t __ret_249; \ - int32x2_t __s0_249 = __p0_249; \ - int8x8_t __s1_249 = __p1_249; \ - int8x16_t __s2_249 = __p2_249; \ - int32x2_t __rev0_249; __rev0_249 = __builtin_shufflevector(__s0_249, __s0_249, __lane_reverse_64_32); \ - int8x8_t __rev1_249; __rev1_249 = __builtin_shufflevector(__s1_249, __s1_249, __lane_reverse_64_8); \ - int8x16_t __rev2_249; __rev2_249 = __builtin_shufflevector(__s2_249, __s2_249, __lane_reverse_128_8); \ - __ret_249 = __noswap_vdot_s32(__rev0_249, __rev1_249, __builtin_bit_cast(int8x8_t, __noswap_splat_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_249), __p3_249))); \ - __ret_249 = __builtin_shufflevector(__ret_249, __ret_249, __lane_reverse_64_32); \ - __ret_249; \ -}) -#endif - #ifdef __LITTLE_ENDIAN__ __ai __attribute__((target("fp16fml,neon"))) float32x4_t vfmlalq_high_f16(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { float32x4_t __ret; @@ -45317,533 +45111,533 @@ __ai __attribute__((target("fullfp16,neon"))) float16x4_t vdiv_f16(float16x4_t _ #endif #ifdef __LITTLE_ENDIAN__ -#define vfmsh_lane_f16(__p0_250, __p1_250, __p2_250, __p3_250) __extension__ ({ \ - float16_t __ret_250; \ - float16_t __s0_250 = __p0_250; \ - float16_t __s1_250 = __p1_250; \ - float16x4_t __s2_250 = __p2_250; \ - __ret_250 = vfmah_lane_f16(__s0_250, -__s1_250, __s2_250, __p3_250); \ +#define vfmsh_lane_f16(__p0_242, __p1_242, __p2_242, __p3_242) __extension__ ({ \ + float16_t __ret_242; \ + float16_t __s0_242 = __p0_242; \ + float16_t __s1_242 = __p1_242; \ + float16x4_t __s2_242 = __p2_242; \ + __ret_242 = vfmah_lane_f16(__s0_242, -__s1_242, __s2_242, __p3_242); \ + __ret_242; \ +}) +#else +#define vfmsh_lane_f16(__p0_243, __p1_243, __p2_243, __p3_243) __extension__ ({ \ + float16_t __ret_243; \ + float16_t __s0_243 = __p0_243; \ + float16_t __s1_243 = __p1_243; \ + float16x4_t __s2_243 = __p2_243; \ + float16x4_t __rev2_243; __rev2_243 = __builtin_shufflevector(__s2_243, __s2_243, __lane_reverse_64_16); \ + __ret_243 = __noswap_vfmah_lane_f16(__s0_243, -__s1_243, __rev2_243, __p3_243); \ + __ret_243; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsq_lane_f16(__p0_244, __p1_244, __p2_244, __p3_244) __extension__ ({ \ + float16x8_t __ret_244; \ + float16x8_t __s0_244 = __p0_244; \ + float16x8_t __s1_244 = __p1_244; \ + float16x4_t __s2_244 = __p2_244; \ + __ret_244 = vfmaq_lane_f16(__s0_244, -__s1_244, __s2_244, __p3_244); \ + __ret_244; \ +}) +#else +#define vfmsq_lane_f16(__p0_245, __p1_245, __p2_245, __p3_245) __extension__ ({ \ + float16x8_t __ret_245; \ + float16x8_t __s0_245 = __p0_245; \ + float16x8_t __s1_245 = __p1_245; \ + float16x4_t __s2_245 = __p2_245; \ + float16x8_t __rev0_245; __rev0_245 = __builtin_shufflevector(__s0_245, __s0_245, __lane_reverse_128_16); \ + float16x8_t __rev1_245; __rev1_245 = __builtin_shufflevector(__s1_245, __s1_245, __lane_reverse_128_16); \ + float16x4_t __rev2_245; __rev2_245 = __builtin_shufflevector(__s2_245, __s2_245, __lane_reverse_64_16); \ + __ret_245 = __noswap_vfmaq_lane_f16(__rev0_245, -__rev1_245, __rev2_245, __p3_245); \ + __ret_245 = __builtin_shufflevector(__ret_245, __ret_245, __lane_reverse_128_16); \ + __ret_245; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfms_lane_f16(__p0_246, __p1_246, __p2_246, __p3_246) __extension__ ({ \ + float16x4_t __ret_246; \ + float16x4_t __s0_246 = __p0_246; \ + float16x4_t __s1_246 = __p1_246; \ + float16x4_t __s2_246 = __p2_246; \ + __ret_246 = vfma_lane_f16(__s0_246, -__s1_246, __s2_246, __p3_246); \ + __ret_246; \ +}) +#else +#define vfms_lane_f16(__p0_247, __p1_247, __p2_247, __p3_247) __extension__ ({ \ + float16x4_t __ret_247; \ + float16x4_t __s0_247 = __p0_247; \ + float16x4_t __s1_247 = __p1_247; \ + float16x4_t __s2_247 = __p2_247; \ + float16x4_t __rev0_247; __rev0_247 = __builtin_shufflevector(__s0_247, __s0_247, __lane_reverse_64_16); \ + float16x4_t __rev1_247; __rev1_247 = __builtin_shufflevector(__s1_247, __s1_247, __lane_reverse_64_16); \ + float16x4_t __rev2_247; __rev2_247 = __builtin_shufflevector(__s2_247, __s2_247, __lane_reverse_64_16); \ + __ret_247 = __noswap_vfma_lane_f16(__rev0_247, -__rev1_247, __rev2_247, __p3_247); \ + __ret_247 = __builtin_shufflevector(__ret_247, __ret_247, __lane_reverse_64_16); \ + __ret_247; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsh_laneq_f16(__p0_248, __p1_248, __p2_248, __p3_248) __extension__ ({ \ + float16_t __ret_248; \ + float16_t __s0_248 = __p0_248; \ + float16_t __s1_248 = __p1_248; \ + float16x8_t __s2_248 = __p2_248; \ + __ret_248 = vfmah_laneq_f16(__s0_248, -__s1_248, __s2_248, __p3_248); \ + __ret_248; \ +}) +#else +#define vfmsh_laneq_f16(__p0_249, __p1_249, __p2_249, __p3_249) __extension__ ({ \ + float16_t __ret_249; \ + float16_t __s0_249 = __p0_249; \ + float16_t __s1_249 = __p1_249; \ + float16x8_t __s2_249 = __p2_249; \ + float16x8_t __rev2_249; __rev2_249 = __builtin_shufflevector(__s2_249, __s2_249, __lane_reverse_128_16); \ + __ret_249 = __noswap_vfmah_laneq_f16(__s0_249, -__s1_249, __rev2_249, __p3_249); \ + __ret_249; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsq_laneq_f16(__p0_250, __p1_250, __p2_250, __p3_250) __extension__ ({ \ + float16x8_t __ret_250; \ + float16x8_t __s0_250 = __p0_250; \ + float16x8_t __s1_250 = __p1_250; \ + float16x8_t __s2_250 = __p2_250; \ + __ret_250 = vfmaq_laneq_f16(__s0_250, -__s1_250, __s2_250, __p3_250); \ __ret_250; \ }) #else -#define vfmsh_lane_f16(__p0_251, __p1_251, __p2_251, __p3_251) __extension__ ({ \ - float16_t __ret_251; \ - float16_t __s0_251 = __p0_251; \ - float16_t __s1_251 = __p1_251; \ - float16x4_t __s2_251 = __p2_251; \ - float16x4_t __rev2_251; __rev2_251 = __builtin_shufflevector(__s2_251, __s2_251, __lane_reverse_64_16); \ - __ret_251 = __noswap_vfmah_lane_f16(__s0_251, -__s1_251, __rev2_251, __p3_251); \ +#define vfmsq_laneq_f16(__p0_251, __p1_251, __p2_251, __p3_251) __extension__ ({ \ + float16x8_t __ret_251; \ + float16x8_t __s0_251 = __p0_251; \ + float16x8_t __s1_251 = __p1_251; \ + float16x8_t __s2_251 = __p2_251; \ + float16x8_t __rev0_251; __rev0_251 = __builtin_shufflevector(__s0_251, __s0_251, __lane_reverse_128_16); \ + float16x8_t __rev1_251; __rev1_251 = __builtin_shufflevector(__s1_251, __s1_251, __lane_reverse_128_16); \ + float16x8_t __rev2_251; __rev2_251 = __builtin_shufflevector(__s2_251, __s2_251, __lane_reverse_128_16); \ + __ret_251 = __noswap_vfmaq_laneq_f16(__rev0_251, -__rev1_251, __rev2_251, __p3_251); \ + __ret_251 = __builtin_shufflevector(__ret_251, __ret_251, __lane_reverse_128_16); \ __ret_251; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vfmsq_lane_f16(__p0_252, __p1_252, __p2_252, __p3_252) __extension__ ({ \ - float16x8_t __ret_252; \ - float16x8_t __s0_252 = __p0_252; \ - float16x8_t __s1_252 = __p1_252; \ - float16x4_t __s2_252 = __p2_252; \ - __ret_252 = vfmaq_lane_f16(__s0_252, -__s1_252, __s2_252, __p3_252); \ +#define vfms_laneq_f16(__p0_252, __p1_252, __p2_252, __p3_252) __extension__ ({ \ + float16x4_t __ret_252; \ + float16x4_t __s0_252 = __p0_252; \ + float16x4_t __s1_252 = __p1_252; \ + float16x8_t __s2_252 = __p2_252; \ + __ret_252 = vfma_laneq_f16(__s0_252, -__s1_252, __s2_252, __p3_252); \ __ret_252; \ }) #else -#define vfmsq_lane_f16(__p0_253, __p1_253, __p2_253, __p3_253) __extension__ ({ \ - float16x8_t __ret_253; \ - float16x8_t __s0_253 = __p0_253; \ - float16x8_t __s1_253 = __p1_253; \ - float16x4_t __s2_253 = __p2_253; \ - float16x8_t __rev0_253; __rev0_253 = __builtin_shufflevector(__s0_253, __s0_253, __lane_reverse_128_16); \ - float16x8_t __rev1_253; __rev1_253 = __builtin_shufflevector(__s1_253, __s1_253, __lane_reverse_128_16); \ - float16x4_t __rev2_253; __rev2_253 = __builtin_shufflevector(__s2_253, __s2_253, __lane_reverse_64_16); \ - __ret_253 = __noswap_vfmaq_lane_f16(__rev0_253, -__rev1_253, __rev2_253, __p3_253); \ - __ret_253 = __builtin_shufflevector(__ret_253, __ret_253, __lane_reverse_128_16); \ +#define vfms_laneq_f16(__p0_253, __p1_253, __p2_253, __p3_253) __extension__ ({ \ + float16x4_t __ret_253; \ + float16x4_t __s0_253 = __p0_253; \ + float16x4_t __s1_253 = __p1_253; \ + float16x8_t __s2_253 = __p2_253; \ + float16x4_t __rev0_253; __rev0_253 = __builtin_shufflevector(__s0_253, __s0_253, __lane_reverse_64_16); \ + float16x4_t __rev1_253; __rev1_253 = __builtin_shufflevector(__s1_253, __s1_253, __lane_reverse_64_16); \ + float16x8_t __rev2_253; __rev2_253 = __builtin_shufflevector(__s2_253, __s2_253, __lane_reverse_128_16); \ + __ret_253 = __noswap_vfma_laneq_f16(__rev0_253, -__rev1_253, __rev2_253, __p3_253); \ + __ret_253 = __builtin_shufflevector(__ret_253, __ret_253, __lane_reverse_64_16); \ __ret_253; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vfms_lane_f16(__p0_254, __p1_254, __p2_254, __p3_254) __extension__ ({ \ - float16x4_t __ret_254; \ - float16x4_t __s0_254 = __p0_254; \ - float16x4_t __s1_254 = __p1_254; \ - float16x4_t __s2_254 = __p2_254; \ - __ret_254 = vfma_lane_f16(__s0_254, -__s1_254, __s2_254, __p3_254); \ +#define vfmsq_n_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __ret; \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16_t __s2 = __p2; \ + __ret = vfmaq_f16(__s0, -__s1, (float16x8_t) {__s2, __s2, __s2, __s2, __s2, __s2, __s2, __s2}); \ + __ret; \ +}) +#else +#define vfmsq_n_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __ret; \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16_t __s2 = __p2; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_16); \ + __ret = __noswap_vfmaq_f16(__rev0, -__rev1, (float16x8_t) {__s2, __s2, __s2, __s2, __s2, __s2, __s2, __s2}); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfms_n_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __ret; \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16_t __s2 = __p2; \ + __ret = vfma_f16(__s0, -__s1, (float16x4_t) {__s2, __s2, __s2, __s2}); \ + __ret; \ +}) +#else +#define vfms_n_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __ret; \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16_t __s2 = __p2; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \ + __ret = __noswap_vfma_f16(__rev0, -__rev1, (float16x4_t) {__s2, __s2, __s2, __s2}); \ + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmaxnmvq_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x8_t __s0 = __p0; \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxnmvq_f16(__builtin_bit_cast(int8x16_t, __s0))); \ + __ret; \ +}) +#else +#define vmaxnmvq_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxnmvq_f16(__builtin_bit_cast(int8x16_t, __rev0))); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmaxnmv_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x4_t __s0 = __p0; \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxnmv_f16(__builtin_bit_cast(int8x8_t, __s0))); \ + __ret; \ +}) +#else +#define vmaxnmv_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxnmv_f16(__builtin_bit_cast(int8x8_t, __rev0))); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmaxvq_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x8_t __s0 = __p0; \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxvq_f16(__builtin_bit_cast(int8x16_t, __s0))); \ + __ret; \ +}) +#else +#define vmaxvq_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxvq_f16(__builtin_bit_cast(int8x16_t, __rev0))); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmaxv_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x4_t __s0 = __p0; \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxv_f16(__builtin_bit_cast(int8x8_t, __s0))); \ + __ret; \ +}) +#else +#define vmaxv_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxv_f16(__builtin_bit_cast(int8x8_t, __rev0))); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vminnmvq_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x8_t __s0 = __p0; \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminnmvq_f16(__builtin_bit_cast(int8x16_t, __s0))); \ + __ret; \ +}) +#else +#define vminnmvq_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminnmvq_f16(__builtin_bit_cast(int8x16_t, __rev0))); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vminnmv_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x4_t __s0 = __p0; \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminnmv_f16(__builtin_bit_cast(int8x8_t, __s0))); \ + __ret; \ +}) +#else +#define vminnmv_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminnmv_f16(__builtin_bit_cast(int8x8_t, __rev0))); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vminvq_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x8_t __s0 = __p0; \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminvq_f16(__builtin_bit_cast(int8x16_t, __s0))); \ + __ret; \ +}) +#else +#define vminvq_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminvq_f16(__builtin_bit_cast(int8x16_t, __rev0))); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vminv_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x4_t __s0 = __p0; \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminv_f16(__builtin_bit_cast(int8x8_t, __s0))); \ + __ret; \ +}) +#else +#define vminv_f16(__p0) __extension__ ({ \ + float16_t __ret; \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminv_f16(__builtin_bit_cast(int8x8_t, __rev0))); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_f16(__p0_254, __p1_254, __p2_254) __extension__ ({ \ + float16x8_t __ret_254; \ + float16x8_t __s0_254 = __p0_254; \ + float16x8_t __s1_254 = __p1_254; \ + __ret_254 = __s0_254 * splatq_laneq_f16(__s1_254, __p2_254); \ __ret_254; \ }) #else -#define vfms_lane_f16(__p0_255, __p1_255, __p2_255, __p3_255) __extension__ ({ \ - float16x4_t __ret_255; \ - float16x4_t __s0_255 = __p0_255; \ - float16x4_t __s1_255 = __p1_255; \ - float16x4_t __s2_255 = __p2_255; \ - float16x4_t __rev0_255; __rev0_255 = __builtin_shufflevector(__s0_255, __s0_255, __lane_reverse_64_16); \ - float16x4_t __rev1_255; __rev1_255 = __builtin_shufflevector(__s1_255, __s1_255, __lane_reverse_64_16); \ - float16x4_t __rev2_255; __rev2_255 = __builtin_shufflevector(__s2_255, __s2_255, __lane_reverse_64_16); \ - __ret_255 = __noswap_vfma_lane_f16(__rev0_255, -__rev1_255, __rev2_255, __p3_255); \ - __ret_255 = __builtin_shufflevector(__ret_255, __ret_255, __lane_reverse_64_16); \ +#define vmulq_laneq_f16(__p0_255, __p1_255, __p2_255) __extension__ ({ \ + float16x8_t __ret_255; \ + float16x8_t __s0_255 = __p0_255; \ + float16x8_t __s1_255 = __p1_255; \ + float16x8_t __rev0_255; __rev0_255 = __builtin_shufflevector(__s0_255, __s0_255, __lane_reverse_128_16); \ + float16x8_t __rev1_255; __rev1_255 = __builtin_shufflevector(__s1_255, __s1_255, __lane_reverse_128_16); \ + __ret_255 = __rev0_255 * __noswap_splatq_laneq_f16(__rev1_255, __p2_255); \ + __ret_255 = __builtin_shufflevector(__ret_255, __ret_255, __lane_reverse_128_16); \ __ret_255; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vfmsh_laneq_f16(__p0_256, __p1_256, __p2_256, __p3_256) __extension__ ({ \ - float16_t __ret_256; \ - float16_t __s0_256 = __p0_256; \ - float16_t __s1_256 = __p1_256; \ - float16x8_t __s2_256 = __p2_256; \ - __ret_256 = vfmah_laneq_f16(__s0_256, -__s1_256, __s2_256, __p3_256); \ +#define vmul_laneq_f16(__p0_256, __p1_256, __p2_256) __extension__ ({ \ + float16x4_t __ret_256; \ + float16x4_t __s0_256 = __p0_256; \ + float16x8_t __s1_256 = __p1_256; \ + __ret_256 = __s0_256 * splat_laneq_f16(__s1_256, __p2_256); \ __ret_256; \ }) #else -#define vfmsh_laneq_f16(__p0_257, __p1_257, __p2_257, __p3_257) __extension__ ({ \ - float16_t __ret_257; \ - float16_t __s0_257 = __p0_257; \ - float16_t __s1_257 = __p1_257; \ - float16x8_t __s2_257 = __p2_257; \ - float16x8_t __rev2_257; __rev2_257 = __builtin_shufflevector(__s2_257, __s2_257, __lane_reverse_128_16); \ - __ret_257 = __noswap_vfmah_laneq_f16(__s0_257, -__s1_257, __rev2_257, __p3_257); \ +#define vmul_laneq_f16(__p0_257, __p1_257, __p2_257) __extension__ ({ \ + float16x4_t __ret_257; \ + float16x4_t __s0_257 = __p0_257; \ + float16x8_t __s1_257 = __p1_257; \ + float16x4_t __rev0_257; __rev0_257 = __builtin_shufflevector(__s0_257, __s0_257, __lane_reverse_64_16); \ + float16x8_t __rev1_257; __rev1_257 = __builtin_shufflevector(__s1_257, __s1_257, __lane_reverse_128_16); \ + __ret_257 = __rev0_257 * __noswap_splat_laneq_f16(__rev1_257, __p2_257); \ + __ret_257 = __builtin_shufflevector(__ret_257, __ret_257, __lane_reverse_64_16); \ __ret_257; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vfmsq_laneq_f16(__p0_258, __p1_258, __p2_258, __p3_258) __extension__ ({ \ +__ai __attribute__((target("fullfp16,neon"))) float16x8_t vmulxq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __builtin_bit_cast(float16x8_t, __builtin_neon_vmulxq_f16(__builtin_bit_cast(int8x16_t, __p0), __builtin_bit_cast(int8x16_t, __p1), 40)); + return __ret; +} +#else +__ai __attribute__((target("fullfp16,neon"))) float16x8_t vmulxq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_16); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, __lane_reverse_128_16); + __ret = __builtin_bit_cast(float16x8_t, __builtin_neon_vmulxq_f16(__builtin_bit_cast(int8x16_t, __rev0), __builtin_bit_cast(int8x16_t, __rev1), 40)); + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); + return __ret; +} +__ai __attribute__((target("fullfp16,neon"))) float16x8_t __noswap_vmulxq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __builtin_bit_cast(float16x8_t, __builtin_neon_vmulxq_f16(__builtin_bit_cast(int8x16_t, __p0), __builtin_bit_cast(int8x16_t, __p1), 40)); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai __attribute__((target("fullfp16,neon"))) float16x4_t vmulx_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __builtin_bit_cast(float16x4_t, __builtin_neon_vmulx_f16(__builtin_bit_cast(int8x8_t, __p0), __builtin_bit_cast(int8x8_t, __p1), 8)); + return __ret; +} +#else +__ai __attribute__((target("fullfp16,neon"))) float16x4_t vmulx_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_64_16); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, __lane_reverse_64_16); + __ret = __builtin_bit_cast(float16x4_t, __builtin_neon_vmulx_f16(__builtin_bit_cast(int8x8_t, __rev0), __builtin_bit_cast(int8x8_t, __rev1), 8)); + __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); + return __ret; +} +__ai __attribute__((target("fullfp16,neon"))) float16x4_t __noswap_vmulx_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __builtin_bit_cast(float16x4_t, __builtin_neon_vmulx_f16(__builtin_bit_cast(int8x8_t, __p0), __builtin_bit_cast(int8x8_t, __p1), 8)); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxh_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16_t __ret; \ + float16_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmulxh_lane_f16(__s0, __s1, __p2)); \ + __ret; \ +}) +#else +#define vmulxh_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16_t __ret; \ + float16_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmulxh_lane_f16(__s0, __rev1, __p2)); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxq_lane_f16(__p0_258, __p1_258, __p2_258) __extension__ ({ \ float16x8_t __ret_258; \ float16x8_t __s0_258 = __p0_258; \ - float16x8_t __s1_258 = __p1_258; \ - float16x8_t __s2_258 = __p2_258; \ - __ret_258 = vfmaq_laneq_f16(__s0_258, -__s1_258, __s2_258, __p3_258); \ + float16x4_t __s1_258 = __p1_258; \ + __ret_258 = vmulxq_f16(__s0_258, splatq_lane_f16(__s1_258, __p2_258)); \ __ret_258; \ }) #else -#define vfmsq_laneq_f16(__p0_259, __p1_259, __p2_259, __p3_259) __extension__ ({ \ +#define vmulxq_lane_f16(__p0_259, __p1_259, __p2_259) __extension__ ({ \ float16x8_t __ret_259; \ float16x8_t __s0_259 = __p0_259; \ - float16x8_t __s1_259 = __p1_259; \ - float16x8_t __s2_259 = __p2_259; \ + float16x4_t __s1_259 = __p1_259; \ float16x8_t __rev0_259; __rev0_259 = __builtin_shufflevector(__s0_259, __s0_259, __lane_reverse_128_16); \ - float16x8_t __rev1_259; __rev1_259 = __builtin_shufflevector(__s1_259, __s1_259, __lane_reverse_128_16); \ - float16x8_t __rev2_259; __rev2_259 = __builtin_shufflevector(__s2_259, __s2_259, __lane_reverse_128_16); \ - __ret_259 = __noswap_vfmaq_laneq_f16(__rev0_259, -__rev1_259, __rev2_259, __p3_259); \ + float16x4_t __rev1_259; __rev1_259 = __builtin_shufflevector(__s1_259, __s1_259, __lane_reverse_64_16); \ + __ret_259 = __noswap_vmulxq_f16(__rev0_259, __noswap_splatq_lane_f16(__rev1_259, __p2_259)); \ __ret_259 = __builtin_shufflevector(__ret_259, __ret_259, __lane_reverse_128_16); \ __ret_259; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vfms_laneq_f16(__p0_260, __p1_260, __p2_260, __p3_260) __extension__ ({ \ +#define vmulx_lane_f16(__p0_260, __p1_260, __p2_260) __extension__ ({ \ float16x4_t __ret_260; \ float16x4_t __s0_260 = __p0_260; \ float16x4_t __s1_260 = __p1_260; \ - float16x8_t __s2_260 = __p2_260; \ - __ret_260 = vfma_laneq_f16(__s0_260, -__s1_260, __s2_260, __p3_260); \ + __ret_260 = vmulx_f16(__s0_260, splat_lane_f16(__s1_260, __p2_260)); \ __ret_260; \ }) #else -#define vfms_laneq_f16(__p0_261, __p1_261, __p2_261, __p3_261) __extension__ ({ \ +#define vmulx_lane_f16(__p0_261, __p1_261, __p2_261) __extension__ ({ \ float16x4_t __ret_261; \ float16x4_t __s0_261 = __p0_261; \ float16x4_t __s1_261 = __p1_261; \ - float16x8_t __s2_261 = __p2_261; \ float16x4_t __rev0_261; __rev0_261 = __builtin_shufflevector(__s0_261, __s0_261, __lane_reverse_64_16); \ float16x4_t __rev1_261; __rev1_261 = __builtin_shufflevector(__s1_261, __s1_261, __lane_reverse_64_16); \ - float16x8_t __rev2_261; __rev2_261 = __builtin_shufflevector(__s2_261, __s2_261, __lane_reverse_128_16); \ - __ret_261 = __noswap_vfma_laneq_f16(__rev0_261, -__rev1_261, __rev2_261, __p3_261); \ + __ret_261 = __noswap_vmulx_f16(__rev0_261, __noswap_splat_lane_f16(__rev1_261, __p2_261)); \ __ret_261 = __builtin_shufflevector(__ret_261, __ret_261, __lane_reverse_64_16); \ __ret_261; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vfmsq_n_f16(__p0, __p1, __p2) __extension__ ({ \ - float16x8_t __ret; \ - float16x8_t __s0 = __p0; \ +#define vmulxh_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16_t __ret; \ + float16_t __s0 = __p0; \ float16x8_t __s1 = __p1; \ - float16_t __s2 = __p2; \ - __ret = vfmaq_f16(__s0, -__s1, (float16x8_t) {__s2, __s2, __s2, __s2, __s2, __s2, __s2, __s2}); \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmulxh_laneq_f16(__s0, __s1, __p2)); \ __ret; \ }) #else -#define vfmsq_n_f16(__p0, __p1, __p2) __extension__ ({ \ - float16x8_t __ret; \ - float16x8_t __s0 = __p0; \ +#define vmulxh_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16_t __ret; \ + float16_t __s0 = __p0; \ float16x8_t __s1 = __p1; \ - float16_t __s2 = __p2; \ - float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_16); \ - __ret = __noswap_vfmaq_f16(__rev0, -__rev1, (float16x8_t) {__s2, __s2, __s2, __s2, __s2, __s2, __s2, __s2}); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \ + __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmulxh_laneq_f16(__s0, __rev1, __p2)); \ __ret; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vfms_n_f16(__p0, __p1, __p2) __extension__ ({ \ - float16x4_t __ret; \ - float16x4_t __s0 = __p0; \ - float16x4_t __s1 = __p1; \ - float16_t __s2 = __p2; \ - __ret = vfma_f16(__s0, -__s1, (float16x4_t) {__s2, __s2, __s2, __s2}); \ - __ret; \ -}) -#else -#define vfms_n_f16(__p0, __p1, __p2) __extension__ ({ \ - float16x4_t __ret; \ - float16x4_t __s0 = __p0; \ - float16x4_t __s1 = __p1; \ - float16_t __s2 = __p2; \ - float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ - float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \ - __ret = __noswap_vfma_f16(__rev0, -__rev1, (float16x4_t) {__s2, __s2, __s2, __s2}); \ - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vmaxnmvq_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x8_t __s0 = __p0; \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxnmvq_f16(__builtin_bit_cast(int8x16_t, __s0))); \ - __ret; \ -}) -#else -#define vmaxnmvq_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x8_t __s0 = __p0; \ - float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxnmvq_f16(__builtin_bit_cast(int8x16_t, __rev0))); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vmaxnmv_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x4_t __s0 = __p0; \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxnmv_f16(__builtin_bit_cast(int8x8_t, __s0))); \ - __ret; \ -}) -#else -#define vmaxnmv_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x4_t __s0 = __p0; \ - float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxnmv_f16(__builtin_bit_cast(int8x8_t, __rev0))); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vmaxvq_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x8_t __s0 = __p0; \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxvq_f16(__builtin_bit_cast(int8x16_t, __s0))); \ - __ret; \ -}) -#else -#define vmaxvq_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x8_t __s0 = __p0; \ - float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxvq_f16(__builtin_bit_cast(int8x16_t, __rev0))); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vmaxv_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x4_t __s0 = __p0; \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxv_f16(__builtin_bit_cast(int8x8_t, __s0))); \ - __ret; \ -}) -#else -#define vmaxv_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x4_t __s0 = __p0; \ - float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmaxv_f16(__builtin_bit_cast(int8x8_t, __rev0))); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vminnmvq_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x8_t __s0 = __p0; \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminnmvq_f16(__builtin_bit_cast(int8x16_t, __s0))); \ - __ret; \ -}) -#else -#define vminnmvq_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x8_t __s0 = __p0; \ - float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminnmvq_f16(__builtin_bit_cast(int8x16_t, __rev0))); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vminnmv_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x4_t __s0 = __p0; \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminnmv_f16(__builtin_bit_cast(int8x8_t, __s0))); \ - __ret; \ -}) -#else -#define vminnmv_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x4_t __s0 = __p0; \ - float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminnmv_f16(__builtin_bit_cast(int8x8_t, __rev0))); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vminvq_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x8_t __s0 = __p0; \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminvq_f16(__builtin_bit_cast(int8x16_t, __s0))); \ - __ret; \ -}) -#else -#define vminvq_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x8_t __s0 = __p0; \ - float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminvq_f16(__builtin_bit_cast(int8x16_t, __rev0))); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vminv_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x4_t __s0 = __p0; \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminv_f16(__builtin_bit_cast(int8x8_t, __s0))); \ - __ret; \ -}) -#else -#define vminv_f16(__p0) __extension__ ({ \ - float16_t __ret; \ - float16x4_t __s0 = __p0; \ - float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vminv_f16(__builtin_bit_cast(int8x8_t, __rev0))); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vmulq_laneq_f16(__p0_262, __p1_262, __p2_262) __extension__ ({ \ +#define vmulxq_laneq_f16(__p0_262, __p1_262, __p2_262) __extension__ ({ \ float16x8_t __ret_262; \ float16x8_t __s0_262 = __p0_262; \ float16x8_t __s1_262 = __p1_262; \ - __ret_262 = __s0_262 * splatq_laneq_f16(__s1_262, __p2_262); \ + __ret_262 = vmulxq_f16(__s0_262, splatq_laneq_f16(__s1_262, __p2_262)); \ __ret_262; \ }) #else -#define vmulq_laneq_f16(__p0_263, __p1_263, __p2_263) __extension__ ({ \ +#define vmulxq_laneq_f16(__p0_263, __p1_263, __p2_263) __extension__ ({ \ float16x8_t __ret_263; \ float16x8_t __s0_263 = __p0_263; \ float16x8_t __s1_263 = __p1_263; \ float16x8_t __rev0_263; __rev0_263 = __builtin_shufflevector(__s0_263, __s0_263, __lane_reverse_128_16); \ float16x8_t __rev1_263; __rev1_263 = __builtin_shufflevector(__s1_263, __s1_263, __lane_reverse_128_16); \ - __ret_263 = __rev0_263 * __noswap_splatq_laneq_f16(__rev1_263, __p2_263); \ + __ret_263 = __noswap_vmulxq_f16(__rev0_263, __noswap_splatq_laneq_f16(__rev1_263, __p2_263)); \ __ret_263 = __builtin_shufflevector(__ret_263, __ret_263, __lane_reverse_128_16); \ __ret_263; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vmul_laneq_f16(__p0_264, __p1_264, __p2_264) __extension__ ({ \ +#define vmulx_laneq_f16(__p0_264, __p1_264, __p2_264) __extension__ ({ \ float16x4_t __ret_264; \ float16x4_t __s0_264 = __p0_264; \ float16x8_t __s1_264 = __p1_264; \ - __ret_264 = __s0_264 * splat_laneq_f16(__s1_264, __p2_264); \ + __ret_264 = vmulx_f16(__s0_264, splat_laneq_f16(__s1_264, __p2_264)); \ __ret_264; \ }) #else -#define vmul_laneq_f16(__p0_265, __p1_265, __p2_265) __extension__ ({ \ +#define vmulx_laneq_f16(__p0_265, __p1_265, __p2_265) __extension__ ({ \ float16x4_t __ret_265; \ float16x4_t __s0_265 = __p0_265; \ float16x8_t __s1_265 = __p1_265; \ float16x4_t __rev0_265; __rev0_265 = __builtin_shufflevector(__s0_265, __s0_265, __lane_reverse_64_16); \ float16x8_t __rev1_265; __rev1_265 = __builtin_shufflevector(__s1_265, __s1_265, __lane_reverse_128_16); \ - __ret_265 = __rev0_265 * __noswap_splat_laneq_f16(__rev1_265, __p2_265); \ + __ret_265 = __noswap_vmulx_f16(__rev0_265, __noswap_splat_laneq_f16(__rev1_265, __p2_265)); \ __ret_265 = __builtin_shufflevector(__ret_265, __ret_265, __lane_reverse_64_16); \ __ret_265; \ }) #endif -#ifdef __LITTLE_ENDIAN__ -__ai __attribute__((target("fullfp16,neon"))) float16x8_t vmulxq_f16(float16x8_t __p0, float16x8_t __p1) { - float16x8_t __ret; - __ret = __builtin_bit_cast(float16x8_t, __builtin_neon_vmulxq_f16(__builtin_bit_cast(int8x16_t, __p0), __builtin_bit_cast(int8x16_t, __p1), 40)); - return __ret; -} -#else -__ai __attribute__((target("fullfp16,neon"))) float16x8_t vmulxq_f16(float16x8_t __p0, float16x8_t __p1) { - float16x8_t __ret; - float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_16); - float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, __lane_reverse_128_16); - __ret = __builtin_bit_cast(float16x8_t, __builtin_neon_vmulxq_f16(__builtin_bit_cast(int8x16_t, __rev0), __builtin_bit_cast(int8x16_t, __rev1), 40)); - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); - return __ret; -} -__ai __attribute__((target("fullfp16,neon"))) float16x8_t __noswap_vmulxq_f16(float16x8_t __p0, float16x8_t __p1) { - float16x8_t __ret; - __ret = __builtin_bit_cast(float16x8_t, __builtin_neon_vmulxq_f16(__builtin_bit_cast(int8x16_t, __p0), __builtin_bit_cast(int8x16_t, __p1), 40)); - return __ret; -} -#endif - -#ifdef __LITTLE_ENDIAN__ -__ai __attribute__((target("fullfp16,neon"))) float16x4_t vmulx_f16(float16x4_t __p0, float16x4_t __p1) { - float16x4_t __ret; - __ret = __builtin_bit_cast(float16x4_t, __builtin_neon_vmulx_f16(__builtin_bit_cast(int8x8_t, __p0), __builtin_bit_cast(int8x8_t, __p1), 8)); - return __ret; -} -#else -__ai __attribute__((target("fullfp16,neon"))) float16x4_t vmulx_f16(float16x4_t __p0, float16x4_t __p1) { - float16x4_t __ret; - float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_64_16); - float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, __lane_reverse_64_16); - __ret = __builtin_bit_cast(float16x4_t, __builtin_neon_vmulx_f16(__builtin_bit_cast(int8x8_t, __rev0), __builtin_bit_cast(int8x8_t, __rev1), 8)); - __ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); - return __ret; -} -__ai __attribute__((target("fullfp16,neon"))) float16x4_t __noswap_vmulx_f16(float16x4_t __p0, float16x4_t __p1) { - float16x4_t __ret; - __ret = __builtin_bit_cast(float16x4_t, __builtin_neon_vmulx_f16(__builtin_bit_cast(int8x8_t, __p0), __builtin_bit_cast(int8x8_t, __p1), 8)); - return __ret; -} -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vmulxh_lane_f16(__p0, __p1, __p2) __extension__ ({ \ - float16_t __ret; \ - float16_t __s0 = __p0; \ - float16x4_t __s1 = __p1; \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmulxh_lane_f16(__s0, __s1, __p2)); \ - __ret; \ -}) -#else -#define vmulxh_lane_f16(__p0, __p1, __p2) __extension__ ({ \ - float16_t __ret; \ - float16_t __s0 = __p0; \ - float16x4_t __s1 = __p1; \ - float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmulxh_lane_f16(__s0, __rev1, __p2)); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vmulxq_lane_f16(__p0_266, __p1_266, __p2_266) __extension__ ({ \ - float16x8_t __ret_266; \ - float16x8_t __s0_266 = __p0_266; \ - float16x4_t __s1_266 = __p1_266; \ - __ret_266 = vmulxq_f16(__s0_266, splatq_lane_f16(__s1_266, __p2_266)); \ - __ret_266; \ -}) -#else -#define vmulxq_lane_f16(__p0_267, __p1_267, __p2_267) __extension__ ({ \ - float16x8_t __ret_267; \ - float16x8_t __s0_267 = __p0_267; \ - float16x4_t __s1_267 = __p1_267; \ - float16x8_t __rev0_267; __rev0_267 = __builtin_shufflevector(__s0_267, __s0_267, __lane_reverse_128_16); \ - float16x4_t __rev1_267; __rev1_267 = __builtin_shufflevector(__s1_267, __s1_267, __lane_reverse_64_16); \ - __ret_267 = __noswap_vmulxq_f16(__rev0_267, __noswap_splatq_lane_f16(__rev1_267, __p2_267)); \ - __ret_267 = __builtin_shufflevector(__ret_267, __ret_267, __lane_reverse_128_16); \ - __ret_267; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vmulx_lane_f16(__p0_268, __p1_268, __p2_268) __extension__ ({ \ - float16x4_t __ret_268; \ - float16x4_t __s0_268 = __p0_268; \ - float16x4_t __s1_268 = __p1_268; \ - __ret_268 = vmulx_f16(__s0_268, splat_lane_f16(__s1_268, __p2_268)); \ - __ret_268; \ -}) -#else -#define vmulx_lane_f16(__p0_269, __p1_269, __p2_269) __extension__ ({ \ - float16x4_t __ret_269; \ - float16x4_t __s0_269 = __p0_269; \ - float16x4_t __s1_269 = __p1_269; \ - float16x4_t __rev0_269; __rev0_269 = __builtin_shufflevector(__s0_269, __s0_269, __lane_reverse_64_16); \ - float16x4_t __rev1_269; __rev1_269 = __builtin_shufflevector(__s1_269, __s1_269, __lane_reverse_64_16); \ - __ret_269 = __noswap_vmulx_f16(__rev0_269, __noswap_splat_lane_f16(__rev1_269, __p2_269)); \ - __ret_269 = __builtin_shufflevector(__ret_269, __ret_269, __lane_reverse_64_16); \ - __ret_269; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vmulxh_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ - float16_t __ret; \ - float16_t __s0 = __p0; \ - float16x8_t __s1 = __p1; \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmulxh_laneq_f16(__s0, __s1, __p2)); \ - __ret; \ -}) -#else -#define vmulxh_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ - float16_t __ret; \ - float16_t __s0 = __p0; \ - float16x8_t __s1 = __p1; \ - float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_16); \ - __ret = __builtin_bit_cast(float16_t, __builtin_neon_vmulxh_laneq_f16(__s0, __rev1, __p2)); \ - __ret; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vmulxq_laneq_f16(__p0_270, __p1_270, __p2_270) __extension__ ({ \ - float16x8_t __ret_270; \ - float16x8_t __s0_270 = __p0_270; \ - float16x8_t __s1_270 = __p1_270; \ - __ret_270 = vmulxq_f16(__s0_270, splatq_laneq_f16(__s1_270, __p2_270)); \ - __ret_270; \ -}) -#else -#define vmulxq_laneq_f16(__p0_271, __p1_271, __p2_271) __extension__ ({ \ - float16x8_t __ret_271; \ - float16x8_t __s0_271 = __p0_271; \ - float16x8_t __s1_271 = __p1_271; \ - float16x8_t __rev0_271; __rev0_271 = __builtin_shufflevector(__s0_271, __s0_271, __lane_reverse_128_16); \ - float16x8_t __rev1_271; __rev1_271 = __builtin_shufflevector(__s1_271, __s1_271, __lane_reverse_128_16); \ - __ret_271 = __noswap_vmulxq_f16(__rev0_271, __noswap_splatq_laneq_f16(__rev1_271, __p2_271)); \ - __ret_271 = __builtin_shufflevector(__ret_271, __ret_271, __lane_reverse_128_16); \ - __ret_271; \ -}) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vmulx_laneq_f16(__p0_272, __p1_272, __p2_272) __extension__ ({ \ - float16x4_t __ret_272; \ - float16x4_t __s0_272 = __p0_272; \ - float16x8_t __s1_272 = __p1_272; \ - __ret_272 = vmulx_f16(__s0_272, splat_laneq_f16(__s1_272, __p2_272)); \ - __ret_272; \ -}) -#else -#define vmulx_laneq_f16(__p0_273, __p1_273, __p2_273) __extension__ ({ \ - float16x4_t __ret_273; \ - float16x4_t __s0_273 = __p0_273; \ - float16x8_t __s1_273 = __p1_273; \ - float16x4_t __rev0_273; __rev0_273 = __builtin_shufflevector(__s0_273, __s0_273, __lane_reverse_64_16); \ - float16x8_t __rev1_273; __rev1_273 = __builtin_shufflevector(__s1_273, __s1_273, __lane_reverse_128_16); \ - __ret_273 = __noswap_vmulx_f16(__rev0_273, __noswap_splat_laneq_f16(__rev1_273, __p2_273)); \ - __ret_273 = __builtin_shufflevector(__ret_273, __ret_273, __lane_reverse_64_16); \ - __ret_273; \ -}) -#endif - #ifdef __LITTLE_ENDIAN__ #define vmulxq_n_f16(__p0, __p1) __extension__ ({ \ float16x8_t __ret; \ @@ -46068,98 +45862,98 @@ __ai __attribute__((target("fullfp16,neon"))) float16x4_t vsqrt_f16(float16x4_t #endif #ifdef __LITTLE_ENDIAN__ -#define vsudotq_laneq_s32(__p0_274, __p1_274, __p2_274, __p3_274) __extension__ ({ \ - int32x4_t __ret_274; \ - int32x4_t __s0_274 = __p0_274; \ - int8x16_t __s1_274 = __p1_274; \ - uint8x16_t __s2_274 = __p2_274; \ - __ret_274 = vusdotq_s32(__s0_274, __builtin_bit_cast(uint8x16_t, splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_274), __p3_274)), __s1_274); \ - __ret_274; \ +#define vsudotq_laneq_s32(__p0_266, __p1_266, __p2_266, __p3_266) __extension__ ({ \ + int32x4_t __ret_266; \ + int32x4_t __s0_266 = __p0_266; \ + int8x16_t __s1_266 = __p1_266; \ + uint8x16_t __s2_266 = __p2_266; \ + __ret_266 = vusdotq_s32(__s0_266, __builtin_bit_cast(uint8x16_t, splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_266), __p3_266)), __s1_266); \ + __ret_266; \ }) #else -#define vsudotq_laneq_s32(__p0_275, __p1_275, __p2_275, __p3_275) __extension__ ({ \ - int32x4_t __ret_275; \ - int32x4_t __s0_275 = __p0_275; \ - int8x16_t __s1_275 = __p1_275; \ - uint8x16_t __s2_275 = __p2_275; \ - int32x4_t __rev0_275; __rev0_275 = __builtin_shufflevector(__s0_275, __s0_275, __lane_reverse_128_32); \ - int8x16_t __rev1_275; __rev1_275 = __builtin_shufflevector(__s1_275, __s1_275, __lane_reverse_128_8); \ - uint8x16_t __rev2_275; __rev2_275 = __builtin_shufflevector(__s2_275, __s2_275, __lane_reverse_128_8); \ - __ret_275 = __noswap_vusdotq_s32(__rev0_275, __builtin_bit_cast(uint8x16_t, __noswap_splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_275), __p3_275)), __rev1_275); \ - __ret_275 = __builtin_shufflevector(__ret_275, __ret_275, __lane_reverse_128_32); \ - __ret_275; \ +#define vsudotq_laneq_s32(__p0_267, __p1_267, __p2_267, __p3_267) __extension__ ({ \ + int32x4_t __ret_267; \ + int32x4_t __s0_267 = __p0_267; \ + int8x16_t __s1_267 = __p1_267; \ + uint8x16_t __s2_267 = __p2_267; \ + int32x4_t __rev0_267; __rev0_267 = __builtin_shufflevector(__s0_267, __s0_267, __lane_reverse_128_32); \ + int8x16_t __rev1_267; __rev1_267 = __builtin_shufflevector(__s1_267, __s1_267, __lane_reverse_128_8); \ + uint8x16_t __rev2_267; __rev2_267 = __builtin_shufflevector(__s2_267, __s2_267, __lane_reverse_128_8); \ + __ret_267 = __noswap_vusdotq_s32(__rev0_267, __builtin_bit_cast(uint8x16_t, __noswap_splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_267), __p3_267)), __rev1_267); \ + __ret_267 = __builtin_shufflevector(__ret_267, __ret_267, __lane_reverse_128_32); \ + __ret_267; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vsudot_laneq_s32(__p0_276, __p1_276, __p2_276, __p3_276) __extension__ ({ \ - int32x2_t __ret_276; \ - int32x2_t __s0_276 = __p0_276; \ - int8x8_t __s1_276 = __p1_276; \ - uint8x16_t __s2_276 = __p2_276; \ - __ret_276 = vusdot_s32(__s0_276, __builtin_bit_cast(uint8x8_t, splat_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_276), __p3_276)), __s1_276); \ - __ret_276; \ +#define vsudot_laneq_s32(__p0_268, __p1_268, __p2_268, __p3_268) __extension__ ({ \ + int32x2_t __ret_268; \ + int32x2_t __s0_268 = __p0_268; \ + int8x8_t __s1_268 = __p1_268; \ + uint8x16_t __s2_268 = __p2_268; \ + __ret_268 = vusdot_s32(__s0_268, __builtin_bit_cast(uint8x8_t, splat_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_268), __p3_268)), __s1_268); \ + __ret_268; \ }) #else -#define vsudot_laneq_s32(__p0_277, __p1_277, __p2_277, __p3_277) __extension__ ({ \ - int32x2_t __ret_277; \ - int32x2_t __s0_277 = __p0_277; \ - int8x8_t __s1_277 = __p1_277; \ - uint8x16_t __s2_277 = __p2_277; \ - int32x2_t __rev0_277; __rev0_277 = __builtin_shufflevector(__s0_277, __s0_277, __lane_reverse_64_32); \ - int8x8_t __rev1_277; __rev1_277 = __builtin_shufflevector(__s1_277, __s1_277, __lane_reverse_64_8); \ - uint8x16_t __rev2_277; __rev2_277 = __builtin_shufflevector(__s2_277, __s2_277, __lane_reverse_128_8); \ - __ret_277 = __noswap_vusdot_s32(__rev0_277, __builtin_bit_cast(uint8x8_t, __noswap_splat_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_277), __p3_277)), __rev1_277); \ - __ret_277 = __builtin_shufflevector(__ret_277, __ret_277, __lane_reverse_64_32); \ - __ret_277; \ +#define vsudot_laneq_s32(__p0_269, __p1_269, __p2_269, __p3_269) __extension__ ({ \ + int32x2_t __ret_269; \ + int32x2_t __s0_269 = __p0_269; \ + int8x8_t __s1_269 = __p1_269; \ + uint8x16_t __s2_269 = __p2_269; \ + int32x2_t __rev0_269; __rev0_269 = __builtin_shufflevector(__s0_269, __s0_269, __lane_reverse_64_32); \ + int8x8_t __rev1_269; __rev1_269 = __builtin_shufflevector(__s1_269, __s1_269, __lane_reverse_64_8); \ + uint8x16_t __rev2_269; __rev2_269 = __builtin_shufflevector(__s2_269, __s2_269, __lane_reverse_128_8); \ + __ret_269 = __noswap_vusdot_s32(__rev0_269, __builtin_bit_cast(uint8x8_t, __noswap_splat_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_269), __p3_269)), __rev1_269); \ + __ret_269 = __builtin_shufflevector(__ret_269, __ret_269, __lane_reverse_64_32); \ + __ret_269; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vusdotq_laneq_s32(__p0_278, __p1_278, __p2_278, __p3_278) __extension__ ({ \ - int32x4_t __ret_278; \ - int32x4_t __s0_278 = __p0_278; \ - uint8x16_t __s1_278 = __p1_278; \ - int8x16_t __s2_278 = __p2_278; \ - __ret_278 = vusdotq_s32(__s0_278, __s1_278, __builtin_bit_cast(int8x16_t, splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_278), __p3_278))); \ - __ret_278; \ +#define vusdotq_laneq_s32(__p0_270, __p1_270, __p2_270, __p3_270) __extension__ ({ \ + int32x4_t __ret_270; \ + int32x4_t __s0_270 = __p0_270; \ + uint8x16_t __s1_270 = __p1_270; \ + int8x16_t __s2_270 = __p2_270; \ + __ret_270 = vusdotq_s32(__s0_270, __s1_270, __builtin_bit_cast(int8x16_t, splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_270), __p3_270))); \ + __ret_270; \ }) #else -#define vusdotq_laneq_s32(__p0_279, __p1_279, __p2_279, __p3_279) __extension__ ({ \ - int32x4_t __ret_279; \ - int32x4_t __s0_279 = __p0_279; \ - uint8x16_t __s1_279 = __p1_279; \ - int8x16_t __s2_279 = __p2_279; \ - int32x4_t __rev0_279; __rev0_279 = __builtin_shufflevector(__s0_279, __s0_279, __lane_reverse_128_32); \ - uint8x16_t __rev1_279; __rev1_279 = __builtin_shufflevector(__s1_279, __s1_279, __lane_reverse_128_8); \ - int8x16_t __rev2_279; __rev2_279 = __builtin_shufflevector(__s2_279, __s2_279, __lane_reverse_128_8); \ - __ret_279 = __noswap_vusdotq_s32(__rev0_279, __rev1_279, __builtin_bit_cast(int8x16_t, __noswap_splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_279), __p3_279))); \ - __ret_279 = __builtin_shufflevector(__ret_279, __ret_279, __lane_reverse_128_32); \ - __ret_279; \ +#define vusdotq_laneq_s32(__p0_271, __p1_271, __p2_271, __p3_271) __extension__ ({ \ + int32x4_t __ret_271; \ + int32x4_t __s0_271 = __p0_271; \ + uint8x16_t __s1_271 = __p1_271; \ + int8x16_t __s2_271 = __p2_271; \ + int32x4_t __rev0_271; __rev0_271 = __builtin_shufflevector(__s0_271, __s0_271, __lane_reverse_128_32); \ + uint8x16_t __rev1_271; __rev1_271 = __builtin_shufflevector(__s1_271, __s1_271, __lane_reverse_128_8); \ + int8x16_t __rev2_271; __rev2_271 = __builtin_shufflevector(__s2_271, __s2_271, __lane_reverse_128_8); \ + __ret_271 = __noswap_vusdotq_s32(__rev0_271, __rev1_271, __builtin_bit_cast(int8x16_t, __noswap_splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_271), __p3_271))); \ + __ret_271 = __builtin_shufflevector(__ret_271, __ret_271, __lane_reverse_128_32); \ + __ret_271; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vusdot_laneq_s32(__p0_280, __p1_280, __p2_280, __p3_280) __extension__ ({ \ - int32x2_t __ret_280; \ - int32x2_t __s0_280 = __p0_280; \ - uint8x8_t __s1_280 = __p1_280; \ - int8x16_t __s2_280 = __p2_280; \ - __ret_280 = vusdot_s32(__s0_280, __s1_280, __builtin_bit_cast(int8x8_t, splat_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_280), __p3_280))); \ - __ret_280; \ +#define vusdot_laneq_s32(__p0_272, __p1_272, __p2_272, __p3_272) __extension__ ({ \ + int32x2_t __ret_272; \ + int32x2_t __s0_272 = __p0_272; \ + uint8x8_t __s1_272 = __p1_272; \ + int8x16_t __s2_272 = __p2_272; \ + __ret_272 = vusdot_s32(__s0_272, __s1_272, __builtin_bit_cast(int8x8_t, splat_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_272), __p3_272))); \ + __ret_272; \ }) #else -#define vusdot_laneq_s32(__p0_281, __p1_281, __p2_281, __p3_281) __extension__ ({ \ - int32x2_t __ret_281; \ - int32x2_t __s0_281 = __p0_281; \ - uint8x8_t __s1_281 = __p1_281; \ - int8x16_t __s2_281 = __p2_281; \ - int32x2_t __rev0_281; __rev0_281 = __builtin_shufflevector(__s0_281, __s0_281, __lane_reverse_64_32); \ - uint8x8_t __rev1_281; __rev1_281 = __builtin_shufflevector(__s1_281, __s1_281, __lane_reverse_64_8); \ - int8x16_t __rev2_281; __rev2_281 = __builtin_shufflevector(__s2_281, __s2_281, __lane_reverse_128_8); \ - __ret_281 = __noswap_vusdot_s32(__rev0_281, __rev1_281, __builtin_bit_cast(int8x8_t, __noswap_splat_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_281), __p3_281))); \ - __ret_281 = __builtin_shufflevector(__ret_281, __ret_281, __lane_reverse_64_32); \ - __ret_281; \ +#define vusdot_laneq_s32(__p0_273, __p1_273, __p2_273, __p3_273) __extension__ ({ \ + int32x2_t __ret_273; \ + int32x2_t __s0_273 = __p0_273; \ + uint8x8_t __s1_273 = __p1_273; \ + int8x16_t __s2_273 = __p2_273; \ + int32x2_t __rev0_273; __rev0_273 = __builtin_shufflevector(__s0_273, __s0_273, __lane_reverse_64_32); \ + uint8x8_t __rev1_273; __rev1_273 = __builtin_shufflevector(__s1_273, __s1_273, __lane_reverse_64_8); \ + int8x16_t __rev2_273; __rev2_273 = __builtin_shufflevector(__s2_273, __s2_273, __lane_reverse_128_8); \ + __ret_273 = __noswap_vusdot_s32(__rev0_273, __rev1_273, __builtin_bit_cast(int8x8_t, __noswap_splat_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_273), __p3_273))); \ + __ret_273 = __builtin_shufflevector(__ret_273, __ret_273, __lane_reverse_64_32); \ + __ret_273; \ }) #endif @@ -48579,974 +48373,1058 @@ __ai __attribute__((target("neon"))) float64x2_t vcombine_f64(float64x1_t __p0, #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_lane_p8(__p0_282, __p1_282, __p2_282, __p3_282) __extension__ ({ \ - poly8x16_t __ret_282; \ - poly8x16_t __s0_282 = __p0_282; \ - poly8x8_t __s2_282 = __p2_282; \ - __ret_282 = vsetq_lane_p8(vget_lane_p8(__s2_282, __p3_282), __s0_282, __p1_282); \ +#define vcopyq_lane_p8(__p0_274, __p1_274, __p2_274, __p3_274) __extension__ ({ \ + poly8x16_t __ret_274; \ + poly8x16_t __s0_274 = __p0_274; \ + poly8x8_t __s2_274 = __p2_274; \ + __ret_274 = vsetq_lane_p8(vget_lane_p8(__s2_274, __p3_274), __s0_274, __p1_274); \ + __ret_274; \ +}) +#else +#define vcopyq_lane_p8(__p0_275, __p1_275, __p2_275, __p3_275) __extension__ ({ \ + poly8x16_t __ret_275; \ + poly8x16_t __s0_275 = __p0_275; \ + poly8x8_t __s2_275 = __p2_275; \ + poly8x16_t __rev0_275; __rev0_275 = __builtin_shufflevector(__s0_275, __s0_275, __lane_reverse_128_8); \ + poly8x8_t __rev2_275; __rev2_275 = __builtin_shufflevector(__s2_275, __s2_275, __lane_reverse_64_8); \ + __ret_275 = __noswap_vsetq_lane_p8(__noswap_vget_lane_p8(__rev2_275, __p3_275), __rev0_275, __p1_275); \ + __ret_275 = __builtin_shufflevector(__ret_275, __ret_275, __lane_reverse_128_8); \ + __ret_275; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_p16(__p0_276, __p1_276, __p2_276, __p3_276) __extension__ ({ \ + poly16x8_t __ret_276; \ + poly16x8_t __s0_276 = __p0_276; \ + poly16x4_t __s2_276 = __p2_276; \ + __ret_276 = vsetq_lane_p16(vget_lane_p16(__s2_276, __p3_276), __s0_276, __p1_276); \ + __ret_276; \ +}) +#else +#define vcopyq_lane_p16(__p0_277, __p1_277, __p2_277, __p3_277) __extension__ ({ \ + poly16x8_t __ret_277; \ + poly16x8_t __s0_277 = __p0_277; \ + poly16x4_t __s2_277 = __p2_277; \ + poly16x8_t __rev0_277; __rev0_277 = __builtin_shufflevector(__s0_277, __s0_277, __lane_reverse_128_16); \ + poly16x4_t __rev2_277; __rev2_277 = __builtin_shufflevector(__s2_277, __s2_277, __lane_reverse_64_16); \ + __ret_277 = __noswap_vsetq_lane_p16(__noswap_vget_lane_p16(__rev2_277, __p3_277), __rev0_277, __p1_277); \ + __ret_277 = __builtin_shufflevector(__ret_277, __ret_277, __lane_reverse_128_16); \ + __ret_277; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_u8(__p0_278, __p1_278, __p2_278, __p3_278) __extension__ ({ \ + uint8x16_t __ret_278; \ + uint8x16_t __s0_278 = __p0_278; \ + uint8x8_t __s2_278 = __p2_278; \ + __ret_278 = vsetq_lane_u8(vget_lane_u8(__s2_278, __p3_278), __s0_278, __p1_278); \ + __ret_278; \ +}) +#else +#define vcopyq_lane_u8(__p0_279, __p1_279, __p2_279, __p3_279) __extension__ ({ \ + uint8x16_t __ret_279; \ + uint8x16_t __s0_279 = __p0_279; \ + uint8x8_t __s2_279 = __p2_279; \ + uint8x16_t __rev0_279; __rev0_279 = __builtin_shufflevector(__s0_279, __s0_279, __lane_reverse_128_8); \ + uint8x8_t __rev2_279; __rev2_279 = __builtin_shufflevector(__s2_279, __s2_279, __lane_reverse_64_8); \ + __ret_279 = __noswap_vsetq_lane_u8(__noswap_vget_lane_u8(__rev2_279, __p3_279), __rev0_279, __p1_279); \ + __ret_279 = __builtin_shufflevector(__ret_279, __ret_279, __lane_reverse_128_8); \ + __ret_279; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_u32(__p0_280, __p1_280, __p2_280, __p3_280) __extension__ ({ \ + uint32x4_t __ret_280; \ + uint32x4_t __s0_280 = __p0_280; \ + uint32x2_t __s2_280 = __p2_280; \ + __ret_280 = vsetq_lane_u32(vget_lane_u32(__s2_280, __p3_280), __s0_280, __p1_280); \ + __ret_280; \ +}) +#else +#define vcopyq_lane_u32(__p0_281, __p1_281, __p2_281, __p3_281) __extension__ ({ \ + uint32x4_t __ret_281; \ + uint32x4_t __s0_281 = __p0_281; \ + uint32x2_t __s2_281 = __p2_281; \ + uint32x4_t __rev0_281; __rev0_281 = __builtin_shufflevector(__s0_281, __s0_281, __lane_reverse_128_32); \ + uint32x2_t __rev2_281; __rev2_281 = __builtin_shufflevector(__s2_281, __s2_281, __lane_reverse_64_32); \ + __ret_281 = __noswap_vsetq_lane_u32(__noswap_vget_lane_u32(__rev2_281, __p3_281), __rev0_281, __p1_281); \ + __ret_281 = __builtin_shufflevector(__ret_281, __ret_281, __lane_reverse_128_32); \ + __ret_281; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_u64(__p0_282, __p1_282, __p2_282, __p3_282) __extension__ ({ \ + uint64x2_t __ret_282; \ + uint64x2_t __s0_282 = __p0_282; \ + uint64x1_t __s2_282 = __p2_282; \ + __ret_282 = vsetq_lane_u64(vget_lane_u64(__s2_282, __p3_282), __s0_282, __p1_282); \ __ret_282; \ }) #else -#define vcopyq_lane_p8(__p0_283, __p1_283, __p2_283, __p3_283) __extension__ ({ \ - poly8x16_t __ret_283; \ - poly8x16_t __s0_283 = __p0_283; \ - poly8x8_t __s2_283 = __p2_283; \ - poly8x16_t __rev0_283; __rev0_283 = __builtin_shufflevector(__s0_283, __s0_283, __lane_reverse_128_8); \ - poly8x8_t __rev2_283; __rev2_283 = __builtin_shufflevector(__s2_283, __s2_283, __lane_reverse_64_8); \ - __ret_283 = __noswap_vsetq_lane_p8(__noswap_vget_lane_p8(__rev2_283, __p3_283), __rev0_283, __p1_283); \ - __ret_283 = __builtin_shufflevector(__ret_283, __ret_283, __lane_reverse_128_8); \ +#define vcopyq_lane_u64(__p0_283, __p1_283, __p2_283, __p3_283) __extension__ ({ \ + uint64x2_t __ret_283; \ + uint64x2_t __s0_283 = __p0_283; \ + uint64x1_t __s2_283 = __p2_283; \ + uint64x2_t __rev0_283; __rev0_283 = __builtin_shufflevector(__s0_283, __s0_283, __lane_reverse_128_64); \ + __ret_283 = __noswap_vsetq_lane_u64(vget_lane_u64(__s2_283, __p3_283), __rev0_283, __p1_283); \ + __ret_283 = __builtin_shufflevector(__ret_283, __ret_283, __lane_reverse_128_64); \ __ret_283; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_lane_p16(__p0_284, __p1_284, __p2_284, __p3_284) __extension__ ({ \ - poly16x8_t __ret_284; \ - poly16x8_t __s0_284 = __p0_284; \ - poly16x4_t __s2_284 = __p2_284; \ - __ret_284 = vsetq_lane_p16(vget_lane_p16(__s2_284, __p3_284), __s0_284, __p1_284); \ +#define vcopyq_lane_u16(__p0_284, __p1_284, __p2_284, __p3_284) __extension__ ({ \ + uint16x8_t __ret_284; \ + uint16x8_t __s0_284 = __p0_284; \ + uint16x4_t __s2_284 = __p2_284; \ + __ret_284 = vsetq_lane_u16(vget_lane_u16(__s2_284, __p3_284), __s0_284, __p1_284); \ __ret_284; \ }) #else -#define vcopyq_lane_p16(__p0_285, __p1_285, __p2_285, __p3_285) __extension__ ({ \ - poly16x8_t __ret_285; \ - poly16x8_t __s0_285 = __p0_285; \ - poly16x4_t __s2_285 = __p2_285; \ - poly16x8_t __rev0_285; __rev0_285 = __builtin_shufflevector(__s0_285, __s0_285, __lane_reverse_128_16); \ - poly16x4_t __rev2_285; __rev2_285 = __builtin_shufflevector(__s2_285, __s2_285, __lane_reverse_64_16); \ - __ret_285 = __noswap_vsetq_lane_p16(__noswap_vget_lane_p16(__rev2_285, __p3_285), __rev0_285, __p1_285); \ +#define vcopyq_lane_u16(__p0_285, __p1_285, __p2_285, __p3_285) __extension__ ({ \ + uint16x8_t __ret_285; \ + uint16x8_t __s0_285 = __p0_285; \ + uint16x4_t __s2_285 = __p2_285; \ + uint16x8_t __rev0_285; __rev0_285 = __builtin_shufflevector(__s0_285, __s0_285, __lane_reverse_128_16); \ + uint16x4_t __rev2_285; __rev2_285 = __builtin_shufflevector(__s2_285, __s2_285, __lane_reverse_64_16); \ + __ret_285 = __noswap_vsetq_lane_u16(__noswap_vget_lane_u16(__rev2_285, __p3_285), __rev0_285, __p1_285); \ __ret_285 = __builtin_shufflevector(__ret_285, __ret_285, __lane_reverse_128_16); \ __ret_285; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_lane_u8(__p0_286, __p1_286, __p2_286, __p3_286) __extension__ ({ \ - uint8x16_t __ret_286; \ - uint8x16_t __s0_286 = __p0_286; \ - uint8x8_t __s2_286 = __p2_286; \ - __ret_286 = vsetq_lane_u8(vget_lane_u8(__s2_286, __p3_286), __s0_286, __p1_286); \ +#define vcopyq_lane_s8(__p0_286, __p1_286, __p2_286, __p3_286) __extension__ ({ \ + int8x16_t __ret_286; \ + int8x16_t __s0_286 = __p0_286; \ + int8x8_t __s2_286 = __p2_286; \ + __ret_286 = vsetq_lane_s8(vget_lane_s8(__s2_286, __p3_286), __s0_286, __p1_286); \ __ret_286; \ }) #else -#define vcopyq_lane_u8(__p0_287, __p1_287, __p2_287, __p3_287) __extension__ ({ \ - uint8x16_t __ret_287; \ - uint8x16_t __s0_287 = __p0_287; \ - uint8x8_t __s2_287 = __p2_287; \ - uint8x16_t __rev0_287; __rev0_287 = __builtin_shufflevector(__s0_287, __s0_287, __lane_reverse_128_8); \ - uint8x8_t __rev2_287; __rev2_287 = __builtin_shufflevector(__s2_287, __s2_287, __lane_reverse_64_8); \ - __ret_287 = __noswap_vsetq_lane_u8(__noswap_vget_lane_u8(__rev2_287, __p3_287), __rev0_287, __p1_287); \ +#define vcopyq_lane_s8(__p0_287, __p1_287, __p2_287, __p3_287) __extension__ ({ \ + int8x16_t __ret_287; \ + int8x16_t __s0_287 = __p0_287; \ + int8x8_t __s2_287 = __p2_287; \ + int8x16_t __rev0_287; __rev0_287 = __builtin_shufflevector(__s0_287, __s0_287, __lane_reverse_128_8); \ + int8x8_t __rev2_287; __rev2_287 = __builtin_shufflevector(__s2_287, __s2_287, __lane_reverse_64_8); \ + __ret_287 = __noswap_vsetq_lane_s8(__noswap_vget_lane_s8(__rev2_287, __p3_287), __rev0_287, __p1_287); \ __ret_287 = __builtin_shufflevector(__ret_287, __ret_287, __lane_reverse_128_8); \ __ret_287; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_lane_u32(__p0_288, __p1_288, __p2_288, __p3_288) __extension__ ({ \ - uint32x4_t __ret_288; \ - uint32x4_t __s0_288 = __p0_288; \ - uint32x2_t __s2_288 = __p2_288; \ - __ret_288 = vsetq_lane_u32(vget_lane_u32(__s2_288, __p3_288), __s0_288, __p1_288); \ +#define vcopyq_lane_f32(__p0_288, __p1_288, __p2_288, __p3_288) __extension__ ({ \ + float32x4_t __ret_288; \ + float32x4_t __s0_288 = __p0_288; \ + float32x2_t __s2_288 = __p2_288; \ + __ret_288 = vsetq_lane_f32(vget_lane_f32(__s2_288, __p3_288), __s0_288, __p1_288); \ __ret_288; \ }) #else -#define vcopyq_lane_u32(__p0_289, __p1_289, __p2_289, __p3_289) __extension__ ({ \ - uint32x4_t __ret_289; \ - uint32x4_t __s0_289 = __p0_289; \ - uint32x2_t __s2_289 = __p2_289; \ - uint32x4_t __rev0_289; __rev0_289 = __builtin_shufflevector(__s0_289, __s0_289, __lane_reverse_128_32); \ - uint32x2_t __rev2_289; __rev2_289 = __builtin_shufflevector(__s2_289, __s2_289, __lane_reverse_64_32); \ - __ret_289 = __noswap_vsetq_lane_u32(__noswap_vget_lane_u32(__rev2_289, __p3_289), __rev0_289, __p1_289); \ +#define vcopyq_lane_f32(__p0_289, __p1_289, __p2_289, __p3_289) __extension__ ({ \ + float32x4_t __ret_289; \ + float32x4_t __s0_289 = __p0_289; \ + float32x2_t __s2_289 = __p2_289; \ + float32x4_t __rev0_289; __rev0_289 = __builtin_shufflevector(__s0_289, __s0_289, __lane_reverse_128_32); \ + float32x2_t __rev2_289; __rev2_289 = __builtin_shufflevector(__s2_289, __s2_289, __lane_reverse_64_32); \ + __ret_289 = __noswap_vsetq_lane_f32(__noswap_vget_lane_f32(__rev2_289, __p3_289), __rev0_289, __p1_289); \ __ret_289 = __builtin_shufflevector(__ret_289, __ret_289, __lane_reverse_128_32); \ __ret_289; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_lane_u64(__p0_290, __p1_290, __p2_290, __p3_290) __extension__ ({ \ - uint64x2_t __ret_290; \ - uint64x2_t __s0_290 = __p0_290; \ - uint64x1_t __s2_290 = __p2_290; \ - __ret_290 = vsetq_lane_u64(vget_lane_u64(__s2_290, __p3_290), __s0_290, __p1_290); \ +#define vcopyq_lane_s32(__p0_290, __p1_290, __p2_290, __p3_290) __extension__ ({ \ + int32x4_t __ret_290; \ + int32x4_t __s0_290 = __p0_290; \ + int32x2_t __s2_290 = __p2_290; \ + __ret_290 = vsetq_lane_s32(vget_lane_s32(__s2_290, __p3_290), __s0_290, __p1_290); \ __ret_290; \ }) #else -#define vcopyq_lane_u64(__p0_291, __p1_291, __p2_291, __p3_291) __extension__ ({ \ - uint64x2_t __ret_291; \ - uint64x2_t __s0_291 = __p0_291; \ - uint64x1_t __s2_291 = __p2_291; \ - uint64x2_t __rev0_291; __rev0_291 = __builtin_shufflevector(__s0_291, __s0_291, __lane_reverse_128_64); \ - __ret_291 = __noswap_vsetq_lane_u64(vget_lane_u64(__s2_291, __p3_291), __rev0_291, __p1_291); \ - __ret_291 = __builtin_shufflevector(__ret_291, __ret_291, __lane_reverse_128_64); \ +#define vcopyq_lane_s32(__p0_291, __p1_291, __p2_291, __p3_291) __extension__ ({ \ + int32x4_t __ret_291; \ + int32x4_t __s0_291 = __p0_291; \ + int32x2_t __s2_291 = __p2_291; \ + int32x4_t __rev0_291; __rev0_291 = __builtin_shufflevector(__s0_291, __s0_291, __lane_reverse_128_32); \ + int32x2_t __rev2_291; __rev2_291 = __builtin_shufflevector(__s2_291, __s2_291, __lane_reverse_64_32); \ + __ret_291 = __noswap_vsetq_lane_s32(__noswap_vget_lane_s32(__rev2_291, __p3_291), __rev0_291, __p1_291); \ + __ret_291 = __builtin_shufflevector(__ret_291, __ret_291, __lane_reverse_128_32); \ __ret_291; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_lane_u16(__p0_292, __p1_292, __p2_292, __p3_292) __extension__ ({ \ - uint16x8_t __ret_292; \ - uint16x8_t __s0_292 = __p0_292; \ - uint16x4_t __s2_292 = __p2_292; \ - __ret_292 = vsetq_lane_u16(vget_lane_u16(__s2_292, __p3_292), __s0_292, __p1_292); \ +#define vcopyq_lane_s64(__p0_292, __p1_292, __p2_292, __p3_292) __extension__ ({ \ + int64x2_t __ret_292; \ + int64x2_t __s0_292 = __p0_292; \ + int64x1_t __s2_292 = __p2_292; \ + __ret_292 = vsetq_lane_s64(vget_lane_s64(__s2_292, __p3_292), __s0_292, __p1_292); \ __ret_292; \ }) #else -#define vcopyq_lane_u16(__p0_293, __p1_293, __p2_293, __p3_293) __extension__ ({ \ - uint16x8_t __ret_293; \ - uint16x8_t __s0_293 = __p0_293; \ - uint16x4_t __s2_293 = __p2_293; \ - uint16x8_t __rev0_293; __rev0_293 = __builtin_shufflevector(__s0_293, __s0_293, __lane_reverse_128_16); \ - uint16x4_t __rev2_293; __rev2_293 = __builtin_shufflevector(__s2_293, __s2_293, __lane_reverse_64_16); \ - __ret_293 = __noswap_vsetq_lane_u16(__noswap_vget_lane_u16(__rev2_293, __p3_293), __rev0_293, __p1_293); \ - __ret_293 = __builtin_shufflevector(__ret_293, __ret_293, __lane_reverse_128_16); \ +#define vcopyq_lane_s64(__p0_293, __p1_293, __p2_293, __p3_293) __extension__ ({ \ + int64x2_t __ret_293; \ + int64x2_t __s0_293 = __p0_293; \ + int64x1_t __s2_293 = __p2_293; \ + int64x2_t __rev0_293; __rev0_293 = __builtin_shufflevector(__s0_293, __s0_293, __lane_reverse_128_64); \ + __ret_293 = __noswap_vsetq_lane_s64(vget_lane_s64(__s2_293, __p3_293), __rev0_293, __p1_293); \ + __ret_293 = __builtin_shufflevector(__ret_293, __ret_293, __lane_reverse_128_64); \ __ret_293; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_lane_s8(__p0_294, __p1_294, __p2_294, __p3_294) __extension__ ({ \ - int8x16_t __ret_294; \ - int8x16_t __s0_294 = __p0_294; \ - int8x8_t __s2_294 = __p2_294; \ - __ret_294 = vsetq_lane_s8(vget_lane_s8(__s2_294, __p3_294), __s0_294, __p1_294); \ +#define vcopyq_lane_mf8(__p0_294, __p1_294, __p2_294, __p3_294) __extension__ ({ \ + mfloat8x16_t __ret_294; \ + mfloat8x16_t __s0_294 = __p0_294; \ + mfloat8x8_t __s2_294 = __p2_294; \ + __ret_294 = vsetq_lane_mf8(vget_lane_mf8(__s2_294, __p3_294), __s0_294, __p1_294); \ __ret_294; \ }) #else -#define vcopyq_lane_s8(__p0_295, __p1_295, __p2_295, __p3_295) __extension__ ({ \ - int8x16_t __ret_295; \ - int8x16_t __s0_295 = __p0_295; \ - int8x8_t __s2_295 = __p2_295; \ - int8x16_t __rev0_295; __rev0_295 = __builtin_shufflevector(__s0_295, __s0_295, __lane_reverse_128_8); \ - int8x8_t __rev2_295; __rev2_295 = __builtin_shufflevector(__s2_295, __s2_295, __lane_reverse_64_8); \ - __ret_295 = __noswap_vsetq_lane_s8(__noswap_vget_lane_s8(__rev2_295, __p3_295), __rev0_295, __p1_295); \ +#define vcopyq_lane_mf8(__p0_295, __p1_295, __p2_295, __p3_295) __extension__ ({ \ + mfloat8x16_t __ret_295; \ + mfloat8x16_t __s0_295 = __p0_295; \ + mfloat8x8_t __s2_295 = __p2_295; \ + mfloat8x16_t __rev0_295; __rev0_295 = __builtin_shufflevector(__s0_295, __s0_295, __lane_reverse_128_8); \ + mfloat8x8_t __rev2_295; __rev2_295 = __builtin_shufflevector(__s2_295, __s2_295, __lane_reverse_64_8); \ + __ret_295 = __noswap_vsetq_lane_mf8(__noswap_vget_lane_mf8(__rev2_295, __p3_295), __rev0_295, __p1_295); \ __ret_295 = __builtin_shufflevector(__ret_295, __ret_295, __lane_reverse_128_8); \ __ret_295; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_lane_f32(__p0_296, __p1_296, __p2_296, __p3_296) __extension__ ({ \ - float32x4_t __ret_296; \ - float32x4_t __s0_296 = __p0_296; \ - float32x2_t __s2_296 = __p2_296; \ - __ret_296 = vsetq_lane_f32(vget_lane_f32(__s2_296, __p3_296), __s0_296, __p1_296); \ +#define vcopyq_lane_s16(__p0_296, __p1_296, __p2_296, __p3_296) __extension__ ({ \ + int16x8_t __ret_296; \ + int16x8_t __s0_296 = __p0_296; \ + int16x4_t __s2_296 = __p2_296; \ + __ret_296 = vsetq_lane_s16(vget_lane_s16(__s2_296, __p3_296), __s0_296, __p1_296); \ __ret_296; \ }) #else -#define vcopyq_lane_f32(__p0_297, __p1_297, __p2_297, __p3_297) __extension__ ({ \ - float32x4_t __ret_297; \ - float32x4_t __s0_297 = __p0_297; \ - float32x2_t __s2_297 = __p2_297; \ - float32x4_t __rev0_297; __rev0_297 = __builtin_shufflevector(__s0_297, __s0_297, __lane_reverse_128_32); \ - float32x2_t __rev2_297; __rev2_297 = __builtin_shufflevector(__s2_297, __s2_297, __lane_reverse_64_32); \ - __ret_297 = __noswap_vsetq_lane_f32(__noswap_vget_lane_f32(__rev2_297, __p3_297), __rev0_297, __p1_297); \ - __ret_297 = __builtin_shufflevector(__ret_297, __ret_297, __lane_reverse_128_32); \ +#define vcopyq_lane_s16(__p0_297, __p1_297, __p2_297, __p3_297) __extension__ ({ \ + int16x8_t __ret_297; \ + int16x8_t __s0_297 = __p0_297; \ + int16x4_t __s2_297 = __p2_297; \ + int16x8_t __rev0_297; __rev0_297 = __builtin_shufflevector(__s0_297, __s0_297, __lane_reverse_128_16); \ + int16x4_t __rev2_297; __rev2_297 = __builtin_shufflevector(__s2_297, __s2_297, __lane_reverse_64_16); \ + __ret_297 = __noswap_vsetq_lane_s16(__noswap_vget_lane_s16(__rev2_297, __p3_297), __rev0_297, __p1_297); \ + __ret_297 = __builtin_shufflevector(__ret_297, __ret_297, __lane_reverse_128_16); \ __ret_297; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_lane_s32(__p0_298, __p1_298, __p2_298, __p3_298) __extension__ ({ \ - int32x4_t __ret_298; \ - int32x4_t __s0_298 = __p0_298; \ - int32x2_t __s2_298 = __p2_298; \ - __ret_298 = vsetq_lane_s32(vget_lane_s32(__s2_298, __p3_298), __s0_298, __p1_298); \ +#define vcopyq_lane_bf16(__p0_298, __p1_298, __p2_298, __p3_298) __extension__ ({ \ + bfloat16x8_t __ret_298; \ + bfloat16x8_t __s0_298 = __p0_298; \ + bfloat16x4_t __s2_298 = __p2_298; \ + __ret_298 = vsetq_lane_bf16(vget_lane_bf16(__s2_298, __p3_298), __s0_298, __p1_298); \ __ret_298; \ }) #else -#define vcopyq_lane_s32(__p0_299, __p1_299, __p2_299, __p3_299) __extension__ ({ \ - int32x4_t __ret_299; \ - int32x4_t __s0_299 = __p0_299; \ - int32x2_t __s2_299 = __p2_299; \ - int32x4_t __rev0_299; __rev0_299 = __builtin_shufflevector(__s0_299, __s0_299, __lane_reverse_128_32); \ - int32x2_t __rev2_299; __rev2_299 = __builtin_shufflevector(__s2_299, __s2_299, __lane_reverse_64_32); \ - __ret_299 = __noswap_vsetq_lane_s32(__noswap_vget_lane_s32(__rev2_299, __p3_299), __rev0_299, __p1_299); \ - __ret_299 = __builtin_shufflevector(__ret_299, __ret_299, __lane_reverse_128_32); \ +#define vcopyq_lane_bf16(__p0_299, __p1_299, __p2_299, __p3_299) __extension__ ({ \ + bfloat16x8_t __ret_299; \ + bfloat16x8_t __s0_299 = __p0_299; \ + bfloat16x4_t __s2_299 = __p2_299; \ + bfloat16x8_t __rev0_299; __rev0_299 = __builtin_shufflevector(__s0_299, __s0_299, __lane_reverse_128_16); \ + bfloat16x4_t __rev2_299; __rev2_299 = __builtin_shufflevector(__s2_299, __s2_299, __lane_reverse_64_16); \ + __ret_299 = __noswap_vsetq_lane_bf16(__noswap_vget_lane_bf16(__rev2_299, __p3_299), __rev0_299, __p1_299); \ + __ret_299 = __builtin_shufflevector(__ret_299, __ret_299, __lane_reverse_128_16); \ __ret_299; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_lane_s64(__p0_300, __p1_300, __p2_300, __p3_300) __extension__ ({ \ - int64x2_t __ret_300; \ - int64x2_t __s0_300 = __p0_300; \ - int64x1_t __s2_300 = __p2_300; \ - __ret_300 = vsetq_lane_s64(vget_lane_s64(__s2_300, __p3_300), __s0_300, __p1_300); \ +#define vcopy_lane_p8(__p0_300, __p1_300, __p2_300, __p3_300) __extension__ ({ \ + poly8x8_t __ret_300; \ + poly8x8_t __s0_300 = __p0_300; \ + poly8x8_t __s2_300 = __p2_300; \ + __ret_300 = vset_lane_p8(vget_lane_p8(__s2_300, __p3_300), __s0_300, __p1_300); \ __ret_300; \ }) #else -#define vcopyq_lane_s64(__p0_301, __p1_301, __p2_301, __p3_301) __extension__ ({ \ - int64x2_t __ret_301; \ - int64x2_t __s0_301 = __p0_301; \ - int64x1_t __s2_301 = __p2_301; \ - int64x2_t __rev0_301; __rev0_301 = __builtin_shufflevector(__s0_301, __s0_301, __lane_reverse_128_64); \ - __ret_301 = __noswap_vsetq_lane_s64(vget_lane_s64(__s2_301, __p3_301), __rev0_301, __p1_301); \ - __ret_301 = __builtin_shufflevector(__ret_301, __ret_301, __lane_reverse_128_64); \ +#define vcopy_lane_p8(__p0_301, __p1_301, __p2_301, __p3_301) __extension__ ({ \ + poly8x8_t __ret_301; \ + poly8x8_t __s0_301 = __p0_301; \ + poly8x8_t __s2_301 = __p2_301; \ + poly8x8_t __rev0_301; __rev0_301 = __builtin_shufflevector(__s0_301, __s0_301, __lane_reverse_64_8); \ + poly8x8_t __rev2_301; __rev2_301 = __builtin_shufflevector(__s2_301, __s2_301, __lane_reverse_64_8); \ + __ret_301 = __noswap_vset_lane_p8(__noswap_vget_lane_p8(__rev2_301, __p3_301), __rev0_301, __p1_301); \ + __ret_301 = __builtin_shufflevector(__ret_301, __ret_301, __lane_reverse_64_8); \ __ret_301; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_lane_mf8(__p0_302, __p1_302, __p2_302, __p3_302) __extension__ ({ \ - mfloat8x16_t __ret_302; \ - mfloat8x16_t __s0_302 = __p0_302; \ - mfloat8x8_t __s2_302 = __p2_302; \ - __ret_302 = vsetq_lane_mf8(vget_lane_mf8(__s2_302, __p3_302), __s0_302, __p1_302); \ +#define vcopy_lane_p16(__p0_302, __p1_302, __p2_302, __p3_302) __extension__ ({ \ + poly16x4_t __ret_302; \ + poly16x4_t __s0_302 = __p0_302; \ + poly16x4_t __s2_302 = __p2_302; \ + __ret_302 = vset_lane_p16(vget_lane_p16(__s2_302, __p3_302), __s0_302, __p1_302); \ __ret_302; \ }) #else -#define vcopyq_lane_mf8(__p0_303, __p1_303, __p2_303, __p3_303) __extension__ ({ \ - mfloat8x16_t __ret_303; \ - mfloat8x16_t __s0_303 = __p0_303; \ - mfloat8x8_t __s2_303 = __p2_303; \ - mfloat8x16_t __rev0_303; __rev0_303 = __builtin_shufflevector(__s0_303, __s0_303, __lane_reverse_128_8); \ - mfloat8x8_t __rev2_303; __rev2_303 = __builtin_shufflevector(__s2_303, __s2_303, __lane_reverse_64_8); \ - __ret_303 = __noswap_vsetq_lane_mf8(__noswap_vget_lane_mf8(__rev2_303, __p3_303), __rev0_303, __p1_303); \ - __ret_303 = __builtin_shufflevector(__ret_303, __ret_303, __lane_reverse_128_8); \ +#define vcopy_lane_p16(__p0_303, __p1_303, __p2_303, __p3_303) __extension__ ({ \ + poly16x4_t __ret_303; \ + poly16x4_t __s0_303 = __p0_303; \ + poly16x4_t __s2_303 = __p2_303; \ + poly16x4_t __rev0_303; __rev0_303 = __builtin_shufflevector(__s0_303, __s0_303, __lane_reverse_64_16); \ + poly16x4_t __rev2_303; __rev2_303 = __builtin_shufflevector(__s2_303, __s2_303, __lane_reverse_64_16); \ + __ret_303 = __noswap_vset_lane_p16(__noswap_vget_lane_p16(__rev2_303, __p3_303), __rev0_303, __p1_303); \ + __ret_303 = __builtin_shufflevector(__ret_303, __ret_303, __lane_reverse_64_16); \ __ret_303; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_lane_s16(__p0_304, __p1_304, __p2_304, __p3_304) __extension__ ({ \ - int16x8_t __ret_304; \ - int16x8_t __s0_304 = __p0_304; \ - int16x4_t __s2_304 = __p2_304; \ - __ret_304 = vsetq_lane_s16(vget_lane_s16(__s2_304, __p3_304), __s0_304, __p1_304); \ +#define vcopy_lane_u8(__p0_304, __p1_304, __p2_304, __p3_304) __extension__ ({ \ + uint8x8_t __ret_304; \ + uint8x8_t __s0_304 = __p0_304; \ + uint8x8_t __s2_304 = __p2_304; \ + __ret_304 = vset_lane_u8(vget_lane_u8(__s2_304, __p3_304), __s0_304, __p1_304); \ __ret_304; \ }) #else -#define vcopyq_lane_s16(__p0_305, __p1_305, __p2_305, __p3_305) __extension__ ({ \ - int16x8_t __ret_305; \ - int16x8_t __s0_305 = __p0_305; \ - int16x4_t __s2_305 = __p2_305; \ - int16x8_t __rev0_305; __rev0_305 = __builtin_shufflevector(__s0_305, __s0_305, __lane_reverse_128_16); \ - int16x4_t __rev2_305; __rev2_305 = __builtin_shufflevector(__s2_305, __s2_305, __lane_reverse_64_16); \ - __ret_305 = __noswap_vsetq_lane_s16(__noswap_vget_lane_s16(__rev2_305, __p3_305), __rev0_305, __p1_305); \ - __ret_305 = __builtin_shufflevector(__ret_305, __ret_305, __lane_reverse_128_16); \ +#define vcopy_lane_u8(__p0_305, __p1_305, __p2_305, __p3_305) __extension__ ({ \ + uint8x8_t __ret_305; \ + uint8x8_t __s0_305 = __p0_305; \ + uint8x8_t __s2_305 = __p2_305; \ + uint8x8_t __rev0_305; __rev0_305 = __builtin_shufflevector(__s0_305, __s0_305, __lane_reverse_64_8); \ + uint8x8_t __rev2_305; __rev2_305 = __builtin_shufflevector(__s2_305, __s2_305, __lane_reverse_64_8); \ + __ret_305 = __noswap_vset_lane_u8(__noswap_vget_lane_u8(__rev2_305, __p3_305), __rev0_305, __p1_305); \ + __ret_305 = __builtin_shufflevector(__ret_305, __ret_305, __lane_reverse_64_8); \ __ret_305; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_lane_p8(__p0_306, __p1_306, __p2_306, __p3_306) __extension__ ({ \ - poly8x8_t __ret_306; \ - poly8x8_t __s0_306 = __p0_306; \ - poly8x8_t __s2_306 = __p2_306; \ - __ret_306 = vset_lane_p8(vget_lane_p8(__s2_306, __p3_306), __s0_306, __p1_306); \ +#define vcopy_lane_u32(__p0_306, __p1_306, __p2_306, __p3_306) __extension__ ({ \ + uint32x2_t __ret_306; \ + uint32x2_t __s0_306 = __p0_306; \ + uint32x2_t __s2_306 = __p2_306; \ + __ret_306 = vset_lane_u32(vget_lane_u32(__s2_306, __p3_306), __s0_306, __p1_306); \ __ret_306; \ }) #else -#define vcopy_lane_p8(__p0_307, __p1_307, __p2_307, __p3_307) __extension__ ({ \ - poly8x8_t __ret_307; \ - poly8x8_t __s0_307 = __p0_307; \ - poly8x8_t __s2_307 = __p2_307; \ - poly8x8_t __rev0_307; __rev0_307 = __builtin_shufflevector(__s0_307, __s0_307, __lane_reverse_64_8); \ - poly8x8_t __rev2_307; __rev2_307 = __builtin_shufflevector(__s2_307, __s2_307, __lane_reverse_64_8); \ - __ret_307 = __noswap_vset_lane_p8(__noswap_vget_lane_p8(__rev2_307, __p3_307), __rev0_307, __p1_307); \ - __ret_307 = __builtin_shufflevector(__ret_307, __ret_307, __lane_reverse_64_8); \ +#define vcopy_lane_u32(__p0_307, __p1_307, __p2_307, __p3_307) __extension__ ({ \ + uint32x2_t __ret_307; \ + uint32x2_t __s0_307 = __p0_307; \ + uint32x2_t __s2_307 = __p2_307; \ + uint32x2_t __rev0_307; __rev0_307 = __builtin_shufflevector(__s0_307, __s0_307, __lane_reverse_64_32); \ + uint32x2_t __rev2_307; __rev2_307 = __builtin_shufflevector(__s2_307, __s2_307, __lane_reverse_64_32); \ + __ret_307 = __noswap_vset_lane_u32(__noswap_vget_lane_u32(__rev2_307, __p3_307), __rev0_307, __p1_307); \ + __ret_307 = __builtin_shufflevector(__ret_307, __ret_307, __lane_reverse_64_32); \ __ret_307; \ }) #endif -#ifdef __LITTLE_ENDIAN__ -#define vcopy_lane_p16(__p0_308, __p1_308, __p2_308, __p3_308) __extension__ ({ \ - poly16x4_t __ret_308; \ - poly16x4_t __s0_308 = __p0_308; \ - poly16x4_t __s2_308 = __p2_308; \ - __ret_308 = vset_lane_p16(vget_lane_p16(__s2_308, __p3_308), __s0_308, __p1_308); \ +#define vcopy_lane_u64(__p0_308, __p1_308, __p2_308, __p3_308) __extension__ ({ \ + uint64x1_t __ret_308; \ + uint64x1_t __s0_308 = __p0_308; \ + uint64x1_t __s2_308 = __p2_308; \ + __ret_308 = vset_lane_u64(vget_lane_u64(__s2_308, __p3_308), __s0_308, __p1_308); \ __ret_308; \ }) -#else -#define vcopy_lane_p16(__p0_309, __p1_309, __p2_309, __p3_309) __extension__ ({ \ - poly16x4_t __ret_309; \ - poly16x4_t __s0_309 = __p0_309; \ - poly16x4_t __s2_309 = __p2_309; \ - poly16x4_t __rev0_309; __rev0_309 = __builtin_shufflevector(__s0_309, __s0_309, __lane_reverse_64_16); \ - poly16x4_t __rev2_309; __rev2_309 = __builtin_shufflevector(__s2_309, __s2_309, __lane_reverse_64_16); \ - __ret_309 = __noswap_vset_lane_p16(__noswap_vget_lane_p16(__rev2_309, __p3_309), __rev0_309, __p1_309); \ - __ret_309 = __builtin_shufflevector(__ret_309, __ret_309, __lane_reverse_64_16); \ +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_u16(__p0_309, __p1_309, __p2_309, __p3_309) __extension__ ({ \ + uint16x4_t __ret_309; \ + uint16x4_t __s0_309 = __p0_309; \ + uint16x4_t __s2_309 = __p2_309; \ + __ret_309 = vset_lane_u16(vget_lane_u16(__s2_309, __p3_309), __s0_309, __p1_309); \ __ret_309; \ }) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vcopy_lane_u8(__p0_310, __p1_310, __p2_310, __p3_310) __extension__ ({ \ - uint8x8_t __ret_310; \ - uint8x8_t __s0_310 = __p0_310; \ - uint8x8_t __s2_310 = __p2_310; \ - __ret_310 = vset_lane_u8(vget_lane_u8(__s2_310, __p3_310), __s0_310, __p1_310); \ +#else +#define vcopy_lane_u16(__p0_310, __p1_310, __p2_310, __p3_310) __extension__ ({ \ + uint16x4_t __ret_310; \ + uint16x4_t __s0_310 = __p0_310; \ + uint16x4_t __s2_310 = __p2_310; \ + uint16x4_t __rev0_310; __rev0_310 = __builtin_shufflevector(__s0_310, __s0_310, __lane_reverse_64_16); \ + uint16x4_t __rev2_310; __rev2_310 = __builtin_shufflevector(__s2_310, __s2_310, __lane_reverse_64_16); \ + __ret_310 = __noswap_vset_lane_u16(__noswap_vget_lane_u16(__rev2_310, __p3_310), __rev0_310, __p1_310); \ + __ret_310 = __builtin_shufflevector(__ret_310, __ret_310, __lane_reverse_64_16); \ __ret_310; \ }) -#else -#define vcopy_lane_u8(__p0_311, __p1_311, __p2_311, __p3_311) __extension__ ({ \ - uint8x8_t __ret_311; \ - uint8x8_t __s0_311 = __p0_311; \ - uint8x8_t __s2_311 = __p2_311; \ - uint8x8_t __rev0_311; __rev0_311 = __builtin_shufflevector(__s0_311, __s0_311, __lane_reverse_64_8); \ - uint8x8_t __rev2_311; __rev2_311 = __builtin_shufflevector(__s2_311, __s2_311, __lane_reverse_64_8); \ - __ret_311 = __noswap_vset_lane_u8(__noswap_vget_lane_u8(__rev2_311, __p3_311), __rev0_311, __p1_311); \ - __ret_311 = __builtin_shufflevector(__ret_311, __ret_311, __lane_reverse_64_8); \ +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_s8(__p0_311, __p1_311, __p2_311, __p3_311) __extension__ ({ \ + int8x8_t __ret_311; \ + int8x8_t __s0_311 = __p0_311; \ + int8x8_t __s2_311 = __p2_311; \ + __ret_311 = vset_lane_s8(vget_lane_s8(__s2_311, __p3_311), __s0_311, __p1_311); \ __ret_311; \ }) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vcopy_lane_u32(__p0_312, __p1_312, __p2_312, __p3_312) __extension__ ({ \ - uint32x2_t __ret_312; \ - uint32x2_t __s0_312 = __p0_312; \ - uint32x2_t __s2_312 = __p2_312; \ - __ret_312 = vset_lane_u32(vget_lane_u32(__s2_312, __p3_312), __s0_312, __p1_312); \ +#else +#define vcopy_lane_s8(__p0_312, __p1_312, __p2_312, __p3_312) __extension__ ({ \ + int8x8_t __ret_312; \ + int8x8_t __s0_312 = __p0_312; \ + int8x8_t __s2_312 = __p2_312; \ + int8x8_t __rev0_312; __rev0_312 = __builtin_shufflevector(__s0_312, __s0_312, __lane_reverse_64_8); \ + int8x8_t __rev2_312; __rev2_312 = __builtin_shufflevector(__s2_312, __s2_312, __lane_reverse_64_8); \ + __ret_312 = __noswap_vset_lane_s8(__noswap_vget_lane_s8(__rev2_312, __p3_312), __rev0_312, __p1_312); \ + __ret_312 = __builtin_shufflevector(__ret_312, __ret_312, __lane_reverse_64_8); \ __ret_312; \ }) -#else -#define vcopy_lane_u32(__p0_313, __p1_313, __p2_313, __p3_313) __extension__ ({ \ - uint32x2_t __ret_313; \ - uint32x2_t __s0_313 = __p0_313; \ - uint32x2_t __s2_313 = __p2_313; \ - uint32x2_t __rev0_313; __rev0_313 = __builtin_shufflevector(__s0_313, __s0_313, __lane_reverse_64_32); \ - uint32x2_t __rev2_313; __rev2_313 = __builtin_shufflevector(__s2_313, __s2_313, __lane_reverse_64_32); \ - __ret_313 = __noswap_vset_lane_u32(__noswap_vget_lane_u32(__rev2_313, __p3_313), __rev0_313, __p1_313); \ - __ret_313 = __builtin_shufflevector(__ret_313, __ret_313, __lane_reverse_64_32); \ +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_f32(__p0_313, __p1_313, __p2_313, __p3_313) __extension__ ({ \ + float32x2_t __ret_313; \ + float32x2_t __s0_313 = __p0_313; \ + float32x2_t __s2_313 = __p2_313; \ + __ret_313 = vset_lane_f32(vget_lane_f32(__s2_313, __p3_313), __s0_313, __p1_313); \ __ret_313; \ }) -#endif - -#define vcopy_lane_u64(__p0_314, __p1_314, __p2_314, __p3_314) __extension__ ({ \ - uint64x1_t __ret_314; \ - uint64x1_t __s0_314 = __p0_314; \ - uint64x1_t __s2_314 = __p2_314; \ - __ret_314 = vset_lane_u64(vget_lane_u64(__s2_314, __p3_314), __s0_314, __p1_314); \ +#else +#define vcopy_lane_f32(__p0_314, __p1_314, __p2_314, __p3_314) __extension__ ({ \ + float32x2_t __ret_314; \ + float32x2_t __s0_314 = __p0_314; \ + float32x2_t __s2_314 = __p2_314; \ + float32x2_t __rev0_314; __rev0_314 = __builtin_shufflevector(__s0_314, __s0_314, __lane_reverse_64_32); \ + float32x2_t __rev2_314; __rev2_314 = __builtin_shufflevector(__s2_314, __s2_314, __lane_reverse_64_32); \ + __ret_314 = __noswap_vset_lane_f32(__noswap_vget_lane_f32(__rev2_314, __p3_314), __rev0_314, __p1_314); \ + __ret_314 = __builtin_shufflevector(__ret_314, __ret_314, __lane_reverse_64_32); \ __ret_314; \ }) +#endif + #ifdef __LITTLE_ENDIAN__ -#define vcopy_lane_u16(__p0_315, __p1_315, __p2_315, __p3_315) __extension__ ({ \ - uint16x4_t __ret_315; \ - uint16x4_t __s0_315 = __p0_315; \ - uint16x4_t __s2_315 = __p2_315; \ - __ret_315 = vset_lane_u16(vget_lane_u16(__s2_315, __p3_315), __s0_315, __p1_315); \ +#define vcopy_lane_s32(__p0_315, __p1_315, __p2_315, __p3_315) __extension__ ({ \ + int32x2_t __ret_315; \ + int32x2_t __s0_315 = __p0_315; \ + int32x2_t __s2_315 = __p2_315; \ + __ret_315 = vset_lane_s32(vget_lane_s32(__s2_315, __p3_315), __s0_315, __p1_315); \ __ret_315; \ }) #else -#define vcopy_lane_u16(__p0_316, __p1_316, __p2_316, __p3_316) __extension__ ({ \ - uint16x4_t __ret_316; \ - uint16x4_t __s0_316 = __p0_316; \ - uint16x4_t __s2_316 = __p2_316; \ - uint16x4_t __rev0_316; __rev0_316 = __builtin_shufflevector(__s0_316, __s0_316, __lane_reverse_64_16); \ - uint16x4_t __rev2_316; __rev2_316 = __builtin_shufflevector(__s2_316, __s2_316, __lane_reverse_64_16); \ - __ret_316 = __noswap_vset_lane_u16(__noswap_vget_lane_u16(__rev2_316, __p3_316), __rev0_316, __p1_316); \ - __ret_316 = __builtin_shufflevector(__ret_316, __ret_316, __lane_reverse_64_16); \ +#define vcopy_lane_s32(__p0_316, __p1_316, __p2_316, __p3_316) __extension__ ({ \ + int32x2_t __ret_316; \ + int32x2_t __s0_316 = __p0_316; \ + int32x2_t __s2_316 = __p2_316; \ + int32x2_t __rev0_316; __rev0_316 = __builtin_shufflevector(__s0_316, __s0_316, __lane_reverse_64_32); \ + int32x2_t __rev2_316; __rev2_316 = __builtin_shufflevector(__s2_316, __s2_316, __lane_reverse_64_32); \ + __ret_316 = __noswap_vset_lane_s32(__noswap_vget_lane_s32(__rev2_316, __p3_316), __rev0_316, __p1_316); \ + __ret_316 = __builtin_shufflevector(__ret_316, __ret_316, __lane_reverse_64_32); \ __ret_316; \ }) #endif -#ifdef __LITTLE_ENDIAN__ -#define vcopy_lane_s8(__p0_317, __p1_317, __p2_317, __p3_317) __extension__ ({ \ - int8x8_t __ret_317; \ - int8x8_t __s0_317 = __p0_317; \ - int8x8_t __s2_317 = __p2_317; \ - __ret_317 = vset_lane_s8(vget_lane_s8(__s2_317, __p3_317), __s0_317, __p1_317); \ +#define vcopy_lane_s64(__p0_317, __p1_317, __p2_317, __p3_317) __extension__ ({ \ + int64x1_t __ret_317; \ + int64x1_t __s0_317 = __p0_317; \ + int64x1_t __s2_317 = __p2_317; \ + __ret_317 = vset_lane_s64(vget_lane_s64(__s2_317, __p3_317), __s0_317, __p1_317); \ __ret_317; \ }) -#else -#define vcopy_lane_s8(__p0_318, __p1_318, __p2_318, __p3_318) __extension__ ({ \ - int8x8_t __ret_318; \ - int8x8_t __s0_318 = __p0_318; \ - int8x8_t __s2_318 = __p2_318; \ - int8x8_t __rev0_318; __rev0_318 = __builtin_shufflevector(__s0_318, __s0_318, __lane_reverse_64_8); \ - int8x8_t __rev2_318; __rev2_318 = __builtin_shufflevector(__s2_318, __s2_318, __lane_reverse_64_8); \ - __ret_318 = __noswap_vset_lane_s8(__noswap_vget_lane_s8(__rev2_318, __p3_318), __rev0_318, __p1_318); \ - __ret_318 = __builtin_shufflevector(__ret_318, __ret_318, __lane_reverse_64_8); \ +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_mf8(__p0_318, __p1_318, __p2_318, __p3_318) __extension__ ({ \ + mfloat8x8_t __ret_318; \ + mfloat8x8_t __s0_318 = __p0_318; \ + mfloat8x8_t __s2_318 = __p2_318; \ + __ret_318 = vset_lane_mf8(vget_lane_mf8(__s2_318, __p3_318), __s0_318, __p1_318); \ __ret_318; \ }) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vcopy_lane_f32(__p0_319, __p1_319, __p2_319, __p3_319) __extension__ ({ \ - float32x2_t __ret_319; \ - float32x2_t __s0_319 = __p0_319; \ - float32x2_t __s2_319 = __p2_319; \ - __ret_319 = vset_lane_f32(vget_lane_f32(__s2_319, __p3_319), __s0_319, __p1_319); \ +#else +#define vcopy_lane_mf8(__p0_319, __p1_319, __p2_319, __p3_319) __extension__ ({ \ + mfloat8x8_t __ret_319; \ + mfloat8x8_t __s0_319 = __p0_319; \ + mfloat8x8_t __s2_319 = __p2_319; \ + mfloat8x8_t __rev0_319; __rev0_319 = __builtin_shufflevector(__s0_319, __s0_319, __lane_reverse_64_8); \ + mfloat8x8_t __rev2_319; __rev2_319 = __builtin_shufflevector(__s2_319, __s2_319, __lane_reverse_64_8); \ + __ret_319 = __noswap_vset_lane_mf8(__noswap_vget_lane_mf8(__rev2_319, __p3_319), __rev0_319, __p1_319); \ + __ret_319 = __builtin_shufflevector(__ret_319, __ret_319, __lane_reverse_64_8); \ __ret_319; \ }) -#else -#define vcopy_lane_f32(__p0_320, __p1_320, __p2_320, __p3_320) __extension__ ({ \ - float32x2_t __ret_320; \ - float32x2_t __s0_320 = __p0_320; \ - float32x2_t __s2_320 = __p2_320; \ - float32x2_t __rev0_320; __rev0_320 = __builtin_shufflevector(__s0_320, __s0_320, __lane_reverse_64_32); \ - float32x2_t __rev2_320; __rev2_320 = __builtin_shufflevector(__s2_320, __s2_320, __lane_reverse_64_32); \ - __ret_320 = __noswap_vset_lane_f32(__noswap_vget_lane_f32(__rev2_320, __p3_320), __rev0_320, __p1_320); \ - __ret_320 = __builtin_shufflevector(__ret_320, __ret_320, __lane_reverse_64_32); \ +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_s16(__p0_320, __p1_320, __p2_320, __p3_320) __extension__ ({ \ + int16x4_t __ret_320; \ + int16x4_t __s0_320 = __p0_320; \ + int16x4_t __s2_320 = __p2_320; \ + __ret_320 = vset_lane_s16(vget_lane_s16(__s2_320, __p3_320), __s0_320, __p1_320); \ __ret_320; \ }) -#endif - -#ifdef __LITTLE_ENDIAN__ -#define vcopy_lane_s32(__p0_321, __p1_321, __p2_321, __p3_321) __extension__ ({ \ - int32x2_t __ret_321; \ - int32x2_t __s0_321 = __p0_321; \ - int32x2_t __s2_321 = __p2_321; \ - __ret_321 = vset_lane_s32(vget_lane_s32(__s2_321, __p3_321), __s0_321, __p1_321); \ +#else +#define vcopy_lane_s16(__p0_321, __p1_321, __p2_321, __p3_321) __extension__ ({ \ + int16x4_t __ret_321; \ + int16x4_t __s0_321 = __p0_321; \ + int16x4_t __s2_321 = __p2_321; \ + int16x4_t __rev0_321; __rev0_321 = __builtin_shufflevector(__s0_321, __s0_321, __lane_reverse_64_16); \ + int16x4_t __rev2_321; __rev2_321 = __builtin_shufflevector(__s2_321, __s2_321, __lane_reverse_64_16); \ + __ret_321 = __noswap_vset_lane_s16(__noswap_vget_lane_s16(__rev2_321, __p3_321), __rev0_321, __p1_321); \ + __ret_321 = __builtin_shufflevector(__ret_321, __ret_321, __lane_reverse_64_16); \ __ret_321; \ }) -#else -#define vcopy_lane_s32(__p0_322, __p1_322, __p2_322, __p3_322) __extension__ ({ \ - int32x2_t __ret_322; \ - int32x2_t __s0_322 = __p0_322; \ - int32x2_t __s2_322 = __p2_322; \ - int32x2_t __rev0_322; __rev0_322 = __builtin_shufflevector(__s0_322, __s0_322, __lane_reverse_64_32); \ - int32x2_t __rev2_322; __rev2_322 = __builtin_shufflevector(__s2_322, __s2_322, __lane_reverse_64_32); \ - __ret_322 = __noswap_vset_lane_s32(__noswap_vget_lane_s32(__rev2_322, __p3_322), __rev0_322, __p1_322); \ - __ret_322 = __builtin_shufflevector(__ret_322, __ret_322, __lane_reverse_64_32); \ +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_bf16(__p0_322, __p1_322, __p2_322, __p3_322) __extension__ ({ \ + bfloat16x4_t __ret_322; \ + bfloat16x4_t __s0_322 = __p0_322; \ + bfloat16x4_t __s2_322 = __p2_322; \ + __ret_322 = vset_lane_bf16(vget_lane_bf16(__s2_322, __p3_322), __s0_322, __p1_322); \ __ret_322; \ }) -#endif - -#define vcopy_lane_s64(__p0_323, __p1_323, __p2_323, __p3_323) __extension__ ({ \ - int64x1_t __ret_323; \ - int64x1_t __s0_323 = __p0_323; \ - int64x1_t __s2_323 = __p2_323; \ - __ret_323 = vset_lane_s64(vget_lane_s64(__s2_323, __p3_323), __s0_323, __p1_323); \ +#else +#define vcopy_lane_bf16(__p0_323, __p1_323, __p2_323, __p3_323) __extension__ ({ \ + bfloat16x4_t __ret_323; \ + bfloat16x4_t __s0_323 = __p0_323; \ + bfloat16x4_t __s2_323 = __p2_323; \ + bfloat16x4_t __rev0_323; __rev0_323 = __builtin_shufflevector(__s0_323, __s0_323, __lane_reverse_64_16); \ + bfloat16x4_t __rev2_323; __rev2_323 = __builtin_shufflevector(__s2_323, __s2_323, __lane_reverse_64_16); \ + __ret_323 = __noswap_vset_lane_bf16(__noswap_vget_lane_bf16(__rev2_323, __p3_323), __rev0_323, __p1_323); \ + __ret_323 = __builtin_shufflevector(__ret_323, __ret_323, __lane_reverse_64_16); \ __ret_323; \ }) +#endif + #ifdef __LITTLE_ENDIAN__ -#define vcopy_lane_mf8(__p0_324, __p1_324, __p2_324, __p3_324) __extension__ ({ \ - mfloat8x8_t __ret_324; \ - mfloat8x8_t __s0_324 = __p0_324; \ - mfloat8x8_t __s2_324 = __p2_324; \ - __ret_324 = vset_lane_mf8(vget_lane_mf8(__s2_324, __p3_324), __s0_324, __p1_324); \ +#define vcopyq_laneq_p8(__p0_324, __p1_324, __p2_324, __p3_324) __extension__ ({ \ + poly8x16_t __ret_324; \ + poly8x16_t __s0_324 = __p0_324; \ + poly8x16_t __s2_324 = __p2_324; \ + __ret_324 = vsetq_lane_p8(vgetq_lane_p8(__s2_324, __p3_324), __s0_324, __p1_324); \ __ret_324; \ }) #else -#define vcopy_lane_mf8(__p0_325, __p1_325, __p2_325, __p3_325) __extension__ ({ \ - mfloat8x8_t __ret_325; \ - mfloat8x8_t __s0_325 = __p0_325; \ - mfloat8x8_t __s2_325 = __p2_325; \ - mfloat8x8_t __rev0_325; __rev0_325 = __builtin_shufflevector(__s0_325, __s0_325, __lane_reverse_64_8); \ - mfloat8x8_t __rev2_325; __rev2_325 = __builtin_shufflevector(__s2_325, __s2_325, __lane_reverse_64_8); \ - __ret_325 = __noswap_vset_lane_mf8(__noswap_vget_lane_mf8(__rev2_325, __p3_325), __rev0_325, __p1_325); \ - __ret_325 = __builtin_shufflevector(__ret_325, __ret_325, __lane_reverse_64_8); \ +#define vcopyq_laneq_p8(__p0_325, __p1_325, __p2_325, __p3_325) __extension__ ({ \ + poly8x16_t __ret_325; \ + poly8x16_t __s0_325 = __p0_325; \ + poly8x16_t __s2_325 = __p2_325; \ + poly8x16_t __rev0_325; __rev0_325 = __builtin_shufflevector(__s0_325, __s0_325, __lane_reverse_128_8); \ + poly8x16_t __rev2_325; __rev2_325 = __builtin_shufflevector(__s2_325, __s2_325, __lane_reverse_128_8); \ + __ret_325 = __noswap_vsetq_lane_p8(__noswap_vgetq_lane_p8(__rev2_325, __p3_325), __rev0_325, __p1_325); \ + __ret_325 = __builtin_shufflevector(__ret_325, __ret_325, __lane_reverse_128_8); \ __ret_325; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_lane_s16(__p0_326, __p1_326, __p2_326, __p3_326) __extension__ ({ \ - int16x4_t __ret_326; \ - int16x4_t __s0_326 = __p0_326; \ - int16x4_t __s2_326 = __p2_326; \ - __ret_326 = vset_lane_s16(vget_lane_s16(__s2_326, __p3_326), __s0_326, __p1_326); \ +#define vcopyq_laneq_p16(__p0_326, __p1_326, __p2_326, __p3_326) __extension__ ({ \ + poly16x8_t __ret_326; \ + poly16x8_t __s0_326 = __p0_326; \ + poly16x8_t __s2_326 = __p2_326; \ + __ret_326 = vsetq_lane_p16(vgetq_lane_p16(__s2_326, __p3_326), __s0_326, __p1_326); \ __ret_326; \ }) #else -#define vcopy_lane_s16(__p0_327, __p1_327, __p2_327, __p3_327) __extension__ ({ \ - int16x4_t __ret_327; \ - int16x4_t __s0_327 = __p0_327; \ - int16x4_t __s2_327 = __p2_327; \ - int16x4_t __rev0_327; __rev0_327 = __builtin_shufflevector(__s0_327, __s0_327, __lane_reverse_64_16); \ - int16x4_t __rev2_327; __rev2_327 = __builtin_shufflevector(__s2_327, __s2_327, __lane_reverse_64_16); \ - __ret_327 = __noswap_vset_lane_s16(__noswap_vget_lane_s16(__rev2_327, __p3_327), __rev0_327, __p1_327); \ - __ret_327 = __builtin_shufflevector(__ret_327, __ret_327, __lane_reverse_64_16); \ +#define vcopyq_laneq_p16(__p0_327, __p1_327, __p2_327, __p3_327) __extension__ ({ \ + poly16x8_t __ret_327; \ + poly16x8_t __s0_327 = __p0_327; \ + poly16x8_t __s2_327 = __p2_327; \ + poly16x8_t __rev0_327; __rev0_327 = __builtin_shufflevector(__s0_327, __s0_327, __lane_reverse_128_16); \ + poly16x8_t __rev2_327; __rev2_327 = __builtin_shufflevector(__s2_327, __s2_327, __lane_reverse_128_16); \ + __ret_327 = __noswap_vsetq_lane_p16(__noswap_vgetq_lane_p16(__rev2_327, __p3_327), __rev0_327, __p1_327); \ + __ret_327 = __builtin_shufflevector(__ret_327, __ret_327, __lane_reverse_128_16); \ __ret_327; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_laneq_p8(__p0_328, __p1_328, __p2_328, __p3_328) __extension__ ({ \ - poly8x16_t __ret_328; \ - poly8x16_t __s0_328 = __p0_328; \ - poly8x16_t __s2_328 = __p2_328; \ - __ret_328 = vsetq_lane_p8(vgetq_lane_p8(__s2_328, __p3_328), __s0_328, __p1_328); \ +#define vcopyq_laneq_u8(__p0_328, __p1_328, __p2_328, __p3_328) __extension__ ({ \ + uint8x16_t __ret_328; \ + uint8x16_t __s0_328 = __p0_328; \ + uint8x16_t __s2_328 = __p2_328; \ + __ret_328 = vsetq_lane_u8(vgetq_lane_u8(__s2_328, __p3_328), __s0_328, __p1_328); \ __ret_328; \ }) #else -#define vcopyq_laneq_p8(__p0_329, __p1_329, __p2_329, __p3_329) __extension__ ({ \ - poly8x16_t __ret_329; \ - poly8x16_t __s0_329 = __p0_329; \ - poly8x16_t __s2_329 = __p2_329; \ - poly8x16_t __rev0_329; __rev0_329 = __builtin_shufflevector(__s0_329, __s0_329, __lane_reverse_128_8); \ - poly8x16_t __rev2_329; __rev2_329 = __builtin_shufflevector(__s2_329, __s2_329, __lane_reverse_128_8); \ - __ret_329 = __noswap_vsetq_lane_p8(__noswap_vgetq_lane_p8(__rev2_329, __p3_329), __rev0_329, __p1_329); \ +#define vcopyq_laneq_u8(__p0_329, __p1_329, __p2_329, __p3_329) __extension__ ({ \ + uint8x16_t __ret_329; \ + uint8x16_t __s0_329 = __p0_329; \ + uint8x16_t __s2_329 = __p2_329; \ + uint8x16_t __rev0_329; __rev0_329 = __builtin_shufflevector(__s0_329, __s0_329, __lane_reverse_128_8); \ + uint8x16_t __rev2_329; __rev2_329 = __builtin_shufflevector(__s2_329, __s2_329, __lane_reverse_128_8); \ + __ret_329 = __noswap_vsetq_lane_u8(__noswap_vgetq_lane_u8(__rev2_329, __p3_329), __rev0_329, __p1_329); \ __ret_329 = __builtin_shufflevector(__ret_329, __ret_329, __lane_reverse_128_8); \ __ret_329; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_laneq_p16(__p0_330, __p1_330, __p2_330, __p3_330) __extension__ ({ \ - poly16x8_t __ret_330; \ - poly16x8_t __s0_330 = __p0_330; \ - poly16x8_t __s2_330 = __p2_330; \ - __ret_330 = vsetq_lane_p16(vgetq_lane_p16(__s2_330, __p3_330), __s0_330, __p1_330); \ +#define vcopyq_laneq_u32(__p0_330, __p1_330, __p2_330, __p3_330) __extension__ ({ \ + uint32x4_t __ret_330; \ + uint32x4_t __s0_330 = __p0_330; \ + uint32x4_t __s2_330 = __p2_330; \ + __ret_330 = vsetq_lane_u32(vgetq_lane_u32(__s2_330, __p3_330), __s0_330, __p1_330); \ __ret_330; \ }) #else -#define vcopyq_laneq_p16(__p0_331, __p1_331, __p2_331, __p3_331) __extension__ ({ \ - poly16x8_t __ret_331; \ - poly16x8_t __s0_331 = __p0_331; \ - poly16x8_t __s2_331 = __p2_331; \ - poly16x8_t __rev0_331; __rev0_331 = __builtin_shufflevector(__s0_331, __s0_331, __lane_reverse_128_16); \ - poly16x8_t __rev2_331; __rev2_331 = __builtin_shufflevector(__s2_331, __s2_331, __lane_reverse_128_16); \ - __ret_331 = __noswap_vsetq_lane_p16(__noswap_vgetq_lane_p16(__rev2_331, __p3_331), __rev0_331, __p1_331); \ - __ret_331 = __builtin_shufflevector(__ret_331, __ret_331, __lane_reverse_128_16); \ +#define vcopyq_laneq_u32(__p0_331, __p1_331, __p2_331, __p3_331) __extension__ ({ \ + uint32x4_t __ret_331; \ + uint32x4_t __s0_331 = __p0_331; \ + uint32x4_t __s2_331 = __p2_331; \ + uint32x4_t __rev0_331; __rev0_331 = __builtin_shufflevector(__s0_331, __s0_331, __lane_reverse_128_32); \ + uint32x4_t __rev2_331; __rev2_331 = __builtin_shufflevector(__s2_331, __s2_331, __lane_reverse_128_32); \ + __ret_331 = __noswap_vsetq_lane_u32(__noswap_vgetq_lane_u32(__rev2_331, __p3_331), __rev0_331, __p1_331); \ + __ret_331 = __builtin_shufflevector(__ret_331, __ret_331, __lane_reverse_128_32); \ __ret_331; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_laneq_u8(__p0_332, __p1_332, __p2_332, __p3_332) __extension__ ({ \ - uint8x16_t __ret_332; \ - uint8x16_t __s0_332 = __p0_332; \ - uint8x16_t __s2_332 = __p2_332; \ - __ret_332 = vsetq_lane_u8(vgetq_lane_u8(__s2_332, __p3_332), __s0_332, __p1_332); \ +#define vcopyq_laneq_u64(__p0_332, __p1_332, __p2_332, __p3_332) __extension__ ({ \ + uint64x2_t __ret_332; \ + uint64x2_t __s0_332 = __p0_332; \ + uint64x2_t __s2_332 = __p2_332; \ + __ret_332 = vsetq_lane_u64(vgetq_lane_u64(__s2_332, __p3_332), __s0_332, __p1_332); \ __ret_332; \ }) #else -#define vcopyq_laneq_u8(__p0_333, __p1_333, __p2_333, __p3_333) __extension__ ({ \ - uint8x16_t __ret_333; \ - uint8x16_t __s0_333 = __p0_333; \ - uint8x16_t __s2_333 = __p2_333; \ - uint8x16_t __rev0_333; __rev0_333 = __builtin_shufflevector(__s0_333, __s0_333, __lane_reverse_128_8); \ - uint8x16_t __rev2_333; __rev2_333 = __builtin_shufflevector(__s2_333, __s2_333, __lane_reverse_128_8); \ - __ret_333 = __noswap_vsetq_lane_u8(__noswap_vgetq_lane_u8(__rev2_333, __p3_333), __rev0_333, __p1_333); \ - __ret_333 = __builtin_shufflevector(__ret_333, __ret_333, __lane_reverse_128_8); \ +#define vcopyq_laneq_u64(__p0_333, __p1_333, __p2_333, __p3_333) __extension__ ({ \ + uint64x2_t __ret_333; \ + uint64x2_t __s0_333 = __p0_333; \ + uint64x2_t __s2_333 = __p2_333; \ + uint64x2_t __rev0_333; __rev0_333 = __builtin_shufflevector(__s0_333, __s0_333, __lane_reverse_128_64); \ + uint64x2_t __rev2_333; __rev2_333 = __builtin_shufflevector(__s2_333, __s2_333, __lane_reverse_128_64); \ + __ret_333 = __noswap_vsetq_lane_u64(__noswap_vgetq_lane_u64(__rev2_333, __p3_333), __rev0_333, __p1_333); \ + __ret_333 = __builtin_shufflevector(__ret_333, __ret_333, __lane_reverse_128_64); \ __ret_333; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_laneq_u32(__p0_334, __p1_334, __p2_334, __p3_334) __extension__ ({ \ - uint32x4_t __ret_334; \ - uint32x4_t __s0_334 = __p0_334; \ - uint32x4_t __s2_334 = __p2_334; \ - __ret_334 = vsetq_lane_u32(vgetq_lane_u32(__s2_334, __p3_334), __s0_334, __p1_334); \ +#define vcopyq_laneq_u16(__p0_334, __p1_334, __p2_334, __p3_334) __extension__ ({ \ + uint16x8_t __ret_334; \ + uint16x8_t __s0_334 = __p0_334; \ + uint16x8_t __s2_334 = __p2_334; \ + __ret_334 = vsetq_lane_u16(vgetq_lane_u16(__s2_334, __p3_334), __s0_334, __p1_334); \ __ret_334; \ }) #else -#define vcopyq_laneq_u32(__p0_335, __p1_335, __p2_335, __p3_335) __extension__ ({ \ - uint32x4_t __ret_335; \ - uint32x4_t __s0_335 = __p0_335; \ - uint32x4_t __s2_335 = __p2_335; \ - uint32x4_t __rev0_335; __rev0_335 = __builtin_shufflevector(__s0_335, __s0_335, __lane_reverse_128_32); \ - uint32x4_t __rev2_335; __rev2_335 = __builtin_shufflevector(__s2_335, __s2_335, __lane_reverse_128_32); \ - __ret_335 = __noswap_vsetq_lane_u32(__noswap_vgetq_lane_u32(__rev2_335, __p3_335), __rev0_335, __p1_335); \ - __ret_335 = __builtin_shufflevector(__ret_335, __ret_335, __lane_reverse_128_32); \ +#define vcopyq_laneq_u16(__p0_335, __p1_335, __p2_335, __p3_335) __extension__ ({ \ + uint16x8_t __ret_335; \ + uint16x8_t __s0_335 = __p0_335; \ + uint16x8_t __s2_335 = __p2_335; \ + uint16x8_t __rev0_335; __rev0_335 = __builtin_shufflevector(__s0_335, __s0_335, __lane_reverse_128_16); \ + uint16x8_t __rev2_335; __rev2_335 = __builtin_shufflevector(__s2_335, __s2_335, __lane_reverse_128_16); \ + __ret_335 = __noswap_vsetq_lane_u16(__noswap_vgetq_lane_u16(__rev2_335, __p3_335), __rev0_335, __p1_335); \ + __ret_335 = __builtin_shufflevector(__ret_335, __ret_335, __lane_reverse_128_16); \ __ret_335; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_laneq_u64(__p0_336, __p1_336, __p2_336, __p3_336) __extension__ ({ \ - uint64x2_t __ret_336; \ - uint64x2_t __s0_336 = __p0_336; \ - uint64x2_t __s2_336 = __p2_336; \ - __ret_336 = vsetq_lane_u64(vgetq_lane_u64(__s2_336, __p3_336), __s0_336, __p1_336); \ +#define vcopyq_laneq_s8(__p0_336, __p1_336, __p2_336, __p3_336) __extension__ ({ \ + int8x16_t __ret_336; \ + int8x16_t __s0_336 = __p0_336; \ + int8x16_t __s2_336 = __p2_336; \ + __ret_336 = vsetq_lane_s8(vgetq_lane_s8(__s2_336, __p3_336), __s0_336, __p1_336); \ __ret_336; \ }) #else -#define vcopyq_laneq_u64(__p0_337, __p1_337, __p2_337, __p3_337) __extension__ ({ \ - uint64x2_t __ret_337; \ - uint64x2_t __s0_337 = __p0_337; \ - uint64x2_t __s2_337 = __p2_337; \ - uint64x2_t __rev0_337; __rev0_337 = __builtin_shufflevector(__s0_337, __s0_337, __lane_reverse_128_64); \ - uint64x2_t __rev2_337; __rev2_337 = __builtin_shufflevector(__s2_337, __s2_337, __lane_reverse_128_64); \ - __ret_337 = __noswap_vsetq_lane_u64(__noswap_vgetq_lane_u64(__rev2_337, __p3_337), __rev0_337, __p1_337); \ - __ret_337 = __builtin_shufflevector(__ret_337, __ret_337, __lane_reverse_128_64); \ +#define vcopyq_laneq_s8(__p0_337, __p1_337, __p2_337, __p3_337) __extension__ ({ \ + int8x16_t __ret_337; \ + int8x16_t __s0_337 = __p0_337; \ + int8x16_t __s2_337 = __p2_337; \ + int8x16_t __rev0_337; __rev0_337 = __builtin_shufflevector(__s0_337, __s0_337, __lane_reverse_128_8); \ + int8x16_t __rev2_337; __rev2_337 = __builtin_shufflevector(__s2_337, __s2_337, __lane_reverse_128_8); \ + __ret_337 = __noswap_vsetq_lane_s8(__noswap_vgetq_lane_s8(__rev2_337, __p3_337), __rev0_337, __p1_337); \ + __ret_337 = __builtin_shufflevector(__ret_337, __ret_337, __lane_reverse_128_8); \ __ret_337; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_laneq_u16(__p0_338, __p1_338, __p2_338, __p3_338) __extension__ ({ \ - uint16x8_t __ret_338; \ - uint16x8_t __s0_338 = __p0_338; \ - uint16x8_t __s2_338 = __p2_338; \ - __ret_338 = vsetq_lane_u16(vgetq_lane_u16(__s2_338, __p3_338), __s0_338, __p1_338); \ +#define vcopyq_laneq_f32(__p0_338, __p1_338, __p2_338, __p3_338) __extension__ ({ \ + float32x4_t __ret_338; \ + float32x4_t __s0_338 = __p0_338; \ + float32x4_t __s2_338 = __p2_338; \ + __ret_338 = vsetq_lane_f32(vgetq_lane_f32(__s2_338, __p3_338), __s0_338, __p1_338); \ __ret_338; \ }) #else -#define vcopyq_laneq_u16(__p0_339, __p1_339, __p2_339, __p3_339) __extension__ ({ \ - uint16x8_t __ret_339; \ - uint16x8_t __s0_339 = __p0_339; \ - uint16x8_t __s2_339 = __p2_339; \ - uint16x8_t __rev0_339; __rev0_339 = __builtin_shufflevector(__s0_339, __s0_339, __lane_reverse_128_16); \ - uint16x8_t __rev2_339; __rev2_339 = __builtin_shufflevector(__s2_339, __s2_339, __lane_reverse_128_16); \ - __ret_339 = __noswap_vsetq_lane_u16(__noswap_vgetq_lane_u16(__rev2_339, __p3_339), __rev0_339, __p1_339); \ - __ret_339 = __builtin_shufflevector(__ret_339, __ret_339, __lane_reverse_128_16); \ +#define vcopyq_laneq_f32(__p0_339, __p1_339, __p2_339, __p3_339) __extension__ ({ \ + float32x4_t __ret_339; \ + float32x4_t __s0_339 = __p0_339; \ + float32x4_t __s2_339 = __p2_339; \ + float32x4_t __rev0_339; __rev0_339 = __builtin_shufflevector(__s0_339, __s0_339, __lane_reverse_128_32); \ + float32x4_t __rev2_339; __rev2_339 = __builtin_shufflevector(__s2_339, __s2_339, __lane_reverse_128_32); \ + __ret_339 = __noswap_vsetq_lane_f32(__noswap_vgetq_lane_f32(__rev2_339, __p3_339), __rev0_339, __p1_339); \ + __ret_339 = __builtin_shufflevector(__ret_339, __ret_339, __lane_reverse_128_32); \ __ret_339; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_laneq_s8(__p0_340, __p1_340, __p2_340, __p3_340) __extension__ ({ \ - int8x16_t __ret_340; \ - int8x16_t __s0_340 = __p0_340; \ - int8x16_t __s2_340 = __p2_340; \ - __ret_340 = vsetq_lane_s8(vgetq_lane_s8(__s2_340, __p3_340), __s0_340, __p1_340); \ +#define vcopyq_laneq_s32(__p0_340, __p1_340, __p2_340, __p3_340) __extension__ ({ \ + int32x4_t __ret_340; \ + int32x4_t __s0_340 = __p0_340; \ + int32x4_t __s2_340 = __p2_340; \ + __ret_340 = vsetq_lane_s32(vgetq_lane_s32(__s2_340, __p3_340), __s0_340, __p1_340); \ __ret_340; \ }) #else -#define vcopyq_laneq_s8(__p0_341, __p1_341, __p2_341, __p3_341) __extension__ ({ \ - int8x16_t __ret_341; \ - int8x16_t __s0_341 = __p0_341; \ - int8x16_t __s2_341 = __p2_341; \ - int8x16_t __rev0_341; __rev0_341 = __builtin_shufflevector(__s0_341, __s0_341, __lane_reverse_128_8); \ - int8x16_t __rev2_341; __rev2_341 = __builtin_shufflevector(__s2_341, __s2_341, __lane_reverse_128_8); \ - __ret_341 = __noswap_vsetq_lane_s8(__noswap_vgetq_lane_s8(__rev2_341, __p3_341), __rev0_341, __p1_341); \ - __ret_341 = __builtin_shufflevector(__ret_341, __ret_341, __lane_reverse_128_8); \ +#define vcopyq_laneq_s32(__p0_341, __p1_341, __p2_341, __p3_341) __extension__ ({ \ + int32x4_t __ret_341; \ + int32x4_t __s0_341 = __p0_341; \ + int32x4_t __s2_341 = __p2_341; \ + int32x4_t __rev0_341; __rev0_341 = __builtin_shufflevector(__s0_341, __s0_341, __lane_reverse_128_32); \ + int32x4_t __rev2_341; __rev2_341 = __builtin_shufflevector(__s2_341, __s2_341, __lane_reverse_128_32); \ + __ret_341 = __noswap_vsetq_lane_s32(__noswap_vgetq_lane_s32(__rev2_341, __p3_341), __rev0_341, __p1_341); \ + __ret_341 = __builtin_shufflevector(__ret_341, __ret_341, __lane_reverse_128_32); \ __ret_341; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_laneq_f32(__p0_342, __p1_342, __p2_342, __p3_342) __extension__ ({ \ - float32x4_t __ret_342; \ - float32x4_t __s0_342 = __p0_342; \ - float32x4_t __s2_342 = __p2_342; \ - __ret_342 = vsetq_lane_f32(vgetq_lane_f32(__s2_342, __p3_342), __s0_342, __p1_342); \ +#define vcopyq_laneq_s64(__p0_342, __p1_342, __p2_342, __p3_342) __extension__ ({ \ + int64x2_t __ret_342; \ + int64x2_t __s0_342 = __p0_342; \ + int64x2_t __s2_342 = __p2_342; \ + __ret_342 = vsetq_lane_s64(vgetq_lane_s64(__s2_342, __p3_342), __s0_342, __p1_342); \ __ret_342; \ }) #else -#define vcopyq_laneq_f32(__p0_343, __p1_343, __p2_343, __p3_343) __extension__ ({ \ - float32x4_t __ret_343; \ - float32x4_t __s0_343 = __p0_343; \ - float32x4_t __s2_343 = __p2_343; \ - float32x4_t __rev0_343; __rev0_343 = __builtin_shufflevector(__s0_343, __s0_343, __lane_reverse_128_32); \ - float32x4_t __rev2_343; __rev2_343 = __builtin_shufflevector(__s2_343, __s2_343, __lane_reverse_128_32); \ - __ret_343 = __noswap_vsetq_lane_f32(__noswap_vgetq_lane_f32(__rev2_343, __p3_343), __rev0_343, __p1_343); \ - __ret_343 = __builtin_shufflevector(__ret_343, __ret_343, __lane_reverse_128_32); \ +#define vcopyq_laneq_s64(__p0_343, __p1_343, __p2_343, __p3_343) __extension__ ({ \ + int64x2_t __ret_343; \ + int64x2_t __s0_343 = __p0_343; \ + int64x2_t __s2_343 = __p2_343; \ + int64x2_t __rev0_343; __rev0_343 = __builtin_shufflevector(__s0_343, __s0_343, __lane_reverse_128_64); \ + int64x2_t __rev2_343; __rev2_343 = __builtin_shufflevector(__s2_343, __s2_343, __lane_reverse_128_64); \ + __ret_343 = __noswap_vsetq_lane_s64(__noswap_vgetq_lane_s64(__rev2_343, __p3_343), __rev0_343, __p1_343); \ + __ret_343 = __builtin_shufflevector(__ret_343, __ret_343, __lane_reverse_128_64); \ __ret_343; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_laneq_s32(__p0_344, __p1_344, __p2_344, __p3_344) __extension__ ({ \ - int32x4_t __ret_344; \ - int32x4_t __s0_344 = __p0_344; \ - int32x4_t __s2_344 = __p2_344; \ - __ret_344 = vsetq_lane_s32(vgetq_lane_s32(__s2_344, __p3_344), __s0_344, __p1_344); \ +#define vcopyq_laneq_mf8(__p0_344, __p1_344, __p2_344, __p3_344) __extension__ ({ \ + mfloat8x16_t __ret_344; \ + mfloat8x16_t __s0_344 = __p0_344; \ + mfloat8x16_t __s2_344 = __p2_344; \ + __ret_344 = vsetq_lane_mf8(vgetq_lane_mf8(__s2_344, __p3_344), __s0_344, __p1_344); \ __ret_344; \ }) #else -#define vcopyq_laneq_s32(__p0_345, __p1_345, __p2_345, __p3_345) __extension__ ({ \ - int32x4_t __ret_345; \ - int32x4_t __s0_345 = __p0_345; \ - int32x4_t __s2_345 = __p2_345; \ - int32x4_t __rev0_345; __rev0_345 = __builtin_shufflevector(__s0_345, __s0_345, __lane_reverse_128_32); \ - int32x4_t __rev2_345; __rev2_345 = __builtin_shufflevector(__s2_345, __s2_345, __lane_reverse_128_32); \ - __ret_345 = __noswap_vsetq_lane_s32(__noswap_vgetq_lane_s32(__rev2_345, __p3_345), __rev0_345, __p1_345); \ - __ret_345 = __builtin_shufflevector(__ret_345, __ret_345, __lane_reverse_128_32); \ +#define vcopyq_laneq_mf8(__p0_345, __p1_345, __p2_345, __p3_345) __extension__ ({ \ + mfloat8x16_t __ret_345; \ + mfloat8x16_t __s0_345 = __p0_345; \ + mfloat8x16_t __s2_345 = __p2_345; \ + mfloat8x16_t __rev0_345; __rev0_345 = __builtin_shufflevector(__s0_345, __s0_345, __lane_reverse_128_8); \ + mfloat8x16_t __rev2_345; __rev2_345 = __builtin_shufflevector(__s2_345, __s2_345, __lane_reverse_128_8); \ + __ret_345 = __noswap_vsetq_lane_mf8(__noswap_vgetq_lane_mf8(__rev2_345, __p3_345), __rev0_345, __p1_345); \ + __ret_345 = __builtin_shufflevector(__ret_345, __ret_345, __lane_reverse_128_8); \ __ret_345; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_laneq_s64(__p0_346, __p1_346, __p2_346, __p3_346) __extension__ ({ \ - int64x2_t __ret_346; \ - int64x2_t __s0_346 = __p0_346; \ - int64x2_t __s2_346 = __p2_346; \ - __ret_346 = vsetq_lane_s64(vgetq_lane_s64(__s2_346, __p3_346), __s0_346, __p1_346); \ +#define vcopyq_laneq_s16(__p0_346, __p1_346, __p2_346, __p3_346) __extension__ ({ \ + int16x8_t __ret_346; \ + int16x8_t __s0_346 = __p0_346; \ + int16x8_t __s2_346 = __p2_346; \ + __ret_346 = vsetq_lane_s16(vgetq_lane_s16(__s2_346, __p3_346), __s0_346, __p1_346); \ __ret_346; \ }) #else -#define vcopyq_laneq_s64(__p0_347, __p1_347, __p2_347, __p3_347) __extension__ ({ \ - int64x2_t __ret_347; \ - int64x2_t __s0_347 = __p0_347; \ - int64x2_t __s2_347 = __p2_347; \ - int64x2_t __rev0_347; __rev0_347 = __builtin_shufflevector(__s0_347, __s0_347, __lane_reverse_128_64); \ - int64x2_t __rev2_347; __rev2_347 = __builtin_shufflevector(__s2_347, __s2_347, __lane_reverse_128_64); \ - __ret_347 = __noswap_vsetq_lane_s64(__noswap_vgetq_lane_s64(__rev2_347, __p3_347), __rev0_347, __p1_347); \ - __ret_347 = __builtin_shufflevector(__ret_347, __ret_347, __lane_reverse_128_64); \ +#define vcopyq_laneq_s16(__p0_347, __p1_347, __p2_347, __p3_347) __extension__ ({ \ + int16x8_t __ret_347; \ + int16x8_t __s0_347 = __p0_347; \ + int16x8_t __s2_347 = __p2_347; \ + int16x8_t __rev0_347; __rev0_347 = __builtin_shufflevector(__s0_347, __s0_347, __lane_reverse_128_16); \ + int16x8_t __rev2_347; __rev2_347 = __builtin_shufflevector(__s2_347, __s2_347, __lane_reverse_128_16); \ + __ret_347 = __noswap_vsetq_lane_s16(__noswap_vgetq_lane_s16(__rev2_347, __p3_347), __rev0_347, __p1_347); \ + __ret_347 = __builtin_shufflevector(__ret_347, __ret_347, __lane_reverse_128_16); \ __ret_347; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_laneq_mf8(__p0_348, __p1_348, __p2_348, __p3_348) __extension__ ({ \ - mfloat8x16_t __ret_348; \ - mfloat8x16_t __s0_348 = __p0_348; \ - mfloat8x16_t __s2_348 = __p2_348; \ - __ret_348 = vsetq_lane_mf8(vgetq_lane_mf8(__s2_348, __p3_348), __s0_348, __p1_348); \ +#define vcopyq_laneq_bf16(__p0_348, __p1_348, __p2_348, __p3_348) __extension__ ({ \ + bfloat16x8_t __ret_348; \ + bfloat16x8_t __s0_348 = __p0_348; \ + bfloat16x8_t __s2_348 = __p2_348; \ + __ret_348 = vsetq_lane_bf16(vgetq_lane_bf16(__s2_348, __p3_348), __s0_348, __p1_348); \ __ret_348; \ }) #else -#define vcopyq_laneq_mf8(__p0_349, __p1_349, __p2_349, __p3_349) __extension__ ({ \ - mfloat8x16_t __ret_349; \ - mfloat8x16_t __s0_349 = __p0_349; \ - mfloat8x16_t __s2_349 = __p2_349; \ - mfloat8x16_t __rev0_349; __rev0_349 = __builtin_shufflevector(__s0_349, __s0_349, __lane_reverse_128_8); \ - mfloat8x16_t __rev2_349; __rev2_349 = __builtin_shufflevector(__s2_349, __s2_349, __lane_reverse_128_8); \ - __ret_349 = __noswap_vsetq_lane_mf8(__noswap_vgetq_lane_mf8(__rev2_349, __p3_349), __rev0_349, __p1_349); \ - __ret_349 = __builtin_shufflevector(__ret_349, __ret_349, __lane_reverse_128_8); \ +#define vcopyq_laneq_bf16(__p0_349, __p1_349, __p2_349, __p3_349) __extension__ ({ \ + bfloat16x8_t __ret_349; \ + bfloat16x8_t __s0_349 = __p0_349; \ + bfloat16x8_t __s2_349 = __p2_349; \ + bfloat16x8_t __rev0_349; __rev0_349 = __builtin_shufflevector(__s0_349, __s0_349, __lane_reverse_128_16); \ + bfloat16x8_t __rev2_349; __rev2_349 = __builtin_shufflevector(__s2_349, __s2_349, __lane_reverse_128_16); \ + __ret_349 = __noswap_vsetq_lane_bf16(__noswap_vgetq_lane_bf16(__rev2_349, __p3_349), __rev0_349, __p1_349); \ + __ret_349 = __builtin_shufflevector(__ret_349, __ret_349, __lane_reverse_128_16); \ __ret_349; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopyq_laneq_s16(__p0_350, __p1_350, __p2_350, __p3_350) __extension__ ({ \ - int16x8_t __ret_350; \ - int16x8_t __s0_350 = __p0_350; \ - int16x8_t __s2_350 = __p2_350; \ - __ret_350 = vsetq_lane_s16(vgetq_lane_s16(__s2_350, __p3_350), __s0_350, __p1_350); \ +#define vcopy_laneq_p8(__p0_350, __p1_350, __p2_350, __p3_350) __extension__ ({ \ + poly8x8_t __ret_350; \ + poly8x8_t __s0_350 = __p0_350; \ + poly8x16_t __s2_350 = __p2_350; \ + __ret_350 = vset_lane_p8(vgetq_lane_p8(__s2_350, __p3_350), __s0_350, __p1_350); \ __ret_350; \ }) #else -#define vcopyq_laneq_s16(__p0_351, __p1_351, __p2_351, __p3_351) __extension__ ({ \ - int16x8_t __ret_351; \ - int16x8_t __s0_351 = __p0_351; \ - int16x8_t __s2_351 = __p2_351; \ - int16x8_t __rev0_351; __rev0_351 = __builtin_shufflevector(__s0_351, __s0_351, __lane_reverse_128_16); \ - int16x8_t __rev2_351; __rev2_351 = __builtin_shufflevector(__s2_351, __s2_351, __lane_reverse_128_16); \ - __ret_351 = __noswap_vsetq_lane_s16(__noswap_vgetq_lane_s16(__rev2_351, __p3_351), __rev0_351, __p1_351); \ - __ret_351 = __builtin_shufflevector(__ret_351, __ret_351, __lane_reverse_128_16); \ +#define vcopy_laneq_p8(__p0_351, __p1_351, __p2_351, __p3_351) __extension__ ({ \ + poly8x8_t __ret_351; \ + poly8x8_t __s0_351 = __p0_351; \ + poly8x16_t __s2_351 = __p2_351; \ + poly8x8_t __rev0_351; __rev0_351 = __builtin_shufflevector(__s0_351, __s0_351, __lane_reverse_64_8); \ + poly8x16_t __rev2_351; __rev2_351 = __builtin_shufflevector(__s2_351, __s2_351, __lane_reverse_128_8); \ + __ret_351 = __noswap_vset_lane_p8(__noswap_vgetq_lane_p8(__rev2_351, __p3_351), __rev0_351, __p1_351); \ + __ret_351 = __builtin_shufflevector(__ret_351, __ret_351, __lane_reverse_64_8); \ __ret_351; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_laneq_p8(__p0_352, __p1_352, __p2_352, __p3_352) __extension__ ({ \ - poly8x8_t __ret_352; \ - poly8x8_t __s0_352 = __p0_352; \ - poly8x16_t __s2_352 = __p2_352; \ - __ret_352 = vset_lane_p8(vgetq_lane_p8(__s2_352, __p3_352), __s0_352, __p1_352); \ +#define vcopy_laneq_p16(__p0_352, __p1_352, __p2_352, __p3_352) __extension__ ({ \ + poly16x4_t __ret_352; \ + poly16x4_t __s0_352 = __p0_352; \ + poly16x8_t __s2_352 = __p2_352; \ + __ret_352 = vset_lane_p16(vgetq_lane_p16(__s2_352, __p3_352), __s0_352, __p1_352); \ __ret_352; \ }) #else -#define vcopy_laneq_p8(__p0_353, __p1_353, __p2_353, __p3_353) __extension__ ({ \ - poly8x8_t __ret_353; \ - poly8x8_t __s0_353 = __p0_353; \ - poly8x16_t __s2_353 = __p2_353; \ - poly8x8_t __rev0_353; __rev0_353 = __builtin_shufflevector(__s0_353, __s0_353, __lane_reverse_64_8); \ - poly8x16_t __rev2_353; __rev2_353 = __builtin_shufflevector(__s2_353, __s2_353, __lane_reverse_128_8); \ - __ret_353 = __noswap_vset_lane_p8(__noswap_vgetq_lane_p8(__rev2_353, __p3_353), __rev0_353, __p1_353); \ - __ret_353 = __builtin_shufflevector(__ret_353, __ret_353, __lane_reverse_64_8); \ +#define vcopy_laneq_p16(__p0_353, __p1_353, __p2_353, __p3_353) __extension__ ({ \ + poly16x4_t __ret_353; \ + poly16x4_t __s0_353 = __p0_353; \ + poly16x8_t __s2_353 = __p2_353; \ + poly16x4_t __rev0_353; __rev0_353 = __builtin_shufflevector(__s0_353, __s0_353, __lane_reverse_64_16); \ + poly16x8_t __rev2_353; __rev2_353 = __builtin_shufflevector(__s2_353, __s2_353, __lane_reverse_128_16); \ + __ret_353 = __noswap_vset_lane_p16(__noswap_vgetq_lane_p16(__rev2_353, __p3_353), __rev0_353, __p1_353); \ + __ret_353 = __builtin_shufflevector(__ret_353, __ret_353, __lane_reverse_64_16); \ __ret_353; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_laneq_p16(__p0_354, __p1_354, __p2_354, __p3_354) __extension__ ({ \ - poly16x4_t __ret_354; \ - poly16x4_t __s0_354 = __p0_354; \ - poly16x8_t __s2_354 = __p2_354; \ - __ret_354 = vset_lane_p16(vgetq_lane_p16(__s2_354, __p3_354), __s0_354, __p1_354); \ +#define vcopy_laneq_u8(__p0_354, __p1_354, __p2_354, __p3_354) __extension__ ({ \ + uint8x8_t __ret_354; \ + uint8x8_t __s0_354 = __p0_354; \ + uint8x16_t __s2_354 = __p2_354; \ + __ret_354 = vset_lane_u8(vgetq_lane_u8(__s2_354, __p3_354), __s0_354, __p1_354); \ __ret_354; \ }) #else -#define vcopy_laneq_p16(__p0_355, __p1_355, __p2_355, __p3_355) __extension__ ({ \ - poly16x4_t __ret_355; \ - poly16x4_t __s0_355 = __p0_355; \ - poly16x8_t __s2_355 = __p2_355; \ - poly16x4_t __rev0_355; __rev0_355 = __builtin_shufflevector(__s0_355, __s0_355, __lane_reverse_64_16); \ - poly16x8_t __rev2_355; __rev2_355 = __builtin_shufflevector(__s2_355, __s2_355, __lane_reverse_128_16); \ - __ret_355 = __noswap_vset_lane_p16(__noswap_vgetq_lane_p16(__rev2_355, __p3_355), __rev0_355, __p1_355); \ - __ret_355 = __builtin_shufflevector(__ret_355, __ret_355, __lane_reverse_64_16); \ +#define vcopy_laneq_u8(__p0_355, __p1_355, __p2_355, __p3_355) __extension__ ({ \ + uint8x8_t __ret_355; \ + uint8x8_t __s0_355 = __p0_355; \ + uint8x16_t __s2_355 = __p2_355; \ + uint8x8_t __rev0_355; __rev0_355 = __builtin_shufflevector(__s0_355, __s0_355, __lane_reverse_64_8); \ + uint8x16_t __rev2_355; __rev2_355 = __builtin_shufflevector(__s2_355, __s2_355, __lane_reverse_128_8); \ + __ret_355 = __noswap_vset_lane_u8(__noswap_vgetq_lane_u8(__rev2_355, __p3_355), __rev0_355, __p1_355); \ + __ret_355 = __builtin_shufflevector(__ret_355, __ret_355, __lane_reverse_64_8); \ __ret_355; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_laneq_u8(__p0_356, __p1_356, __p2_356, __p3_356) __extension__ ({ \ - uint8x8_t __ret_356; \ - uint8x8_t __s0_356 = __p0_356; \ - uint8x16_t __s2_356 = __p2_356; \ - __ret_356 = vset_lane_u8(vgetq_lane_u8(__s2_356, __p3_356), __s0_356, __p1_356); \ +#define vcopy_laneq_u32(__p0_356, __p1_356, __p2_356, __p3_356) __extension__ ({ \ + uint32x2_t __ret_356; \ + uint32x2_t __s0_356 = __p0_356; \ + uint32x4_t __s2_356 = __p2_356; \ + __ret_356 = vset_lane_u32(vgetq_lane_u32(__s2_356, __p3_356), __s0_356, __p1_356); \ __ret_356; \ }) #else -#define vcopy_laneq_u8(__p0_357, __p1_357, __p2_357, __p3_357) __extension__ ({ \ - uint8x8_t __ret_357; \ - uint8x8_t __s0_357 = __p0_357; \ - uint8x16_t __s2_357 = __p2_357; \ - uint8x8_t __rev0_357; __rev0_357 = __builtin_shufflevector(__s0_357, __s0_357, __lane_reverse_64_8); \ - uint8x16_t __rev2_357; __rev2_357 = __builtin_shufflevector(__s2_357, __s2_357, __lane_reverse_128_8); \ - __ret_357 = __noswap_vset_lane_u8(__noswap_vgetq_lane_u8(__rev2_357, __p3_357), __rev0_357, __p1_357); \ - __ret_357 = __builtin_shufflevector(__ret_357, __ret_357, __lane_reverse_64_8); \ +#define vcopy_laneq_u32(__p0_357, __p1_357, __p2_357, __p3_357) __extension__ ({ \ + uint32x2_t __ret_357; \ + uint32x2_t __s0_357 = __p0_357; \ + uint32x4_t __s2_357 = __p2_357; \ + uint32x2_t __rev0_357; __rev0_357 = __builtin_shufflevector(__s0_357, __s0_357, __lane_reverse_64_32); \ + uint32x4_t __rev2_357; __rev2_357 = __builtin_shufflevector(__s2_357, __s2_357, __lane_reverse_128_32); \ + __ret_357 = __noswap_vset_lane_u32(__noswap_vgetq_lane_u32(__rev2_357, __p3_357), __rev0_357, __p1_357); \ + __ret_357 = __builtin_shufflevector(__ret_357, __ret_357, __lane_reverse_64_32); \ __ret_357; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_laneq_u32(__p0_358, __p1_358, __p2_358, __p3_358) __extension__ ({ \ - uint32x2_t __ret_358; \ - uint32x2_t __s0_358 = __p0_358; \ - uint32x4_t __s2_358 = __p2_358; \ - __ret_358 = vset_lane_u32(vgetq_lane_u32(__s2_358, __p3_358), __s0_358, __p1_358); \ +#define vcopy_laneq_u64(__p0_358, __p1_358, __p2_358, __p3_358) __extension__ ({ \ + uint64x1_t __ret_358; \ + uint64x1_t __s0_358 = __p0_358; \ + uint64x2_t __s2_358 = __p2_358; \ + __ret_358 = vset_lane_u64(vgetq_lane_u64(__s2_358, __p3_358), __s0_358, __p1_358); \ __ret_358; \ }) #else -#define vcopy_laneq_u32(__p0_359, __p1_359, __p2_359, __p3_359) __extension__ ({ \ - uint32x2_t __ret_359; \ - uint32x2_t __s0_359 = __p0_359; \ - uint32x4_t __s2_359 = __p2_359; \ - uint32x2_t __rev0_359; __rev0_359 = __builtin_shufflevector(__s0_359, __s0_359, __lane_reverse_64_32); \ - uint32x4_t __rev2_359; __rev2_359 = __builtin_shufflevector(__s2_359, __s2_359, __lane_reverse_128_32); \ - __ret_359 = __noswap_vset_lane_u32(__noswap_vgetq_lane_u32(__rev2_359, __p3_359), __rev0_359, __p1_359); \ - __ret_359 = __builtin_shufflevector(__ret_359, __ret_359, __lane_reverse_64_32); \ +#define vcopy_laneq_u64(__p0_359, __p1_359, __p2_359, __p3_359) __extension__ ({ \ + uint64x1_t __ret_359; \ + uint64x1_t __s0_359 = __p0_359; \ + uint64x2_t __s2_359 = __p2_359; \ + uint64x2_t __rev2_359; __rev2_359 = __builtin_shufflevector(__s2_359, __s2_359, __lane_reverse_128_64); \ + __ret_359 = vset_lane_u64(__noswap_vgetq_lane_u64(__rev2_359, __p3_359), __s0_359, __p1_359); \ __ret_359; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_laneq_u64(__p0_360, __p1_360, __p2_360, __p3_360) __extension__ ({ \ - uint64x1_t __ret_360; \ - uint64x1_t __s0_360 = __p0_360; \ - uint64x2_t __s2_360 = __p2_360; \ - __ret_360 = vset_lane_u64(vgetq_lane_u64(__s2_360, __p3_360), __s0_360, __p1_360); \ +#define vcopy_laneq_u16(__p0_360, __p1_360, __p2_360, __p3_360) __extension__ ({ \ + uint16x4_t __ret_360; \ + uint16x4_t __s0_360 = __p0_360; \ + uint16x8_t __s2_360 = __p2_360; \ + __ret_360 = vset_lane_u16(vgetq_lane_u16(__s2_360, __p3_360), __s0_360, __p1_360); \ __ret_360; \ }) #else -#define vcopy_laneq_u64(__p0_361, __p1_361, __p2_361, __p3_361) __extension__ ({ \ - uint64x1_t __ret_361; \ - uint64x1_t __s0_361 = __p0_361; \ - uint64x2_t __s2_361 = __p2_361; \ - uint64x2_t __rev2_361; __rev2_361 = __builtin_shufflevector(__s2_361, __s2_361, __lane_reverse_128_64); \ - __ret_361 = vset_lane_u64(__noswap_vgetq_lane_u64(__rev2_361, __p3_361), __s0_361, __p1_361); \ +#define vcopy_laneq_u16(__p0_361, __p1_361, __p2_361, __p3_361) __extension__ ({ \ + uint16x4_t __ret_361; \ + uint16x4_t __s0_361 = __p0_361; \ + uint16x8_t __s2_361 = __p2_361; \ + uint16x4_t __rev0_361; __rev0_361 = __builtin_shufflevector(__s0_361, __s0_361, __lane_reverse_64_16); \ + uint16x8_t __rev2_361; __rev2_361 = __builtin_shufflevector(__s2_361, __s2_361, __lane_reverse_128_16); \ + __ret_361 = __noswap_vset_lane_u16(__noswap_vgetq_lane_u16(__rev2_361, __p3_361), __rev0_361, __p1_361); \ + __ret_361 = __builtin_shufflevector(__ret_361, __ret_361, __lane_reverse_64_16); \ __ret_361; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_laneq_u16(__p0_362, __p1_362, __p2_362, __p3_362) __extension__ ({ \ - uint16x4_t __ret_362; \ - uint16x4_t __s0_362 = __p0_362; \ - uint16x8_t __s2_362 = __p2_362; \ - __ret_362 = vset_lane_u16(vgetq_lane_u16(__s2_362, __p3_362), __s0_362, __p1_362); \ +#define vcopy_laneq_s8(__p0_362, __p1_362, __p2_362, __p3_362) __extension__ ({ \ + int8x8_t __ret_362; \ + int8x8_t __s0_362 = __p0_362; \ + int8x16_t __s2_362 = __p2_362; \ + __ret_362 = vset_lane_s8(vgetq_lane_s8(__s2_362, __p3_362), __s0_362, __p1_362); \ __ret_362; \ }) #else -#define vcopy_laneq_u16(__p0_363, __p1_363, __p2_363, __p3_363) __extension__ ({ \ - uint16x4_t __ret_363; \ - uint16x4_t __s0_363 = __p0_363; \ - uint16x8_t __s2_363 = __p2_363; \ - uint16x4_t __rev0_363; __rev0_363 = __builtin_shufflevector(__s0_363, __s0_363, __lane_reverse_64_16); \ - uint16x8_t __rev2_363; __rev2_363 = __builtin_shufflevector(__s2_363, __s2_363, __lane_reverse_128_16); \ - __ret_363 = __noswap_vset_lane_u16(__noswap_vgetq_lane_u16(__rev2_363, __p3_363), __rev0_363, __p1_363); \ - __ret_363 = __builtin_shufflevector(__ret_363, __ret_363, __lane_reverse_64_16); \ +#define vcopy_laneq_s8(__p0_363, __p1_363, __p2_363, __p3_363) __extension__ ({ \ + int8x8_t __ret_363; \ + int8x8_t __s0_363 = __p0_363; \ + int8x16_t __s2_363 = __p2_363; \ + int8x8_t __rev0_363; __rev0_363 = __builtin_shufflevector(__s0_363, __s0_363, __lane_reverse_64_8); \ + int8x16_t __rev2_363; __rev2_363 = __builtin_shufflevector(__s2_363, __s2_363, __lane_reverse_128_8); \ + __ret_363 = __noswap_vset_lane_s8(__noswap_vgetq_lane_s8(__rev2_363, __p3_363), __rev0_363, __p1_363); \ + __ret_363 = __builtin_shufflevector(__ret_363, __ret_363, __lane_reverse_64_8); \ __ret_363; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_laneq_s8(__p0_364, __p1_364, __p2_364, __p3_364) __extension__ ({ \ - int8x8_t __ret_364; \ - int8x8_t __s0_364 = __p0_364; \ - int8x16_t __s2_364 = __p2_364; \ - __ret_364 = vset_lane_s8(vgetq_lane_s8(__s2_364, __p3_364), __s0_364, __p1_364); \ +#define vcopy_laneq_f32(__p0_364, __p1_364, __p2_364, __p3_364) __extension__ ({ \ + float32x2_t __ret_364; \ + float32x2_t __s0_364 = __p0_364; \ + float32x4_t __s2_364 = __p2_364; \ + __ret_364 = vset_lane_f32(vgetq_lane_f32(__s2_364, __p3_364), __s0_364, __p1_364); \ __ret_364; \ }) #else -#define vcopy_laneq_s8(__p0_365, __p1_365, __p2_365, __p3_365) __extension__ ({ \ - int8x8_t __ret_365; \ - int8x8_t __s0_365 = __p0_365; \ - int8x16_t __s2_365 = __p2_365; \ - int8x8_t __rev0_365; __rev0_365 = __builtin_shufflevector(__s0_365, __s0_365, __lane_reverse_64_8); \ - int8x16_t __rev2_365; __rev2_365 = __builtin_shufflevector(__s2_365, __s2_365, __lane_reverse_128_8); \ - __ret_365 = __noswap_vset_lane_s8(__noswap_vgetq_lane_s8(__rev2_365, __p3_365), __rev0_365, __p1_365); \ - __ret_365 = __builtin_shufflevector(__ret_365, __ret_365, __lane_reverse_64_8); \ +#define vcopy_laneq_f32(__p0_365, __p1_365, __p2_365, __p3_365) __extension__ ({ \ + float32x2_t __ret_365; \ + float32x2_t __s0_365 = __p0_365; \ + float32x4_t __s2_365 = __p2_365; \ + float32x2_t __rev0_365; __rev0_365 = __builtin_shufflevector(__s0_365, __s0_365, __lane_reverse_64_32); \ + float32x4_t __rev2_365; __rev2_365 = __builtin_shufflevector(__s2_365, __s2_365, __lane_reverse_128_32); \ + __ret_365 = __noswap_vset_lane_f32(__noswap_vgetq_lane_f32(__rev2_365, __p3_365), __rev0_365, __p1_365); \ + __ret_365 = __builtin_shufflevector(__ret_365, __ret_365, __lane_reverse_64_32); \ __ret_365; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_laneq_f32(__p0_366, __p1_366, __p2_366, __p3_366) __extension__ ({ \ - float32x2_t __ret_366; \ - float32x2_t __s0_366 = __p0_366; \ - float32x4_t __s2_366 = __p2_366; \ - __ret_366 = vset_lane_f32(vgetq_lane_f32(__s2_366, __p3_366), __s0_366, __p1_366); \ +#define vcopy_laneq_s32(__p0_366, __p1_366, __p2_366, __p3_366) __extension__ ({ \ + int32x2_t __ret_366; \ + int32x2_t __s0_366 = __p0_366; \ + int32x4_t __s2_366 = __p2_366; \ + __ret_366 = vset_lane_s32(vgetq_lane_s32(__s2_366, __p3_366), __s0_366, __p1_366); \ __ret_366; \ }) #else -#define vcopy_laneq_f32(__p0_367, __p1_367, __p2_367, __p3_367) __extension__ ({ \ - float32x2_t __ret_367; \ - float32x2_t __s0_367 = __p0_367; \ - float32x4_t __s2_367 = __p2_367; \ - float32x2_t __rev0_367; __rev0_367 = __builtin_shufflevector(__s0_367, __s0_367, __lane_reverse_64_32); \ - float32x4_t __rev2_367; __rev2_367 = __builtin_shufflevector(__s2_367, __s2_367, __lane_reverse_128_32); \ - __ret_367 = __noswap_vset_lane_f32(__noswap_vgetq_lane_f32(__rev2_367, __p3_367), __rev0_367, __p1_367); \ +#define vcopy_laneq_s32(__p0_367, __p1_367, __p2_367, __p3_367) __extension__ ({ \ + int32x2_t __ret_367; \ + int32x2_t __s0_367 = __p0_367; \ + int32x4_t __s2_367 = __p2_367; \ + int32x2_t __rev0_367; __rev0_367 = __builtin_shufflevector(__s0_367, __s0_367, __lane_reverse_64_32); \ + int32x4_t __rev2_367; __rev2_367 = __builtin_shufflevector(__s2_367, __s2_367, __lane_reverse_128_32); \ + __ret_367 = __noswap_vset_lane_s32(__noswap_vgetq_lane_s32(__rev2_367, __p3_367), __rev0_367, __p1_367); \ __ret_367 = __builtin_shufflevector(__ret_367, __ret_367, __lane_reverse_64_32); \ __ret_367; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_laneq_s32(__p0_368, __p1_368, __p2_368, __p3_368) __extension__ ({ \ - int32x2_t __ret_368; \ - int32x2_t __s0_368 = __p0_368; \ - int32x4_t __s2_368 = __p2_368; \ - __ret_368 = vset_lane_s32(vgetq_lane_s32(__s2_368, __p3_368), __s0_368, __p1_368); \ +#define vcopy_laneq_s64(__p0_368, __p1_368, __p2_368, __p3_368) __extension__ ({ \ + int64x1_t __ret_368; \ + int64x1_t __s0_368 = __p0_368; \ + int64x2_t __s2_368 = __p2_368; \ + __ret_368 = vset_lane_s64(vgetq_lane_s64(__s2_368, __p3_368), __s0_368, __p1_368); \ __ret_368; \ }) #else -#define vcopy_laneq_s32(__p0_369, __p1_369, __p2_369, __p3_369) __extension__ ({ \ - int32x2_t __ret_369; \ - int32x2_t __s0_369 = __p0_369; \ - int32x4_t __s2_369 = __p2_369; \ - int32x2_t __rev0_369; __rev0_369 = __builtin_shufflevector(__s0_369, __s0_369, __lane_reverse_64_32); \ - int32x4_t __rev2_369; __rev2_369 = __builtin_shufflevector(__s2_369, __s2_369, __lane_reverse_128_32); \ - __ret_369 = __noswap_vset_lane_s32(__noswap_vgetq_lane_s32(__rev2_369, __p3_369), __rev0_369, __p1_369); \ - __ret_369 = __builtin_shufflevector(__ret_369, __ret_369, __lane_reverse_64_32); \ +#define vcopy_laneq_s64(__p0_369, __p1_369, __p2_369, __p3_369) __extension__ ({ \ + int64x1_t __ret_369; \ + int64x1_t __s0_369 = __p0_369; \ + int64x2_t __s2_369 = __p2_369; \ + int64x2_t __rev2_369; __rev2_369 = __builtin_shufflevector(__s2_369, __s2_369, __lane_reverse_128_64); \ + __ret_369 = vset_lane_s64(__noswap_vgetq_lane_s64(__rev2_369, __p3_369), __s0_369, __p1_369); \ __ret_369; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_laneq_s64(__p0_370, __p1_370, __p2_370, __p3_370) __extension__ ({ \ - int64x1_t __ret_370; \ - int64x1_t __s0_370 = __p0_370; \ - int64x2_t __s2_370 = __p2_370; \ - __ret_370 = vset_lane_s64(vgetq_lane_s64(__s2_370, __p3_370), __s0_370, __p1_370); \ +#define vcopy_laneq_mf8(__p0_370, __p1_370, __p2_370, __p3_370) __extension__ ({ \ + mfloat8x8_t __ret_370; \ + mfloat8x8_t __s0_370 = __p0_370; \ + mfloat8x16_t __s2_370 = __p2_370; \ + __ret_370 = vset_lane_mf8(vgetq_lane_mf8(__s2_370, __p3_370), __s0_370, __p1_370); \ __ret_370; \ }) #else -#define vcopy_laneq_s64(__p0_371, __p1_371, __p2_371, __p3_371) __extension__ ({ \ - int64x1_t __ret_371; \ - int64x1_t __s0_371 = __p0_371; \ - int64x2_t __s2_371 = __p2_371; \ - int64x2_t __rev2_371; __rev2_371 = __builtin_shufflevector(__s2_371, __s2_371, __lane_reverse_128_64); \ - __ret_371 = vset_lane_s64(__noswap_vgetq_lane_s64(__rev2_371, __p3_371), __s0_371, __p1_371); \ +#define vcopy_laneq_mf8(__p0_371, __p1_371, __p2_371, __p3_371) __extension__ ({ \ + mfloat8x8_t __ret_371; \ + mfloat8x8_t __s0_371 = __p0_371; \ + mfloat8x16_t __s2_371 = __p2_371; \ + mfloat8x8_t __rev0_371; __rev0_371 = __builtin_shufflevector(__s0_371, __s0_371, __lane_reverse_64_8); \ + mfloat8x16_t __rev2_371; __rev2_371 = __builtin_shufflevector(__s2_371, __s2_371, __lane_reverse_128_8); \ + __ret_371 = __noswap_vset_lane_mf8(__noswap_vgetq_lane_mf8(__rev2_371, __p3_371), __rev0_371, __p1_371); \ + __ret_371 = __builtin_shufflevector(__ret_371, __ret_371, __lane_reverse_64_8); \ __ret_371; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_laneq_mf8(__p0_372, __p1_372, __p2_372, __p3_372) __extension__ ({ \ - mfloat8x8_t __ret_372; \ - mfloat8x8_t __s0_372 = __p0_372; \ - mfloat8x16_t __s2_372 = __p2_372; \ - __ret_372 = vset_lane_mf8(vgetq_lane_mf8(__s2_372, __p3_372), __s0_372, __p1_372); \ +#define vcopy_laneq_s16(__p0_372, __p1_372, __p2_372, __p3_372) __extension__ ({ \ + int16x4_t __ret_372; \ + int16x4_t __s0_372 = __p0_372; \ + int16x8_t __s2_372 = __p2_372; \ + __ret_372 = vset_lane_s16(vgetq_lane_s16(__s2_372, __p3_372), __s0_372, __p1_372); \ __ret_372; \ }) #else -#define vcopy_laneq_mf8(__p0_373, __p1_373, __p2_373, __p3_373) __extension__ ({ \ - mfloat8x8_t __ret_373; \ - mfloat8x8_t __s0_373 = __p0_373; \ - mfloat8x16_t __s2_373 = __p2_373; \ - mfloat8x8_t __rev0_373; __rev0_373 = __builtin_shufflevector(__s0_373, __s0_373, __lane_reverse_64_8); \ - mfloat8x16_t __rev2_373; __rev2_373 = __builtin_shufflevector(__s2_373, __s2_373, __lane_reverse_128_8); \ - __ret_373 = __noswap_vset_lane_mf8(__noswap_vgetq_lane_mf8(__rev2_373, __p3_373), __rev0_373, __p1_373); \ - __ret_373 = __builtin_shufflevector(__ret_373, __ret_373, __lane_reverse_64_8); \ +#define vcopy_laneq_s16(__p0_373, __p1_373, __p2_373, __p3_373) __extension__ ({ \ + int16x4_t __ret_373; \ + int16x4_t __s0_373 = __p0_373; \ + int16x8_t __s2_373 = __p2_373; \ + int16x4_t __rev0_373; __rev0_373 = __builtin_shufflevector(__s0_373, __s0_373, __lane_reverse_64_16); \ + int16x8_t __rev2_373; __rev2_373 = __builtin_shufflevector(__s2_373, __s2_373, __lane_reverse_128_16); \ + __ret_373 = __noswap_vset_lane_s16(__noswap_vgetq_lane_s16(__rev2_373, __p3_373), __rev0_373, __p1_373); \ + __ret_373 = __builtin_shufflevector(__ret_373, __ret_373, __lane_reverse_64_16); \ __ret_373; \ }) #endif #ifdef __LITTLE_ENDIAN__ -#define vcopy_laneq_s16(__p0_374, __p1_374, __p2_374, __p3_374) __extension__ ({ \ - int16x4_t __ret_374; \ - int16x4_t __s0_374 = __p0_374; \ - int16x8_t __s2_374 = __p2_374; \ - __ret_374 = vset_lane_s16(vgetq_lane_s16(__s2_374, __p3_374), __s0_374, __p1_374); \ +#define vcopy_laneq_bf16(__p0_374, __p1_374, __p2_374, __p3_374) __extension__ ({ \ + bfloat16x4_t __ret_374; \ + bfloat16x4_t __s0_374 = __p0_374; \ + bfloat16x8_t __s2_374 = __p2_374; \ + __ret_374 = vset_lane_bf16(vgetq_lane_bf16(__s2_374, __p3_374), __s0_374, __p1_374); \ __ret_374; \ }) #else -#define vcopy_laneq_s16(__p0_375, __p1_375, __p2_375, __p3_375) __extension__ ({ \ - int16x4_t __ret_375; \ - int16x4_t __s0_375 = __p0_375; \ - int16x8_t __s2_375 = __p2_375; \ - int16x4_t __rev0_375; __rev0_375 = __builtin_shufflevector(__s0_375, __s0_375, __lane_reverse_64_16); \ - int16x8_t __rev2_375; __rev2_375 = __builtin_shufflevector(__s2_375, __s2_375, __lane_reverse_128_16); \ - __ret_375 = __noswap_vset_lane_s16(__noswap_vgetq_lane_s16(__rev2_375, __p3_375), __rev0_375, __p1_375); \ +#define vcopy_laneq_bf16(__p0_375, __p1_375, __p2_375, __p3_375) __extension__ ({ \ + bfloat16x4_t __ret_375; \ + bfloat16x4_t __s0_375 = __p0_375; \ + bfloat16x8_t __s2_375 = __p2_375; \ + bfloat16x4_t __rev0_375; __rev0_375 = __builtin_shufflevector(__s0_375, __s0_375, __lane_reverse_64_16); \ + bfloat16x8_t __rev2_375; __rev2_375 = __builtin_shufflevector(__s2_375, __s2_375, __lane_reverse_128_16); \ + __ret_375 = __noswap_vset_lane_bf16(__noswap_vgetq_lane_bf16(__rev2_375, __p3_375), __rev0_375, __p1_375); \ __ret_375 = __builtin_shufflevector(__ret_375, __ret_375, __lane_reverse_64_16); \ __ret_375; \ }) @@ -64144,6 +64022,296 @@ __ai __attribute__((target("neon"))) int16x4_t vreinterpret_s16_mf8(mfloat8x8_t __ret = __builtin_bit_cast(int16x4_t, __p0); return __ret; } +__ai __attribute__((target("neon"))) poly8x8_t vreinterpret_p8_bf16(bfloat16x4_t __p0) { + poly8x8_t __ret; + __ret = __builtin_bit_cast(poly8x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) poly64x1_t vreinterpret_p64_bf16(bfloat16x4_t __p0) { + poly64x1_t __ret; + __ret = __builtin_bit_cast(poly64x1_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) poly16x4_t vreinterpret_p16_bf16(bfloat16x4_t __p0) { + poly16x4_t __ret; + __ret = __builtin_bit_cast(poly16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) poly8x16_t vreinterpretq_p8_bf16(bfloat16x8_t __p0) { + poly8x16_t __ret; + __ret = __builtin_bit_cast(poly8x16_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) poly128_t vreinterpretq_p128_bf16(bfloat16x8_t __p0) { + poly128_t __ret; + __ret = __builtin_bit_cast(poly128_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) poly64x2_t vreinterpretq_p64_bf16(bfloat16x8_t __p0) { + poly64x2_t __ret; + __ret = __builtin_bit_cast(poly64x2_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) poly16x8_t vreinterpretq_p16_bf16(bfloat16x8_t __p0) { + poly16x8_t __ret; + __ret = __builtin_bit_cast(poly16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint8x16_t vreinterpretq_u8_bf16(bfloat16x8_t __p0) { + uint8x16_t __ret; + __ret = __builtin_bit_cast(uint8x16_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint32x4_t vreinterpretq_u32_bf16(bfloat16x8_t __p0) { + uint32x4_t __ret; + __ret = __builtin_bit_cast(uint32x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint64x2_t vreinterpretq_u64_bf16(bfloat16x8_t __p0) { + uint64x2_t __ret; + __ret = __builtin_bit_cast(uint64x2_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint16x8_t vreinterpretq_u16_bf16(bfloat16x8_t __p0) { + uint16x8_t __ret; + __ret = __builtin_bit_cast(uint16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int8x16_t vreinterpretq_s8_bf16(bfloat16x8_t __p0) { + int8x16_t __ret; + __ret = __builtin_bit_cast(int8x16_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) float64x2_t vreinterpretq_f64_bf16(bfloat16x8_t __p0) { + float64x2_t __ret; + __ret = __builtin_bit_cast(float64x2_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) float32x4_t vreinterpretq_f32_bf16(bfloat16x8_t __p0) { + float32x4_t __ret; + __ret = __builtin_bit_cast(float32x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) float16x8_t vreinterpretq_f16_bf16(bfloat16x8_t __p0) { + float16x8_t __ret; + __ret = __builtin_bit_cast(float16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int32x4_t vreinterpretq_s32_bf16(bfloat16x8_t __p0) { + int32x4_t __ret; + __ret = __builtin_bit_cast(int32x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int64x2_t vreinterpretq_s64_bf16(bfloat16x8_t __p0) { + int64x2_t __ret; + __ret = __builtin_bit_cast(int64x2_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int16x8_t vreinterpretq_s16_bf16(bfloat16x8_t __p0) { + int16x8_t __ret; + __ret = __builtin_bit_cast(int16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint8x8_t vreinterpret_u8_bf16(bfloat16x4_t __p0) { + uint8x8_t __ret; + __ret = __builtin_bit_cast(uint8x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint32x2_t vreinterpret_u32_bf16(bfloat16x4_t __p0) { + uint32x2_t __ret; + __ret = __builtin_bit_cast(uint32x2_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint64x1_t vreinterpret_u64_bf16(bfloat16x4_t __p0) { + uint64x1_t __ret; + __ret = __builtin_bit_cast(uint64x1_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) uint16x4_t vreinterpret_u16_bf16(bfloat16x4_t __p0) { + uint16x4_t __ret; + __ret = __builtin_bit_cast(uint16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int8x8_t vreinterpret_s8_bf16(bfloat16x4_t __p0) { + int8x8_t __ret; + __ret = __builtin_bit_cast(int8x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) float64x1_t vreinterpret_f64_bf16(bfloat16x4_t __p0) { + float64x1_t __ret; + __ret = __builtin_bit_cast(float64x1_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) float32x2_t vreinterpret_f32_bf16(bfloat16x4_t __p0) { + float32x2_t __ret; + __ret = __builtin_bit_cast(float32x2_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) float16x4_t vreinterpret_f16_bf16(bfloat16x4_t __p0) { + float16x4_t __ret; + __ret = __builtin_bit_cast(float16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int32x2_t vreinterpret_s32_bf16(bfloat16x4_t __p0) { + int32x2_t __ret; + __ret = __builtin_bit_cast(int32x2_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int64x1_t vreinterpret_s64_bf16(bfloat16x4_t __p0) { + int64x1_t __ret; + __ret = __builtin_bit_cast(int64x1_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) int16x4_t vreinterpret_s16_bf16(bfloat16x4_t __p0) { + int16x4_t __ret; + __ret = __builtin_bit_cast(int16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_p8(poly8x16_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_p128(poly128_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_p64(poly64x2_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_p16(poly16x8_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_u8(uint8x16_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_u32(uint32x4_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_u64(uint64x2_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_u16(uint16x8_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_s8(int8x16_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_f64(float64x2_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_f32(float32x4_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_f16(float16x8_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_s32(int32x4_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_s64(int64x2_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x8_t vreinterpretq_bf16_s16(int16x8_t __p0) { + bfloat16x8_t __ret; + __ret = __builtin_bit_cast(bfloat16x8_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_p8(poly8x8_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_p64(poly64x1_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_p16(poly16x4_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_u8(uint8x8_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_u32(uint32x2_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_u64(uint64x1_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_u16(uint16x4_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_s8(int8x8_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_f64(float64x1_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_f32(float32x2_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_f16(float16x4_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_s32(int32x2_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_s64(int64x1_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} +__ai __attribute__((target("neon"))) bfloat16x4_t vreinterpret_bf16_s16(int16x4_t __p0) { + bfloat16x4_t __ret; + __ret = __builtin_bit_cast(bfloat16x4_t, __p0); + return __ret; +} __ai __attribute__((target("neon"))) uint64_t vrshld_u64(uint64_t __p0, int64_t __p1) { uint64_t __ret; __ret = __builtin_bit_cast(uint64_t, __builtin_neon_vrshld_u64(__p0, __p1)); diff --git a/lib/include/arm_sme.h b/lib/include/arm_sme.h index 0983e4a58d7240eeab6bd4ed52288805c30828c4..98d5951fc0fc5be438dafb4d07599cfab890acb7 100644 --- a/lib/include/arm_sme.h +++ b/lib/include/arm_sme.h @@ -3732,6 +3732,18 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svzero_za64_vg4x2))) void svzero_za64_vg4x2(uint32_t); __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svzero_za64_vg4x4))) void svzero_za64_vg4x4(uint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti6_zt_u8))) +svuint8_t svluti6_zt_u8(uint64_t, svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti6_zt_s8))) +svint8_t svluti6_zt_s8(uint64_t, svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti6_zt_mf8))) +svmfloat8_t svluti6_zt_mf8(uint64_t, svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti6_zt_u8_x4))) +svuint8x4_t svluti6_zt_u8_x4(uint64_t, svuint8x3_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti6_zt_s8_x4))) +svint8x4_t svluti6_zt_s8_x4(uint64_t, svuint8x3_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti6_zt_mf8_x4))) +svmfloat8x4_t svluti6_zt_mf8_x4(uint64_t, svuint8x3_t); #ifdef __cplusplus } // extern "C" #endif diff --git a/lib/include/arm_sve.h b/lib/include/arm_sve.h index d56bd345306552f8d1b7bf4be24283a01f5df60d..c2af3c58526cc9f72aa3358240e56418b2026529 100644 --- a/lib/include/arm_sve.h +++ b/lib/include/arm_sve.h @@ -4697,6 +4697,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f32_x4))) svfloat32x4_t svmul(svfloat32x4_t, svfloat32x4_t); __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f16_x4))) svfloat16x4_t svmul(svfloat16x4_t, svfloat16x4_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_u16_x4_u16_x2_u8_x2))) +svuint16x4_t svluti6_lane_u16_x4_u16_x2_u8_x2(svuint16x2_t, svuint8x2_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_bf16_x4_bf16_x2_u8_x2))) +svbfloat16x4_t svluti6_lane_bf16_x4_bf16_x2_u8_x2(svbfloat16x2_t, svuint8x2_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_f16_x4_f16_x2_u8_x2))) +svfloat16x4_t svluti6_lane_f16_x4_f16_x2_u8_x2(svfloat16x2_t, svuint8x2_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_s16_x4_s16_x2_u8_x2))) +svint16x4_t svluti6_lane_s16_x4_s16_x2_u8_x2(svint16x2_t, svuint8x2_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_u16_x4_u16_x2_u8_x3))) +svuint16x4_t svluti6_lane_u16_x4_u16_x2_u8_x3(svuint16x2_t, svuint8x3_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_bf16_x4_bf16_x2_u8_x3))) +svbfloat16x4_t svluti6_lane_bf16_x4_bf16_x2_u8_x3(svbfloat16x2_t, svuint8x3_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_f16_x4_f16_x2_u8_x3))) +svfloat16x4_t svluti6_lane_f16_x4_f16_x2_u8_x3(svfloat16x2_t, svuint8x3_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_s16_x4_s16_x2_u8_x3))) +svint16x4_t svluti6_lane_s16_x4_s16_x2_u8_x3(svint16x2_t, svuint8x3_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_u16_x4_u16_x2_u8_x2))) +svuint16x4_t svluti6_lane_u16_x4(svuint16x2_t, svuint8x2_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_bf16_x4_bf16_x2_u8_x2))) +svbfloat16x4_t svluti6_lane_bf16_x4(svbfloat16x2_t, svuint8x2_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_f16_x4_f16_x2_u8_x2))) +svfloat16x4_t svluti6_lane_f16_x4(svfloat16x2_t, svuint8x2_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_s16_x4_s16_x2_u8_x2))) +svint16x4_t svluti6_lane_s16_x4(svint16x2_t, svuint8x2_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_u16_x4_u16_x2_u8_x3))) +svuint16x4_t svluti6_lane_u16_x4(svuint16x2_t, svuint8x3_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_bf16_x4_bf16_x2_u8_x3))) +svbfloat16x4_t svluti6_lane_bf16_x4(svbfloat16x2_t, svuint8x3_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_f16_x4_f16_x2_u8_x3))) +svfloat16x4_t svluti6_lane_f16_x4(svfloat16x2_t, svuint8x3_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_s16_x4_s16_x2_u8_x3))) +svint16x4_t svluti6_lane_s16_x4(svint16x2_t, svuint8x3_t, uint64_t); __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f64))) float64_t svadda_f64(svbool_t, float64_t, svfloat64_t); __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f32))) @@ -7223,6 +7255,12 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcreate2_b))) svboolx2_t svcreate2_b(svbool_t, svbool_t); __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcreate4_b))) svboolx4_t svcreate4_b(svbool_t, svbool_t, svbool_t, svbool_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f32_f16))) +svfloat32_t svdot_n_f32_f16(svfloat32_t, svfloat16_t, float16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_s32_s16))) +svint32_t svdot_n_s32_s16(svint32_t, svint16_t, int16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_u32_u16))) +svuint32_t svdot_n_u32_u16(svuint32_t, svuint16_t, uint16_t); __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f32_f16))) svfloat32_t svdot_f32_f16(svfloat32_t, svfloat16_t, svfloat16_t); __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_s32_s16))) @@ -7341,6 +7379,12 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcreate2_b))) svboolx2_t svcreate2(svbool_t, svbool_t); __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcreate4_b))) svboolx4_t svcreate4(svbool_t, svbool_t, svbool_t, svbool_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f32_f16))) +svfloat32_t svdot(svfloat32_t, svfloat16_t, float16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_s32_s16))) +svint32_t svdot(svint32_t, svint16_t, int16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_u32_u16))) +svuint32_t svdot(svuint32_t, svuint16_t, uint16_t); __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f32_f16))) svfloat32_t svdot(svfloat32_t, svfloat16_t, svfloat16_t); __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_s32_s16))) @@ -9049,6 +9093,346 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_mf8))) svmfloat8_t svexpand(svbool_t, svmfloat8_t); __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_s16))) svint16_t svexpand(svbool_t, svint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_s32))) +svint32_t svabal_n_s32(svint32_t, svint16_t, int16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_s64))) +svint64_t svabal_n_s64(svint64_t, svint32_t, int32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_s16))) +svint16_t svabal_n_s16(svint16_t, svint8_t, int8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_u32))) +svuint32_t svabal_n_u32(svuint32_t, svuint16_t, uint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_u64))) +svuint64_t svabal_n_u64(svuint64_t, svuint32_t, uint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_u16))) +svuint16_t svabal_n_u16(svuint16_t, svuint8_t, uint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_s32))) +svint32_t svabal_s32(svint32_t, svint16_t, svint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_s64))) +svint64_t svabal_s64(svint64_t, svint32_t, svint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_s16))) +svint16_t svabal_s16(svint16_t, svint8_t, svint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_u32))) +svuint32_t svabal_u32(svuint32_t, svuint16_t, svuint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_u64))) +svuint64_t svabal_u64(svuint64_t, svuint32_t, svuint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_u16))) +svuint16_t svabal_u16(svuint16_t, svuint8_t, svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u8))) +svuint8_t svaddqp_u8(svuint8_t, svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u32))) +svuint32_t svaddqp_u32(svuint32_t, svuint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u64))) +svuint64_t svaddqp_u64(svuint64_t, svuint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u16))) +svuint16_t svaddqp_u16(svuint16_t, svuint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s8))) +svint8_t svaddqp_s8(svint8_t, svint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s32))) +svint32_t svaddqp_s32(svint32_t, svint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s64))) +svint64_t svaddqp_s64(svint64_t, svint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s16))) +svint16_t svaddqp_s16(svint16_t, svint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u8))) +svuint8_t svaddsubp_u8(svuint8_t, svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u32))) +svuint32_t svaddsubp_u32(svuint32_t, svuint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u64))) +svuint64_t svaddsubp_u64(svuint64_t, svuint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u16))) +svuint16_t svaddsubp_u16(svuint16_t, svuint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s8))) +svint8_t svaddsubp_s8(svint8_t, svint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s32))) +svint32_t svaddsubp_s32(svint32_t, svint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s64))) +svint64_t svaddsubp_s64(svint64_t, svint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s16))) +svint16_t svaddsubp_s16(svint16_t, svint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f64_s32))) +svfloat64_t svcvtb_f64_s32(svint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f32_s16))) +svfloat32_t svcvtb_f32_s16(svint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f16_s8))) +svfloat16_t svcvtb_f16_s8(svint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f64_u32))) +svfloat64_t svcvtb_f64_u32(svuint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f32_u16))) +svfloat32_t svcvtb_f32_u16(svuint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f16_u8))) +svfloat16_t svcvtb_f16_u8(svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_s32_f64_x2))) +svint32_t svcvtn_s32_f64_x2(svfloat64x2_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_s16_f32_x2))) +svint16_t svcvtn_s16_f32_x2(svfloat32x2_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_s8_f16_x2))) +svint8_t svcvtn_s8_f16_x2(svfloat16x2_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_u32_f64_x2))) +svuint32_t svcvtn_u32_f64_x2(svfloat64x2_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_u16_f32_x2))) +svuint16_t svcvtn_u16_f32_x2(svfloat32x2_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_u8_f16_x2))) +svuint8_t svcvtn_u8_f16_x2(svfloat16x2_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f64_s32))) +svfloat64_t svcvtt_f64_s32(svint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f32_s16))) +svfloat32_t svcvtt_f32_s16(svint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f16_s8))) +svfloat16_t svcvtt_f16_s8(svint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f64_u32))) +svfloat64_t svcvtt_f64_u32(svuint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f32_u16))) +svfloat32_t svcvtt_f32_u16(svuint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f16_u8))) +svfloat16_t svcvtt_f16_u8(svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_s16_s8))) +svint16_t svdot_n_s16_s8(svint16_t, svint8_t, int8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_u16_u8))) +svuint16_t svdot_n_u16_u8(svuint16_t, svuint8_t, uint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_s16_s8))) +svint16_t svdot_s16_s8(svint16_t, svint8_t, svint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_u16_u8))) +svuint16_t svdot_u16_u8(svuint16_t, svuint8_t, svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_s16_s8))) +svint16_t svdot_lane_s16_s8(svint16_t, svint8_t, svint8_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_u16_u8))) +svuint16_t svdot_lane_u16_u8(svuint16_t, svuint8_t, svuint8_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqrshrn_n_s8_s16_x2))) +svint8_t svqrshrn_n_s8_s16_x2(svint16x2_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqrshrn_n_u8_u16_x2))) +svuint8_t svqrshrn_n_u8_u16_x2(svuint16x2_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqrshrun_n_u8_s16_x2))) +svuint8_t svqrshrun_n_u8_s16_x2(svint16x2_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_s16_s32_x2))) +svint16_t svqshrn_n_s16_s32_x2(svint32x2_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_s8_s16_x2))) +svint8_t svqshrn_n_s8_s16_x2(svint16x2_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_u16_u32_x2))) +svuint16_t svqshrn_n_u16_u32_x2(svuint32x2_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_u8_u16_x2))) +svuint8_t svqshrn_n_u8_u16_x2(svuint16x2_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrun_n_u16_s32_x2))) +svuint16_t svqshrun_n_u16_s32_x2(svint32x2_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrun_n_u8_s16_x2))) +svuint8_t svqshrun_n_u8_s16_x2(svint16x2_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u8_m))) +svuint8_t svsubp_u8_m(svbool_t, svuint8_t, svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u32_m))) +svuint32_t svsubp_u32_m(svbool_t, svuint32_t, svuint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u64_m))) +svuint64_t svsubp_u64_m(svbool_t, svuint64_t, svuint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u16_m))) +svuint16_t svsubp_u16_m(svbool_t, svuint16_t, svuint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s8_m))) +svint8_t svsubp_s8_m(svbool_t, svint8_t, svint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s32_m))) +svint32_t svsubp_s32_m(svbool_t, svint32_t, svint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s64_m))) +svint64_t svsubp_s64_m(svbool_t, svint64_t, svint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s16_m))) +svint16_t svsubp_s16_m(svbool_t, svint16_t, svint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u8_x))) +svuint8_t svsubp_u8_x(svbool_t, svuint8_t, svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u32_x))) +svuint32_t svsubp_u32_x(svbool_t, svuint32_t, svuint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u64_x))) +svuint64_t svsubp_u64_x(svbool_t, svuint64_t, svuint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u16_x))) +svuint16_t svsubp_u16_x(svbool_t, svuint16_t, svuint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s8_x))) +svint8_t svsubp_s8_x(svbool_t, svint8_t, svint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s32_x))) +svint32_t svsubp_s32_x(svbool_t, svint32_t, svint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s64_x))) +svint64_t svsubp_s64_x(svbool_t, svint64_t, svint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s16_x))) +svint16_t svsubp_s16_x(svbool_t, svint16_t, svint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u8_z))) +svuint8_t svsubp_u8_z(svbool_t, svuint8_t, svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u32_z))) +svuint32_t svsubp_u32_z(svbool_t, svuint32_t, svuint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u64_z))) +svuint64_t svsubp_u64_z(svbool_t, svuint64_t, svuint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u16_z))) +svuint16_t svsubp_u16_z(svbool_t, svuint16_t, svuint16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s8_z))) +svint8_t svsubp_s8_z(svbool_t, svint8_t, svint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s32_z))) +svint32_t svsubp_s32_z(svbool_t, svint32_t, svint32_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s64_z))) +svint64_t svsubp_s64_z(svbool_t, svint64_t, svint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s16_z))) +svint16_t svsubp_s16_z(svbool_t, svint16_t, svint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_s32))) +svint32_t svabal(svint32_t, svint16_t, int16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_s64))) +svint64_t svabal(svint64_t, svint32_t, int32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_s16))) +svint16_t svabal(svint16_t, svint8_t, int8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_u32))) +svuint32_t svabal(svuint32_t, svuint16_t, uint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_u64))) +svuint64_t svabal(svuint64_t, svuint32_t, uint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_n_u16))) +svuint16_t svabal(svuint16_t, svuint8_t, uint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_s32))) +svint32_t svabal(svint32_t, svint16_t, svint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_s64))) +svint64_t svabal(svint64_t, svint32_t, svint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_s16))) +svint16_t svabal(svint16_t, svint8_t, svint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_u32))) +svuint32_t svabal(svuint32_t, svuint16_t, svuint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_u64))) +svuint64_t svabal(svuint64_t, svuint32_t, svuint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabal_u16))) +svuint16_t svabal(svuint16_t, svuint8_t, svuint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u8))) +svuint8_t svaddqp(svuint8_t, svuint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u32))) +svuint32_t svaddqp(svuint32_t, svuint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u64))) +svuint64_t svaddqp(svuint64_t, svuint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_u16))) +svuint16_t svaddqp(svuint16_t, svuint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s8))) +svint8_t svaddqp(svint8_t, svint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s32))) +svint32_t svaddqp(svint32_t, svint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s64))) +svint64_t svaddqp(svint64_t, svint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqp_s16))) +svint16_t svaddqp(svint16_t, svint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u8))) +svuint8_t svaddsubp(svuint8_t, svuint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u32))) +svuint32_t svaddsubp(svuint32_t, svuint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u64))) +svuint64_t svaddsubp(svuint64_t, svuint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_u16))) +svuint16_t svaddsubp(svuint16_t, svuint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s8))) +svint8_t svaddsubp(svint8_t, svint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s32))) +svint32_t svaddsubp(svint32_t, svint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s64))) +svint64_t svaddsubp(svint64_t, svint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddsubp_s16))) +svint16_t svaddsubp(svint16_t, svint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f64_s32))) +svfloat64_t svcvtb_f64(svint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f32_s16))) +svfloat32_t svcvtb_f32(svint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f16_s8))) +svfloat16_t svcvtb_f16(svint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f64_u32))) +svfloat64_t svcvtb_f64(svuint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f32_u16))) +svfloat32_t svcvtb_f32(svuint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtb_f16_u8))) +svfloat16_t svcvtb_f16(svuint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_s32_f64_x2))) +svint32_t svcvtn_s32(svfloat64x2_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_s16_f32_x2))) +svint16_t svcvtn_s16(svfloat32x2_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_s8_f16_x2))) +svint8_t svcvtn_s8(svfloat16x2_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_u32_f64_x2))) +svuint32_t svcvtn_u32(svfloat64x2_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_u16_f32_x2))) +svuint16_t svcvtn_u16(svfloat32x2_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_u8_f16_x2))) +svuint8_t svcvtn_u8(svfloat16x2_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f64_s32))) +svfloat64_t svcvtt_f64(svint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f32_s16))) +svfloat32_t svcvtt_f32(svint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f16_s8))) +svfloat16_t svcvtt_f16(svint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f64_u32))) +svfloat64_t svcvtt_f64(svuint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f32_u16))) +svfloat32_t svcvtt_f32(svuint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtt_f16_u8))) +svfloat16_t svcvtt_f16(svuint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_s16_s8))) +svint16_t svdot(svint16_t, svint8_t, int8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_u16_u8))) +svuint16_t svdot(svuint16_t, svuint8_t, uint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_s16_s8))) +svint16_t svdot(svint16_t, svint8_t, svint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_u16_u8))) +svuint16_t svdot(svuint16_t, svuint8_t, svuint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_s16_s8))) +svint16_t svdot_lane(svint16_t, svint8_t, svint8_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_u16_u8))) +svuint16_t svdot_lane(svuint16_t, svuint8_t, svuint8_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqrshrn_n_s8_s16_x2))) +svint8_t svqrshrn_s8(svint16x2_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqrshrn_n_u8_u16_x2))) +svuint8_t svqrshrn_u8(svuint16x2_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqrshrun_n_u8_s16_x2))) +svuint8_t svqrshrun_u8(svint16x2_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_s16_s32_x2))) +svint16_t svqshrn_s16(svint32x2_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_s8_s16_x2))) +svint8_t svqshrn_s8(svint16x2_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_u16_u32_x2))) +svuint16_t svqshrn_u16(svuint32x2_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrn_n_u8_u16_x2))) +svuint8_t svqshrn_u8(svuint16x2_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrun_n_u16_s32_x2))) +svuint16_t svqshrun_u16(svint32x2_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svqshrun_n_u8_s16_x2))) +svuint8_t svqshrun_u8(svint16x2_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u8_m))) +svuint8_t svsubp_m(svbool_t, svuint8_t, svuint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u32_m))) +svuint32_t svsubp_m(svbool_t, svuint32_t, svuint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u64_m))) +svuint64_t svsubp_m(svbool_t, svuint64_t, svuint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u16_m))) +svuint16_t svsubp_m(svbool_t, svuint16_t, svuint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s8_m))) +svint8_t svsubp_m(svbool_t, svint8_t, svint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s32_m))) +svint32_t svsubp_m(svbool_t, svint32_t, svint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s64_m))) +svint64_t svsubp_m(svbool_t, svint64_t, svint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s16_m))) +svint16_t svsubp_m(svbool_t, svint16_t, svint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u8_x))) +svuint8_t svsubp_x(svbool_t, svuint8_t, svuint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u32_x))) +svuint32_t svsubp_x(svbool_t, svuint32_t, svuint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u64_x))) +svuint64_t svsubp_x(svbool_t, svuint64_t, svuint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u16_x))) +svuint16_t svsubp_x(svbool_t, svuint16_t, svuint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s8_x))) +svint8_t svsubp_x(svbool_t, svint8_t, svint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s32_x))) +svint32_t svsubp_x(svbool_t, svint32_t, svint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s64_x))) +svint64_t svsubp_x(svbool_t, svint64_t, svint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s16_x))) +svint16_t svsubp_x(svbool_t, svint16_t, svint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u8_z))) +svuint8_t svsubp_z(svbool_t, svuint8_t, svuint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u32_z))) +svuint32_t svsubp_z(svbool_t, svuint32_t, svuint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u64_z))) +svuint64_t svsubp_z(svbool_t, svuint64_t, svuint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_u16_z))) +svuint16_t svsubp_z(svbool_t, svuint16_t, svuint16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s8_z))) +svint8_t svsubp_z(svbool_t, svint8_t, svint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s32_z))) +svint32_t svsubp_z(svbool_t, svint32_t, svint32_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s64_z))) +svint64_t svsubp_z(svbool_t, svint64_t, svint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsubp_s16_z))) +svint16_t svsubp_z(svbool_t, svint16_t, svint16_t); __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaba_n_s8))) svint8_t svaba_n_s8(svint8_t, svint8_t, int8_t); __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaba_n_s32))) @@ -16581,6 +16965,10 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_x))) svbfloat16_t svsub_x(svbool_t, svbfloat16_t, svbfloat16_t); __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_z))) svbfloat16_t svsub_z(svbool_t, svbfloat16_t, svbfloat16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_bf16))) +svbfloat16_t svmmla_bf16(svbfloat16_t, svbfloat16_t, svbfloat16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_bf16))) +svbfloat16_t svmmla(svbfloat16_t, svbfloat16_t, svbfloat16_t); __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_n_bf16_m))) svbfloat16_t svscale_n_bf16_m(svbool_t, svbfloat16_t, int16_t); __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_n_bf16_x))) @@ -19415,6 +19803,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c64_s64)) svcount_t svwhilelt_c64(int64_t, int64_t, uint64_t); __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c16_s64))) svcount_t svwhilelt_c16(int64_t, int64_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f16))) +svfloat16_t svmmla_f16(svfloat16_t, svfloat16_t, svfloat16_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f16))) +svfloat16_t svmmla(svfloat16_t, svfloat16_t, svfloat16_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_u8_x2))) +svuint8_t svluti6_u8_x2(svuint8x2_t, svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_s8_x2))) +svint8_t svluti6_s8_x2(svint8x2_t, svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_mf8_x2))) +svmfloat8_t svluti6_mf8_x2(svmfloat8x2_t, svuint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_u8_x2))) +svuint8_t svluti6(svuint8x2_t, svuint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_s8_x2))) +svint8_t svluti6(svint8x2_t, svuint8_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_mf8_x2))) +svmfloat8_t svluti6(svmfloat8x2_t, svuint8_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_u16_x2))) +svuint16_t svluti6_lane_u16_x2(svuint16x2_t, svuint8_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_bf16_x2))) +svbfloat16_t svluti6_lane_bf16_x2(svbfloat16x2_t, svuint8_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_f16_x2))) +svfloat16_t svluti6_lane_f16_x2(svfloat16x2_t, svuint8_t, uint64_t); +__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_s16_x2))) +svint16_t svluti6_lane_s16_x2(svint16x2_t, svuint8_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_u16_x2))) +svuint16_t svluti6_lane(svuint16x2_t, svuint8_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_bf16_x2))) +svbfloat16_t svluti6_lane(svbfloat16x2_t, svuint8_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_f16_x2))) +svfloat16_t svluti6_lane(svfloat16x2_t, svuint8_t, uint64_t); +__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti6_lane_s16_x2))) +svint16_t svluti6_lane(svint16x2_t, svuint8_t, uint64_t); __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabd_n_f64_m))) svfloat64_t svabd_n_f64_m(svbool_t, svfloat64_t, float64_t); __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabd_n_f32_m))) diff --git a/lib/include/avx2intrin.h b/lib/include/avx2intrin.h index d3ceb2327ac62769d6fea651ed2edf49327164d1..d262a53a6a402969ed945f97de321f63ce5bf30e 100644 --- a/lib/include/avx2intrin.h +++ b/lib/include/avx2intrin.h @@ -1810,10 +1810,9 @@ _mm256_or_si256(__m256i __a, __m256i __b) /// \param __b /// A 256-bit integer vector. /// \returns A 256-bit integer vector containing the result. -static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_sad_epu8(__m256i __a, __m256i __b) -{ - return __builtin_ia32_psadbw256((__v32qi)__a, (__v32qi)__b); +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_sad_epu8(__m256i __a, __m256i __b) { + return __builtin_ia32_psadbw256((__v32qu)__a, (__v32qu)__b); } /// Shuffles 8-bit integers in the 256-bit integer vector \a __a according @@ -3058,7 +3057,7 @@ _mm256_broadcastsi128_si256(__m128i __X) { /// An immediate 8-bit integer operand, with bits [7:0] specifying the /// source for each element of the result. The position of the mask bit /// corresponds to the index of a copied value. When a mask bit is 0, the -/// element is copied from \a V1; otherwise, it is is copied from \a V2. +/// element is copied from \a V1; otherwise, it is copied from \a V2. /// \returns A 256-bit vector of [8 x i32] containing the result. #define _mm256_blend_epi32(V1, V2, M) \ ((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(V1), \ diff --git a/lib/include/avx512bmmintrin.h b/lib/include/avx512bmmintrin.h new file mode 100644 index 0000000000000000000000000000000000000000..97d6e03e972625c27a9dcbcd473d3d32c41f77fa --- /dev/null +++ b/lib/include/avx512bmmintrin.h @@ -0,0 +1,174 @@ +/*===-------- avx512bmmintrin.h - AVX512BMM intrinsics *------------------=== + * + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===---------------------------------------------------------------------=== + */ + +#ifndef __IMMINTRIN_H +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512BMMINTRIN_H +#define _AVX512BMMINTRIN_H + +/* Define the default attributes for the functions in this file. */ +#define __DEFAULT_FN_ATTRS \ + __attribute__((__always_inline__, __nodebug__, __target__("avx512bmm"), \ + __min_vector_width__(512))) + +#if defined(__cplusplus) && (__cplusplus >= 201103L) +#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr +#else +#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS +#endif + +/// Multiplies two 16x16 bit matrices using OR reduction and ORs the product +/// into a third 16x16 bit matrix (which is also the destination). +/// +/// For the 512-bit ZMM form, each register contains two 16x16 (256-bit) +/// matrices in bits [255:0] and [511:256]. The operation performs: +/// \code{.operation} +/// for i in 0 to 15 +/// for j in 0 to 15 +/// reduction_bit = __C[16*i+j] +/// for k in 0 to 15 +/// reduction_bit |= __A[16*i+k] & __B[16*k+j] +/// end for k +/// dest[16*i+j] = reduction_bit +/// end for j +/// end for i +/// \endcode +/// +/// \headerfile +/// +/// This intrinsic corresponds to the VBMACOR16X16X16 instruction. +/// +/// \param __A +/// A 512-bit vector containing two 16x16 bit matrices (one per 256-bit +/// lane). +/// \param __B +/// A 512-bit vector containing two 16x16 bit matrices (one per 256-bit +/// lane). +/// \param __C +/// A 512-bit accumulator vector containing the initial values to OR with. +/// \returns A 512-bit vector containing the accumulated result for each lane. +/// \note This instruction does not support masking. +static __inline __m512i __DEFAULT_FN_ATTRS_CONSTEXPR +_mm512_bmacor16x16x16(__m512i __A, __m512i __B, __m512i __C) { + return (__m512i)__builtin_ia32_bmacor16x16x16_v32hi( + (__v32hi)__A, (__v32hi)__B, (__v32hi)__C); +} + +/// Multiplies two 16x16 bit matrices using XOR reduction and XORs the product +/// into a third 16x16 bit matrix (which is also the destination). +/// +/// For the 512-bit ZMM form, each register contains two 16x16 (256-bit) +/// matrices in bits [255:0] and [511:256]. The operation performs: +/// \code{.operation} +/// for i in 0 to 15 +/// for j in 0 to 15 +/// reduction_bit = __C[16*i+j] +/// for k in 0 to 15 +/// reduction_bit ^= __A[16*i+k] & __B[16*k+j] +/// end for k +/// dest[16*i+j] = reduction_bit +/// end for j +/// end for i +/// \endcode +/// +/// \headerfile +/// +/// This intrinsic corresponds to the VBMACXOR16X16X16 instruction. +/// +/// \param __A +/// A 512-bit vector containing two 16x16 bit matrices (one per 256-bit +/// lane). +/// \param __B +/// A 512-bit vector containing two 16x16 bit matrices (one per 256-bit +/// lane). +/// \param __C +/// A 512-bit accumulator vector containing the initial values to XOR with. +/// \returns A 512-bit vector containing the accumulated result for each lane. +/// \note This instruction does not support masking. +static __inline __m512i __DEFAULT_FN_ATTRS_CONSTEXPR +_mm512_bmacxor16x16x16(__m512i __A, __m512i __B, __m512i __C) { + return (__m512i)__builtin_ia32_bmacxor16x16x16_v32hi( + (__v32hi)__A, (__v32hi)__B, (__v32hi)__C); +} + +/// Reverses the bits within each byte of the source vector. +/// +/// For each byte in the source, reverses the order of its 8 bits to generate +/// the corresponding destination byte. For example, 0b10110001 becomes +/// 0b10001101. +/// +/// \headerfile +/// +/// This intrinsic corresponds to the VBITREV instruction. +/// +/// \param __A +/// A 512-bit vector of [64 x i8] where each byte will have its bits +/// reversed. +/// \returns A 512-bit vector of [64 x i8] with bit-reversed bytes. +static __inline __m512i __DEFAULT_FN_ATTRS_CONSTEXPR +_mm512_bitrev_epi8(__m512i __A) { + return (__m512i)__builtin_elementwise_bitreverse((__v64qi)__A); +} + +/// Reverses the bits within each byte of the source vector, using a writemask +/// to conditionally select elements. +/// +/// For each byte position, if the corresponding mask bit is 1, the byte from +/// \a A has its bits reversed and stored in the result. If the mask bit is 0, +/// the corresponding byte from \a B is copied to the result (merge masking). +/// +/// \headerfile +/// +/// This intrinsic corresponds to the VBITREV instruction. +/// +/// \param __U +/// A 64-bit mask value where each bit controls one byte (per 8-bit element). +/// A 1 performs bit reversal; a 0 selects the passthrough byte from __B. +/// \param __A +/// A 512-bit vector of [64 x i8] to be bit-reversed. +/// \param __B +/// A 512-bit vector of [64 x i8] providing passthrough values. +/// \returns A 512-bit vector combining bit-reversed and passthrough bytes. +static __inline __m512i __DEFAULT_FN_ATTRS_CONSTEXPR +_mm512_mask_bitrev_epi8(__mmask64 __U, __m512i __A, __m512i __B) { + return (__m512i)__builtin_ia32_selectb_512( + (__mmask64)__U, (__v64qi)_mm512_bitrev_epi8(__A), (__v64qi)__B); +} + +/// Reverses the bits within each byte of the source vector, zeroing elements +/// based on the writemask. +/// +/// For each byte position, if the corresponding mask bit is 1, the byte from +/// \a A has its bits reversed and stored in the result. If the mask bit is 0, +/// the result byte is set to zero (zero masking). +/// +/// \headerfile +/// +/// This intrinsic corresponds to the VBITREV instruction. +/// +/// \param __U +/// A 64-bit mask value where each bit controls one byte (per 8-bit element). +/// A 1 performs bit reversal; a 0 sets the byte to zero. +/// \param __A +/// A 512-bit vector of [64 x i8] to be bit-reversed. +/// \returns A 512-bit vector with bit-reversed or zeroed bytes. +static __inline __m512i __DEFAULT_FN_ATTRS_CONSTEXPR +_mm512_maskz_bitrev_epi8(__mmask64 __U, __m512i __A) { + return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, + (__v64qi)_mm512_bitrev_epi8(__A), + (__v64qi)_mm512_setzero_si512()); +} + +#undef __DEFAULT_FN_ATTRS +#undef __DEFAULT_FN_ATTRS_CONSTEXPR + +#endif diff --git a/lib/include/avx512bmmvlintrin.h b/lib/include/avx512bmmvlintrin.h new file mode 100644 index 0000000000000000000000000000000000000000..b9bad4c844f546e31c08be365ede427c2987fcb8 --- /dev/null +++ b/lib/include/avx512bmmvlintrin.h @@ -0,0 +1,245 @@ +/*===------------- avx512bmmvlintrin.h - BMM intrinsics ------------------=== + * + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===-----------------------------------------------------------------------=== + */ +#ifndef __IMMINTRIN_H +#error \ + "Never use directly; include instead." +#endif + +#ifndef __BMMVLINTRIN_H +#define __BMMVLINTRIN_H + +/* Define the default attributes for the functions in this file. */ +#define __DEFAULT_FN_ATTRS128 \ + __attribute__((__always_inline__, __nodebug__, \ + __target__("avx512bmm,avx512vl"), __min_vector_width__(128))) +#define __DEFAULT_FN_ATTRS256 \ + __attribute__((__always_inline__, __nodebug__, \ + __target__("avx512bmm,avx512vl"), __min_vector_width__(256))) + +#if defined(__cplusplus) && (__cplusplus >= 201103L) +#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr +#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr +#else +#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 +#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 +#endif + +/// Multiplies two 16x16 bit matrices using OR reduction and ORs the product +/// into a third 16x16 bit matrix (which is also the destination). +/// +/// For the 256-bit YMM form, the source registers/memory each contain a single +/// 16x16 (256-bit) matrix in bits [255:0]. The operation performs: +/// \code{.operation} +/// for i in 0 to 15 +/// for j in 0 to 15 +/// reduction_bit = __C[16*i+j] +/// for k in 0 to 15 +/// reduction_bit |= __A[16*i+k] & __B[16*k+j] +/// end for k +/// dest[16*i+j] = reduction_bit +/// end for j +/// end for i +/// \endcode +/// +/// \headerfile +/// +/// This intrinsic corresponds to the VBMACOR16X16X16 instruction. +/// +/// \param __A +/// A 256-bit vector containing a 16x16 bit matrix. +/// \param __B +/// A 256-bit vector containing a 16x16 bit matrix. +/// \param __C +/// A 256-bit accumulator vector containing the initial values to OR with. +/// \returns A 256-bit vector containing the accumulated result. +/// \note This instruction does not support masking. +static __inline __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_bmacor16x16x16(__m256i __A, __m256i __B, __m256i __C) { + return (__m256i)__builtin_ia32_bmacor16x16x16_v16hi( + (__v16hi)__A, (__v16hi)__B, (__v16hi)__C); +} + +/// Multiplies two 16x16 bit matrices using XOR reduction and XORs the product +/// into a third 16x16 bit matrix (which is also the destination). +/// +/// For the 256-bit YMM form, the source registers/memory each contain a single +/// 16x16 (256-bit) matrix in bits [255:0]. The operation performs: +/// \code{.operation} +/// for i in 0 to 15 +/// for j in 0 to 15 +/// reduction_bit = __C[16*i+j] +/// for k in 0 to 15 +/// reduction_bit ^= __A[16*i+k] & __B[16*k+j] +/// end for k +/// dest[16*i+j] = reduction_bit +/// end for j +/// end for i +/// \endcode +/// +/// \headerfile +/// +/// This intrinsic corresponds to the VBMACXOR16X16X16 instruction. +/// +/// \param __A +/// A 256-bit vector containing a 16x16 bit matrix. +/// \param __B +/// A 256-bit vector containing a 16x16 bit matrix. +/// \param __C +/// A 256-bit accumulator vector containing the initial values to XOR with. +/// \returns A 256-bit vector containing the accumulated result. +/// \note This instruction does not support masking. +static __inline __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_bmacxor16x16x16(__m256i __A, __m256i __B, __m256i __C) { + return (__m256i)__builtin_ia32_bmacxor16x16x16_v16hi( + (__v16hi)__A, (__v16hi)__B, (__v16hi)__C); +} + +/// Reverses the bits within each byte of the source vector. +/// +/// For each byte in the source, reverses the order of its 8 bits to generate +/// the corresponding destination byte. For example, 0b10110001 becomes +/// 0b10001101. +/// +/// \headerfile +/// +/// This intrinsic corresponds to the VBITREV instruction. +/// +/// \param __A +/// A 128-bit vector of [16 x i8] where each byte will have its bits +/// reversed. +/// \returns A 128-bit vector of [16 x i8] with bit-reversed bytes. +static __inline __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm128_bitrev_epi8(__m128i __A) { + return (__m128i)__builtin_elementwise_bitreverse((__v16qi)__A); +} + +/// Reverses the bits within each byte of the source vector. +/// +/// For each byte in the source, reverses the order of its 8 bits to generate +/// the corresponding destination byte. For example, 0b10110001 becomes +/// 0b10001101. +/// +/// \headerfile +/// +/// This intrinsic corresponds to the VBITREV instruction. +/// +/// \param __A +/// A 256-bit vector of [32 x i8] where each byte will have its bits +/// reversed. +/// \returns A 256-bit vector of [32 x i8] with bit-reversed bytes. +static __inline __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_bitrev_epi8(__m256i __A) { + return (__m256i)__builtin_elementwise_bitreverse((__v32qi)__A); +} + +/// Reverses the bits within each byte of the source vector, using a writemask +/// to conditionally select elements. +/// +/// For each byte position, if the corresponding mask bit is 1, the byte from +/// \a A has its bits reversed and stored in the result. If the mask bit is 0, +/// the corresponding byte from \a B is copied to the result (merge masking). +/// +/// \headerfile +/// +/// This intrinsic corresponds to the VBITREV instruction. +/// +/// \param __U +/// A 16-bit mask value where each bit controls one byte (per 8-bit element). +/// A 1 performs bit reversal; a 0 selects the passthrough byte from __B. +/// \param __A +/// A 128-bit vector of [16 x i8] to be bit-reversed. +/// \param __B +/// A 128-bit vector of [16 x i8] providing passthrough values. +/// \returns A 128-bit vector combining bit-reversed and passthrough bytes. +static __inline __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm128_mask_bitrev_epi8(__mmask16 __U, __m128i __A, __m128i __B) { + return (__m128i)__builtin_ia32_selectb_128( + (__mmask16)__U, (__v16qi)_mm128_bitrev_epi8(__A), (__v16qi)__B); +} + +/// Reverses the bits within each byte of the source vector, using a writemask +/// to conditionally select elements. +/// +/// For each byte position, if the corresponding mask bit is 1, the byte from +/// \a A has its bits reversed and stored in the result. If the mask bit is 0, +/// the corresponding byte from \a B is copied to the result (merge masking). +/// +/// \headerfile +/// +/// This intrinsic corresponds to the VBITREV instruction. +/// +/// \param __U +/// A 32-bit mask value where each bit controls one byte (per 8-bit element). +/// A 1 performs bit reversal; a 0 selects the passthrough byte from __B. +/// \param __A +/// A 256-bit vector of [32 x i8] to be bit-reversed. +/// \param __B +/// A 256-bit vector of [32 x i8] providing passthrough values. +/// \returns A 256-bit vector combining bit-reversed and passthrough bytes. +static __inline __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_bitrev_epi8(__mmask32 __U, __m256i __A, __m256i __B) { + return (__m256i)__builtin_ia32_selectb_256( + (__mmask32)__U, (__v32qi)_mm256_bitrev_epi8(__A), (__v32qi)__B); +} + +/// Reverses the bits within each byte of the source vector, zeroing elements +/// based on the writemask. +/// +/// For each byte position, if the corresponding mask bit is 1, the byte from +/// \a A has its bits reversed and stored in the result. If the mask bit is 0, +/// the result byte is set to zero (zero masking). +/// +/// \headerfile +/// +/// This intrinsic corresponds to the VBITREV instruction. +/// +/// \param __U +/// A 16-bit mask value where each bit controls one byte (per 8-bit element). +/// A 1 performs bit reversal; a 0 sets the byte to zero. +/// \param __A +/// A 128-bit vector of [16 x i8] to be bit-reversed. +/// \returns A 128-bit vector with bit-reversed or zeroed bytes. +static __inline __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm128_maskz_bitrev_epi8(__mmask16 __U, __m128i __A) { + return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U, + (__v16qi)_mm128_bitrev_epi8(__A), + (__v16qi)_mm_setzero_si128()); +} + +/// Reverses the bits within each byte of the source vector, zeroing elements +/// based on the writemask. +/// +/// For each byte position, if the corresponding mask bit is 1, the byte from +/// \a A has its bits reversed and stored in the result. If the mask bit is 0, +/// the result byte is set to zero (zero masking). +/// +/// \headerfile +/// +/// This intrinsic corresponds to the VBITREV instruction. +/// +/// \param __U +/// A 32-bit mask value where each bit controls one byte (per 8-bit element). +/// A 1 performs bit reversal; a 0 sets the byte to zero. +/// \param __A +/// A 256-bit vector of [32 x i8] to be bit-reversed. +/// \returns A 256-bit vector with bit-reversed or zeroed bytes. +static __inline __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_maskz_bitrev_epi8(__mmask32 __U, __m256i __A) { + return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U, + (__v32qi)_mm256_bitrev_epi8(__A), + (__v32qi)_mm256_setzero_si256()); +} + +#undef __DEFAULT_FN_ATTRS128_CONSTEXPR +#undef __DEFAULT_FN_ATTRS256_CONSTEXPR +#undef __DEFAULT_FN_ATTRS128 +#undef __DEFAULT_FN_ATTRS256 + +#endif diff --git a/lib/include/avx512bwintrin.h b/lib/include/avx512bwintrin.h index cd4663abe7d9e0b1fc4c518a28eb08e4066014cf..5917eeaaa38fe7c6c8881f018b0462a6ca1ac9d2 100644 --- a/lib/include/avx512bwintrin.h +++ b/lib/include/avx512bwintrin.h @@ -1695,57 +1695,49 @@ _mm512_mask_storeu_epi8 (void *__P, __mmask64 __U, __m512i __A) (__mmask64) __U); } -static __inline__ __mmask64 __DEFAULT_FN_ATTRS512 -_mm512_test_epi8_mask (__m512i __A, __m512i __B) -{ +static __inline__ __mmask64 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_test_epi8_mask(__m512i __A, __m512i __B) { return _mm512_cmpneq_epi8_mask (_mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } -static __inline__ __mmask64 __DEFAULT_FN_ATTRS512 -_mm512_mask_test_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B) -{ +static __inline__ __mmask64 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_mask_test_epi8_mask(__mmask64 __U, __m512i __A, __m512i __B) { return _mm512_mask_cmpneq_epi8_mask (__U, _mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } -static __inline__ __mmask32 __DEFAULT_FN_ATTRS512 -_mm512_test_epi16_mask (__m512i __A, __m512i __B) -{ +static __inline__ __mmask32 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_test_epi16_mask(__m512i __A, __m512i __B) { return _mm512_cmpneq_epi16_mask (_mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } -static __inline__ __mmask32 __DEFAULT_FN_ATTRS512 -_mm512_mask_test_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B) -{ +static __inline__ __mmask32 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_mask_test_epi16_mask(__mmask32 __U, __m512i __A, __m512i __B) { return _mm512_mask_cmpneq_epi16_mask (__U, _mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } -static __inline__ __mmask64 __DEFAULT_FN_ATTRS512 -_mm512_testn_epi8_mask (__m512i __A, __m512i __B) -{ +static __inline__ __mmask64 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_testn_epi8_mask(__m512i __A, __m512i __B) { return _mm512_cmpeq_epi8_mask (_mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } -static __inline__ __mmask64 __DEFAULT_FN_ATTRS512 -_mm512_mask_testn_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B) -{ +static __inline__ __mmask64 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_mask_testn_epi8_mask(__mmask64 __U, __m512i __A, __m512i __B) { return _mm512_mask_cmpeq_epi8_mask (__U, _mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } -static __inline__ __mmask32 __DEFAULT_FN_ATTRS512 -_mm512_testn_epi16_mask (__m512i __A, __m512i __B) -{ +static __inline__ __mmask32 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_testn_epi16_mask(__m512i __A, __m512i __B) { return _mm512_cmpeq_epi16_mask (_mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } -static __inline__ __mmask32 __DEFAULT_FN_ATTRS512 -_mm512_mask_testn_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B) -{ +static __inline__ __mmask32 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_mask_testn_epi16_mask(__mmask32 __U, __m512i __A, __m512i __B) { return _mm512_mask_cmpeq_epi16_mask (__U, _mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } @@ -1880,11 +1872,9 @@ _mm512_mask_permutexvar_epi16(__m512i __W, __mmask32 __M, __m512i __A, (__v32hi)_mm512_dbsad_epu8((A), (B), (imm)), \ (__v32hi)_mm512_setzero_si512())) -static __inline__ __m512i __DEFAULT_FN_ATTRS512 -_mm512_sad_epu8 (__m512i __A, __m512i __B) -{ - return (__m512i) __builtin_ia32_psadbw512 ((__v64qi) __A, - (__v64qi) __B); +static __inline__ __m512i + __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_sad_epu8(__m512i __A, __m512i __B) { + return (__m512i)__builtin_ia32_psadbw512((__v64qu)__A, (__v64qu)__B); } #undef __DEFAULT_FN_ATTRS512 diff --git a/lib/include/avx512fintrin.h b/lib/include/avx512fintrin.h index 942ed72686740a059b5aee91be5ea11440f6a8ba..2dfcb7fd159f8c628f5bb221c7bb8b5fffb06101 100644 --- a/lib/include/avx512fintrin.h +++ b/lib/include/avx512fintrin.h @@ -42,11 +42,12 @@ typedef unsigned char __mmask8; typedef unsigned short __mmask16; /* Rounding mode macros. */ -#define _MM_FROUND_TO_NEAREST_INT 0x00 -#define _MM_FROUND_TO_NEG_INF 0x01 -#define _MM_FROUND_TO_POS_INF 0x02 -#define _MM_FROUND_TO_ZERO 0x03 -#define _MM_FROUND_CUR_DIRECTION 0x04 +#define _MM_FROUND_TO_NEAREST_INT 0x00 +#define _MM_FROUND_TO_NEAREST_TIES_EVEN 0x00 +#define _MM_FROUND_TO_NEG_INF 0x01 +#define _MM_FROUND_TO_POS_INF 0x02 +#define _MM_FROUND_TO_ZERO 0x03 +#define _MM_FROUND_CUR_DIRECTION 0x04 /* Constants for integer comparison predicates */ typedef enum { @@ -986,10 +987,8 @@ _mm512_maskz_max_ps(__mmask16 __U, __m512 __A, __m512 __B) { (__v16sf)_mm512_setzero_ps()); } -static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_mask_max_ss(__m128 __W, - __mmask8 __U, - __m128 __A, - __m128 __B) { +static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_max_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) { return (__m128) __builtin_ia32_maxss_round_mask ((__v4sf) __A, (__v4sf) __B, (__v4sf) __W, @@ -997,9 +996,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_mask_max_ss(__m128 __W, _MM_FROUND_CUR_DIRECTION); } -static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_maskz_max_ss(__mmask8 __U, - __m128 __A, - __m128 __B) { +static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_maskz_max_ss(__mmask8 __U, __m128 __A, __m128 __B) { return (__m128) __builtin_ia32_maxss_round_mask ((__v4sf) __A, (__v4sf) __B, (__v4sf) _mm_setzero_ps (), @@ -1025,10 +1023,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_maskz_max_ss(__mmask8 __U, (__v4sf)_mm_setzero_ps(), \ (__mmask8)(U), (int)(R))) -static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_mask_max_sd(__m128d __W, - __mmask8 __U, - __m128d __A, - __m128d __B) { +static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_max_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) { return (__m128d) __builtin_ia32_maxsd_round_mask ((__v2df) __A, (__v2df) __B, (__v2df) __W, @@ -1036,9 +1032,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_mask_max_sd(__m128d __W, _MM_FROUND_CUR_DIRECTION); } -static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_maskz_max_sd(__mmask8 __U, - __m128d __A, - __m128d __B) { +static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_maskz_max_sd(__mmask8 __U, __m128d __A, __m128d __B) { return (__m128d) __builtin_ia32_maxsd_round_mask ((__v2df) __A, (__v2df) __B, (__v2df) _mm_setzero_pd (), @@ -1208,10 +1203,8 @@ _mm512_maskz_min_ps(__mmask16 __U, __m512 __A, __m512 __B) { (__v16sf)_mm512_setzero_ps()); } -static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_mask_min_ss(__m128 __W, - __mmask8 __U, - __m128 __A, - __m128 __B) { +static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_min_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) { return (__m128) __builtin_ia32_minss_round_mask ((__v4sf) __A, (__v4sf) __B, (__v4sf) __W, @@ -1219,9 +1212,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_mask_min_ss(__m128 __W, _MM_FROUND_CUR_DIRECTION); } -static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_maskz_min_ss(__mmask8 __U, - __m128 __A, - __m128 __B) { +static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_maskz_min_ss(__mmask8 __U, __m128 __A, __m128 __B) { return (__m128) __builtin_ia32_minss_round_mask ((__v4sf) __A, (__v4sf) __B, (__v4sf) _mm_setzero_ps (), @@ -1247,10 +1239,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_maskz_min_ss(__mmask8 __U, (__v4sf)_mm_setzero_ps(), \ (__mmask8)(U), (int)(R))) -static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_mask_min_sd(__m128d __W, - __mmask8 __U, - __m128d __A, - __m128d __B) { +static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_min_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) { return (__m128d) __builtin_ia32_minsd_round_mask ((__v2df) __A, (__v2df) __B, (__v2df) __W, @@ -1258,9 +1248,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_mask_min_sd(__m128d __W, _MM_FROUND_CUR_DIRECTION); } -static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_maskz_min_sd(__mmask8 __U, - __m128d __A, - __m128d __B) { +static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_maskz_min_sd(__mmask8 __U, __m128d __A, __m128d __B) { return (__m128d) __builtin_ia32_minsd_round_mask ((__v2df) __A, (__v2df) __B, (__v2df) _mm_setzero_pd (), @@ -8108,68 +8097,60 @@ _mm512_stream_ps (void *__P, __m512 __A) __builtin_nontemporal_store((__v16sf_aligned)__A, (__v16sf_aligned*)__P); } -static __inline__ __m512d __DEFAULT_FN_ATTRS512 -_mm512_mask_compress_pd (__m512d __W, __mmask8 __U, __m512d __A) -{ +static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_mask_compress_pd(__m512d __W, __mmask8 __U, __m512d __A) { return (__m512d) __builtin_ia32_compressdf512_mask ((__v8df) __A, (__v8df) __W, (__mmask8) __U); } -static __inline__ __m512d __DEFAULT_FN_ATTRS512 -_mm512_maskz_compress_pd (__mmask8 __U, __m512d __A) -{ +static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_maskz_compress_pd(__mmask8 __U, __m512d __A) { return (__m512d) __builtin_ia32_compressdf512_mask ((__v8df) __A, (__v8df) _mm512_setzero_pd (), (__mmask8) __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 -_mm512_mask_compress_epi64 (__m512i __W, __mmask8 __U, __m512i __A) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_mask_compress_epi64(__m512i __W, __mmask8 __U, __m512i __A) { return (__m512i) __builtin_ia32_compressdi512_mask ((__v8di) __A, (__v8di) __W, (__mmask8) __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 -_mm512_maskz_compress_epi64 (__mmask8 __U, __m512i __A) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_maskz_compress_epi64(__mmask8 __U, __m512i __A) { return (__m512i) __builtin_ia32_compressdi512_mask ((__v8di) __A, (__v8di) _mm512_setzero_si512 (), (__mmask8) __U); } -static __inline__ __m512 __DEFAULT_FN_ATTRS512 -_mm512_mask_compress_ps (__m512 __W, __mmask16 __U, __m512 __A) -{ +static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_mask_compress_ps(__m512 __W, __mmask16 __U, __m512 __A) { return (__m512) __builtin_ia32_compresssf512_mask ((__v16sf) __A, (__v16sf) __W, (__mmask16) __U); } -static __inline__ __m512 __DEFAULT_FN_ATTRS512 -_mm512_maskz_compress_ps (__mmask16 __U, __m512 __A) -{ +static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_maskz_compress_ps(__mmask16 __U, __m512 __A) { return (__m512) __builtin_ia32_compresssf512_mask ((__v16sf) __A, (__v16sf) _mm512_setzero_ps (), (__mmask16) __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 -_mm512_mask_compress_epi32 (__m512i __W, __mmask16 __U, __m512i __A) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_mask_compress_epi32(__m512i __W, __mmask16 __U, __m512i __A) { return (__m512i) __builtin_ia32_compresssi512_mask ((__v16si) __A, (__v16si) __W, (__mmask16) __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 -_mm512_maskz_compress_epi32 (__mmask16 __U, __m512i __A) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_maskz_compress_epi32(__mmask16 __U, __m512i __A) { return (__m512i) __builtin_ia32_compresssi512_mask ((__v16si) __A, (__v16si) _mm512_setzero_si512 (), @@ -8222,58 +8203,50 @@ _mm512_maskz_compress_epi32 (__mmask16 __U, __m512i __A) /* Bit Test */ -static __inline __mmask16 __DEFAULT_FN_ATTRS512 -_mm512_test_epi32_mask (__m512i __A, __m512i __B) -{ +static __inline __mmask16 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_test_epi32_mask(__m512i __A, __m512i __B) { return _mm512_cmpneq_epi32_mask (_mm512_and_epi32(__A, __B), _mm512_setzero_si512()); } -static __inline__ __mmask16 __DEFAULT_FN_ATTRS512 -_mm512_mask_test_epi32_mask (__mmask16 __U, __m512i __A, __m512i __B) -{ +static __inline__ __mmask16 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_mask_test_epi32_mask(__mmask16 __U, __m512i __A, __m512i __B) { return _mm512_mask_cmpneq_epi32_mask (__U, _mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } -static __inline __mmask8 __DEFAULT_FN_ATTRS512 -_mm512_test_epi64_mask (__m512i __A, __m512i __B) -{ +static __inline __mmask8 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_test_epi64_mask(__m512i __A, __m512i __B) { return _mm512_cmpneq_epi64_mask (_mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS512 -_mm512_mask_test_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_mask_test_epi64_mask(__mmask8 __U, __m512i __A, __m512i __B) { return _mm512_mask_cmpneq_epi64_mask (__U, _mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } -static __inline__ __mmask16 __DEFAULT_FN_ATTRS512 -_mm512_testn_epi32_mask (__m512i __A, __m512i __B) -{ +static __inline__ __mmask16 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_testn_epi32_mask(__m512i __A, __m512i __B) { return _mm512_cmpeq_epi32_mask (_mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } -static __inline__ __mmask16 __DEFAULT_FN_ATTRS512 -_mm512_mask_testn_epi32_mask (__mmask16 __U, __m512i __A, __m512i __B) -{ +static __inline__ __mmask16 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_mask_testn_epi32_mask(__mmask16 __U, __m512i __A, __m512i __B) { return _mm512_mask_cmpeq_epi32_mask (__U, _mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS512 -_mm512_testn_epi64_mask (__m512i __A, __m512i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_testn_epi64_mask(__m512i __A, __m512i __B) { return _mm512_cmpeq_epi64_mask (_mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS512 -_mm512_mask_testn_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS512_CONSTEXPR +_mm512_mask_testn_epi64_mask(__mmask8 __U, __m512i __A, __m512i __B) { return _mm512_mask_cmpeq_epi64_mask (__U, _mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); } @@ -8403,7 +8376,7 @@ _mm_maskz_load_sd (__mmask8 __U, const double* __A) (__v16si)_mm512_shuffle_epi32((A), (I)), \ (__v16si)_mm512_setzero_si512())) -static __inline__ __m512d __DEFAULT_FN_ATTRS512 +static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_mask_expand_pd (__m512d __W, __mmask8 __U, __m512d __A) { return (__m512d) __builtin_ia32_expanddf512_mask ((__v8df) __A, @@ -8411,7 +8384,7 @@ _mm512_mask_expand_pd (__m512d __W, __mmask8 __U, __m512d __A) (__mmask8) __U); } -static __inline__ __m512d __DEFAULT_FN_ATTRS512 +static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_maskz_expand_pd (__mmask8 __U, __m512d __A) { return (__m512d) __builtin_ia32_expanddf512_mask ((__v8df) __A, @@ -8419,7 +8392,7 @@ _mm512_maskz_expand_pd (__mmask8 __U, __m512d __A) (__mmask8) __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_mask_expand_epi64 (__m512i __W, __mmask8 __U, __m512i __A) { return (__m512i) __builtin_ia32_expanddi512_mask ((__v8di) __A, @@ -8427,7 +8400,7 @@ _mm512_mask_expand_epi64 (__m512i __W, __mmask8 __U, __m512i __A) (__mmask8) __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_maskz_expand_epi64 ( __mmask8 __U, __m512i __A) { return (__m512i) __builtin_ia32_expanddi512_mask ((__v8di) __A, @@ -8499,7 +8472,7 @@ _mm512_maskz_expandloadu_epi32(__mmask16 __U, void const *__P) (__mmask16) __U); } -static __inline__ __m512 __DEFAULT_FN_ATTRS512 +static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_mask_expand_ps (__m512 __W, __mmask16 __U, __m512 __A) { return (__m512) __builtin_ia32_expandsf512_mask ((__v16sf) __A, @@ -8507,7 +8480,7 @@ _mm512_mask_expand_ps (__m512 __W, __mmask16 __U, __m512 __A) (__mmask16) __U); } -static __inline__ __m512 __DEFAULT_FN_ATTRS512 +static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_maskz_expand_ps (__mmask16 __U, __m512 __A) { return (__m512) __builtin_ia32_expandsf512_mask ((__v16sf) __A, @@ -8515,7 +8488,7 @@ _mm512_maskz_expand_ps (__mmask16 __U, __m512 __A) (__mmask16) __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_mask_expand_epi32 (__m512i __W, __mmask16 __U, __m512i __A) { return (__m512i) __builtin_ia32_expandsi512_mask ((__v16si) __A, @@ -8523,7 +8496,7 @@ _mm512_mask_expand_epi32 (__m512i __W, __mmask16 __U, __m512i __A) (__mmask16) __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS512 +static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_maskz_expand_epi32 (__mmask16 __U, __m512i __A) { return (__m512i) __builtin_ia32_expandsi512_mask ((__v16si) __A, diff --git a/lib/include/avx512fp16intrin.h b/lib/include/avx512fp16intrin.h index 9a1d1930f66b61b99368aadd3cf5d3465fec419e..4268104c3b619d76e02b8191e01e1b54b0bd4f7b 100644 --- a/lib/include/avx512fp16intrin.h +++ b/lib/include/avx512fp16intrin.h @@ -720,25 +720,22 @@ _mm_maskz_div_sh(__mmask8 __U, __m128h __A, __m128h __B) { (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (__v8hf)_mm_setzero_ph(), \ (__mmask8)(U), (int)(R))) -static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_min_sh(__m128h __A, - __m128h __B) { +static __inline__ __m128h + __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_min_sh(__m128h __A, __m128h __B) { return (__m128h)__builtin_ia32_minsh_round_mask( (__v8hf)__A, (__v8hf)__B, (__v8hf)_mm_setzero_ph(), (__mmask8)-1, _MM_FROUND_CUR_DIRECTION); } -static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_min_sh(__m128h __W, - __mmask8 __U, - __m128h __A, - __m128h __B) { +static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_min_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) { return (__m128h)__builtin_ia32_minsh_round_mask((__v8hf)__A, (__v8hf)__B, (__v8hf)__W, (__mmask8)__U, _MM_FROUND_CUR_DIRECTION); } -static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_min_sh(__mmask8 __U, - __m128h __A, - __m128h __B) { +static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_maskz_min_sh(__mmask8 __U, __m128h __A, __m128h __B) { return (__m128h)__builtin_ia32_minsh_round_mask( (__v8hf)__A, (__v8hf)__B, (__v8hf)_mm_setzero_ph(), (__mmask8)__U, _MM_FROUND_CUR_DIRECTION); @@ -759,25 +756,22 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_min_sh(__mmask8 __U, (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (__v8hf)_mm_setzero_ph(), \ (__mmask8)(U), (int)(R))) -static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_max_sh(__m128h __A, - __m128h __B) { +static __inline__ __m128h + __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_max_sh(__m128h __A, __m128h __B) { return (__m128h)__builtin_ia32_maxsh_round_mask( (__v8hf)__A, (__v8hf)__B, (__v8hf)_mm_setzero_ph(), (__mmask8)-1, _MM_FROUND_CUR_DIRECTION); } -static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_max_sh(__m128h __W, - __mmask8 __U, - __m128h __A, - __m128h __B) { +static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_max_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) { return (__m128h)__builtin_ia32_maxsh_round_mask((__v8hf)__A, (__v8hf)__B, (__v8hf)__W, (__mmask8)__U, _MM_FROUND_CUR_DIRECTION); } -static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_max_sh(__mmask8 __U, - __m128h __A, - __m128h __B) { +static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_maskz_max_sh(__mmask8 __U, __m128h __A, __m128h __B) { return (__m128h)__builtin_ia32_maxsh_round_mask( (__v8hf)__A, (__v8hf)__B, (__v8hf)_mm_setzero_ph(), (__mmask8)__U, _MM_FROUND_CUR_DIRECTION); diff --git a/lib/include/avx512vbmi2intrin.h b/lib/include/avx512vbmi2intrin.h index a24b6e592197cc6307090b5a4c591989a0fd5165..f46a42c2172e2a5a5f2741ba432fdca804cc8c72 100644 --- a/lib/include/avx512vbmi2intrin.h +++ b/lib/include/avx512vbmi2intrin.h @@ -25,33 +25,29 @@ #define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS #endif -static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_mask_compress_epi16(__m512i __S, __mmask32 __U, __m512i __D) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR +_mm512_mask_compress_epi16(__m512i __S, __mmask32 __U, __m512i __D) { return (__m512i) __builtin_ia32_compresshi512_mask ((__v32hi) __D, (__v32hi) __S, __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_maskz_compress_epi16(__mmask32 __U, __m512i __D) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR +_mm512_maskz_compress_epi16(__mmask32 __U, __m512i __D) { return (__m512i) __builtin_ia32_compresshi512_mask ((__v32hi) __D, (__v32hi) _mm512_setzero_si512(), __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_mask_compress_epi8(__m512i __S, __mmask64 __U, __m512i __D) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR +_mm512_mask_compress_epi8(__m512i __S, __mmask64 __U, __m512i __D) { return (__m512i) __builtin_ia32_compressqi512_mask ((__v64qi) __D, (__v64qi) __S, __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_maskz_compress_epi8(__mmask64 __U, __m512i __D) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR +_mm512_maskz_compress_epi8(__mmask64 __U, __m512i __D) { return (__m512i) __builtin_ia32_compressqi512_mask ((__v64qi) __D, (__v64qi) _mm512_setzero_si512(), __U); @@ -71,7 +67,7 @@ _mm512_mask_compressstoreu_epi8(void *__P, __mmask64 __U, __m512i __D) __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS +static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR _mm512_mask_expand_epi16(__m512i __S, __mmask32 __U, __m512i __D) { return (__m512i) __builtin_ia32_expandhi512_mask ((__v32hi) __D, @@ -79,7 +75,7 @@ _mm512_mask_expand_epi16(__m512i __S, __mmask32 __U, __m512i __D) __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS +static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR _mm512_maskz_expand_epi16(__mmask32 __U, __m512i __D) { return (__m512i) __builtin_ia32_expandhi512_mask ((__v32hi) __D, @@ -87,7 +83,7 @@ _mm512_maskz_expand_epi16(__mmask32 __U, __m512i __D) __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS +static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR _mm512_mask_expand_epi8(__m512i __S, __mmask64 __U, __m512i __D) { return (__m512i) __builtin_ia32_expandqi512_mask ((__v64qi) __D, @@ -95,7 +91,7 @@ _mm512_mask_expand_epi8(__m512i __S, __mmask64 __U, __m512i __D) __U); } -static __inline__ __m512i __DEFAULT_FN_ATTRS +static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR _mm512_maskz_expand_epi8(__mmask64 __U, __m512i __D) { return (__m512i) __builtin_ia32_expandqi512_mask ((__v64qi) __D, diff --git a/lib/include/avx512vlbwintrin.h b/lib/include/avx512vlbwintrin.h index fb5d9d4dcc90417ea3c9243f8a3272f1b37217f0..930261d981585b2e0042d97779ae6e44dd9127c4 100644 --- a/lib/include/avx512vlbwintrin.h +++ b/lib/include/avx512vlbwintrin.h @@ -2394,36 +2394,31 @@ _mm256_test_epi8_mask(__m256i __A, __m256i __B) { _mm256_setzero_si256()); } -static __inline__ __mmask32 __DEFAULT_FN_ATTRS256 -_mm256_mask_test_epi8_mask (__mmask32 __U, __m256i __A, __m256i __B) -{ +static __inline__ __mmask32 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_test_epi8_mask(__mmask32 __U, __m256i __A, __m256i __B) { return _mm256_mask_cmpneq_epi8_mask (__U, _mm256_and_si256(__A, __B), _mm256_setzero_si256()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS128 -_mm_test_epi16_mask (__m128i __A, __m128i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_test_epi16_mask(__m128i __A, __m128i __B) { return _mm_cmpneq_epi16_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS128 -_mm_mask_test_epi16_mask (__mmask8 __U, __m128i __A, __m128i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_test_epi16_mask(__mmask8 __U, __m128i __A, __m128i __B) { return _mm_mask_cmpneq_epi16_mask (__U, _mm_and_si128 (__A, __B), _mm_setzero_si128()); } -static __inline__ __mmask16 __DEFAULT_FN_ATTRS256 -_mm256_test_epi16_mask (__m256i __A, __m256i __B) -{ +static __inline__ __mmask16 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_test_epi16_mask(__m256i __A, __m256i __B) { return _mm256_cmpneq_epi16_mask (_mm256_and_si256 (__A, __B), _mm256_setzero_si256 ()); } -static __inline__ __mmask16 __DEFAULT_FN_ATTRS256 -_mm256_mask_test_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B) -{ +static __inline__ __mmask16 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_test_epi16_mask(__mmask16 __U, __m256i __A, __m256i __B) { return _mm256_mask_cmpneq_epi16_mask (__U, _mm256_and_si256(__A, __B), _mm256_setzero_si256()); } @@ -2433,49 +2428,42 @@ _mm_testn_epi8_mask(__m128i __A, __m128i __B) { return _mm_cmpeq_epi8_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128()); } -static __inline__ __mmask16 __DEFAULT_FN_ATTRS128 -_mm_mask_testn_epi8_mask (__mmask16 __U, __m128i __A, __m128i __B) -{ +static __inline__ __mmask16 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_testn_epi8_mask(__mmask16 __U, __m128i __A, __m128i __B) { return _mm_mask_cmpeq_epi8_mask (__U, _mm_and_si128 (__A, __B), _mm_setzero_si128()); } -static __inline__ __mmask32 __DEFAULT_FN_ATTRS256 -_mm256_testn_epi8_mask (__m256i __A, __m256i __B) -{ +static __inline__ __mmask32 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_testn_epi8_mask(__m256i __A, __m256i __B) { return _mm256_cmpeq_epi8_mask (_mm256_and_si256 (__A, __B), _mm256_setzero_si256()); } -static __inline__ __mmask32 __DEFAULT_FN_ATTRS256 -_mm256_mask_testn_epi8_mask (__mmask32 __U, __m256i __A, __m256i __B) -{ +static __inline__ __mmask32 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_testn_epi8_mask(__mmask32 __U, __m256i __A, __m256i __B) { return _mm256_mask_cmpeq_epi8_mask (__U, _mm256_and_si256 (__A, __B), _mm256_setzero_si256()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS128 -_mm_testn_epi16_mask (__m128i __A, __m128i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_testn_epi16_mask(__m128i __A, __m128i __B) { return _mm_cmpeq_epi16_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS128 -_mm_mask_testn_epi16_mask (__mmask8 __U, __m128i __A, __m128i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_testn_epi16_mask(__mmask8 __U, __m128i __A, __m128i __B) { return _mm_mask_cmpeq_epi16_mask (__U, _mm_and_si128(__A, __B), _mm_setzero_si128()); } -static __inline__ __mmask16 __DEFAULT_FN_ATTRS256 -_mm256_testn_epi16_mask (__m256i __A, __m256i __B) -{ +static __inline__ __mmask16 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_testn_epi16_mask(__m256i __A, __m256i __B) { return _mm256_cmpeq_epi16_mask (_mm256_and_si256(__A, __B), _mm256_setzero_si256()); } -static __inline__ __mmask16 __DEFAULT_FN_ATTRS256 -_mm256_mask_testn_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B) -{ +static __inline__ __mmask16 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_testn_epi16_mask(__mmask16 __U, __m256i __A, __m256i __B) { return _mm256_mask_cmpeq_epi16_mask (__U, _mm256_and_si256 (__A, __B), _mm256_setzero_si256()); } diff --git a/lib/include/avx512vlintrin.h b/lib/include/avx512vlintrin.h index ea43046240cc00fc5fc86788fa0314fba482c715..fe80857f171ab8a2cfb32346c314e2181254e4ea 100644 --- a/lib/include/avx512vlintrin.h +++ b/lib/include/avx512vlintrin.h @@ -1530,120 +1530,120 @@ _mm256_mask_blend_epi64(__mmask8 __U, __m256i __A, __m256i __W) { (__v4di) __A); } -static __inline__ __m128d __DEFAULT_FN_ATTRS128 -_mm_mask_compress_pd (__m128d __W, __mmask8 __U, __m128d __A) { +static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_compress_pd(__m128d __W, __mmask8 __U, __m128d __A) { return (__m128d) __builtin_ia32_compressdf128_mask ((__v2df) __A, (__v2df) __W, (__mmask8) __U); } -static __inline__ __m128d __DEFAULT_FN_ATTRS128 -_mm_maskz_compress_pd (__mmask8 __U, __m128d __A) { +static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_maskz_compress_pd(__mmask8 __U, __m128d __A) { return (__m128d) __builtin_ia32_compressdf128_mask ((__v2df) __A, (__v2df) _mm_setzero_pd (), (__mmask8) __U); } -static __inline__ __m256d __DEFAULT_FN_ATTRS256 -_mm256_mask_compress_pd (__m256d __W, __mmask8 __U, __m256d __A) { +static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_compress_pd(__m256d __W, __mmask8 __U, __m256d __A) { return (__m256d) __builtin_ia32_compressdf256_mask ((__v4df) __A, (__v4df) __W, (__mmask8) __U); } -static __inline__ __m256d __DEFAULT_FN_ATTRS256 -_mm256_maskz_compress_pd (__mmask8 __U, __m256d __A) { +static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_maskz_compress_pd(__mmask8 __U, __m256d __A) { return (__m256d) __builtin_ia32_compressdf256_mask ((__v4df) __A, (__v4df) _mm256_setzero_pd (), (__mmask8) __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_mask_compress_epi64 (__m128i __W, __mmask8 __U, __m128i __A) { +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_compress_epi64(__m128i __W, __mmask8 __U, __m128i __A) { return (__m128i) __builtin_ia32_compressdi128_mask ((__v2di) __A, (__v2di) __W, (__mmask8) __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_maskz_compress_epi64 (__mmask8 __U, __m128i __A) { +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_maskz_compress_epi64(__mmask8 __U, __m128i __A) { return (__m128i) __builtin_ia32_compressdi128_mask ((__v2di) __A, (__v2di) _mm_setzero_si128 (), (__mmask8) __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_mask_compress_epi64 (__m256i __W, __mmask8 __U, __m256i __A) { +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_compress_epi64(__m256i __W, __mmask8 __U, __m256i __A) { return (__m256i) __builtin_ia32_compressdi256_mask ((__v4di) __A, (__v4di) __W, (__mmask8) __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_maskz_compress_epi64 (__mmask8 __U, __m256i __A) { +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_maskz_compress_epi64(__mmask8 __U, __m256i __A) { return (__m256i) __builtin_ia32_compressdi256_mask ((__v4di) __A, (__v4di) _mm256_setzero_si256 (), (__mmask8) __U); } -static __inline__ __m128 __DEFAULT_FN_ATTRS128 -_mm_mask_compress_ps (__m128 __W, __mmask8 __U, __m128 __A) { +static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_compress_ps(__m128 __W, __mmask8 __U, __m128 __A) { return (__m128) __builtin_ia32_compresssf128_mask ((__v4sf) __A, (__v4sf) __W, (__mmask8) __U); } -static __inline__ __m128 __DEFAULT_FN_ATTRS128 -_mm_maskz_compress_ps (__mmask8 __U, __m128 __A) { +static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_maskz_compress_ps(__mmask8 __U, __m128 __A) { return (__m128) __builtin_ia32_compresssf128_mask ((__v4sf) __A, (__v4sf) _mm_setzero_ps (), (__mmask8) __U); } -static __inline__ __m256 __DEFAULT_FN_ATTRS256 -_mm256_mask_compress_ps (__m256 __W, __mmask8 __U, __m256 __A) { +static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_compress_ps(__m256 __W, __mmask8 __U, __m256 __A) { return (__m256) __builtin_ia32_compresssf256_mask ((__v8sf) __A, (__v8sf) __W, (__mmask8) __U); } -static __inline__ __m256 __DEFAULT_FN_ATTRS256 -_mm256_maskz_compress_ps (__mmask8 __U, __m256 __A) { +static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_maskz_compress_ps(__mmask8 __U, __m256 __A) { return (__m256) __builtin_ia32_compresssf256_mask ((__v8sf) __A, (__v8sf) _mm256_setzero_ps (), (__mmask8) __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_mask_compress_epi32 (__m128i __W, __mmask8 __U, __m128i __A) { +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_compress_epi32(__m128i __W, __mmask8 __U, __m128i __A) { return (__m128i) __builtin_ia32_compresssi128_mask ((__v4si) __A, (__v4si) __W, (__mmask8) __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_maskz_compress_epi32 (__mmask8 __U, __m128i __A) { +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_maskz_compress_epi32(__mmask8 __U, __m128i __A) { return (__m128i) __builtin_ia32_compresssi128_mask ((__v4si) __A, (__v4si) _mm_setzero_si128 (), (__mmask8) __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_mask_compress_epi32 (__m256i __W, __mmask8 __U, __m256i __A) { +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_compress_epi32(__m256i __W, __mmask8 __U, __m256i __A) { return (__m256i) __builtin_ia32_compresssi256_mask ((__v8si) __A, (__v8si) __W, (__mmask8) __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_maskz_compress_epi32 (__mmask8 __U, __m256i __A) { +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_maskz_compress_epi32(__mmask8 __U, __m256i __A) { return (__m256i) __builtin_ia32_compresssi256_mask ((__v8si) __A, (__v8si) _mm256_setzero_si256 (), @@ -2250,14 +2250,14 @@ _mm256_maskz_div_ps(__mmask8 __U, __m256 __A, __m256 __B) { (__v8sf)_mm256_setzero_ps()); } -static __inline__ __m128d __DEFAULT_FN_ATTRS128 +static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_mask_expand_pd (__m128d __W, __mmask8 __U, __m128d __A) { return (__m128d) __builtin_ia32_expanddf128_mask ((__v2df) __A, (__v2df) __W, (__mmask8) __U); } -static __inline__ __m128d __DEFAULT_FN_ATTRS128 +static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_maskz_expand_pd (__mmask8 __U, __m128d __A) { return (__m128d) __builtin_ia32_expanddf128_mask ((__v2df) __A, (__v2df) @@ -2265,14 +2265,14 @@ _mm_maskz_expand_pd (__mmask8 __U, __m128d __A) { (__mmask8) __U); } -static __inline__ __m256d __DEFAULT_FN_ATTRS256 +static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_mask_expand_pd (__m256d __W, __mmask8 __U, __m256d __A) { return (__m256d) __builtin_ia32_expanddf256_mask ((__v4df) __A, (__v4df) __W, (__mmask8) __U); } -static __inline__ __m256d __DEFAULT_FN_ATTRS256 +static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_maskz_expand_pd (__mmask8 __U, __m256d __A) { return (__m256d) __builtin_ia32_expanddf256_mask ((__v4df) __A, (__v4df) @@ -2280,14 +2280,14 @@ _mm256_maskz_expand_pd (__mmask8 __U, __m256d __A) { (__mmask8) __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_mask_expand_epi64 (__m128i __W, __mmask8 __U, __m128i __A) { return (__m128i) __builtin_ia32_expanddi128_mask ((__v2di) __A, (__v2di) __W, (__mmask8) __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_maskz_expand_epi64 (__mmask8 __U, __m128i __A) { return (__m128i) __builtin_ia32_expanddi128_mask ((__v2di) __A, (__v2di) @@ -2295,14 +2295,14 @@ _mm_maskz_expand_epi64 (__mmask8 __U, __m128i __A) { (__mmask8) __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_mask_expand_epi64 (__m256i __W, __mmask8 __U, __m256i __A) { return (__m256i) __builtin_ia32_expanddi256_mask ((__v4di) __A, (__v4di) __W, (__mmask8) __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_maskz_expand_epi64 (__mmask8 __U, __m256i __A) { return (__m256i) __builtin_ia32_expanddi256_mask ((__v4di) __A, (__v4di) @@ -2445,14 +2445,14 @@ _mm256_maskz_expandloadu_epi32 (__mmask8 __U, void const *__P) { __U); } -static __inline__ __m128 __DEFAULT_FN_ATTRS128 +static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_mask_expand_ps (__m128 __W, __mmask8 __U, __m128 __A) { return (__m128) __builtin_ia32_expandsf128_mask ((__v4sf) __A, (__v4sf) __W, (__mmask8) __U); } -static __inline__ __m128 __DEFAULT_FN_ATTRS128 +static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_maskz_expand_ps (__mmask8 __U, __m128 __A) { return (__m128) __builtin_ia32_expandsf128_mask ((__v4sf) __A, (__v4sf) @@ -2460,14 +2460,14 @@ _mm_maskz_expand_ps (__mmask8 __U, __m128 __A) { (__mmask8) __U); } -static __inline__ __m256 __DEFAULT_FN_ATTRS256 +static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_mask_expand_ps (__m256 __W, __mmask8 __U, __m256 __A) { return (__m256) __builtin_ia32_expandsf256_mask ((__v8sf) __A, (__v8sf) __W, (__mmask8) __U); } -static __inline__ __m256 __DEFAULT_FN_ATTRS256 +static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_maskz_expand_ps (__mmask8 __U, __m256 __A) { return (__m256) __builtin_ia32_expandsf256_mask ((__v8sf) __A, (__v8sf) @@ -2475,14 +2475,14 @@ _mm256_maskz_expand_ps (__mmask8 __U, __m256 __A) { (__mmask8) __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_mask_expand_epi32 (__m128i __W, __mmask8 __U, __m128i __A) { return (__m128i) __builtin_ia32_expandsi128_mask ((__v4si) __A, (__v4si) __W, (__mmask8) __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_maskz_expand_epi32 (__mmask8 __U, __m128i __A) { return (__m128i) __builtin_ia32_expandsi128_mask ((__v4si) __A, (__v4si) @@ -2490,14 +2490,14 @@ _mm_maskz_expand_epi32 (__mmask8 __U, __m128i __A) { (__mmask8) __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_mask_expand_epi32 (__m256i __W, __mmask8 __U, __m256i __A) { return (__m256i) __builtin_ia32_expandsi256_mask ((__v8si) __A, (__v8si) __W, (__mmask8) __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_maskz_expand_epi32 (__mmask8 __U, __m256i __A) { return (__m256i) __builtin_ia32_expandsi256_mask ((__v8si) __A, (__v8si) @@ -5887,110 +5887,94 @@ _mm256_maskz_permutevar_ps(__mmask8 __U, __m256 __A, __m256i __C) { (__v8sf)_mm256_setzero_ps()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS128 -_mm_test_epi32_mask (__m128i __A, __m128i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_test_epi32_mask(__m128i __A, __m128i __B) { return _mm_cmpneq_epi32_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS128 -_mm_mask_test_epi32_mask (__mmask8 __U, __m128i __A, __m128i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_test_epi32_mask(__mmask8 __U, __m128i __A, __m128i __B) { return _mm_mask_cmpneq_epi32_mask (__U, _mm_and_si128 (__A, __B), _mm_setzero_si128()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS256 -_mm256_test_epi32_mask (__m256i __A, __m256i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_test_epi32_mask(__m256i __A, __m256i __B) { return _mm256_cmpneq_epi32_mask (_mm256_and_si256 (__A, __B), _mm256_setzero_si256()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS256 -_mm256_mask_test_epi32_mask (__mmask8 __U, __m256i __A, __m256i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_test_epi32_mask(__mmask8 __U, __m256i __A, __m256i __B) { return _mm256_mask_cmpneq_epi32_mask (__U, _mm256_and_si256 (__A, __B), _mm256_setzero_si256()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS128 -_mm_test_epi64_mask (__m128i __A, __m128i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_test_epi64_mask(__m128i __A, __m128i __B) { return _mm_cmpneq_epi64_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS128 -_mm_mask_test_epi64_mask (__mmask8 __U, __m128i __A, __m128i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_test_epi64_mask(__mmask8 __U, __m128i __A, __m128i __B) { return _mm_mask_cmpneq_epi64_mask (__U, _mm_and_si128 (__A, __B), _mm_setzero_si128()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS256 -_mm256_test_epi64_mask (__m256i __A, __m256i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_test_epi64_mask(__m256i __A, __m256i __B) { return _mm256_cmpneq_epi64_mask (_mm256_and_si256 (__A, __B), _mm256_setzero_si256()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS256 -_mm256_mask_test_epi64_mask (__mmask8 __U, __m256i __A, __m256i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_test_epi64_mask(__mmask8 __U, __m256i __A, __m256i __B) { return _mm256_mask_cmpneq_epi64_mask (__U, _mm256_and_si256 (__A, __B), _mm256_setzero_si256()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS128 -_mm_testn_epi32_mask (__m128i __A, __m128i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_testn_epi32_mask(__m128i __A, __m128i __B) { return _mm_cmpeq_epi32_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS128 -_mm_mask_testn_epi32_mask (__mmask8 __U, __m128i __A, __m128i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_testn_epi32_mask(__mmask8 __U, __m128i __A, __m128i __B) { return _mm_mask_cmpeq_epi32_mask (__U, _mm_and_si128 (__A, __B), _mm_setzero_si128()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS256 -_mm256_testn_epi32_mask (__m256i __A, __m256i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_testn_epi32_mask(__m256i __A, __m256i __B) { return _mm256_cmpeq_epi32_mask (_mm256_and_si256 (__A, __B), _mm256_setzero_si256()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS256 -_mm256_mask_testn_epi32_mask (__mmask8 __U, __m256i __A, __m256i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_testn_epi32_mask(__mmask8 __U, __m256i __A, __m256i __B) { return _mm256_mask_cmpeq_epi32_mask (__U, _mm256_and_si256 (__A, __B), _mm256_setzero_si256()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS128 -_mm_testn_epi64_mask (__m128i __A, __m128i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_testn_epi64_mask(__m128i __A, __m128i __B) { return _mm_cmpeq_epi64_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS128 -_mm_mask_testn_epi64_mask (__mmask8 __U, __m128i __A, __m128i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_testn_epi64_mask(__mmask8 __U, __m128i __A, __m128i __B) { return _mm_mask_cmpeq_epi64_mask (__U, _mm_and_si128 (__A, __B), _mm_setzero_si128()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS256 -_mm256_testn_epi64_mask (__m256i __A, __m256i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_testn_epi64_mask(__m256i __A, __m256i __B) { return _mm256_cmpeq_epi64_mask (_mm256_and_si256 (__A, __B), _mm256_setzero_si256()); } -static __inline__ __mmask8 __DEFAULT_FN_ATTRS256 -_mm256_mask_testn_epi64_mask (__mmask8 __U, __m256i __A, __m256i __B) -{ +static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_testn_epi64_mask(__mmask8 __U, __m256i __A, __m256i __B) { return _mm256_mask_cmpeq_epi64_mask (__U, _mm256_and_si256 (__A, __B), _mm256_setzero_si256()); } diff --git a/lib/include/avx512vlvbmi2intrin.h b/lib/include/avx512vlvbmi2intrin.h index da295d2a12e628add4b2516b2f03725748d155a7..2cf4b2dfc9a8733959aae29674a2b647961fb895 100644 --- a/lib/include/avx512vlvbmi2intrin.h +++ b/lib/include/avx512vlvbmi2intrin.h @@ -32,33 +32,29 @@ #define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 #endif -static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_mask_compress_epi16(__m128i __S, __mmask8 __U, __m128i __D) -{ +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_compress_epi16(__m128i __S, __mmask8 __U, __m128i __D) { return (__m128i) __builtin_ia32_compresshi128_mask ((__v8hi) __D, (__v8hi) __S, __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_maskz_compress_epi16(__mmask8 __U, __m128i __D) -{ +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_maskz_compress_epi16(__mmask8 __U, __m128i __D) { return (__m128i) __builtin_ia32_compresshi128_mask ((__v8hi) __D, (__v8hi) _mm_setzero_si128(), __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_mask_compress_epi8(__m128i __S, __mmask16 __U, __m128i __D) -{ +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_mask_compress_epi8(__m128i __S, __mmask16 __U, __m128i __D) { return (__m128i) __builtin_ia32_compressqi128_mask ((__v16qi) __D, (__v16qi) __S, __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_maskz_compress_epi8(__mmask16 __U, __m128i __D) -{ +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR +_mm_maskz_compress_epi8(__mmask16 __U, __m128i __D) { return (__m128i) __builtin_ia32_compressqi128_mask ((__v16qi) __D, (__v16qi) _mm_setzero_si128(), __U); @@ -78,7 +74,7 @@ _mm_mask_compressstoreu_epi8(void *__P, __mmask16 __U, __m128i __D) __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_mask_expand_epi16(__m128i __S, __mmask8 __U, __m128i __D) { return (__m128i) __builtin_ia32_expandhi128_mask ((__v8hi) __D, @@ -86,7 +82,7 @@ _mm_mask_expand_epi16(__m128i __S, __mmask8 __U, __m128i __D) __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_maskz_expand_epi16(__mmask8 __U, __m128i __D) { return (__m128i) __builtin_ia32_expandhi128_mask ((__v8hi) __D, @@ -94,7 +90,7 @@ _mm_maskz_expand_epi16(__mmask8 __U, __m128i __D) __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_mask_expand_epi8(__m128i __S, __mmask16 __U, __m128i __D) { return (__m128i) __builtin_ia32_expandqi128_mask ((__v16qi) __D, @@ -102,7 +98,7 @@ _mm_mask_expand_epi8(__m128i __S, __mmask16 __U, __m128i __D) __U); } -static __inline__ __m128i __DEFAULT_FN_ATTRS128 +static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_maskz_expand_epi8(__mmask16 __U, __m128i __D) { return (__m128i) __builtin_ia32_expandqi128_mask ((__v16qi) __D, @@ -142,33 +138,29 @@ _mm_maskz_expandloadu_epi8(__mmask16 __U, void const *__P) __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_mask_compress_epi16(__m256i __S, __mmask16 __U, __m256i __D) -{ +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_compress_epi16(__m256i __S, __mmask16 __U, __m256i __D) { return (__m256i) __builtin_ia32_compresshi256_mask ((__v16hi) __D, (__v16hi) __S, __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_maskz_compress_epi16(__mmask16 __U, __m256i __D) -{ +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_maskz_compress_epi16(__mmask16 __U, __m256i __D) { return (__m256i) __builtin_ia32_compresshi256_mask ((__v16hi) __D, (__v16hi) _mm256_setzero_si256(), __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_mask_compress_epi8(__m256i __S, __mmask32 __U, __m256i __D) -{ +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_mask_compress_epi8(__m256i __S, __mmask32 __U, __m256i __D) { return (__m256i) __builtin_ia32_compressqi256_mask ((__v32qi) __D, (__v32qi) __S, __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_maskz_compress_epi8(__mmask32 __U, __m256i __D) -{ +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR +_mm256_maskz_compress_epi8(__mmask32 __U, __m256i __D) { return (__m256i) __builtin_ia32_compressqi256_mask ((__v32qi) __D, (__v32qi) _mm256_setzero_si256(), __U); @@ -188,7 +180,7 @@ _mm256_mask_compressstoreu_epi8(void *__P, __mmask32 __U, __m256i __D) __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_mask_expand_epi16(__m256i __S, __mmask16 __U, __m256i __D) { return (__m256i) __builtin_ia32_expandhi256_mask ((__v16hi) __D, @@ -196,7 +188,7 @@ _mm256_mask_expand_epi16(__m256i __S, __mmask16 __U, __m256i __D) __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_maskz_expand_epi16(__mmask16 __U, __m256i __D) { return (__m256i) __builtin_ia32_expandhi256_mask ((__v16hi) __D, @@ -204,7 +196,7 @@ _mm256_maskz_expand_epi16(__mmask16 __U, __m256i __D) __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_mask_expand_epi8(__m256i __S, __mmask32 __U, __m256i __D) { return (__m256i) __builtin_ia32_expandqi256_mask ((__v32qi) __D, @@ -212,7 +204,7 @@ _mm256_mask_expand_epi8(__m256i __S, __mmask32 __U, __m256i __D) __U); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 +static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_maskz_expand_epi8(__mmask32 __U, __m256i __D) { return (__m256i) __builtin_ia32_expandqi256_mask ((__v32qi) __D, diff --git a/lib/include/avx512vlvnniintrin.h b/lib/include/avx512vlvnniintrin.h index 4b8a199af32e59d4d04ea07e75bb2c94606db79c..8dfcab3f016061a41f90962ec4fdb4fe0c292a01 100644 --- a/lib/include/avx512vlvnniintrin.h +++ b/lib/include/avx512vlvnniintrin.h @@ -15,6 +15,16 @@ #define __AVX512VLVNNIINTRIN_H /* Define the default attributes for the functions in this file. */ +#if defined(__cplusplus) && (__cplusplus >= 201103L) +#define __DEFAULT_FN_ATTRS128 \ + __attribute__((__always_inline__, __nodebug__, \ + __target__("avx512vl,avx512vnni"), \ + __min_vector_width__(128))) constexpr +#define __DEFAULT_FN_ATTRS256 \ + __attribute__((__always_inline__, __nodebug__, \ + __target__("avx512vl,avx512vnni"), \ + __min_vector_width__(256))) constexpr +#else #define __DEFAULT_FN_ATTRS128 \ __attribute__((__always_inline__, __nodebug__, \ __target__("avx512vl,avx512vnni"), \ @@ -23,6 +33,7 @@ __attribute__((__always_inline__, __nodebug__, \ __target__("avx512vl,avx512vnni"), \ __min_vector_width__(256))) +#endif /// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a A with /// corresponding signed 8-bit integers in \a B, producing 4 intermediate signed @@ -180,128 +191,112 @@ ((__m128i)__builtin_ia32_vpdpwssds128((__v4si)(S), (__v8hi)(A), (__v8hi)(B))) static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_mask_dpbusd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) -{ +_mm256_mask_dpbusd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) { return (__m256i)__builtin_ia32_selectd_256(__U, (__v8si)_mm256_dpbusd_epi32(__S, __A, __B), (__v8si)__S); } static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_maskz_dpbusd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) -{ +_mm256_maskz_dpbusd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) { return (__m256i)__builtin_ia32_selectd_256(__U, (__v8si)_mm256_dpbusd_epi32(__S, __A, __B), (__v8si)_mm256_setzero_si256()); } static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_mask_dpbusds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) -{ +_mm256_mask_dpbusds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) { return (__m256i)__builtin_ia32_selectd_256(__U, (__v8si)_mm256_dpbusds_epi32(__S, __A, __B), (__v8si)__S); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_maskz_dpbusds_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) -{ +static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpbusds_epi32( + __mmask8 __U, __m256i __S, __m256i __A, __m256i __B) { return (__m256i)__builtin_ia32_selectd_256(__U, (__v8si)_mm256_dpbusds_epi32(__S, __A, __B), (__v8si)_mm256_setzero_si256()); } static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_mask_dpwssd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) -{ +_mm256_mask_dpwssd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) { return (__m256i)__builtin_ia32_selectd_256(__U, (__v8si)_mm256_dpwssd_epi32(__S, __A, __B), (__v8si)__S); } static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_maskz_dpwssd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) -{ +_mm256_maskz_dpwssd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) { return (__m256i)__builtin_ia32_selectd_256(__U, (__v8si)_mm256_dpwssd_epi32(__S, __A, __B), (__v8si)_mm256_setzero_si256()); } static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_mask_dpwssds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) -{ +_mm256_mask_dpwssds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) { return (__m256i)__builtin_ia32_selectd_256(__U, (__v8si)_mm256_dpwssds_epi32(__S, __A, __B), (__v8si)__S); } -static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_maskz_dpwssds_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) -{ +static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpwssds_epi32( + __mmask8 __U, __m256i __S, __m256i __A, __m256i __B) { return (__m256i)__builtin_ia32_selectd_256(__U, (__v8si)_mm256_dpwssds_epi32(__S, __A, __B), (__v8si)_mm256_setzero_si256()); } static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_mask_dpbusd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) -{ +_mm_mask_dpbusd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) { return (__m128i)__builtin_ia32_selectd_128(__U, (__v4si)_mm_dpbusd_epi32(__S, __A, __B), (__v4si)__S); } static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_maskz_dpbusd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) -{ +_mm_maskz_dpbusd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) { return (__m128i)__builtin_ia32_selectd_128(__U, (__v4si)_mm_dpbusd_epi32(__S, __A, __B), (__v4si)_mm_setzero_si128()); } static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_mask_dpbusds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) -{ +_mm_mask_dpbusds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) { return (__m128i)__builtin_ia32_selectd_128(__U, (__v4si)_mm_dpbusds_epi32(__S, __A, __B), (__v4si)__S); } static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_maskz_dpbusds_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) -{ +_mm_maskz_dpbusds_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) { return (__m128i)__builtin_ia32_selectd_128(__U, (__v4si)_mm_dpbusds_epi32(__S, __A, __B), (__v4si)_mm_setzero_si128()); } static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_mask_dpwssd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) -{ +_mm_mask_dpwssd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) { return (__m128i)__builtin_ia32_selectd_128(__U, (__v4si)_mm_dpwssd_epi32(__S, __A, __B), (__v4si)__S); } static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_maskz_dpwssd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) -{ +_mm_maskz_dpwssd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) { return (__m128i)__builtin_ia32_selectd_128(__U, (__v4si)_mm_dpwssd_epi32(__S, __A, __B), (__v4si)_mm_setzero_si128()); } static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_mask_dpwssds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) -{ +_mm_mask_dpwssds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) { return (__m128i)__builtin_ia32_selectd_128(__U, (__v4si)_mm_dpwssds_epi32(__S, __A, __B), (__v4si)__S); } static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_maskz_dpwssds_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) -{ +_mm_maskz_dpwssds_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) { return (__m128i)__builtin_ia32_selectd_128(__U, (__v4si)_mm_dpwssds_epi32(__S, __A, __B), (__v4si)_mm_setzero_si128()); diff --git a/lib/include/avx512vnniintrin.h b/lib/include/avx512vnniintrin.h index 2ce88efe4a04f2b6f4b790b2183d2f598f22e736..1e245292ab8699767195571e6cd583b27d6856c7 100644 --- a/lib/include/avx512vnniintrin.h +++ b/lib/include/avx512vnniintrin.h @@ -15,102 +15,99 @@ #define __AVX512VNNIINTRIN_H /* Define the default attributes for the functions in this file. */ +#if defined(__cplusplus) && (__cplusplus >= 201103L) +#define __DEFAULT_FN_ATTRS \ + __attribute__((__always_inline__, __nodebug__, __target__("avx512vnni"), \ + __min_vector_width__(512))) constexpr +#else #define __DEFAULT_FN_ATTRS \ __attribute__((__always_inline__, __nodebug__, __target__("avx512vnni"), \ __min_vector_width__(512))) +#endif -static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_dpbusd_epi32(__m512i __S, __m512i __A, __m512i __B) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbusd_epi32(__m512i __S, + __m512i __A, + __m512i __B) { return (__m512i)__builtin_ia32_vpdpbusd512((__v16si)__S, (__v64qu)__A, (__v64qi)__B); } static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_mask_dpbusd_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) -{ +_mm512_mask_dpbusd_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) { return (__m512i)__builtin_ia32_selectd_512(__U, (__v16si)_mm512_dpbusd_epi32(__S, __A, __B), (__v16si)__S); } -static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_maskz_dpbusd_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbusd_epi32( + __mmask16 __U, __m512i __S, __m512i __A, __m512i __B) { return (__m512i)__builtin_ia32_selectd_512(__U, (__v16si)_mm512_dpbusd_epi32(__S, __A, __B), (__v16si)_mm512_setzero_si512()); } -static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_dpbusds_epi32(__m512i __S, __m512i __A, __m512i __B) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbusds_epi32(__m512i __S, + __m512i __A, + __m512i __B) { return (__m512i)__builtin_ia32_vpdpbusds512((__v16si)__S, (__v64qu)__A, (__v64qi)__B); } -static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_mask_dpbusds_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpbusds_epi32( + __m512i __S, __mmask16 __U, __m512i __A, __m512i __B) { return (__m512i)__builtin_ia32_selectd_512(__U, (__v16si)_mm512_dpbusds_epi32(__S, __A, __B), (__v16si)__S); } -static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_maskz_dpbusds_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbusds_epi32( + __mmask16 __U, __m512i __S, __m512i __A, __m512i __B) { return (__m512i)__builtin_ia32_selectd_512(__U, (__v16si)_mm512_dpbusds_epi32(__S, __A, __B), (__v16si)_mm512_setzero_si512()); } -static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_dpwssd_epi32(__m512i __S, __m512i __A, __m512i __B) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwssd_epi32(__m512i __S, + __m512i __A, + __m512i __B) { return (__m512i)__builtin_ia32_vpdpwssd512((__v16si)__S, (__v32hi)__A, (__v32hi)__B); } static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_mask_dpwssd_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) -{ +_mm512_mask_dpwssd_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) { return (__m512i)__builtin_ia32_selectd_512(__U, (__v16si)_mm512_dpwssd_epi32(__S, __A, __B), (__v16si)__S); } -static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_maskz_dpwssd_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwssd_epi32( + __mmask16 __U, __m512i __S, __m512i __A, __m512i __B) { return (__m512i)__builtin_ia32_selectd_512(__U, (__v16si)_mm512_dpwssd_epi32(__S, __A, __B), (__v16si)_mm512_setzero_si512()); } -static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_dpwssds_epi32(__m512i __S, __m512i __A, __m512i __B) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwssds_epi32(__m512i __S, + __m512i __A, + __m512i __B) { return (__m512i)__builtin_ia32_vpdpwssds512((__v16si)__S, (__v32hi)__A, (__v32hi)__B); } -static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_mask_dpwssds_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwssds_epi32( + __m512i __S, __mmask16 __U, __m512i __A, __m512i __B) { return (__m512i)__builtin_ia32_selectd_512(__U, (__v16si)_mm512_dpwssds_epi32(__S, __A, __B), (__v16si)__S); } -static __inline__ __m512i __DEFAULT_FN_ATTRS -_mm512_maskz_dpwssds_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) -{ +static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwssds_epi32( + __mmask16 __U, __m512i __S, __m512i __A, __m512i __B) { return (__m512i)__builtin_ia32_selectd_512(__U, (__v16si)_mm512_dpwssds_epi32(__S, __A, __B), (__v16si)_mm512_setzero_si512()); } #undef __DEFAULT_FN_ATTRS - #endif diff --git a/lib/include/avx512vp2intersectintrin.h b/lib/include/avx512vp2intersectintrin.h index 7d999960a574bdf844b85b81c7b0d937680bb185..f98e84031fc35b1fd892d807a0c47f9455f95e85 100644 --- a/lib/include/avx512vp2intersectintrin.h +++ b/lib/include/avx512vp2intersectintrin.h @@ -22,7 +22,7 @@ *===-----------------------------------------------------------------------=== */ #ifndef __IMMINTRIN_H -#error "Never use directly; include instead." +#error "Never use directly; include instead." #endif #ifndef _AVX512VP2INTERSECT_H diff --git a/lib/include/avxvnniintrin.h b/lib/include/avxvnniintrin.h index 1d2e8c906effc63b979f5899efcc678a714e213c..c92790b69ffbe26a62da03b0f8be7fd5823f91b2 100644 --- a/lib/include/avxvnniintrin.h +++ b/lib/include/avxvnniintrin.h @@ -40,8 +40,21 @@ /* Intrinsics with _avx_ prefix are for compatibility with msvc. */ /* Define the default attributes for the functions in this file. */ -#define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), __min_vector_width__(256))) -#define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), __min_vector_width__(128))) +#if defined(__cplusplus) && (__cplusplus >= 201103L) +#define __DEFAULT_FN_ATTRS256 \ + __attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), \ + __min_vector_width__(256))) constexpr +#define __DEFAULT_FN_ATTRS128 \ + __attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), \ + __min_vector_width__(128))) constexpr +#else +#define __DEFAULT_FN_ATTRS256 \ + __attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), \ + __min_vector_width__(256))) +#define __DEFAULT_FN_ATTRS128 \ + __attribute__((__always_inline__, __nodebug__, __target__("avxvnni"), \ + __min_vector_width__(128))) +#endif /// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with /// corresponding signed 8-bit integers in \a __B, producing 4 intermediate signed @@ -61,8 +74,7 @@ /// DST[MAX:256] := 0 /// \endcode static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_dpbusd_avx_epi32(__m256i __S, __m256i __A, __m256i __B) -{ +_mm256_dpbusd_avx_epi32(__m256i __S, __m256i __A, __m256i __B) { return (__m256i)__builtin_ia32_vpdpbusd256((__v8si)__S, (__v32qu)__A, (__v32qi)__B); } @@ -85,8 +97,7 @@ _mm256_dpbusd_avx_epi32(__m256i __S, __m256i __A, __m256i __B) /// DST[MAX:256] := 0 /// \endcode static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_dpbusds_avx_epi32(__m256i __S, __m256i __A, __m256i __B) -{ +_mm256_dpbusds_avx_epi32(__m256i __S, __m256i __A, __m256i __B) { return (__m256i)__builtin_ia32_vpdpbusds256((__v8si)__S, (__v32qu)__A, (__v32qi)__B); } @@ -107,8 +118,7 @@ _mm256_dpbusds_avx_epi32(__m256i __S, __m256i __A, __m256i __B) /// DST[MAX:256] := 0 /// \endcode static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_dpwssd_avx_epi32(__m256i __S, __m256i __A, __m256i __B) -{ +_mm256_dpwssd_avx_epi32(__m256i __S, __m256i __A, __m256i __B) { return (__m256i)__builtin_ia32_vpdpwssd256((__v8si)__S, (__v16hi)__A, (__v16hi)__B); } @@ -129,8 +139,7 @@ _mm256_dpwssd_avx_epi32(__m256i __S, __m256i __A, __m256i __B) /// DST[MAX:256] := 0 /// \endcode static __inline__ __m256i __DEFAULT_FN_ATTRS256 -_mm256_dpwssds_avx_epi32(__m256i __S, __m256i __A, __m256i __B) -{ +_mm256_dpwssds_avx_epi32(__m256i __S, __m256i __A, __m256i __B) { return (__m256i)__builtin_ia32_vpdpwssds256((__v8si)__S, (__v16hi)__A, (__v16hi)__B); } @@ -153,8 +162,7 @@ _mm256_dpwssds_avx_epi32(__m256i __S, __m256i __A, __m256i __B) /// DST[MAX:128] := 0 /// \endcode static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_dpbusd_avx_epi32(__m128i __S, __m128i __A, __m128i __B) -{ +_mm_dpbusd_avx_epi32(__m128i __S, __m128i __A, __m128i __B) { return (__m128i)__builtin_ia32_vpdpbusd128((__v4si)__S, (__v16qu)__A, (__v16qi)__B); } @@ -177,8 +185,7 @@ _mm_dpbusd_avx_epi32(__m128i __S, __m128i __A, __m128i __B) /// DST[MAX:128] := 0 /// \endcode static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_dpbusds_avx_epi32(__m128i __S, __m128i __A, __m128i __B) -{ +_mm_dpbusds_avx_epi32(__m128i __S, __m128i __A, __m128i __B) { return (__m128i)__builtin_ia32_vpdpbusds128((__v4si)__S, (__v16qu)__A, (__v16qi)__B); } @@ -199,8 +206,7 @@ _mm_dpbusds_avx_epi32(__m128i __S, __m128i __A, __m128i __B) /// DST[MAX:128] := 0 /// \endcode static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_dpwssd_avx_epi32(__m128i __S, __m128i __A, __m128i __B) -{ +_mm_dpwssd_avx_epi32(__m128i __S, __m128i __A, __m128i __B) { return (__m128i)__builtin_ia32_vpdpwssd128((__v4si)__S, (__v8hi)__A, (__v8hi)__B); } @@ -221,8 +227,7 @@ _mm_dpwssd_avx_epi32(__m128i __S, __m128i __A, __m128i __B) /// DST[MAX:128] := 0 /// \endcode static __inline__ __m128i __DEFAULT_FN_ATTRS128 -_mm_dpwssds_avx_epi32(__m128i __S, __m128i __A, __m128i __B) -{ +_mm_dpwssds_avx_epi32(__m128i __S, __m128i __A, __m128i __B) { return (__m128i)__builtin_ia32_vpdpwssds128((__v4si)__S, (__v8hi)__A, (__v8hi)__B); } diff --git a/lib/include/crc32intrin.h b/lib/include/crc32intrin.h index a0bd99d1b57254c13cba60fb8c16a0dd8ecc76ba..9bf8b2edb6e9fb9a2262de9cf8b1cf3ce62ecdc8 100644 --- a/lib/include/crc32intrin.h +++ b/lib/include/crc32intrin.h @@ -10,8 +10,14 @@ #ifndef __CRC32INTRIN_H #define __CRC32INTRIN_H +/// We only declare crc32 as a constexpr if we are compiling C++ code +#if defined(__cplusplus) && (__cplusplus >= 201103L) +#define __DEFAULT_FN_ATTRS \ + __attribute__((__always_inline__, __nodebug__, __target__("crc32"))) constexpr +#else #define __DEFAULT_FN_ATTRS \ __attribute__((__always_inline__, __nodebug__, __target__("crc32"))) +#endif /// Adds the unsigned integer operand to the CRC-32C checksum of the /// unsigned char operand. @@ -28,8 +34,7 @@ /// \returns The result of adding operand \a __C to the CRC-32C checksum of /// operand \a __D. static __inline__ unsigned int __DEFAULT_FN_ATTRS -_mm_crc32_u8(unsigned int __C, unsigned char __D) -{ +_mm_crc32_u8(unsigned int __C, unsigned char __D) { return __builtin_ia32_crc32qi(__C, __D); } @@ -48,8 +53,7 @@ _mm_crc32_u8(unsigned int __C, unsigned char __D) /// \returns The result of adding operand \a __C to the CRC-32C checksum of /// operand \a __D. static __inline__ unsigned int __DEFAULT_FN_ATTRS -_mm_crc32_u16(unsigned int __C, unsigned short __D) -{ +_mm_crc32_u16(unsigned int __C, unsigned short __D) { return __builtin_ia32_crc32hi(__C, __D); } @@ -68,8 +72,7 @@ _mm_crc32_u16(unsigned int __C, unsigned short __D) /// \returns The result of adding operand \a __C to the CRC-32C checksum of /// operand \a __D. static __inline__ unsigned int __DEFAULT_FN_ATTRS -_mm_crc32_u32(unsigned int __C, unsigned int __D) -{ +_mm_crc32_u32(unsigned int __C, unsigned int __D) { return __builtin_ia32_crc32si(__C, __D); } @@ -89,8 +92,7 @@ _mm_crc32_u32(unsigned int __C, unsigned int __D) /// \returns The result of adding operand \a __C to the CRC-32C checksum of /// operand \a __D. static __inline__ unsigned long long __DEFAULT_FN_ATTRS -_mm_crc32_u64(unsigned long long __C, unsigned long long __D) -{ +_mm_crc32_u64(unsigned long long __C, unsigned long long __D) { return __builtin_ia32_crc32di(__C, __D); } #endif /* __x86_64__ */ diff --git a/lib/include/emmintrin.h b/lib/include/emmintrin.h index 61b35e97314fd4976448ccdd19800f4d1455f23e..3f039edf87e8106b6db730b71a750561dc458fa6 100644 --- a/lib/include/emmintrin.h +++ b/lib/include/emmintrin.h @@ -279,8 +279,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sqrt_pd(__m128d __a) { /// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the /// minimum value between both operands. The upper 64 bits are copied from /// the upper 64 bits of the first source operand. -static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_min_sd(__m128d __a, - __m128d __b) { +static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_min_sd(__m128d __a, + __m128d __b) { return __builtin_ia32_minsd((__v2df)__a, (__v2df)__b); } @@ -325,8 +325,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_min_pd(__m128d __a, /// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the /// maximum value between both operands. The upper 64 bits are copied from /// the upper 64 bits of the first source operand. -static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_max_sd(__m128d __a, - __m128d __b) { +static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_max_sd(__m128d __a, + __m128d __b) { return __builtin_ia32_maxsd((__v2df)__a, (__v2df)__b); } @@ -1343,7 +1343,7 @@ _mm_cvtepi32_pd(__m128i __a) { /// \returns A 128-bit vector of [4 x i32] whose lower 64 bits contain the /// converted values. The upper 64 bits are set to zero. static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtpd_epi32(__m128d __a) { - return __builtin_ia32_cvtpd2dq((__v2df)__a); + return (__m128i)__builtin_ia32_cvtpd2dq((__v2df)__a); } /// Converts the low-order element of a 128-bit vector of [2 x double] @@ -2481,9 +2481,9 @@ _mm_mul_epu32(__m128i __a, __m128i __b) { /// A 128-bit integer vector containing one of the source operands. /// \returns A [2 x i64] vector containing the sums of the sets of absolute /// differences between both operands. -static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sad_epu8(__m128i __a, - __m128i __b) { - return __builtin_ia32_psadbw128((__v16qi)__a, (__v16qi)__b); +static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR +_mm_sad_epu8(__m128i __a, __m128i __b) { + return __builtin_ia32_psadbw128((__v16qu)__a, (__v16qu)__b); } /// Subtracts the corresponding 8-bit integer values in the operands. diff --git a/lib/include/endian.h b/lib/include/endian.h new file mode 100644 index 0000000000000000000000000000000000000000..aa30d8fc5c49b19d23b897f924d7b349892d5b0c --- /dev/null +++ b/lib/include/endian.h @@ -0,0 +1,91 @@ +//===-- Definition of macros from endian.h --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLANG_ENDIAN_H +#define __CLANG_ENDIAN_H + +// If the system has an endian.h, let's use that instead. +#if __has_include_next() +#include_next +#else + +#include + +// Implementation taken from llvm libc endian-macros.h. +#ifdef __cplusplus +#define __CLANG_ENDIAN_CAST(cast, type, value) (cast(value)) +#else +#define __CLANG_ENDIAN_CAST(cast, type, value) ((type)(value)) +#endif + +#define LITTLE_ENDIAN __ORDER_LITTLE_ENDIAN__ +#define BIG_ENDIAN __ORDER_BIG_ENDIAN__ +#define PDP_ENDIAN __ORDER_PDP_ENDIAN__ +#define BYTE_ORDER __BYTE_ORDER__ + +// Define some compatibility macros if they are not defined. +#ifndef __BYTE_ORDER +#define __BYTE_ORDER BYTE_ORDER +#endif +#ifndef __LITTLE_ENDIAN +#define __LITTLE_ENDIAN LITTLE_ENDIAN +#endif +#ifndef __BIG_ENDIAN +#define __BIG_ENDIAN BIG_ENDIAN +#endif +#ifndef __PDP_ENDIAN +#define __PDP_ENDIAN PDP_ENDIAN +#endif + +#if BYTE_ORDER == LITTLE_ENDIAN + +#define htobe16(x) \ + __builtin_bswap16(__CLANG_ENDIAN_CAST(static_cast, uint16_t, x)) +#define htobe32(x) \ + __builtin_bswap32(__CLANG_ENDIAN_CAST(static_cast, uint32_t, x)) +#define htobe64(x) \ + __builtin_bswap64(__CLANG_ENDIAN_CAST(static_cast, uint64_t, x)) +#define htole16(x) __CLANG_ENDIAN_CAST(static_cast, uint16_t, x) +#define htole32(x) __CLANG_ENDIAN_CAST(static_cast, uint32_t, x) +#define htole64(x) __CLANG_ENDIAN_CAST(static_cast, uint64_t, x) +#define be16toh(x) \ + __builtin_bswap16(__CLANG_ENDIAN_CAST(static_cast, uint16_t, x)) +#define be32toh(x) \ + __builtin_bswap32(__CLANG_ENDIAN_CAST(static_cast, uint32_t, x)) +#define be64toh(x) \ + __builtin_bswap64(__CLANG_ENDIAN_CAST(static_cast, uint64_t, x)) +#define le16toh(x) __CLANG_ENDIAN_CAST(static_cast, uint16_t, x) +#define le32toh(x) __CLANG_ENDIAN_CAST(static_cast, uint32_t, x) +#define le64toh(x) __CLANG_ENDIAN_CAST(static_cast, uint64_t, x) + +#elif BYTE_ORDER == BIG_ENDIAN + +#define htobe16(x) __CLANG_ENDIAN_CAST(static_cast, uint16_t, x) +#define htobe32(x) __CLANG_ENDIAN_CAST(static_cast, uint32_t, x) +#define htobe64(x) __CLANG_ENDIAN_CAST(static_cast, uint64_t, x) +#define htole16(x) \ + __builtin_bswap16(__CLANG_ENDIAN_CAST(static_cast, uint16_t, x)) +#define htole32(x) \ + __builtin_bswap32(__CLANG_ENDIAN_CAST(static_cast, uint32_t, x)) +#define htole64(x) \ + __builtin_bswap64(__CLANG_ENDIAN_CAST(static_cast, uint64_t, x)) +#define be16toh(x) __CLANG_ENDIAN_CAST(static_cast, uint16_t, x) +#define be32toh(x) __CLANG_ENDIAN_CAST(static_cast, uint32_t, x) +#define be64toh(x) __CLANG_ENDIAN_CAST(static_cast, uint64_t, x) +#define le16toh(x) \ + __builtin_bswap16(__CLANG_ENDIAN_CAST(static_cast, uint16_t, x)) +#define le32toh(x) \ + __builtin_bswap32(__CLANG_ENDIAN_CAST(static_cast, uint32_t, x)) +#define le64toh(x) \ + __builtin_bswap64(__CLANG_ENDIAN_CAST(static_cast, uint64_t, x)) + +#else +#error "Unsupported endianness" +#endif +#endif // __has_include_next +#endif // __CLANG_ENDIAN_H diff --git a/lib/include/hvx_hexagon_protos.h b/lib/include/hvx_hexagon_protos.h index 981fbd1a12f7edbbc87874a912a4f2c8cc9ed530..2f4110401d011cc29b676432c5057ecc8768f014 100644 --- a/lib/include/hvx_hexagon_protos.h +++ b/lib/include/hvx_hexagon_protos.h @@ -4577,8 +4577,9 @@ #if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__ /* ========================================================================== Assembly Syntax: Vdd32.sf=vadd(Vu32.bf,Vv32.bf) - C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vadd_VbfVbf(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vadd_VbfVbf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Wsf_vadd_VbfVbf(Vu, Vv) \ @@ -4636,8 +4637,9 @@ #if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__ /* ========================================================================== Assembly Syntax: Vd32.bf=vcvt(Vu32.sf,Vv32.sf) - C Intrinsic Prototype: HVX_Vector Q6_Vbf_vcvt_VsfVsf(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VX Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_Vector Q6_Vbf_vcvt_VsfVsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Vbf_vcvt_VsfVsf(Vu, Vv) \ @@ -4647,8 +4649,9 @@ #if __HVX_ARCH__ >= 73 /* ========================================================================== Assembly Syntax: Qd4=vcmp.gt(Vu32.bf,Vv32.bf) - C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VbfVbf(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VA Execution Slots: SLOT0123 + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VbfVbf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_Q_vcmp_gt_VbfVbf(Vu, Vv) \ @@ -4659,9 +4662,9 @@ #if __HVX_ARCH__ >= 73 /* ========================================================================== Assembly Syntax: Qx4&=vcmp.gt(Vu32.bf,Vv32.bf) - C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVbfVbf(HVX_VectorPred - Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution - Slots: SLOT0123 + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVbfVbf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_Q_vcmp_gtand_QVbfVbf(Qx, Vu, Vv) \ @@ -4675,9 +4678,9 @@ #if __HVX_ARCH__ >= 73 /* ========================================================================== Assembly Syntax: Qx4|=vcmp.gt(Vu32.bf,Vv32.bf) - C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVbfVbf(HVX_VectorPred - Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution - Slots: SLOT0123 + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVbfVbf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_Q_vcmp_gtor_QVbfVbf(Qx, Vu, Vv) \ @@ -4691,9 +4694,9 @@ #if __HVX_ARCH__ >= 73 /* ========================================================================== Assembly Syntax: Qx4^=vcmp.gt(Vu32.bf,Vv32.bf) - C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVbfVbf(HVX_VectorPred - Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution - Slots: SLOT0123 + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVbfVbf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_Q_vcmp_gtxacc_QVbfVbf(Qx, Vu, Vv) \ @@ -4707,8 +4710,9 @@ #if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__ /* ========================================================================== Assembly Syntax: Vd32.bf=vmax(Vu32.bf,Vv32.bf) - C Intrinsic Prototype: HVX_Vector Q6_Vbf_vmax_VbfVbf(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VX_LATE Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_Vector Q6_Vbf_vmax_VbfVbf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_LATE + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Vbf_vmax_VbfVbf(Vu, Vv) \ @@ -4718,8 +4722,9 @@ #if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__ /* ========================================================================== Assembly Syntax: Vd32.bf=vmin(Vu32.bf,Vv32.bf) - C Intrinsic Prototype: HVX_Vector Q6_Vbf_vmin_VbfVbf(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VX_LATE Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_Vector Q6_Vbf_vmin_VbfVbf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_LATE + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Vbf_vmin_VbfVbf(Vu, Vv) \ @@ -4729,8 +4734,9 @@ #if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__ /* ========================================================================== Assembly Syntax: Vdd32.sf=vmpy(Vu32.bf,Vv32.bf) - C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpy_VbfVbf(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpy_VbfVbf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Wsf_vmpy_VbfVbf(Vu, Vv) \ @@ -4740,9 +4746,9 @@ #if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__ /* ========================================================================== Assembly Syntax: Vxx32.sf+=vmpy(Vu32.bf,Vv32.bf) - C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpyacc_WsfVbfVbf(HVX_VectorPair - Vxx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution - Slots: SLOT23 + C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpyacc_WsfVbfVbf(HVX_VectorPair Vxx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Wsf_vmpyacc_WsfVbfVbf(Vxx, Vu, Vv) \ @@ -4752,8 +4758,9 @@ #if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__ /* ========================================================================== Assembly Syntax: Vdd32.sf=vsub(Vu32.bf,Vv32.bf) - C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vsub_VbfVbf(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vsub_VbfVbf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Wsf_vsub_VbfVbf(Vu, Vv) \ @@ -4775,9 +4782,9 @@ #if __HVX_ARCH__ >= 79 /* ========================================================================== Assembly Syntax: Vx32|=vgetqfext(Vu32.x,Rt32) - C Intrinsic Prototype: HVX_Vector Q6_V_vgetqfextor_VVR(HVX_Vector Vx, - HVX_Vector Vu, Word32 Rt) Instruction Type: CVI_VX Execution Slots: - SLOT23 + C Intrinsic Prototype: HVX_Vector Q6_V_vgetqfextor_VVR(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt) + Instruction Type: CVI_VX + Execution Slots: SLOT23 ========================================================================== */ #define Q6_V_vgetqfextor_VVR(Vx, Vu, Rt) \ @@ -4810,8 +4817,9 @@ #if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__ /* ========================================================================== Assembly Syntax: Vdd32.hf=vadd(Vu32.f8,Vv32.f8) - C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vadd_VV(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vadd_VV(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Whf_vadd_VV(Vu, Vv) \ @@ -4821,8 +4829,9 @@ #if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__ /* ========================================================================== Assembly Syntax: Vd32.b=vcvt2(Vu32.hf,Vv32.hf) - C Intrinsic Prototype: HVX_Vector Q6_Vb_vcvt2_VhfVhf(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VX Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_Vector Q6_Vb_vcvt2_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Vb_vcvt2_VhfVhf(Vu, Vv) \ @@ -4856,8 +4865,9 @@ #if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__ /* ========================================================================== Assembly Syntax: Vd32.ub=vcvt2(Vu32.hf,Vv32.hf) - C Intrinsic Prototype: HVX_Vector Q6_Vub_vcvt2_VhfVhf(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VX Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_Vector Q6_Vub_vcvt2_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Vub_vcvt2_VhfVhf(Vu, Vv) \ @@ -4867,8 +4877,9 @@ #if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__ /* ========================================================================== Assembly Syntax: Vd32.f8=vcvt(Vu32.hf,Vv32.hf) - C Intrinsic Prototype: HVX_Vector Q6_V_vcvt_VhfVhf(HVX_Vector Vu, HVX_Vector - Vv) Instruction Type: CVI_VX Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_Vector Q6_V_vcvt_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX + Execution Slots: SLOT23 ========================================================================== */ #define Q6_V_vcvt_VhfVhf(Vu, Vv) \ @@ -4926,8 +4937,9 @@ #if __HVX_ARCH__ >= 79 /* ========================================================================== Assembly Syntax: Vd32=vmerge(Vu32.x,Vv32.w) - C Intrinsic Prototype: HVX_Vector Q6_V_vmerge_VVw(HVX_Vector Vu, HVX_Vector - Vv) Instruction Type: CVI_VS Execution Slots: SLOT0123 + C Intrinsic Prototype: HVX_Vector Q6_V_vmerge_VVw(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_V_vmerge_VVw(Vu, Vv) \ @@ -4937,8 +4949,9 @@ #if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__ /* ========================================================================== Assembly Syntax: Vdd32.hf=vmpy(Vu32.f8,Vv32.f8) - C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vmpy_VV(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vmpy_VV(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Whf_vmpy_VV(Vu, Vv) \ @@ -4948,9 +4961,9 @@ #if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__ /* ========================================================================== Assembly Syntax: Vxx32.hf+=vmpy(Vu32.f8,Vv32.f8) - C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vmpyacc_WhfVV(HVX_VectorPair - Vxx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution - Slots: SLOT23 + C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vmpyacc_WhfVV(HVX_VectorPair Vxx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Whf_vmpyacc_WhfVV(Vxx, Vu, Vv) \ @@ -4960,8 +4973,9 @@ #if __HVX_ARCH__ >= 79 /* ========================================================================== Assembly Syntax: Vd32.qf16=vmpy(Vu32.hf,Rt32.hf) - C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_VhfRhf(HVX_Vector Vu, Word32 - Rt) Instruction Type: CVI_VX_DV Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_VhfRhf(HVX_Vector Vu, Word32 Rt) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Vqf16_vmpy_VhfRhf(Vu, Rt) \ @@ -4971,8 +4985,9 @@ #if __HVX_ARCH__ >= 79 /* ========================================================================== Assembly Syntax: Vd32.qf16=vmpy(Vu32.qf16,Rt32.hf) - C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_Vqf16Rhf(HVX_Vector Vu, - Word32 Rt) Instruction Type: CVI_VX_DV Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_Vqf16Rhf(HVX_Vector Vu, Word32 Rt) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Vqf16_vmpy_Vqf16Rhf(Vu, Rt) \ @@ -4982,8 +4997,9 @@ #if __HVX_ARCH__ >= 79 /* ========================================================================== Assembly Syntax: Vd32.qf32=vmpy(Vu32.sf,Rt32.sf) - C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vmpy_VsfRsf(HVX_Vector Vu, Word32 - Rt) Instruction Type: CVI_VX_DV Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vmpy_VsfRsf(HVX_Vector Vu, Word32 Rt) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Vqf32_vmpy_VsfRsf(Vu, Rt) \ @@ -4993,8 +5009,9 @@ #if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__ /* ========================================================================== Assembly Syntax: Vdd32.hf=vsub(Vu32.f8,Vv32.f8) - C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vsub_VV(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23 + C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vsub_VV(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VX_DV + Execution Slots: SLOT23 ========================================================================== */ #define Q6_Whf_vsub_VV(Vu, Vv) \ @@ -5052,8 +5069,9 @@ #if __HVX_ARCH__ >= 81 /* ========================================================================== Assembly Syntax: Vd32=valign4(Vu32,Vv32,Rt8) - C Intrinsic Prototype: HVX_Vector Q6_V_valign4_VVR(HVX_Vector Vu, HVX_Vector - Vv, Word32 Rt) Instruction Type: CVI_VA Execution Slots: SLOT0123 + C Intrinsic Prototype: HVX_Vector Q6_V_valign4_VVR(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_V_valign4_VVR(Vu, Vv, Rt) \ @@ -5159,8 +5177,9 @@ #if __HVX_ARCH__ >= 81 /* ========================================================================== Assembly Syntax: Qd4=vcmp.eq(Vu32.hf,Vv32.hf) - C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eq_VhfVhf(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VA Execution Slots: SLOT0123 + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eq_VhfVhf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_Q_vcmp_eq_VhfVhf(Vu, Vv) \ @@ -5171,9 +5190,9 @@ #if __HVX_ARCH__ >= 81 /* ========================================================================== Assembly Syntax: Qx4&=vcmp.eq(Vu32.hf,Vv32.hf) - C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqand_QVhfVhf(HVX_VectorPred - Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution - Slots: SLOT0123 + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqand_QVhfVhf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_Q_vcmp_eqand_QVhfVhf(Qx, Vu, Vv) \ @@ -5187,9 +5206,9 @@ #if __HVX_ARCH__ >= 81 /* ========================================================================== Assembly Syntax: Qx4|=vcmp.eq(Vu32.hf,Vv32.hf) - C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqor_QVhfVhf(HVX_VectorPred - Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution - Slots: SLOT0123 + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqor_QVhfVhf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_Q_vcmp_eqor_QVhfVhf(Qx, Vu, Vv) \ @@ -5203,9 +5222,9 @@ #if __HVX_ARCH__ >= 81 /* ========================================================================== Assembly Syntax: Qx4^=vcmp.eq(Vu32.hf,Vv32.hf) - C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqxacc_QVhfVhf(HVX_VectorPred - Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution - Slots: SLOT0123 + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqxacc_QVhfVhf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_Q_vcmp_eqxacc_QVhfVhf(Qx, Vu, Vv) \ @@ -5219,8 +5238,9 @@ #if __HVX_ARCH__ >= 81 /* ========================================================================== Assembly Syntax: Qd4=vcmp.eq(Vu32.sf,Vv32.sf) - C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eq_VsfVsf(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VA Execution Slots: SLOT0123 + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eq_VsfVsf(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_Q_vcmp_eq_VsfVsf(Vu, Vv) \ @@ -5231,9 +5251,9 @@ #if __HVX_ARCH__ >= 81 /* ========================================================================== Assembly Syntax: Qx4&=vcmp.eq(Vu32.sf,Vv32.sf) - C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqand_QVsfVsf(HVX_VectorPred - Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution - Slots: SLOT0123 + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqand_QVsfVsf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_Q_vcmp_eqand_QVsfVsf(Qx, Vu, Vv) \ @@ -5247,9 +5267,9 @@ #if __HVX_ARCH__ >= 81 /* ========================================================================== Assembly Syntax: Qx4|=vcmp.eq(Vu32.sf,Vv32.sf) - C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqor_QVsfVsf(HVX_VectorPred - Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution - Slots: SLOT0123 + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqor_QVsfVsf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_Q_vcmp_eqor_QVsfVsf(Qx, Vu, Vv) \ @@ -5263,9 +5283,9 @@ #if __HVX_ARCH__ >= 81 /* ========================================================================== Assembly Syntax: Qx4^=vcmp.eq(Vu32.sf,Vv32.sf) - C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqxacc_QVsfVsf(HVX_VectorPred - Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution - Slots: SLOT0123 + C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqxacc_QVsfVsf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VA + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_Q_vcmp_eqxacc_QVsfVsf(Qx, Vu, Vv) \ @@ -5375,8 +5395,9 @@ #if __HVX_ARCH__ >= 81 /* ========================================================================== Assembly Syntax: Vd32.qf16=vsub(Vu32.hf,Vv32.qf16) - C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vsub_VhfVqf16(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VS Execution Slots: SLOT0123 + C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vsub_VhfVqf16(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_Vqf16_vsub_VhfVqf16(Vu, Vv) \ @@ -5386,8 +5407,9 @@ #if __HVX_ARCH__ >= 81 /* ========================================================================== Assembly Syntax: Vd32.qf32=vsub(Vu32.sf,Vv32.qf32) - C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vsub_VsfVqf32(HVX_Vector Vu, - HVX_Vector Vv) Instruction Type: CVI_VS Execution Slots: SLOT0123 + C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vsub_VsfVqf32(HVX_Vector Vu, HVX_Vector Vv) + Instruction Type: CVI_VS + Execution Slots: SLOT0123 ========================================================================== */ #define Q6_Vqf32_vsub_VsfVqf32(Vu, Vv) \ diff --git a/lib/include/immintrin.h b/lib/include/immintrin.h index 19064a4ff5cea31127cf5a93708665d2cd2a7855..c9be46af22c2978e39cfd52bf9c352ee65601e39 100644 --- a/lib/include/immintrin.h +++ b/lib/include/immintrin.h @@ -58,6 +58,10 @@ #include +#include + +#include + #include #include @@ -479,8 +483,6 @@ _storebe_i64(void * __P, long long __D) { #include -#include - #include #include diff --git a/lib/include/intrin.h b/lib/include/intrin.h index 210ed0c1f773b618c9dccb1ec7da92a01c75f626..4cb8cac960bcf5974bdd9bb654471da5952251c5 100644 --- a/lib/include/intrin.h +++ b/lib/include/intrin.h @@ -44,8 +44,10 @@ #if __x86_64__ #define __LPTRINT_TYPE__ __int64 +#define __IPTRINT_TYPE__ __int64 #else #define __LPTRINT_TYPE__ long +#define __IPTRINT_TYPE__ int #endif #ifdef __cplusplus @@ -95,12 +97,12 @@ void __outdword(unsigned short, unsigned long); void __outdwordstring(unsigned short, unsigned long *, unsigned long); void __outword(unsigned short, unsigned short); void __outwordstring(unsigned short, unsigned short *, unsigned long); -unsigned long __readcr0(void); -unsigned long __readcr2(void); +unsigned __LPTRINT_TYPE__ __readcr0(void); +unsigned __LPTRINT_TYPE__ __readcr2(void); unsigned __LPTRINT_TYPE__ __readcr3(void); unsigned __LPTRINT_TYPE__ __readcr4(void); -unsigned __int64 __readcr8(void); -unsigned int __readdr(unsigned int); +unsigned __LPTRINT_TYPE__ __readcr8(void); +unsigned __IPTRINT_TYPE__ __readdr(unsigned int); #ifdef __i386__ unsigned char __readfsbyte(unsigned long); unsigned short __readfsword(unsigned long); @@ -126,11 +128,12 @@ unsigned __int64 __ull_rshift(unsigned __int64, int); void __vmx_off(void); void __vmx_vmptrst(unsigned __int64 *); void __wbinvd(void); -void __writecr0(unsigned int); -void __writecr3(unsigned __INTPTR_TYPE__); -void __writecr4(unsigned __INTPTR_TYPE__); -void __writecr8(unsigned __int64); -void __writedr(unsigned int, unsigned int); +void __writecr0(unsigned __IPTRINT_TYPE__); +void __writecr2(unsigned __IPTRINT_TYPE__); +void __writecr3(unsigned __IPTRINT_TYPE__); +void __writecr4(unsigned __IPTRINT_TYPE__); +void __writecr8(unsigned __IPTRINT_TYPE__); +void __writedr(unsigned int, unsigned __IPTRINT_TYPE__); void __writefsbyte(unsigned long, unsigned char); void __writefsdword(unsigned long, unsigned long); void __writefsqword(unsigned long, unsigned __int64); @@ -374,6 +377,9 @@ static __inline__ void __DEFAULT_FN_ATTRS __nop(void) { \*----------------------------------------------------------------------------*/ #if defined(__aarch64__) || defined(__arm64ec__) unsigned __int64 __getReg(int); +double __getRegFp(int _Reg); +void __setReg(int, unsigned __int64); +void __setRegFp(int, double); unsigned char _interlockedbittestandreset_acq(long volatile *, long); unsigned char _interlockedbittestandreset_nf(long volatile *, long); unsigned char _interlockedbittestandreset_rel(long volatile *, long); @@ -438,10 +444,13 @@ unsigned int _CountLeadingSigns(long); unsigned int _CountLeadingSigns64(__int64); unsigned int _CountOneBits(unsigned long); unsigned int _CountOneBits64(unsigned __int64); +unsigned int _CountTrailingZeros(unsigned long); +unsigned int _CountTrailingZeros64(unsigned __int64); unsigned int __hlt(unsigned int, ...); void __cdecl __prefetch(const void *); +void __cdecl __prefetch2(const void *, unsigned char); #endif @@ -474,7 +483,7 @@ static __inline__ unsigned __LPTRINT_TYPE__ __DEFAULT_FN_ATTRS __readcr3(void) { } static __inline__ void __DEFAULT_FN_ATTRS -__writecr3(unsigned __INTPTR_TYPE__ __cr3_val) { +__writecr3(unsigned __IPTRINT_TYPE__ __cr3_val) { __asm__ ("mov {%0, %%cr3|cr3, %0}" : : "r"(__cr3_val) : "memory"); } #endif @@ -484,6 +493,7 @@ __writecr3(unsigned __INTPTR_TYPE__ __cr3_val) { #endif #undef __LPTRINT_TYPE__ +#undef __IPTRINT_TYPE__ #undef __DEFAULT_FN_ATTRS diff --git a/lib/include/larchintrin.h b/lib/include/larchintrin.h index a1247d12e21f8fe3bda852287671a0d7abed6f7f..3f0fab2b570e039748ee998149e435639a824f39 100644 --- a/lib/include/larchintrin.h +++ b/lib/include/larchintrin.h @@ -120,10 +120,12 @@ extern __inline int #define __ibar(/*ui15*/ _1) __builtin_loongarch_ibar((_1)) +#ifdef __loongarch_hard_float #define __movfcsr2gr(/*ui5*/ _1) __builtin_loongarch_movfcsr2gr((_1)); #define __movgr2fcsr(/*ui5*/ _1, _2) \ __builtin_loongarch_movgr2fcsr((_1), (unsigned int)_2); +#endif #define __syscall(/*ui15*/ _1) __builtin_loongarch_syscall((_1)) diff --git a/lib/include/llvm_libc_wrappers/ctype.h b/lib/include/llvm_libc_wrappers/ctype.h index 79b0c1e9be953e53f93f17c732b0ea86502d686e..8d7bb608d4c5b9312d38af290fd75c5263521cec 100644 --- a/lib/include/llvm_libc_wrappers/ctype.h +++ b/lib/include/llvm_libc_wrappers/ctype.h @@ -9,13 +9,14 @@ #ifndef __CLANG_LLVM_LIBC_WRAPPERS_CTYPE_H__ #define __CLANG_LLVM_LIBC_WRAPPERS_CTYPE_H__ -#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__) +#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__) && \ + !defined(__SPIRV__) #error "This file is for GPU offloading compilation only" #endif #include_next -#if defined(__HIP__) || defined(__CUDA__) +#if defined(__HIP__) || defined(__CUDA__) || defined(__SPIRV__) #define __LIBC_ATTRS __attribute__((device)) #else #define __LIBC_ATTRS diff --git a/lib/include/llvm_libc_wrappers/string.h b/lib/include/llvm_libc_wrappers/string.h index 766a58f5b6db472db59fae6895389f54c63f01c9..5edb86ac9d5204004b25cf728b9ad69e09ba922b 100644 --- a/lib/include/llvm_libc_wrappers/string.h +++ b/lib/include/llvm_libc_wrappers/string.h @@ -9,13 +9,14 @@ #ifndef __CLANG_LLVM_LIBC_WRAPPERS_STRING_H__ #define __CLANG_LLVM_LIBC_WRAPPERS_STRING_H__ -#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__) +#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__) && \ + !defined(__SPIRV__) #error "This file is for GPU offloading compilation only" #endif #include_next -#if defined(__HIP__) || defined(__CUDA__) +#if defined(__HIP__) || defined(__CUDA__) || defined(__SPIRV__) #define __LIBC_ATTRS __attribute__((device)) #else #define __LIBC_ATTRS diff --git a/lib/include/module.modulemap b/lib/include/module.modulemap index c13dd3fd48ac8888343e132026ba3da4fbe03302..c8f96df1672c1e51b1aea08dcd23110d0fb2c713 100644 --- a/lib/include/module.modulemap +++ b/lib/include/module.modulemap @@ -21,13 +21,22 @@ module _Builtin_intrinsics [system] [extern_c] { export * } - explicit module neon { - requires neon - header "arm_neon.h" + explicit module bf16 { + header "arm_bf16.h" + export * + } + + explicit module fp16 { header "arm_fp16.h" export * } + explicit module neon { + requires neon + header "arm_neon.h" + export * + } + explicit module sve { requires sve header "arm_sve.h" @@ -41,6 +50,12 @@ module _Builtin_intrinsics [system] [extern_c] { header "arm64intr.h" export * + + explicit module neon { + requires neon + header "arm64_neon.h" + export * + } } explicit module intel { diff --git a/lib/include/openmp_wrappers/__clang_openmp_device_functions.h b/lib/include/openmp_wrappers/__clang_openmp_device_functions.h index 3e354c63efc6681c79780f770d4066b4217a5710..2e99992500620d6c27a8ebe4385e9663b39ba383 100644 --- a/lib/include/openmp_wrappers/__clang_openmp_device_functions.h +++ b/lib/include/openmp_wrappers/__clang_openmp_device_functions.h @@ -55,6 +55,20 @@ extern "C" { #pragma omp end declare variant #endif +#ifdef __SPIRV__ +#pragma omp begin declare variant match( \ + device = {arch(spirv64)}, implementation = {extension(match_any)}) + +#define __OPENMP_SPIRV__ + +/// Include declarations for libdevice functions. +#include <__clang_spirv_libdevice_declares.h> + +#undef __OPENMP_SPIRV__ + +#pragma omp end declare variant +#endif + #ifdef __cplusplus } // extern "C" #endif diff --git a/lib/include/openmp_wrappers/complex b/lib/include/openmp_wrappers/complex index 1ceecc1af8aeca00045e3736e6cb4941d808814f..afa8e3eb3d835f319779c9cf305c09d6bea97f36 100644 --- a/lib/include/openmp_wrappers/complex +++ b/lib/include/openmp_wrappers/complex @@ -29,6 +29,12 @@ #undef __OPENMP_AMDGCN__ #endif // __AMDGCN__ +#ifdef __SPIRV__ +#define __OPENMP_SPIRV__ +#include <__clang_cuda_complex_builtins.h> +#undef __OPENMP_SPIRV__ +#endif // __SPIRV__ + #endif // Grab the host header too. @@ -45,7 +51,7 @@ #ifndef _LIBCPP_STD_VER #pragma omp begin declare variant match( \ - device = {arch(amdgcn, nvptx, nvptx64)}, \ + device = {arch(amdgcn, nvptx, nvptx64, spirv64)}, \ implementation = {extension(match_any, allow_templates)}) #include diff --git a/lib/include/openmp_wrappers/complex.h b/lib/include/openmp_wrappers/complex.h index 7e7c0866426bc9ec9ffb9e41f11c1c8b6c079243..92839088b47d5cea61f65491dd4a5f4e11836f43 100644 --- a/lib/include/openmp_wrappers/complex.h +++ b/lib/include/openmp_wrappers/complex.h @@ -29,6 +29,12 @@ #undef __OPENMP_AMDGCN__ #endif +#ifdef __SPIRV__ +#define __OPENMP_SPIRV__ +#include <__clang_cuda_complex_builtins.h> +#undef __OPENMP_SPIRV__ +#endif // __SPIRV__ + #endif // Grab the host header too. diff --git a/lib/include/openmp_wrappers/math.h b/lib/include/openmp_wrappers/math.h index 1e3c07cfdb8cd53da2e7ed1117073c22533ada1f..024762ba127cd5c9ed2d92aa548935668f36d435 100644 --- a/lib/include/openmp_wrappers/math.h +++ b/lib/include/openmp_wrappers/math.h @@ -58,4 +58,14 @@ #pragma omp end declare variant #endif +#ifdef __SPIRV__ +#pragma omp begin declare variant match(device = {arch(spirv64)}) + +#define __OPENMP_SPIRV__ +#include <__clang_spirv_math.h> +#undef __OPENMP_SPIRV__ + +#pragma omp end declare variant +#endif + #endif diff --git a/lib/include/ptrauth.h b/lib/include/ptrauth.h index ad28f06f0930cd72c6fb4226875233f14ed755d3..32c089d830715356ef783cde6e32825dd6ccfe0f 100644 --- a/lib/include/ptrauth.h +++ b/lib/include/ptrauth.h @@ -38,8 +38,7 @@ typedef enum { The extra data is always 0. */ ptrauth_key_function_pointer = ptrauth_key_process_independent_code, - /* The key used to sign C++ v-table pointers. - The extra data is always 0. */ + /* The key used to sign pointers to C++ v-tables. */ ptrauth_key_cxx_vtable_pointer = ptrauth_key_process_independent_data, /* The key used to sign metadata pointers to Objective-C method-lists. */ @@ -178,6 +177,56 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t; __builtin_ptrauth_auth_and_resign(__value, __old_key, __old_data, __new_key, \ __new_data) +/* Authenticate a pointer using a PC-based signature scheme and resign + it using a different scheme. + + If the result is subsequently authenticated using the new scheme, that + authentication is guaranteed to fail if and only if the initial + authentication failed. + + The value must be an expression of pointer type. + The key must be a constant expression of type ptrauth_key. + The extra data must be an expression of pointer or integer type; + if an integer, it will be coerced to ptrauth_extra_data_t. + The oldpc must be an expression of pointer or integer type representing + the PC value where the original signature was created. + The result will have the same type as the original value. + + This operation is guaranteed to not leave the intermediate value + available for attack before it is re-signed. + + Do not pass a null pointer to this function. A null pointer + will not successfully authenticate. */ +#define ptrauth_auth_with_pc_and_resign(__value, __old_key, __old_data, \ + __old_pc, __new_key, __new_data) \ + __builtin_ptrauth_auth_with_pc_and_resign(__value, __old_key, __old_data, \ + __old_pc, __new_key, __new_data) + +/* Authenticate a pointer using one scheme, load 32bit value at offset addend + from the pointer, and add this value to the pointer, sign using specified + scheme. + + If the result is subsequently authenticated using the new scheme, that + authentication is guaranteed to fail if and only if the initial + authentication failed. + + The value must be an expression of pointer type. + The key must be a constant expression of type ptrauth_key. + The extra data must be an expression of pointer or integer type; + if an integer, it will be coerced to ptrauth_extra_data_t. + The addend must be an immediate ptrdiff_t value. + The result will have the same type as the original value. + + This operation is guaranteed to not leave the intermediate value + available for attack before it is re-signed. + + Do not pass a null pointer to this function. A null pointer + will not successfully authenticate. */ +#define ptrauth_auth_load_relative_and_sign(__value, __old_key, __old_data, \ + __new_key, __new_data, __offset) \ + __builtin_ptrauth_auth_load_relative_and_sign( \ + __value, __old_key, __old_data, __new_key, __new_data, __offset) + /* Authenticate a pointer using one scheme and resign it as a C function pointer. @@ -324,30 +373,38 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t; __ptrauth(ptrauth_key_objc_class_ro_pointer, 1, \ __ptrauth_objc_class_ro_discriminator) +#if __has_feature(ptrauth_init_fini_address_discrimination) +#define __ptrauth_init_fini_pointer \ + __ptrauth(ptrauth_key_init_fini_pointer, 1, __ptrauth_init_fini_discriminator) +#else +#define __ptrauth_init_fini_pointer \ + __ptrauth(ptrauth_key_init_fini_pointer, 0, __ptrauth_init_fini_discriminator) +#endif + #else #define ptrauth_strip(__value, __key) \ - ({ \ + __extension__({ \ (void)__key; \ __value; \ }) #define ptrauth_blend_discriminator(__pointer, __integer) \ - ({ \ + __extension__({ \ (void)__pointer; \ (void)__integer; \ ((ptrauth_extra_data_t)0); \ }) #define ptrauth_sign_constant(__value, __key, __data) \ - ({ \ + __extension__({ \ (void)__key; \ (void)__data; \ __value; \ }) #define ptrauth_sign_unauthenticated(__value, __key, __data) \ - ({ \ + __extension__({ \ (void)__key; \ (void)__data; \ __value; \ @@ -355,7 +412,7 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t; #define ptrauth_auth_and_resign(__value, __old_key, __old_data, __new_key, \ __new_data) \ - ({ \ + __extension__({ \ (void)__old_key; \ (void)__old_data; \ (void)__new_key; \ @@ -363,22 +420,44 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t; __value; \ }) +#define ptrauth_auth_with_pc_and_resign(__value, __old_key, __old_data, \ + __old_pc, __new_key, __new_data) \ + __extension__({ \ + (void)__old_key; \ + (void)__old_data; \ + (void)__old_pc; \ + (void)__new_key; \ + (void)__new_data; \ + __value; \ + }) + +#define ptrauth_auth_load_relative_and_sign(__value, __old_key, __old_data, \ + __new_key, __new_data, __offset) \ + __extension__({ \ + (void)__old_key; \ + (void)__old_data; \ + (void)__new_key; \ + (void)__new_data; \ + const char *__value_tmp = (const char *)(__value); \ + (void *)(__value_tmp + *(const int *)(__value_tmp + (__offset))); \ + }) + #define ptrauth_auth_function(__value, __old_key, __old_data) \ - ({ \ + __extension__({ \ (void)__old_key; \ (void)__old_data; \ __value; \ }) #define ptrauth_auth_data(__value, __old_key, __old_data) \ - ({ \ + __extension__({ \ (void)__old_key; \ (void)__old_data; \ __value; \ }) #define ptrauth_string_discriminator(__string) \ - ({ \ + __extension__({ \ (void)__string; \ ((ptrauth_extra_data_t)0); \ }) @@ -388,13 +467,12 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t; ((ptrauth_extra_data_t)0) #define ptrauth_sign_generic_data(__value, __data) \ - ({ \ + __extension__({ \ (void)__value; \ (void)__data; \ ((ptrauth_generic_signature_t)0); \ }) - #define ptrauth_cxx_vtable_pointer(key, address_discrimination, \ extra_discrimination...) diff --git a/lib/include/riscv_bitmanip.h b/lib/include/riscv_bitmanip.h index 2bc7ee022a96bdf22729911982a004c91183008f..083a509c23c5820c5e5d10b4c90089103cb12c47 100644 --- a/lib/include/riscv_bitmanip.h +++ b/lib/include/riscv_bitmanip.h @@ -16,6 +16,13 @@ extern "C" { #endif +#define __riscv_intrinsic_b 1 +#define __riscv_intrinsic_zbb 1 +#define __riscv_intrinsic_zbc 1 +#define __riscv_intrinsic_zbkb 1 +#define __riscv_intrinsic_zbkc 1 +#define __riscv_intrinsic_zbkx 1 + #if defined(__riscv_zbb) static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__)) __riscv_orc_b_32(uint32_t __x) { diff --git a/lib/include/riscv_corev_alu.h b/lib/include/riscv_corev_alu.h index 84f4d087e4863fb29e5c89c2950ff2185759a056..d6fba94790aab11c11f534b378790c7a96be8575 100644 --- a/lib/include/riscv_corev_alu.h +++ b/lib/include/riscv_corev_alu.h @@ -16,6 +16,8 @@ extern "C" { #endif +#define __riscv_intrinsic_xcvalu 1 + #if defined(__riscv_xcvalu) #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) diff --git a/lib/include/riscv_crypto.h b/lib/include/riscv_crypto.h index 7cd2a708f5575635ef9273921ef842ef3f05e1aa..ea9f29c0dd31b4fd4321aa5d73007cb3c62816ee 100644 --- a/lib/include/riscv_crypto.h +++ b/lib/include/riscv_crypto.h @@ -16,6 +16,14 @@ extern "C" { #endif +#define __riscv_intrinsic_zkn 1 +#define __riscv_intrinsic_zknd 1 +#define __riscv_intrinsic_zkne 1 +#define __riscv_intrinsic_zknh 1 +#define __riscv_intrinsic_zks 1 +#define __riscv_intrinsic_zksed 1 +#define __riscv_intrinsic_zksh 1 + #if defined(__riscv_zknd) #if __riscv_xlen == 32 #define __riscv_aes32dsi(x, y, bs) __builtin_riscv_aes32dsi(x, y, bs) diff --git a/lib/include/riscv_mips.h b/lib/include/riscv_mips.h index 124a989280ed432dc9f792fc2cdc5bdb3d506792..afd390b174bf4ce8532227b467518ab4155f6149 100644 --- a/lib/include/riscv_mips.h +++ b/lib/include/riscv_mips.h @@ -14,6 +14,8 @@ #error "This header is only meant to be used on riscv architecture" #endif +#define __riscv_intrinsic_xmipsexectl 1 + #define __DEFAULT_FN_ATTRS \ __attribute__((__always_inline__, __nodebug__, __target__("xmipsexectl"))) diff --git a/lib/include/riscv_nds.h b/lib/include/riscv_nds.h index 29734c43834c7d1137cff0aa3d822075e3754d5e..c321e0b26338005ea938e792606259693765bc1f 100644 --- a/lib/include/riscv_nds.h +++ b/lib/include/riscv_nds.h @@ -16,6 +16,9 @@ extern "C" { #endif +#define __riscv_intrinsic_xandesbfhcvt 1 +#define __riscv_intrinsic_xandesperf 1 + #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) #if defined(__riscv_xandesperf) diff --git a/lib/include/riscv_ntlh.h b/lib/include/riscv_ntlh.h index c92e580a0a63171cdb9949ee47432592c4eae73b..10540d95eb164e7ead860bccc9821e7e88e116ba 100644 --- a/lib/include/riscv_ntlh.h +++ b/lib/include/riscv_ntlh.h @@ -10,6 +10,8 @@ #ifndef __RISCV_NTLH_H #define __RISCV_NTLH_H +#define __riscv_intrinsic_zihintntl 1 + #ifndef __riscv_zihintntl #error "NTLH intrinsics require the NTLH extension." #endif diff --git a/lib/include/riscv_packed_simd.h b/lib/include/riscv_packed_simd.h new file mode 100644 index 0000000000000000000000000000000000000000..21953fdee1fceac518e85a7b5fa48b830c03c646 --- /dev/null +++ b/lib/include/riscv_packed_simd.h @@ -0,0 +1,644 @@ +/*===---- riscv_packed_simd.h - RISC-V P intrinsics ------------------------=== + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===-----------------------------------------------------------------------=== + */ + +#ifndef __RISCV_PACKED_SIMD_H +#define __RISCV_PACKED_SIMD_H + +#include + +#if defined(__cplusplus) +extern "C" { +#endif + +/* Packed SIMD Types */ + +typedef int8_t int8x4_t __attribute__((__vector_size__(4))); +typedef uint8_t uint8x4_t __attribute__((__vector_size__(4))); +typedef int16_t int16x2_t __attribute__((__vector_size__(4))); +typedef uint16_t uint16x2_t __attribute__((__vector_size__(4))); + +typedef int8_t int8x8_t __attribute__((__vector_size__(8))); +typedef uint8_t uint8x8_t __attribute__((__vector_size__(8))); +typedef int16_t int16x4_t __attribute__((__vector_size__(8))); +typedef uint16_t uint16x4_t __attribute__((__vector_size__(8))); +typedef int32_t int32x2_t __attribute__((__vector_size__(8))); +typedef uint32_t uint32x2_t __attribute__((__vector_size__(8))); + +#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) + +#define __packed_splat2(ty, x) ((ty){(x), (x)}) +#define __packed_splat4(ty, x) ((ty){(x), (x), (x), (x)}) +#define __packed_splat8(ty, x) ((ty){(x), (x), (x), (x), (x), (x), (x), (x)}) + +#define __packed_splat(name, ty, scalar_ty, splat) \ + static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(scalar_ty __x) { \ + return splat(ty, __x); \ + } + +#define __packed_shift(name, ty, op, mask) \ + static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \ + unsigned __rs2) { \ + return __rs1 op(__rs2 & (mask)); \ + } +#define __packed_shift8(name, ty, op) __packed_shift(name, ty, op, 0x7) +#define __packed_shift16(name, ty, op) __packed_shift(name, ty, op, 0xf) +#define __packed_shift32(name, ty, op) __packed_shift(name, ty, op, 0x1f) + +#define __packed_scalar_binary_op(name, ty, scalar_ty, op, splat) \ + static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \ + scalar_ty __rs2) { \ + return __rs1 op splat(ty, __rs2); \ + } + +#define __packed_binary_op(name, ty, op) \ + static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, ty __rs2) { \ + return __rs1 op __rs2; \ + } + +#define __packed_unary_op(name, ty, op) \ + static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \ + return op __rs1; \ + } + +#define __packed_binary_builtin(name, ty, builtin) \ + static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, ty __rs2) { \ + return builtin(__rs1, __rs2); \ + } + +#define __packed_sh1add(name, ty) \ + static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, ty __rs2) { \ + return (__rs1 << 1) + __rs2; \ + } + +/* TODO: switch to sadd_sat(__builtin_elementwise_shl_sat(a, 1), b) once a + * generic elementwise shl_sat builtin exists. sadd_sat(a, a) is equivalent + * for signed types and the backend's saturating_shl1 PatFrags matches both + * shapes. */ +#define __packed_sh1sadd(name, ty) \ + static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, ty __rs2) { \ + return __builtin_elementwise_add_sat( \ + __builtin_elementwise_add_sat(__rs1, __rs1), __rs2); \ + } + +#define __packed_cmp(name, ty, rty, op) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \ + ty __rs2) { \ + return (rty)(__rs1 op __rs2); \ + } + +#define __packed_pabs(name, ty, rty) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \ + return (rty)__builtin_elementwise_abs(__rs1); \ + } + +#define __packed_binary_builtin_cast(name, ty, rty, builtin) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \ + ty __rs2) { \ + return (rty)builtin(__rs1, __rs2); \ + } + +#define __packed_reduction(name, rty, ty, builtin) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \ + rty __rs2) { \ + return builtin(__rs1, __rs2); \ + } + +#define __packed_merge_builtin(name, ty, mask_ty, builtin) \ + static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, ty __rs2, \ + mask_ty __rd) { \ + return (ty)builtin(__rs1, __rs2, __rd); \ + } + +#define __packed_psabs(name, ty, builtin) \ + static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \ + return builtin(__rs1); \ + } + +#define __packed_widen_convert(name, rty, ty) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \ + return __builtin_convertvector(__rs1, rty); \ + } +#define __packed_widen_high2(name, rty, ty) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \ + return (rty)__builtin_shufflevector((ty){0}, __rs1, 0, 2, 1, 3); \ + } +#define __packed_widen_high4(name, rty, ty) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \ + return (rty)__builtin_shufflevector((ty){0}, __rs1, 0, 4, 1, 5, 2, 6, 3, \ + 7); \ + } + +/* Packed Reverse: reverse the order of the elements. Lowered to a single + * rev8/rev16/ppairoe.* by the backend's packed reverse-shuffle handling. */ +#define __packed_reverse2(name, ty) \ + static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \ + return __builtin_shufflevector(__rs1, __rs1, 1, 0); \ + } +#define __packed_reverse4(name, ty) \ + static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \ + return __builtin_shufflevector(__rs1, __rs1, 3, 2, 1, 0); \ + } +#define __packed_reverse8(name, ty) \ + static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \ + return __builtin_shufflevector(__rs1, __rs1, 7, 6, 5, 4, 3, 2, 1, 0); \ + } + +#define __packed_zip2(name, rty, ty) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \ + ty __rs2) { \ + return __builtin_shufflevector(__rs1, __rs2, 0, 2, 1, 3); \ + } +#define __packed_zip4(name, rty, ty) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \ + ty __rs2) { \ + return __builtin_shufflevector(__rs1, __rs2, 0, 4, 1, 5, 2, 6, 3, 7); \ + } +#define __packed_unzipe2(name, rty, ty) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \ + return __builtin_shufflevector(__rs1, __rs1, 0, 2); \ + } +#define __packed_unzipe4(name, rty, ty) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \ + return __builtin_shufflevector(__rs1, __rs1, 0, 2, 4, 6); \ + } +#define __packed_unzipo2(name, rty, ty) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \ + return __builtin_shufflevector(__rs1, __rs1, 1, 3); \ + } +#define __packed_unzipo4(name, rty, ty) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1) { \ + return __builtin_shufflevector(__rs1, __rs1, 1, 3, 5, 7); \ + } + +#define __packed_abdsum(name, rty, ty, builtin) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, \ + ty __rs2) { \ + return builtin(__rs1, __rs2); \ + } + +#define __packed_abdsum_acc(name, rty, ty, builtin) \ + static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(rty __rd, ty __rs1, \ + ty __rs2) { \ + return builtin(__rd, __rs1, __rs2); \ + } + +// clang-format off: macro call sites have no trailing semicolons, which +// confuses clang-format into a deeply nested expression. + +/* Packed Splat (32-bit) */ +__packed_splat(pmv_s_u8x4, uint8x4_t, uint8_t, __packed_splat4) +__packed_splat(pmv_s_i8x4, int8x4_t, int8_t, __packed_splat4) +__packed_splat(pmv_s_u16x2, uint16x2_t, uint16_t, __packed_splat2) +__packed_splat(pmv_s_i16x2, int16x2_t, int16_t, __packed_splat2) + +/* Packed Splat (64-bit) */ +__packed_splat(pmv_s_u8x8, uint8x8_t, uint8_t, __packed_splat8) +__packed_splat(pmv_s_i8x8, int8x8_t, int8_t, __packed_splat8) +__packed_splat(pmv_s_u16x4, uint16x4_t, uint16_t, __packed_splat4) +__packed_splat(pmv_s_i16x4, int16x4_t, int16_t, __packed_splat4) +__packed_splat(pmv_s_u32x2, uint32x2_t, uint32_t, __packed_splat2) +__packed_splat(pmv_s_i32x2, int32x2_t, int32_t, __packed_splat2) + +/* Packed Addition and Subtraction (32-bit) */ +__packed_binary_op(padd_i8x4, int8x4_t, +) +__packed_binary_op(padd_u8x4, uint8x4_t, +) +__packed_binary_op(padd_i16x2, int16x2_t, +) +__packed_binary_op(padd_u16x2, uint16x2_t, +) +__packed_binary_op(psub_i8x4, int8x4_t, -) +__packed_binary_op(psub_u8x4, uint8x4_t, -) +__packed_binary_op(psub_i16x2, int16x2_t, -) +__packed_binary_op(psub_u16x2, uint16x2_t, -) +__packed_unary_op(pneg_i8x4, int8x4_t, -) +__packed_unary_op(pneg_i16x2, int16x2_t, -) + +/* Packed Addition and Subtraction (64-bit) */ +__packed_binary_op(padd_i8x8, int8x8_t, +) +__packed_binary_op(padd_u8x8, uint8x8_t, +) +__packed_binary_op(padd_i16x4, int16x4_t, +) +__packed_binary_op(padd_u16x4, uint16x4_t, +) +__packed_binary_op(padd_i32x2, int32x2_t, +) +__packed_binary_op(padd_u32x2, uint32x2_t, +) +__packed_binary_op(psub_i8x8, int8x8_t, -) +__packed_binary_op(psub_u8x8, uint8x8_t, -) +__packed_binary_op(psub_i16x4, int16x4_t, -) +__packed_binary_op(psub_u16x4, uint16x4_t, -) +__packed_binary_op(psub_i32x2, int32x2_t, -) +__packed_binary_op(psub_u32x2, uint32x2_t, -) +__packed_unary_op(pneg_i8x8, int8x8_t, -) +__packed_unary_op(pneg_i16x4, int16x4_t, -) +__packed_unary_op(pneg_i32x2, int32x2_t, -) + +/* Packed Addition with Scalar (32-bit) */ +__packed_scalar_binary_op(padd_s_u8x4, uint8x4_t, uint8_t, +, __packed_splat4) +__packed_scalar_binary_op(padd_s_i8x4, int8x4_t, int8_t, +, __packed_splat4) +__packed_scalar_binary_op(padd_s_u16x2, uint16x2_t, uint16_t, +, + __packed_splat2) +__packed_scalar_binary_op(padd_s_i16x2, int16x2_t, int16_t, +, + __packed_splat2) + +/* Packed Addition with Scalar (64-bit) */ +__packed_scalar_binary_op(padd_s_u8x8, uint8x8_t, uint8_t, +, __packed_splat8) +__packed_scalar_binary_op(padd_s_i8x8, int8x8_t, int8_t, +, __packed_splat8) +__packed_scalar_binary_op(padd_s_u16x4, uint16x4_t, uint16_t, +, + __packed_splat4) +__packed_scalar_binary_op(padd_s_i16x4, int16x4_t, int16_t, +, + __packed_splat4) +__packed_scalar_binary_op(padd_s_u32x2, uint32x2_t, uint32_t, +, + __packed_splat2) +__packed_scalar_binary_op(padd_s_i32x2, int32x2_t, int32_t, +, + __packed_splat2) + +/* Packed Saturating Addition and Subtraction (32-bit) */ +__packed_binary_builtin(psadd_i8x4, int8x4_t, __builtin_elementwise_add_sat) +__packed_binary_builtin(psadd_i16x2, int16x2_t, __builtin_elementwise_add_sat) +__packed_binary_builtin(psaddu_u8x4, uint8x4_t, __builtin_elementwise_add_sat) +__packed_binary_builtin(psaddu_u16x2, uint16x2_t, __builtin_elementwise_add_sat) +__packed_binary_builtin(pssub_i8x4, int8x4_t, __builtin_elementwise_sub_sat) +__packed_binary_builtin(pssub_i16x2, int16x2_t, __builtin_elementwise_sub_sat) +__packed_binary_builtin(pssubu_u8x4, uint8x4_t, __builtin_elementwise_sub_sat) +__packed_binary_builtin(pssubu_u16x2, uint16x2_t, __builtin_elementwise_sub_sat) + +/* Packed Saturating Addition and Subtraction (64-bit) */ +__packed_binary_builtin(psadd_i8x8, int8x8_t, __builtin_elementwise_add_sat) +__packed_binary_builtin(psadd_i16x4, int16x4_t, __builtin_elementwise_add_sat) +__packed_binary_builtin(psadd_i32x2, int32x2_t, __builtin_elementwise_add_sat) +__packed_binary_builtin(psaddu_u8x8, uint8x8_t, __builtin_elementwise_add_sat) +__packed_binary_builtin(psaddu_u16x4, uint16x4_t, __builtin_elementwise_add_sat) +__packed_binary_builtin(psaddu_u32x2, uint32x2_t, __builtin_elementwise_add_sat) +__packed_binary_builtin(pssub_i8x8, int8x8_t, __builtin_elementwise_sub_sat) +__packed_binary_builtin(pssub_i16x4, int16x4_t, __builtin_elementwise_sub_sat) +__packed_binary_builtin(pssub_i32x2, int32x2_t, __builtin_elementwise_sub_sat) +__packed_binary_builtin(pssubu_u8x8, uint8x8_t, __builtin_elementwise_sub_sat) +__packed_binary_builtin(pssubu_u16x4, uint16x4_t, __builtin_elementwise_sub_sat) +__packed_binary_builtin(pssubu_u32x2, uint32x2_t, __builtin_elementwise_sub_sat) + +/* Packed Shift-Add (32-bit) */ +__packed_sh1add(psh1add_i16x2, int16x2_t) +__packed_sh1add(psh1add_u16x2, uint16x2_t) +__packed_sh1sadd(pssh1sadd_i16x2, int16x2_t) + +/* Packed Shift-Add (64-bit) */ +__packed_sh1add(psh1add_i16x4, int16x4_t) +__packed_sh1add(psh1add_u16x4, uint16x4_t) +__packed_sh1add(psh1add_i32x2, int32x2_t) +__packed_sh1add(psh1add_u32x2, uint32x2_t) +__packed_sh1sadd(pssh1sadd_i16x4, int16x4_t) +__packed_sh1sadd(pssh1sadd_i32x2, int32x2_t) + +/* Packed Exchanged Addition and Subtraction (32-bit) */ +__packed_binary_builtin(pas_x_i16x2, int16x2_t, __builtin_riscv_pas_x_i16x2) +__packed_binary_builtin(psa_x_i16x2, int16x2_t, __builtin_riscv_psa_x_i16x2) +__packed_binary_builtin(psas_x_i16x2, int16x2_t, __builtin_riscv_psas_x_i16x2) +__packed_binary_builtin(pssa_x_i16x2, int16x2_t, __builtin_riscv_pssa_x_i16x2) +__packed_binary_builtin(paas_x_i16x2, int16x2_t, __builtin_riscv_paas_x_i16x2) +__packed_binary_builtin(pasa_x_i16x2, int16x2_t, __builtin_riscv_pasa_x_i16x2) + +/* Packed Exchanged Addition and Subtraction (64-bit) */ +__packed_binary_builtin(pas_x_i16x4, int16x4_t, __builtin_riscv_pas_x_i16x4) +__packed_binary_builtin(psa_x_i16x4, int16x4_t, __builtin_riscv_psa_x_i16x4) +__packed_binary_builtin(psas_x_i16x4, int16x4_t, __builtin_riscv_psas_x_i16x4) +__packed_binary_builtin(pssa_x_i16x4, int16x4_t, __builtin_riscv_pssa_x_i16x4) +__packed_binary_builtin(paas_x_i16x4, int16x4_t, __builtin_riscv_paas_x_i16x4) +__packed_binary_builtin(pasa_x_i16x4, int16x4_t, __builtin_riscv_pasa_x_i16x4) +__packed_binary_builtin(pas_x_i32x2, int32x2_t, __builtin_riscv_pas_x_i32x2) +__packed_binary_builtin(psa_x_i32x2, int32x2_t, __builtin_riscv_psa_x_i32x2) +__packed_binary_builtin(psas_x_i32x2, int32x2_t, __builtin_riscv_psas_x_i32x2) +__packed_binary_builtin(pssa_x_i32x2, int32x2_t, __builtin_riscv_pssa_x_i32x2) +__packed_binary_builtin(paas_x_i32x2, int32x2_t, __builtin_riscv_paas_x_i32x2) +__packed_binary_builtin(pasa_x_i32x2, int32x2_t, __builtin_riscv_pasa_x_i32x2) + +/* Packed Minimum and Maximum (32-bit) */ +__packed_binary_builtin(pmin_i8x4, int8x4_t, __builtin_elementwise_min) +__packed_binary_builtin(pmin_i16x2, int16x2_t, __builtin_elementwise_min) +__packed_binary_builtin(pminu_u8x4, uint8x4_t, __builtin_elementwise_min) +__packed_binary_builtin(pminu_u16x2, uint16x2_t, __builtin_elementwise_min) +__packed_binary_builtin(pmax_i8x4, int8x4_t, __builtin_elementwise_max) +__packed_binary_builtin(pmax_i16x2, int16x2_t, __builtin_elementwise_max) +__packed_binary_builtin(pmaxu_u8x4, uint8x4_t, __builtin_elementwise_max) +__packed_binary_builtin(pmaxu_u16x2, uint16x2_t, __builtin_elementwise_max) + +/* Packed Minimum and Maximum (64-bit) */ +__packed_binary_builtin(pmin_i8x8, int8x8_t, __builtin_elementwise_min) +__packed_binary_builtin(pmin_i16x4, int16x4_t, __builtin_elementwise_min) +__packed_binary_builtin(pmin_i32x2, int32x2_t, __builtin_elementwise_min) +__packed_binary_builtin(pminu_u8x8, uint8x8_t, __builtin_elementwise_min) +__packed_binary_builtin(pminu_u16x4, uint16x4_t, __builtin_elementwise_min) +__packed_binary_builtin(pminu_u32x2, uint32x2_t, __builtin_elementwise_min) +__packed_binary_builtin(pmax_i8x8, int8x8_t, __builtin_elementwise_max) +__packed_binary_builtin(pmax_i16x4, int16x4_t, __builtin_elementwise_max) +__packed_binary_builtin(pmax_i32x2, int32x2_t, __builtin_elementwise_max) +__packed_binary_builtin(pmaxu_u8x8, uint8x8_t, __builtin_elementwise_max) +__packed_binary_builtin(pmaxu_u16x4, uint16x4_t, __builtin_elementwise_max) +__packed_binary_builtin(pmaxu_u32x2, uint32x2_t, __builtin_elementwise_max) + +/* Packed Comparison (32-bit) */ +__packed_cmp(pmseq_i8x4_u8x4, int8x4_t, uint8x4_t, ==) +__packed_cmp(pmseq_u8x4_u8x4, uint8x4_t, uint8x4_t, ==) +__packed_cmp(pmsne_i8x4_u8x4, int8x4_t, uint8x4_t, !=) +__packed_cmp(pmsne_u8x4_u8x4, uint8x4_t, uint8x4_t, !=) +__packed_cmp(pmslt_u8x4, int8x4_t, uint8x4_t, <) +__packed_cmp(pmsltu_u8x4, uint8x4_t, uint8x4_t, <) +__packed_cmp(pmsgt_u8x4, int8x4_t, uint8x4_t, >) +__packed_cmp(pmsgtu_u8x4, uint8x4_t, uint8x4_t, >) +__packed_cmp(pmsge_u8x4, int8x4_t, uint8x4_t, >=) +__packed_cmp(pmsgeu_u8x4, uint8x4_t, uint8x4_t, >=) +__packed_cmp(pmsle_u8x4, int8x4_t, uint8x4_t, <=) +__packed_cmp(pmsleu_u8x4, uint8x4_t, uint8x4_t, <=) +__packed_cmp(pmseq_i16x2_u16x2, int16x2_t, uint16x2_t, ==) +__packed_cmp(pmseq_u16x2_u16x2, uint16x2_t, uint16x2_t, ==) +__packed_cmp(pmsne_i16x2_u16x2, int16x2_t, uint16x2_t, !=) +__packed_cmp(pmsne_u16x2_u16x2, uint16x2_t, uint16x2_t, !=) +__packed_cmp(pmslt_u16x2, int16x2_t, uint16x2_t, <) +__packed_cmp(pmsltu_u16x2, uint16x2_t, uint16x2_t, <) +__packed_cmp(pmsgt_u16x2, int16x2_t, uint16x2_t, >) +__packed_cmp(pmsgtu_u16x2, uint16x2_t, uint16x2_t, >) +__packed_cmp(pmsge_u16x2, int16x2_t, uint16x2_t, >=) +__packed_cmp(pmsgeu_u16x2, uint16x2_t, uint16x2_t, >=) +__packed_cmp(pmsle_u16x2, int16x2_t, uint16x2_t, <=) +__packed_cmp(pmsleu_u16x2, uint16x2_t, uint16x2_t, <=) + +/* Packed Comparison (64-bit) */ +__packed_cmp(pmseq_i8x8_u8x8, int8x8_t, uint8x8_t, ==) +__packed_cmp(pmseq_u8x8_u8x8, uint8x8_t, uint8x8_t, ==) +__packed_cmp(pmsne_i8x8_u8x8, int8x8_t, uint8x8_t, !=) +__packed_cmp(pmsne_u8x8_u8x8, uint8x8_t, uint8x8_t, !=) +__packed_cmp(pmslt_u8x8, int8x8_t, uint8x8_t, <) +__packed_cmp(pmsltu_u8x8, uint8x8_t, uint8x8_t, <) +__packed_cmp(pmsgt_u8x8, int8x8_t, uint8x8_t, >) +__packed_cmp(pmsgtu_u8x8, uint8x8_t, uint8x8_t, >) +__packed_cmp(pmsge_u8x8, int8x8_t, uint8x8_t, >=) +__packed_cmp(pmsgeu_u8x8, uint8x8_t, uint8x8_t, >=) +__packed_cmp(pmsle_u8x8, int8x8_t, uint8x8_t, <=) +__packed_cmp(pmsleu_u8x8, uint8x8_t, uint8x8_t, <=) +__packed_cmp(pmseq_i16x4_u16x4, int16x4_t, uint16x4_t, ==) +__packed_cmp(pmseq_u16x4_u16x4, uint16x4_t, uint16x4_t, ==) +__packed_cmp(pmsne_i16x4_u16x4, int16x4_t, uint16x4_t, !=) +__packed_cmp(pmsne_u16x4_u16x4, uint16x4_t, uint16x4_t, !=) +__packed_cmp(pmslt_u16x4, int16x4_t, uint16x4_t, <) +__packed_cmp(pmsltu_u16x4, uint16x4_t, uint16x4_t, <) +__packed_cmp(pmsgt_u16x4, int16x4_t, uint16x4_t, >) +__packed_cmp(pmsgtu_u16x4, uint16x4_t, uint16x4_t, >) +__packed_cmp(pmsge_u16x4, int16x4_t, uint16x4_t, >=) +__packed_cmp(pmsgeu_u16x4, uint16x4_t, uint16x4_t, >=) +__packed_cmp(pmsle_u16x4, int16x4_t, uint16x4_t, <=) +__packed_cmp(pmsleu_u16x4, uint16x4_t, uint16x4_t, <=) +__packed_cmp(pmseq_i32x2_u32x2, int32x2_t, uint32x2_t, ==) +__packed_cmp(pmseq_u32x2_u32x2, uint32x2_t, uint32x2_t, ==) +__packed_cmp(pmsne_i32x2_u32x2, int32x2_t, uint32x2_t, !=) +__packed_cmp(pmsne_u32x2_u32x2, uint32x2_t, uint32x2_t, !=) +__packed_cmp(pmslt_u32x2, int32x2_t, uint32x2_t, <) +__packed_cmp(pmsltu_u32x2, uint32x2_t, uint32x2_t, <) +__packed_cmp(pmsgt_u32x2, int32x2_t, uint32x2_t, >) +__packed_cmp(pmsgtu_u32x2, uint32x2_t, uint32x2_t, >) +__packed_cmp(pmsge_u32x2, int32x2_t, uint32x2_t, >=) +__packed_cmp(pmsgeu_u32x2, uint32x2_t, uint32x2_t, >=) +__packed_cmp(pmsle_u32x2, int32x2_t, uint32x2_t, <=) +__packed_cmp(pmsleu_u32x2, uint32x2_t, uint32x2_t, <=) + +/* Packed Shifts (32-bit) */ +__packed_shift8(psll_s_u8x4, uint8x4_t, <<) +__packed_shift8(psll_s_i8x4, int8x4_t, <<) +__packed_shift16(psll_s_u16x2, uint16x2_t, <<) +__packed_shift16(psll_s_i16x2, int16x2_t, <<) +__packed_shift8(psrl_s_u8x4, uint8x4_t, >>) +__packed_shift16(psrl_s_u16x2, uint16x2_t, >>) +__packed_shift8(psra_s_i8x4, int8x4_t, >>) +__packed_shift16(psra_s_i16x2, int16x2_t, >>) + +/* Packed Shifts (64-bit) */ +__packed_shift8(psll_s_u8x8, uint8x8_t, <<) +__packed_shift8(psll_s_i8x8, int8x8_t, <<) +__packed_shift16(psll_s_u16x4, uint16x4_t, <<) +__packed_shift16(psll_s_i16x4, int16x4_t, <<) +__packed_shift32(psll_s_u32x2, uint32x2_t, <<) +__packed_shift32(psll_s_i32x2, int32x2_t, <<) +__packed_shift8(psrl_s_u8x8, uint8x8_t, >>) +__packed_shift16(psrl_s_u16x4, uint16x4_t, >>) +__packed_shift32(psrl_s_u32x2, uint32x2_t, >>) +__packed_shift8(psra_s_i8x8, int8x8_t, >>) +__packed_shift16(psra_s_i16x4, int16x4_t, >>) +__packed_shift32(psra_s_i32x2, int32x2_t, >>) + +/* Packed Logical Operations (32-bit) */ +__packed_binary_op(pand_i8x4, int8x4_t, &) +__packed_binary_op(pand_u8x4, uint8x4_t, &) +__packed_binary_op(pand_i16x2, int16x2_t, &) +__packed_binary_op(pand_u16x2, uint16x2_t, &) +__packed_binary_op(por_i8x4, int8x4_t, |) +__packed_binary_op(por_u8x4, uint8x4_t, |) +__packed_binary_op(por_i16x2, int16x2_t, |) +__packed_binary_op(por_u16x2, uint16x2_t, |) +__packed_binary_op(pxor_i8x4, int8x4_t, ^) +__packed_binary_op(pxor_u8x4, uint8x4_t, ^) +__packed_binary_op(pxor_i16x2, int16x2_t, ^) +__packed_binary_op(pxor_u16x2, uint16x2_t, ^) +__packed_unary_op(pnot_i8x4, int8x4_t, ~) +__packed_unary_op(pnot_u8x4, uint8x4_t, ~) +__packed_unary_op(pnot_i16x2, int16x2_t, ~) +__packed_unary_op(pnot_u16x2, uint16x2_t, ~) + +/* Packed Logical Operations (64-bit) */ +__packed_binary_op(pand_i8x8, int8x8_t, &) +__packed_binary_op(pand_u8x8, uint8x8_t, &) +__packed_binary_op(pand_i16x4, int16x4_t, &) +__packed_binary_op(pand_u16x4, uint16x4_t, &) +__packed_binary_op(pand_i32x2, int32x2_t, &) +__packed_binary_op(pand_u32x2, uint32x2_t, &) +__packed_binary_op(por_i8x8, int8x8_t, |) +__packed_binary_op(por_u8x8, uint8x8_t, |) +__packed_binary_op(por_i16x4, int16x4_t, |) +__packed_binary_op(por_u16x4, uint16x4_t, |) +__packed_binary_op(por_i32x2, int32x2_t, |) +__packed_binary_op(por_u32x2, uint32x2_t, |) +__packed_binary_op(pxor_i8x8, int8x8_t, ^) +__packed_binary_op(pxor_u8x8, uint8x8_t, ^) +__packed_binary_op(pxor_i16x4, int16x4_t, ^) +__packed_binary_op(pxor_u16x4, uint16x4_t, ^) +__packed_binary_op(pxor_i32x2, int32x2_t, ^) +__packed_binary_op(pxor_u32x2, uint32x2_t, ^) +__packed_unary_op(pnot_i8x8, int8x8_t, ~) +__packed_unary_op(pnot_u8x8, uint8x8_t, ~) +__packed_unary_op(pnot_i16x4, int16x4_t, ~) +__packed_unary_op(pnot_u16x4, uint16x4_t, ~) +__packed_unary_op(pnot_i32x2, int32x2_t, ~) +__packed_unary_op(pnot_u32x2, uint32x2_t, ~) + +/* Packed Widening Convert */ +__packed_widen_convert(pwcvt_i16x4, int16x4_t, int8x4_t) +__packed_widen_convert(pwcvt_i32x2, int32x2_t, int16x2_t) +__packed_widen_convert(pwcvtu_u16x4, uint16x4_t, uint8x4_t) +__packed_widen_convert(pwcvtu_u32x2, uint32x2_t, uint16x2_t) +__packed_widen_high4(pwcvth_i16x4, int16x4_t, int8x4_t) +__packed_widen_high4(pwcvth_u16x4, uint16x4_t, uint8x4_t) +__packed_widen_high2(pwcvth_i32x2, int32x2_t, int16x2_t) +__packed_widen_high2(pwcvth_u32x2, uint32x2_t, uint16x2_t) + +/* Packed Reverse (32-bit) */ +__packed_reverse4(prev_i8x4, int8x4_t) +__packed_reverse4(prev_u8x4, uint8x4_t) +__packed_reverse2(prev_i16x2, int16x2_t) +__packed_reverse2(prev_u16x2, uint16x2_t) + +/* Packed Reverse (64-bit) */ +__packed_reverse8(prev_i8x8, int8x8_t) +__packed_reverse8(prev_u8x8, uint8x8_t) +__packed_reverse4(prev_i16x4, int16x4_t) +__packed_reverse4(prev_u16x4, uint16x4_t) +__packed_reverse2(prev_i32x2, int32x2_t) +__packed_reverse2(prev_u32x2, uint32x2_t) + +/* Packed Zip */ +__packed_zip4(pzip_i8x8, int8x8_t, int8x4_t) +__packed_zip4(pzip_u8x8, uint8x8_t, uint8x4_t) +__packed_zip2(pzip_i16x4, int16x4_t, int16x2_t) +__packed_zip2(pzip_u16x4, uint16x4_t, uint16x2_t) + +/* Packed Unzip */ +__packed_unzipe4(punzipe_i8x4, int8x4_t, int8x8_t) +__packed_unzipo4(punzipo_i8x4, int8x4_t, int8x8_t) +__packed_unzipe4(punzipe_u8x4, uint8x4_t, uint8x8_t) +__packed_unzipo4(punzipo_u8x4, uint8x4_t, uint8x8_t) +__packed_unzipe2(punzipe_i16x2, int16x2_t, int16x4_t) +__packed_unzipo2(punzipo_i16x2, int16x2_t, int16x4_t) +__packed_unzipe2(punzipe_u16x2, uint16x2_t, uint16x4_t) +__packed_unzipo2(punzipo_u16x2, uint16x2_t, uint16x4_t) + +/* Packed Averaging Addition and Subtraction (32-bit) */ +__packed_binary_builtin(paadd_i8x4, int8x4_t, __builtin_riscv_paadd_i8x4) +__packed_binary_builtin(paadd_i16x2, int16x2_t, __builtin_riscv_paadd_i16x2) +__packed_binary_builtin(paaddu_u8x4, uint8x4_t, __builtin_riscv_paaddu_u8x4) +__packed_binary_builtin(paaddu_u16x2, uint16x2_t, __builtin_riscv_paaddu_u16x2) +__packed_binary_builtin(pasub_i8x4, int8x4_t, __builtin_riscv_pasub_i8x4) +__packed_binary_builtin(pasub_i16x2, int16x2_t, __builtin_riscv_pasub_i16x2) +__packed_binary_builtin(pasubu_u8x4, uint8x4_t, __builtin_riscv_pasubu_u8x4) +__packed_binary_builtin(pasubu_u16x2, uint16x2_t, __builtin_riscv_pasubu_u16x2) + +/* Packed Averaging Addition and Subtraction (64-bit) */ +__packed_binary_builtin(paadd_i8x8, int8x8_t, __builtin_riscv_paadd_i8x8) +__packed_binary_builtin(paadd_i16x4, int16x4_t, __builtin_riscv_paadd_i16x4) +__packed_binary_builtin(paadd_i32x2, int32x2_t, __builtin_riscv_paadd_i32x2) +__packed_binary_builtin(paaddu_u8x8, uint8x8_t, __builtin_riscv_paaddu_u8x8) +__packed_binary_builtin(paaddu_u16x4, uint16x4_t, __builtin_riscv_paaddu_u16x4) +__packed_binary_builtin(paaddu_u32x2, uint32x2_t, __builtin_riscv_paaddu_u32x2) +__packed_binary_builtin(pasub_i8x8, int8x8_t, __builtin_riscv_pasub_i8x8) +__packed_binary_builtin(pasub_i16x4, int16x4_t, __builtin_riscv_pasub_i16x4) +__packed_binary_builtin(pasub_i32x2, int32x2_t, __builtin_riscv_pasub_i32x2) +__packed_binary_builtin(pasubu_u8x8, uint8x8_t, __builtin_riscv_pasubu_u8x8) +__packed_binary_builtin(pasubu_u16x4, uint16x4_t, __builtin_riscv_pasubu_u16x4) +__packed_binary_builtin(pasubu_u32x2, uint32x2_t, __builtin_riscv_pasubu_u32x2) + +/* Packed Absolute Value and Absolute Difference (32-bit) */ +__packed_pabs(pabs_i8x4, int8x4_t, uint8x4_t) +__packed_pabs(pabs_i16x2, int16x2_t, uint16x2_t) +__packed_binary_builtin_cast(pabd_i8x4, int8x4_t, uint8x4_t, __builtin_riscv_pabd_i8x4) +__packed_binary_builtin_cast(pabd_i16x2, int16x2_t, uint16x2_t, __builtin_riscv_pabd_i16x2) +__packed_binary_builtin_cast(pabdu_u8x4, uint8x4_t, uint8x4_t, __builtin_riscv_pabdu_u8x4) +__packed_binary_builtin_cast(pabdu_u16x2, uint16x2_t, uint16x2_t, __builtin_riscv_pabdu_u16x2) + +/* Packed Absolute Value and Absolute Difference (64-bit) */ +__packed_pabs(pabs_i8x8, int8x8_t, uint8x8_t) +__packed_pabs(pabs_i16x4, int16x4_t, uint16x4_t) +__packed_binary_builtin_cast(pabd_i8x8, int8x8_t, uint8x8_t, __builtin_riscv_pabd_i8x8) +__packed_binary_builtin_cast(pabd_i16x4, int16x4_t, uint16x4_t, __builtin_riscv_pabd_i16x4) +__packed_binary_builtin_cast(pabdu_u8x8, uint8x8_t, uint8x8_t, __builtin_riscv_pabdu_u8x8) +__packed_binary_builtin_cast(pabdu_u16x4, uint16x4_t, uint16x4_t, __builtin_riscv_pabdu_u16x4) + +/* Packed Reduction Sum (32-bit) */ +__packed_reduction(predsum_i8x4_i32, int32_t, int8x4_t, __builtin_riscv_predsum_i8x4_i32) +__packed_reduction(predsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_predsumu_u8x4_u32) +__packed_reduction(predsum_i16x2_i32, int32_t, int16x2_t, __builtin_riscv_predsum_i16x2_i32) +__packed_reduction(predsumu_u16x2_u32, uint32_t, uint16x2_t, __builtin_riscv_predsumu_u16x2_u32) + +/* Packed Reduction Sum (64-bit) */ +__packed_reduction(predsum_i8x8_i32, int32_t, int8x8_t, __builtin_riscv_predsum_i8x8_i32) +__packed_reduction(predsumu_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_predsumu_u8x8_u32) +__packed_reduction(predsum_i16x4_i32, int32_t, int16x4_t, __builtin_riscv_predsum_i16x4_i32) +__packed_reduction(predsumu_u16x4_u32, uint32_t, uint16x4_t, __builtin_riscv_predsumu_u16x4_u32) +__packed_reduction(predsum_i8x8_i64, int64_t, int8x8_t, __builtin_riscv_predsum_i8x8_i64) +__packed_reduction(predsumu_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_predsumu_u8x8_u64) +__packed_reduction(predsum_i16x4_i64, int64_t, int16x4_t, __builtin_riscv_predsum_i16x4_i64) +__packed_reduction(predsumu_u16x4_u64, uint64_t, uint16x4_t, __builtin_riscv_predsumu_u16x4_u64) +__packed_reduction(predsum_i32x2_i64, int64_t, int32x2_t, __builtin_riscv_predsum_i32x2_i64) +__packed_reduction(predsumu_u32x2_u64, uint64_t, uint32x2_t, __builtin_riscv_predsumu_u32x2_u64) + +/* Packed Merge (32-bit) */ +__packed_merge_builtin(pmerge_u8x4, uint8x4_t, uint8x4_t, __builtin_riscv_pmerge_u8x4) +__packed_merge_builtin(pmerge_i8x4, int8x4_t, uint8x4_t, __builtin_riscv_pmerge_i8x4) +__packed_merge_builtin(pmerge_u16x2, uint16x2_t, uint16x2_t, __builtin_riscv_pmerge_u16x2) +__packed_merge_builtin(pmerge_i16x2, int16x2_t, uint16x2_t, __builtin_riscv_pmerge_i16x2) + +/* Packed Merge (64-bit) */ +__packed_merge_builtin(pmerge_u8x8, uint8x8_t, uint8x8_t, __builtin_riscv_pmerge_u8x8) +__packed_merge_builtin(pmerge_i8x8, int8x8_t, uint8x8_t, __builtin_riscv_pmerge_i8x8) +__packed_merge_builtin(pmerge_u16x4, uint16x4_t, uint16x4_t, __builtin_riscv_pmerge_u16x4) +__packed_merge_builtin(pmerge_i16x4, int16x4_t, uint16x4_t, __builtin_riscv_pmerge_i16x4) +__packed_merge_builtin(pmerge_u32x2, uint32x2_t, uint32x2_t, __builtin_riscv_pmerge_u32x2) +__packed_merge_builtin(pmerge_i32x2, int32x2_t, uint32x2_t, __builtin_riscv_pmerge_i32x2) + +/* Packed Absolute Difference Sum (32-bit) */ +__packed_abdsum(pabdsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumu_u8x4_u32) +__packed_abdsum_acc(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32) + +/* Packed Absolute Difference Sum (64-bit) */ +__packed_abdsum(pabdsumu_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumu_u8x8_u32) +__packed_abdsum(pabdsumu_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumu_u8x8_u64) +__packed_abdsum_acc(pabdsumau_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u32) +__packed_abdsum_acc(pabdsumau_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u64) + +/* Packed Saturating Absolute Value (32-bit) */ +__packed_psabs(psabs_i8x4, int8x4_t, __builtin_riscv_psabs_i8x4) +__packed_psabs(psabs_i16x2, int16x2_t, __builtin_riscv_psabs_i16x2) + +/* Packed Saturating Absolute Value (64-bit) */ +__packed_psabs(psabs_i8x8, int8x8_t, __builtin_riscv_psabs_i8x8) +__packed_psabs(psabs_i16x4, int16x4_t, __builtin_riscv_psabs_i16x4) + +// clang-format on + +#undef __packed_splat2 +#undef __packed_splat4 +#undef __packed_splat8 +#undef __packed_splat +#undef __packed_shift +#undef __packed_shift8 +#undef __packed_shift16 +#undef __packed_shift32 +#undef __packed_scalar_binary_op +#undef __packed_binary_op +#undef __packed_unary_op +#undef __packed_binary_builtin +#undef __packed_sh1add +#undef __packed_sh1sadd +#undef __packed_cmp +#undef __packed_pabs +#undef __packed_binary_builtin_cast +#undef __packed_reduction +#undef __packed_merge_builtin +#undef __packed_psabs +#undef __packed_widen_convert +#undef __packed_widen_high2 +#undef __packed_widen_high4 +#undef __packed_reverse2 +#undef __packed_reverse4 +#undef __packed_reverse8 +#undef __packed_zip2 +#undef __packed_zip4 +#undef __packed_unzipe2 +#undef __packed_unzipe4 +#undef __packed_unzipo2 +#undef __packed_unzipo4 +#undef __packed_abdsum +#undef __packed_abdsum_acc +#undef __DEFAULT_FN_ATTRS + +#if defined(__cplusplus) +} +#endif + +#endif /* __RISCV_PACKED_SIMD_H */ diff --git a/lib/include/riscv_vector.h b/lib/include/riscv_vector.h index f94f95800db92f611517d0a400a8ff7f3737aeaa..eb56f7b952fcc6f7acc116208547205f7b8788c3 100644 --- a/lib/include/riscv_vector.h +++ b/lib/include/riscv_vector.h @@ -20,6 +20,35 @@ extern "C" { #pragma clang riscv intrinsic vector +#define __riscv_intrinsic_v 1 +#define __riscv_intrinsic_zvabd 1 +#define __riscv_intrinsic_zvbb 1 +#define __riscv_intrinsic_zvbc 1 +#define __riscv_intrinsic_zvdot4a8i 1 +#define __riscv_intrinsic_zve32f 1 +#define __riscv_intrinsic_zve32x 1 +#define __riscv_intrinsic_zve64d 1 +#define __riscv_intrinsic_zve64f 1 +#define __riscv_intrinsic_zve64x 1 +#define __riscv_intrinsic_zvfbfa 1 +#define __riscv_intrinsic_zvfbfmin 1 +#define __riscv_intrinsic_zvfbfwma 1 +#define __riscv_intrinsic_zvfh 1 +#define __riscv_intrinsic_zvfhmin 1 +#define __riscv_intrinsic_zvfofp8min 1 +#define __riscv_intrinsic_zvkb 1 +#define __riscv_intrinsic_zvkg 1 +#define __riscv_intrinsic_zvkn 1 +#define __riscv_intrinsic_zvknc 1 +#define __riscv_intrinsic_zvkned 1 +#define __riscv_intrinsic_zvkng 1 +#define __riscv_intrinsic_zvknha 1 +#define __riscv_intrinsic_zvknhb 1 +#define __riscv_intrinsic_zvks 1 +#define __riscv_intrinsic_zvksc 1 +#define __riscv_intrinsic_zvksed 1 +#define __riscv_intrinsic_zvksg 1 +#define __riscv_intrinsic_zvksh 1 enum __RISCV_FRM { __RISCV_FRM_RNE = 0, @@ -415,6 +444,20 @@ typedef __rvv_bfloat16m2x4_t vbfloat16m2x4_t; typedef __rvv_bfloat16m4_t vbfloat16m4_t; typedef __rvv_bfloat16m4x2_t vbfloat16m4x2_t; typedef __rvv_bfloat16m8_t vbfloat16m8_t; +typedef __rvv_float8e4m3mf8_t vfloat8e4m3mf8_t; +typedef __rvv_float8e4m3mf4_t vfloat8e4m3mf4_t; +typedef __rvv_float8e4m3mf2_t vfloat8e4m3mf2_t; +typedef __rvv_float8e4m3m1_t vfloat8e4m3m1_t; +typedef __rvv_float8e4m3m2_t vfloat8e4m3m2_t; +typedef __rvv_float8e4m3m4_t vfloat8e4m3m4_t; +typedef __rvv_float8e4m3m8_t vfloat8e4m3m8_t; +typedef __rvv_float8e5m2mf8_t vfloat8e5m2mf8_t; +typedef __rvv_float8e5m2mf4_t vfloat8e5m2mf4_t; +typedef __rvv_float8e5m2mf2_t vfloat8e5m2mf2_t; +typedef __rvv_float8e5m2m1_t vfloat8e5m2m1_t; +typedef __rvv_float8e5m2m2_t vfloat8e5m2m2_t; +typedef __rvv_float8e5m2m4_t vfloat8e5m2m4_t; +typedef __rvv_float8e5m2m8_t vfloat8e5m2m8_t; #ifdef __cplusplus } diff --git a/lib/include/sifive_vector.h b/lib/include/sifive_vector.h index d315eb960982156115c7143e7cfc87f891b70c30..79882d987bf5ef9abf88f367a738bcf9b7a74504 100644 --- a/lib/include/sifive_vector.h +++ b/lib/include/sifive_vector.h @@ -13,6 +13,24 @@ #pragma clang riscv intrinsic sifive_vector +#define __riscv_intrinsic_xsfmm32a16f 1 +#define __riscv_intrinsic_xsfmm32a32f 1 +#define __riscv_intrinsic_xsfmm32a8f 1 +#define __riscv_intrinsic_xsfmm32a8i 1 +#define __riscv_intrinsic_xsfmm32a 1 +#define __riscv_intrinsic_xsfmm64a64f 1 +#define __riscv_intrinsic_xsfmmbase 1 +#define __riscv_intrinsic_xsfvcp 1 +#define __riscv_intrinsic_xsfvfbfexp16e 1 +#define __riscv_intrinsic_xsfvfexp16e 1 +#define __riscv_intrinsic_xsfvfexp32e 1 +#define __riscv_intrinsic_xsfvfexpa 1 +#define __riscv_intrinsic_xsfvfexpa64e 1 +#define __riscv_intrinsic_xsfvfnrclipxfqf 1 +#define __riscv_intrinsic_xsfvfwmaccqqq 1 +#define __riscv_intrinsic_xsfvqmaccdod 1 +#define __riscv_intrinsic_xsfvqmaccqoq 1 + #define __riscv_sf_vc_x_se_u8mf4(p27_26, p24_20, p11_7, rs1, vl) \ __riscv_sf_vc_x_se(p27_26, p24_20, p11_7, (uint8_t)rs1, 8, 6, vl) #define __riscv_sf_vc_x_se_u8mf2(p27_26, p24_20, p11_7, rs1, vl) \ diff --git a/lib/include/spirvintrin.h b/lib/include/spirvintrin.h index 2a10a47adedde9f018b65ef239c9531c0790bd86..0c61a3f9cc8e34ac0e1265ff241b0d3c2193c4a9 100644 --- a/lib/include/spirvintrin.h +++ b/lib/include/spirvintrin.h @@ -27,9 +27,6 @@ _Pragma("omp begin declare variant match(device = {arch(spirv64)})"); #define __gpu_global __attribute__((address_space(1))) #define __gpu_generic __attribute__((address_space(4))) -// Attribute to declare a function as a kernel. -#define __gpu_kernel __attribute__((device_kernel, visibility("protected"))) - // Returns the number of workgroups in the 'x' dimension of the grid. _DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_blocks_x(void) { return __builtin_spirv_num_workgroups(0); @@ -146,37 +143,13 @@ __gpu_shuffle_idx_u32(uint64_t __lane_mask, uint32_t __idx, uint32_t __x, return __builtin_spirv_subgroup_shuffle(__x, __lane); } -// Returns a bitmask marking all lanes that have the same value of __x. -_DEFAULT_FN_ATTRS static __inline__ uint64_t -__gpu_match_any_u32(uint64_t __lane_mask, uint32_t __x) { - return __gpu_match_any_u32_impl(__lane_mask, __x); -} - -// Returns a bitmask marking all lanes that have the same value of __x. -_DEFAULT_FN_ATTRS static __inline__ uint64_t -__gpu_match_any_u64(uint64_t __lane_mask, uint64_t __x) { - return __gpu_match_any_u64_impl(__lane_mask, __x); -} - -// Returns the current lane mask if every lane contains __x. -_DEFAULT_FN_ATTRS static __inline__ uint64_t -__gpu_match_all_u32(uint64_t __lane_mask, uint32_t __x) { - return __gpu_match_all_u32_impl(__lane_mask, __x); -} - -// Returns the current lane mask if every lane contains __x. -_DEFAULT_FN_ATTRS static __inline__ uint64_t -__gpu_match_all_u64(uint64_t __lane_mask, uint64_t __x) { - return __gpu_match_all_u64_impl(__lane_mask, __x); -} - // SPIR-V does not expose this, always return false. -_DEFAULT_FN_ATTRS static __inline__ bool __gpu_is_ptr_local(void *ptr) { +_DEFAULT_FN_ATTRS static __inline__ bool __gpu_is_ptr_local(const void *ptr) { return 0; } // SPIR-V does not expose this, always return false. -_DEFAULT_FN_ATTRS static __inline__ bool __gpu_is_ptr_private(void *ptr) { +_DEFAULT_FN_ATTRS static __inline__ bool __gpu_is_ptr_private(const void *ptr) { return 0; } diff --git a/lib/include/stdint.h b/lib/include/stdint.h index 96c2ccace13d09b57eacb9982f052b4cf94f768c..07c32542198c32a5b5d5754cc80937d4b8736690 100644 --- a/lib/include/stdint.h +++ b/lib/include/stdint.h @@ -805,25 +805,16 @@ typedef __UINTMAX_TYPE__ uintmax_t; #endif /* C99 7.18.3 Limits of other integer types. */ -#define SIG_ATOMIC_MIN __INTN_MIN(__SIG_ATOMIC_WIDTH__) -#define SIG_ATOMIC_MAX __INTN_MAX(__SIG_ATOMIC_WIDTH__) -#ifdef __WINT_UNSIGNED__ -# define WINT_MIN __UINTN_C(__WINT_WIDTH__, 0) -# define WINT_MAX __UINTN_MAX(__WINT_WIDTH__) -#else -# define WINT_MIN __INTN_MIN(__WINT_WIDTH__) -# define WINT_MAX __INTN_MAX(__WINT_WIDTH__) -#endif +#define SIG_ATOMIC_MIN __SIG_ATOMIC_MIN__ +#define SIG_ATOMIC_MAX __SIG_ATOMIC_MAX__ +#define WINT_MIN __WINT_MIN__ +#define WINT_MAX __WINT_MAX__ #ifndef WCHAR_MAX # define WCHAR_MAX __WCHAR_MAX__ #endif #ifndef WCHAR_MIN -# if __WCHAR_MAX__ == __INTN_MAX(__WCHAR_WIDTH__) -# define WCHAR_MIN __INTN_MIN(__WCHAR_WIDTH__) -# else -# define WCHAR_MIN __UINTN_C(__WCHAR_WIDTH__, 0) -# endif +#define WCHAR_MIN __WCHAR_MIN__ #endif /* 7.18.4.2 Macros for greatest-width integer constants. */ diff --git a/lib/include/vecintrin.h b/lib/include/vecintrin.h index 338ea51ce8863d81114ff81e5c5634fa396669d9..4e567b9adc265962960521354455a0df9fb21ac1 100644 --- a/lib/include/vecintrin.h +++ b/lib/include/vecintrin.h @@ -207,7 +207,7 @@ vec_insert(unsigned long long __scalar, __vector unsigned long long __vec, #if __ARCH__ >= 12 static inline __ATTRS_o_ai __vector float vec_insert(float __scalar, __vector float __vec, int __index) { - __vec[__index & 1] = __scalar; + __vec[__index & 3] = __scalar; return __vec; } #endif diff --git a/lib/include/wasm_simd128.h b/lib/include/wasm_simd128.h index 08e39bf1a79b4ffe01fcfc3c2ac29048ee961975..b4b8e4667a8ae3d7a0117465e809494bd7947f74 100644 --- a/lib/include/wasm_simd128.h +++ b/lib/include/wasm_simd128.h @@ -45,6 +45,7 @@ typedef int __i32x2 __attribute__((__vector_size__(8), __aligned__(8))); typedef unsigned int __u32x2 __attribute__((__vector_size__(8), __aligned__(8))); typedef float __f32x2 __attribute__((__vector_size__(8), __aligned__(8))); +typedef __fp16 __f16x4 __attribute__((__vector_size__(8), __aligned__(8))); #define __DEFAULT_FN_ATTRS \ __attribute__((__always_inline__, __nodebug__, __target__("simd128"), \ @@ -2010,6 +2011,22 @@ static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_convert_u16x8(v128_t __a) { return (v128_t) __builtin_convertvector((__u16x8)__a, __f16x8); } +static __inline__ v128_t __FP16_FN_ATTRS +wasm_f32x4_promote_low_f16x8(v128_t __a) { + return (v128_t) __builtin_convertvector( + (__f16x4){((__f16x8)__a)[0], ((__f16x8)__a)[1], ((__f16x8)__a)[2], + ((__f16x8)__a)[3]}, + __f32x4); +} + +static __inline__ v128_t __FP16_FN_ATTRS +wasm_f16x8_demote_f32x4_zero(v128_t __a) { + return (v128_t) __builtin_convertvector( + __builtin_shufflevector((__f32x4)__a, (__f32x4){0, 0, 0, 0}, 0, 1, 2, 3, + 4, 5, 6, 7), + __f16x8); +} + static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_relaxed_madd(v128_t __a, v128_t __b, v128_t __c) { diff --git a/lib/include/xmmintrin.h b/lib/include/xmmintrin.h index ab0f0c1690759e7bc6d7d357cbf3d271a98056f6..73eab9e460ca5a192d06f470793fe33d74aae2e0 100644 --- a/lib/include/xmmintrin.h +++ b/lib/include/xmmintrin.h @@ -341,7 +341,8 @@ _mm_rsqrt_ps(__m128 __a) /// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the /// minimum value between both operands. The upper 96 bits are copied from /// the upper 96 bits of the first source operand. -static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_min_ss(__m128 __a, __m128 __b) { +static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_min_ss(__m128 __a, + __m128 __b) { return __builtin_ia32_minss((__v4sf)__a, (__v4sf)__b); } @@ -384,7 +385,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_min_ps(__m128 __a, /// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the /// maximum value between both operands. The upper 96 bits are copied from /// the upper 96 bits of the first source operand. -static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_max_ss(__m128 __a, __m128 __b) { +static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_max_ss(__m128 __a, + __m128 __b) { return __builtin_ia32_maxss((__v4sf)__a, (__v4sf)__b); } @@ -2569,11 +2571,10 @@ _mm_avg_pu16(__m64 __a, __m64 __b) { /// \returns A 64-bit integer vector whose lower 16 bits contain the sums of the /// sets of absolute differences between both operands. The upper bits are /// cleared. -static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 -_mm_sad_pu8(__m64 __a, __m64 __b) -{ - return __trunc64(__builtin_ia32_psadbw128((__v16qi)__zext128(__a), - (__v16qi)__zext128(__b))); +static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR +_mm_sad_pu8(__m64 __a, __m64 __b) { + return __trunc64(__builtin_ia32_psadbw128((__v16qu)__zext128(__a), + (__v16qu)__zext128(__b))); } #if defined(__cplusplus) -- 2.54.0