| author |  Andrew Kelley <andrew@ziglang.org> 2019-02-07 18:49:42-05:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2019-02-07 18:49:42-05:00 |
| log | 48c1e235cb620dacc30254d0898390b4d97f3e73 |
| tree | 7c61685b609041e3a2bce0ba8db1ea71b62ba24b |
| parent | 00902ff9b0931a85ab87649efc8b4e4027ab8358 |
| signature | Commit is signed but in an unrecognized format. |
update clang headers to 8.0.0rc2
24 files changed, 4290 insertions(+), 2548 deletions(-)
c_headers
- __clang_cuda_runtime_wrapper.h
- adxintrin.h
- altivec.h
- arm_fp16.h
- arm_neon.h
- avx512bwintrin.h
- avx512dqintrin.h
- avx512fintrin.h
- avx512pfintrin.h
- avx512vbmi2intrin.h
- avx512vbmiintrin.h
- avx512vbmivlintrin.h
- avx512vlbwintrin.h
- avx512vlintrin.h
- avx512vlvbmi2intrin.h
- bmiintrin.h
cuda_wrappers
- emmintrin.h
- float.h
- immintrin.h
- intrin.h
- lzcntintrin.h
- opencl-c.h
- vecintrin.h
c_headers/__clang_cuda_runtime_wrapper.h+7-1
| ... | @@ -62,10 +62,15 @@ | ... | @@ -62,10 +62,15 @@ |
| 62 | #include "cuda.h" | 62 | #include "cuda.h" |
| 63 | #if !defined(CUDA_VERSION) | 63 | #if !defined(CUDA_VERSION) |
| 64 | #error "cuda.h did not define CUDA_VERSION" | 64 | #error "cuda.h did not define CUDA_VERSION" |
| 65 | #elif CUDA_VERSION < 7000 || CUDA_VERSION > 9020 | 65 | #elif CUDA_VERSION < 7000 || CUDA_VERSION > 10000 |
| 66 | #error "Unsupported CUDA version!" | 66 | #error "Unsupported CUDA version!" |
| 67 | #endif | 67 | #endif |
| 68 | 68 | ||
| 69 | #pragma push_macro("__CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__") | ||
| 70 | #if CUDA_VERSION >= 10000 | ||
| 71 | #define __CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__ | ||
| 72 | #endif | ||
| 73 | |||
| 69 | // Make largest subset of device functions available during host | 74 | // Make largest subset of device functions available during host |
| 70 | // compilation -- SM_35 for the time being. | 75 | // compilation -- SM_35 for the time being. |
| 71 | #ifndef __CUDA_ARCH__ | 76 | #ifndef __CUDA_ARCH__ |
| ... | @@ -419,6 +424,7 @@ __device__ inline __cuda_builtin_gridDim_t::operator dim3() const { | ... | @@ -419,6 +424,7 @@ __device__ inline __cuda_builtin_gridDim_t::operator dim3() const { |
| 419 | #pragma pop_macro("dim3") | 424 | #pragma pop_macro("dim3") |
| 420 | #pragma pop_macro("uint3") | 425 | #pragma pop_macro("uint3") |
| 421 | #pragma pop_macro("__USE_FAST_MATH__") | 426 | #pragma pop_macro("__USE_FAST_MATH__") |
| 427 | #pragma pop_macro("__CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__") | ||
| 422 | 428 | ||
| 423 | #endif // __CUDA__ | 429 | #endif // __CUDA__ |
| 424 | #endif // __CLANG_CUDA_RUNTIME_WRAPPER_H__ | 430 | #endif // __CLANG_CUDA_RUNTIME_WRAPPER_H__ |
c_headers/adxintrin.h+2-2
| ... | @@ -53,7 +53,7 @@ static __inline unsigned char __DEFAULT_FN_ATTRS | ... | @@ -53,7 +53,7 @@ static __inline unsigned char __DEFAULT_FN_ATTRS |
| 53 | _addcarry_u32(unsigned char __cf, unsigned int __x, unsigned int __y, | 53 | _addcarry_u32(unsigned char __cf, unsigned int __x, unsigned int __y, |
| 54 | unsigned int *__p) | 54 | unsigned int *__p) |
| 55 | { | 55 | { |
| 56 | return __builtin_ia32_addcarry_u32(__cf, __x, __y, __p); | 56 | return __builtin_ia32_addcarryx_u32(__cf, __x, __y, __p); |
| 57 | } | 57 | } |
| 58 | 58 | ||
| 59 | #ifdef __x86_64__ | 59 | #ifdef __x86_64__ |
| ... | @@ -61,7 +61,7 @@ static __inline unsigned char __DEFAULT_FN_ATTRS | ... | @@ -61,7 +61,7 @@ static __inline unsigned char __DEFAULT_FN_ATTRS |
| 61 | _addcarry_u64(unsigned char __cf, unsigned long long __x, | 61 | _addcarry_u64(unsigned char __cf, unsigned long long __x, |
| 62 | unsigned long long __y, unsigned long long *__p) | 62 | unsigned long long __y, unsigned long long *__p) |
| 63 | { | 63 | { |
| 64 | return __builtin_ia32_addcarry_u64(__cf, __x, __y, __p); | 64 | return __builtin_ia32_addcarryx_u64(__cf, __x, __y, __p); |
| 65 | } | 65 | } |
| 66 | #endif | 66 | #endif |
| 67 | 67 |
c_headers/altivec.h+70-53
| ... | @@ -9492,49 +9492,51 @@ vec_splat_u32(signed char __a) { | ... | @@ -9492,49 +9492,51 @@ vec_splat_u32(signed char __a) { |
| 9492 | 9492 | ||
| 9493 | /* vec_sr */ | 9493 | /* vec_sr */ |
| 9494 | 9494 | ||
| 9495 | static __inline__ vector signed char __ATTRS_o_ai | 9495 | // vec_sr does modulo arithmetic on __b first, so __b is allowed to be more |
| 9496 | vec_sr(vector signed char __a, vector unsigned char __b) { | 9496 | // than the length of __a. |
| 9497 | vector unsigned char __res = (vector unsigned char)__a >> __b; | ||
| 9498 | return (vector signed char)__res; | ||
| 9499 | } | ||
| 9500 | |||
| 9501 | static __inline__ vector unsigned char __ATTRS_o_ai | 9497 | static __inline__ vector unsigned char __ATTRS_o_ai |
| 9502 | vec_sr(vector unsigned char __a, vector unsigned char __b) { | 9498 | vec_sr(vector unsigned char __a, vector unsigned char __b) { |
| 9503 | return __a >> __b; | 9499 | return __a >> |
| 9500 | (__b % (vector unsigned char)(sizeof(unsigned char) * __CHAR_BIT__)); | ||
| 9504 | } | 9501 | } |
| 9505 | 9502 | ||
| 9506 | static __inline__ vector signed short __ATTRS_o_ai | 9503 | static __inline__ vector signed char __ATTRS_o_ai |
| 9507 | vec_sr(vector signed short __a, vector unsigned short __b) { | 9504 | vec_sr(vector signed char __a, vector unsigned char __b) { |
| 9508 | vector unsigned short __res = (vector unsigned short)__a >> __b; | 9505 | return (vector signed char)vec_sr((vector unsigned char)__a, __b); |
| 9509 | return (vector signed short)__res; | ||
| 9510 | } | 9506 | } |
| 9511 | 9507 | ||
| 9512 | static __inline__ vector unsigned short __ATTRS_o_ai | 9508 | static __inline__ vector unsigned short __ATTRS_o_ai |
| 9513 | vec_sr(vector unsigned short __a, vector unsigned short __b) { | 9509 | vec_sr(vector unsigned short __a, vector unsigned short __b) { |
| 9514 | return __a >> __b; | 9510 | return __a >> |
| 9511 | (__b % (vector unsigned short)(sizeof(unsigned short) * __CHAR_BIT__)); | ||
| 9515 | } | 9512 | } |
| 9516 | 9513 | ||
| 9517 | static __inline__ vector signed int __ATTRS_o_ai | 9514 | static __inline__ vector short __ATTRS_o_ai vec_sr(vector short __a, |
| 9518 | vec_sr(vector signed int __a, vector unsigned int __b) { | 9515 | vector unsigned short __b) { |
| 9519 | vector unsigned int __res = (vector unsigned int)__a >> __b; | 9516 | return (vector short)vec_sr((vector unsigned short)__a, __b); |
| 9520 | return (vector signed int)__res; | ||
| 9521 | } | 9517 | } |
| 9522 | 9518 | ||
| 9523 | static __inline__ vector unsigned int __ATTRS_o_ai | 9519 | static __inline__ vector unsigned int __ATTRS_o_ai |
| 9524 | vec_sr(vector unsigned int __a, vector unsigned int __b) { | 9520 | vec_sr(vector unsigned int __a, vector unsigned int __b) { |
| 9525 | return __a >> __b; | 9521 | return __a >> |
| 9522 | (__b % (vector unsigned int)(sizeof(unsigned int) * __CHAR_BIT__)); | ||
| 9526 | } | 9523 | } |
| 9527 | 9524 | ||
| 9528 | #ifdef __POWER8_VECTOR__ | 9525 | static __inline__ vector int __ATTRS_o_ai vec_sr(vector int __a, |
| 9529 | static __inline__ vector signed long long __ATTRS_o_ai | 9526 | vector unsigned int __b) { |
| 9530 | vec_sr(vector signed long long __a, vector unsigned long long __b) { | 9527 | return (vector int)vec_sr((vector unsigned int)__a, __b); |
| 9531 | vector unsigned long long __res = (vector unsigned long long)__a >> __b; | ||
| 9532 | return (vector signed long long)__res; | ||
| 9533 | } | 9528 | } |
| 9534 | 9529 | ||
| 9530 | #ifdef __POWER8_VECTOR__ | ||
| 9535 | static __inline__ vector unsigned long long __ATTRS_o_ai | 9531 | static __inline__ vector unsigned long long __ATTRS_o_ai |
| 9536 | vec_sr(vector unsigned long long __a, vector unsigned long long __b) { | 9532 | vec_sr(vector unsigned long long __a, vector unsigned long long __b) { |
| 9537 | return __a >> __b; | 9533 | return __a >> (__b % (vector unsigned long long)(sizeof(unsigned long long) * |
| 9534 | __CHAR_BIT__)); | ||
| 9535 | } | ||
| 9536 | |||
| 9537 | static __inline__ vector long long __ATTRS_o_ai | ||
| 9538 | vec_sr(vector long long __a, vector unsigned long long __b) { | ||
| 9539 | return (vector long long)vec_sr((vector unsigned long long)__a, __b); | ||
| 9538 | } | 9540 | } |
| 9539 | #endif | 9541 | #endif |
| 9540 | 9542 | ||
| ... | @@ -9544,12 +9546,12 @@ vec_sr(vector unsigned long long __a, vector unsigned long long __b) { | ... | @@ -9544,12 +9546,12 @@ vec_sr(vector unsigned long long __a, vector unsigned long long __b) { |
| 9544 | 9546 | ||
| 9545 | static __inline__ vector signed char __ATTRS_o_ai | 9547 | static __inline__ vector signed char __ATTRS_o_ai |
| 9546 | vec_vsrb(vector signed char __a, vector unsigned char __b) { | 9548 | vec_vsrb(vector signed char __a, vector unsigned char __b) { |
| 9547 | return __a >> (vector signed char)__b; | 9549 | return vec_sr(__a, __b); |
| 9548 | } | 9550 | } |
| 9549 | 9551 | ||
| 9550 | static __inline__ vector unsigned char __ATTRS_o_ai | 9552 | static __inline__ vector unsigned char __ATTRS_o_ai |
| 9551 | vec_vsrb(vector unsigned char __a, vector unsigned char __b) { | 9553 | vec_vsrb(vector unsigned char __a, vector unsigned char __b) { |
| 9552 | return __a >> __b; | 9554 | return vec_sr(__a, __b); |
| 9553 | } | 9555 | } |
| 9554 | 9556 | ||
| 9555 | /* vec_vsrh */ | 9557 | /* vec_vsrh */ |
| ... | @@ -9558,12 +9560,12 @@ vec_vsrb(vector unsigned char __a, vector unsigned char __b) { | ... | @@ -9558,12 +9560,12 @@ vec_vsrb(vector unsigned char __a, vector unsigned char __b) { |
| 9558 | 9560 | ||
| 9559 | static __inline__ vector short __ATTRS_o_ai | 9561 | static __inline__ vector short __ATTRS_o_ai |
| 9560 | vec_vsrh(vector short __a, vector unsigned short __b) { | 9562 | vec_vsrh(vector short __a, vector unsigned short __b) { |
| 9561 | return __a >> (vector short)__b; | 9563 | return vec_sr(__a, __b); |
| 9562 | } | 9564 | } |
| 9563 | 9565 | ||
| 9564 | static __inline__ vector unsigned short __ATTRS_o_ai | 9566 | static __inline__ vector unsigned short __ATTRS_o_ai |
| 9565 | vec_vsrh(vector unsigned short __a, vector unsigned short __b) { | 9567 | vec_vsrh(vector unsigned short __a, vector unsigned short __b) { |
| 9566 | return __a >> __b; | 9568 | return vec_sr(__a, __b); |
| 9567 | } | 9569 | } |
| 9568 | 9570 | ||
| 9569 | /* vec_vsrw */ | 9571 | /* vec_vsrw */ |
| ... | @@ -9572,12 +9574,12 @@ vec_vsrh(vector unsigned short __a, vector unsigned short __b) { | ... | @@ -9572,12 +9574,12 @@ vec_vsrh(vector unsigned short __a, vector unsigned short __b) { |
| 9572 | 9574 | ||
| 9573 | static __inline__ vector int __ATTRS_o_ai vec_vsrw(vector int __a, | 9575 | static __inline__ vector int __ATTRS_o_ai vec_vsrw(vector int __a, |
| 9574 | vector unsigned int __b) { | 9576 | vector unsigned int __b) { |
| 9575 | return __a >> (vector int)__b; | 9577 | return vec_sr(__a, __b); |
| 9576 | } | 9578 | } |
| 9577 | 9579 | ||
| 9578 | static __inline__ vector unsigned int __ATTRS_o_ai | 9580 | static __inline__ vector unsigned int __ATTRS_o_ai |
| 9579 | vec_vsrw(vector unsigned int __a, vector unsigned int __b) { | 9581 | vec_vsrw(vector unsigned int __a, vector unsigned int __b) { |
| 9580 | return __a >> __b; | 9582 | return vec_sr(__a, __b); |
| 9581 | } | 9583 | } |
| 9582 | 9584 | ||
| 9583 | /* vec_sra */ | 9585 | /* vec_sra */ |
| ... | @@ -16353,67 +16355,82 @@ vec_revb(vector unsigned __int128 __a) { | ... | @@ -16353,67 +16355,82 @@ vec_revb(vector unsigned __int128 __a) { |
| 16353 | 16355 | ||
| 16354 | /* vec_xl */ | 16356 | /* vec_xl */ |
| 16355 | 16357 | ||
| 16358 | typedef vector signed char unaligned_vec_schar __attribute__((aligned(1))); | ||
| 16359 | typedef vector unsigned char unaligned_vec_uchar __attribute__((aligned(1))); | ||
| 16360 | typedef vector signed short unaligned_vec_sshort __attribute__((aligned(1))); | ||
| 16361 | typedef vector unsigned short unaligned_vec_ushort __attribute__((aligned(1))); | ||
| 16362 | typedef vector signed int unaligned_vec_sint __attribute__((aligned(1))); | ||
| 16363 | typedef vector unsigned int unaligned_vec_uint __attribute__((aligned(1))); | ||
| 16364 | typedef vector float unaligned_vec_float __attribute__((aligned(1))); | ||
| 16365 | |||
| 16356 | static inline __ATTRS_o_ai vector signed char vec_xl(signed long long __offset, | 16366 | static inline __ATTRS_o_ai vector signed char vec_xl(signed long long __offset, |
| 16357 | signed char *__ptr) { | 16367 | signed char *__ptr) { |
| 16358 | return *(vector signed char *)(__ptr + __offset); | 16368 | return *(unaligned_vec_schar *)(__ptr + __offset); |
| 16359 | } | 16369 | } |
| 16360 | 16370 | ||
| 16361 | static inline __ATTRS_o_ai vector unsigned char | 16371 | static inline __ATTRS_o_ai vector unsigned char |
| 16362 | vec_xl(signed long long __offset, unsigned char *__ptr) { | 16372 | vec_xl(signed long long __offset, unsigned char *__ptr) { |
| 16363 | return *(vector unsigned char *)(__ptr + __offset); | 16373 | return *(unaligned_vec_uchar*)(__ptr + __offset); |
| 16364 | } | 16374 | } |
| 16365 | 16375 | ||
| 16366 | static inline __ATTRS_o_ai vector signed short vec_xl(signed long long __offset, | 16376 | static inline __ATTRS_o_ai vector signed short vec_xl(signed long long __offset, |
| 16367 | signed short *__ptr) { | 16377 | signed short *__ptr) { |
| 16368 | return *(vector signed short *)(__ptr + __offset); | 16378 | return *(unaligned_vec_sshort *)(__ptr + __offset); |
| 16369 | } | 16379 | } |
| 16370 | 16380 | ||
| 16371 | static inline __ATTRS_o_ai vector unsigned short | 16381 | static inline __ATTRS_o_ai vector unsigned short |
| 16372 | vec_xl(signed long long __offset, unsigned short *__ptr) { | 16382 | vec_xl(signed long long __offset, unsigned short *__ptr) { |
| 16373 | return *(vector unsigned short *)(__ptr + __offset); | 16383 | return *(unaligned_vec_ushort *)(__ptr + __offset); |
| 16374 | } | 16384 | } |
| 16375 | 16385 | ||
| 16376 | static inline __ATTRS_o_ai vector signed int vec_xl(signed long long __offset, | 16386 | static inline __ATTRS_o_ai vector signed int vec_xl(signed long long __offset, |
| 16377 | signed int *__ptr) { | 16387 | signed int *__ptr) { |
| 16378 | return *(vector signed int *)(__ptr + __offset); | 16388 | return *(unaligned_vec_sint *)(__ptr + __offset); |
| 16379 | } | 16389 | } |
| 16380 | 16390 | ||
| 16381 | static inline __ATTRS_o_ai vector unsigned int vec_xl(signed long long __offset, | 16391 | static inline __ATTRS_o_ai vector unsigned int vec_xl(signed long long __offset, |
| 16382 | unsigned int *__ptr) { | 16392 | unsigned int *__ptr) { |
| 16383 | return *(vector unsigned int *)(__ptr + __offset); | 16393 | return *(unaligned_vec_uint *)(__ptr + __offset); |
| 16384 | } | 16394 | } |
| 16385 | 16395 | ||
| 16386 | static inline __ATTRS_o_ai vector float vec_xl(signed long long __offset, | 16396 | static inline __ATTRS_o_ai vector float vec_xl(signed long long __offset, |
| 16387 | float *__ptr) { | 16397 | float *__ptr) { |
| 16388 | return *(vector float *)(__ptr + __offset); | 16398 | return *(unaligned_vec_float *)(__ptr + __offset); |
| 16389 | } | 16399 | } |
| 16390 | 16400 | ||
| 16391 | #ifdef __VSX__ | 16401 | #ifdef __VSX__ |
| 16402 | typedef vector signed long long unaligned_vec_sll __attribute__((aligned(1))); | ||
| 16403 | typedef vector unsigned long long unaligned_vec_ull __attribute__((aligned(1))); | ||
| 16404 | typedef vector double unaligned_vec_double __attribute__((aligned(1))); | ||
| 16405 | |||
| 16392 | static inline __ATTRS_o_ai vector signed long long | 16406 | static inline __ATTRS_o_ai vector signed long long |
| 16393 | vec_xl(signed long long __offset, signed long long *__ptr) { | 16407 | vec_xl(signed long long __offset, signed long long *__ptr) { |
| 16394 | return *(vector signed long long *)(__ptr + __offset); | 16408 | return *(unaligned_vec_sll *)(__ptr + __offset); |
| 16395 | } | 16409 | } |
| 16396 | 16410 | ||
| 16397 | static inline __ATTRS_o_ai vector unsigned long long | 16411 | static inline __ATTRS_o_ai vector unsigned long long |
| 16398 | vec_xl(signed long long __offset, unsigned long long *__ptr) { | 16412 | vec_xl(signed long long __offset, unsigned long long *__ptr) { |
| 16399 | return *(vector unsigned long long *)(__ptr + __offset); | 16413 | return *(unaligned_vec_ull *)(__ptr + __offset); |
| 16400 | } | 16414 | } |
| 16401 | 16415 | ||
| 16402 | static inline __ATTRS_o_ai vector double vec_xl(signed long long __offset, | 16416 | static inline __ATTRS_o_ai vector double vec_xl(signed long long __offset, |
| 16403 | double *__ptr) { | 16417 | double *__ptr) { |
| 16404 | return *(vector double *)(__ptr + __offset); | 16418 | return *(unaligned_vec_double *)(__ptr + __offset); |
| 16405 | } | 16419 | } |
| 16406 | #endif | 16420 | #endif |
| 16407 | 16421 | ||
| 16408 | #if defined(__POWER8_VECTOR__) && defined(__powerpc64__) | 16422 | #if defined(__POWER8_VECTOR__) && defined(__powerpc64__) |
| 16423 | typedef vector signed __int128 unaligned_vec_si128 __attribute__((aligned(1))); | ||
| 16424 | typedef vector unsigned __int128 unaligned_vec_ui128 | ||
| 16425 | __attribute__((aligned(1))); | ||
| 16409 | static inline __ATTRS_o_ai vector signed __int128 | 16426 | static inline __ATTRS_o_ai vector signed __int128 |
| 16410 | vec_xl(signed long long __offset, signed __int128 *__ptr) { | 16427 | vec_xl(signed long long __offset, signed __int128 *__ptr) { |
| 16411 | return *(vector signed __int128 *)(__ptr + __offset); | 16428 | return *(unaligned_vec_si128 *)(__ptr + __offset); |
| 16412 | } | 16429 | } |
| 16413 | 16430 | ||
| 16414 | static inline __ATTRS_o_ai vector unsigned __int128 | 16431 | static inline __ATTRS_o_ai vector unsigned __int128 |
| 16415 | vec_xl(signed long long __offset, unsigned __int128 *__ptr) { | 16432 | vec_xl(signed long long __offset, unsigned __int128 *__ptr) { |
| 16416 | return *(vector unsigned __int128 *)(__ptr + __offset); | 16433 | return *(unaligned_vec_ui128 *)(__ptr + __offset); |
| 16417 | } | 16434 | } |
| 16418 | #endif | 16435 | #endif |
| 16419 | 16436 | ||
| ... | @@ -16498,62 +16515,62 @@ vec_xl_be(signed long long __offset, unsigned __int128 *__ptr) { | ... | @@ -16498,62 +16515,62 @@ vec_xl_be(signed long long __offset, unsigned __int128 *__ptr) { |
| 16498 | static inline __ATTRS_o_ai void vec_xst(vector signed char __vec, | 16515 | static inline __ATTRS_o_ai void vec_xst(vector signed char __vec, |
| 16499 | signed long long __offset, | 16516 | signed long long __offset, |
| 16500 | signed char *__ptr) { | 16517 | signed char *__ptr) { |
| 16501 | *(vector signed char *)(__ptr + __offset) = __vec; | 16518 | *(unaligned_vec_schar *)(__ptr + __offset) = __vec; |
| 16502 | } | 16519 | } |
| 16503 | 16520 | ||
| 16504 | static inline __ATTRS_o_ai void vec_xst(vector unsigned char __vec, | 16521 | static inline __ATTRS_o_ai void vec_xst(vector unsigned char __vec, |
| 16505 | signed long long __offset, | 16522 | signed long long __offset, |
| 16506 | unsigned char *__ptr) { | 16523 | unsigned char *__ptr) { |
| 16507 | *(vector unsigned char *)(__ptr + __offset) = __vec; | 16524 | *(unaligned_vec_uchar *)(__ptr + __offset) = __vec; |
| 16508 | } | 16525 | } |
| 16509 | 16526 | ||
| 16510 | static inline __ATTRS_o_ai void vec_xst(vector signed short __vec, | 16527 | static inline __ATTRS_o_ai void vec_xst(vector signed short __vec, |
| 16511 | signed long long __offset, | 16528 | signed long long __offset, |
| 16512 | signed short *__ptr) { | 16529 | signed short *__ptr) { |
| 16513 | *(vector signed short *)(__ptr + __offset) = __vec; | 16530 | *(unaligned_vec_sshort *)(__ptr + __offset) = __vec; |
| 16514 | } | 16531 | } |
| 16515 | 16532 | ||
| 16516 | static inline __ATTRS_o_ai void vec_xst(vector unsigned short __vec, | 16533 | static inline __ATTRS_o_ai void vec_xst(vector unsigned short __vec, |
| 16517 | signed long long __offset, | 16534 | signed long long __offset, |
| 16518 | unsigned short *__ptr) { | 16535 | unsigned short *__ptr) { |
| 16519 | *(vector unsigned short *)(__ptr + __offset) = __vec; | 16536 | *(unaligned_vec_ushort *)(__ptr + __offset) = __vec; |
| 16520 | } | 16537 | } |
| 16521 | 16538 | ||
| 16522 | static inline __ATTRS_o_ai void vec_xst(vector signed int __vec, | 16539 | static inline __ATTRS_o_ai void vec_xst(vector signed int __vec, |
| 16523 | signed long long __offset, | 16540 | signed long long __offset, |
| 16524 | signed int *__ptr) { | 16541 | signed int *__ptr) { |
| 16525 | *(vector signed int *)(__ptr + __offset) = __vec; | 16542 | *(unaligned_vec_sint *)(__ptr + __offset) = __vec; |
| 16526 | } | 16543 | } |
| 16527 | 16544 | ||
| 16528 | static inline __ATTRS_o_ai void vec_xst(vector unsigned int __vec, | 16545 | static inline __ATTRS_o_ai void vec_xst(vector unsigned int __vec, |
| 16529 | signed long long __offset, | 16546 | signed long long __offset, |
| 16530 | unsigned int *__ptr) { | 16547 | unsigned int *__ptr) { |
| 16531 | *(vector unsigned int *)(__ptr + __offset) = __vec; | 16548 | *(unaligned_vec_uint *)(__ptr + __offset) = __vec; |
| 16532 | } | 16549 | } |
| 16533 | 16550 | ||
| 16534 | static inline __ATTRS_o_ai void vec_xst(vector float __vec, | 16551 | static inline __ATTRS_o_ai void vec_xst(vector float __vec, |
| 16535 | signed long long __offset, | 16552 | signed long long __offset, |
| 16536 | float *__ptr) { | 16553 | float *__ptr) { |
| 16537 | *(vector float *)(__ptr + __offset) = __vec; | 16554 | *(unaligned_vec_float *)(__ptr + __offset) = __vec; |
| 16538 | } | 16555 | } |
| 16539 | 16556 | ||
| 16540 | #ifdef __VSX__ | 16557 | #ifdef __VSX__ |
| 16541 | static inline __ATTRS_o_ai void vec_xst(vector signed long long __vec, | 16558 | static inline __ATTRS_o_ai void vec_xst(vector signed long long __vec, |
| 16542 | signed long long __offset, | 16559 | signed long long __offset, |
| 16543 | signed long long *__ptr) { | 16560 | signed long long *__ptr) { |
| 16544 | *(vector signed long long *)(__ptr + __offset) = __vec; | 16561 | *(unaligned_vec_sll *)(__ptr + __offset) = __vec; |
| 16545 | } | 16562 | } |
| 16546 | 16563 | ||
| 16547 | static inline __ATTRS_o_ai void vec_xst(vector unsigned long long __vec, | 16564 | static inline __ATTRS_o_ai void vec_xst(vector unsigned long long __vec, |
| 16548 | signed long long __offset, | 16565 | signed long long __offset, |
| 16549 | unsigned long long *__ptr) { | 16566 | unsigned long long *__ptr) { |
| 16550 | *(vector unsigned long long *)(__ptr + __offset) = __vec; | 16567 | *(unaligned_vec_ull *)(__ptr + __offset) = __vec; |
| 16551 | } | 16568 | } |
| 16552 | 16569 | ||
| 16553 | static inline __ATTRS_o_ai void vec_xst(vector double __vec, | 16570 | static inline __ATTRS_o_ai void vec_xst(vector double __vec, |
| 16554 | signed long long __offset, | 16571 | signed long long __offset, |
| 16555 | double *__ptr) { | 16572 | double *__ptr) { |
| 16556 | *(vector double *)(__ptr + __offset) = __vec; | 16573 | *(unaligned_vec_double *)(__ptr + __offset) = __vec; |
| 16557 | } | 16574 | } |
| 16558 | #endif | 16575 | #endif |
| 16559 | 16576 | ||
| ... | @@ -16561,13 +16578,13 @@ static inline __ATTRS_o_ai void vec_xst(vector double __vec, | ... | @@ -16561,13 +16578,13 @@ static inline __ATTRS_o_ai void vec_xst(vector double __vec, |
| 16561 | static inline __ATTRS_o_ai void vec_xst(vector signed __int128 __vec, | 16578 | static inline __ATTRS_o_ai void vec_xst(vector signed __int128 __vec, |
| 16562 | signed long long __offset, | 16579 | signed long long __offset, |
| 16563 | signed __int128 *__ptr) { | 16580 | signed __int128 *__ptr) { |
| 16564 | *(vector signed __int128 *)(__ptr + __offset) = __vec; | 16581 | *(unaligned_vec_si128 *)(__ptr + __offset) = __vec; |
| 16565 | } | 16582 | } |
| 16566 | 16583 | ||
| 16567 | static inline __ATTRS_o_ai void vec_xst(vector unsigned __int128 __vec, | 16584 | static inline __ATTRS_o_ai void vec_xst(vector unsigned __int128 __vec, |
| 16568 | signed long long __offset, | 16585 | signed long long __offset, |
| 16569 | unsigned __int128 *__ptr) { | 16586 | unsigned __int128 *__ptr) { |
| 16570 | *(vector unsigned __int128 *)(__ptr + __offset) = __vec; | 16587 | *(unaligned_vec_ui128 *)(__ptr + __offset) = __vec; |
| 16571 | } | 16588 | } |
| 16572 | #endif | 16589 | #endif |
| 16573 | 16590 |
c_headers/arm_fp16.h+1-1
| ... | @@ -27,7 +27,7 @@ | ... | @@ -27,7 +27,7 @@ |
| 27 | #include <stdint.h> | 27 | #include <stdint.h> |
| 28 | 28 | ||
| 29 | typedef __fp16 float16_t; | 29 | typedef __fp16 float16_t; |
| 30 | #define __ai static inline __attribute__((__always_inline__, __nodebug__)) | 30 | #define __ai static __inline__ __attribute__((__always_inline__, __nodebug__)) |
| 31 | 31 | ||
| 32 | #if defined(__ARM_FEATURE_FP16_SCALAR_ARITHMETIC) && defined(__aarch64__) | 32 | #if defined(__ARM_FEATURE_FP16_SCALAR_ARITHMETIC) && defined(__aarch64__) |
| 33 | #ifdef __LITTLE_ENDIAN__ | 33 | #ifdef __LITTLE_ENDIAN__ |
c_headers/arm_neon.h+1765-1193
| ... | @@ -427,7 +427,7 @@ typedef struct poly64x2x4_t { | ... | @@ -427,7 +427,7 @@ typedef struct poly64x2x4_t { |
| 427 | 427 | ||
| 428 | #endif | 428 | #endif |
| 429 | 429 | ||
| 430 | #define __ai static inline __attribute__((__always_inline__, __nodebug__)) | 430 | #define __ai static __inline__ __attribute__((__always_inline__, __nodebug__)) |
| 431 | 431 | ||
| 432 | #ifdef __LITTLE_ENDIAN__ | 432 | #ifdef __LITTLE_ENDIAN__ |
| 433 | __ai uint8x16_t vabdq_u8(uint8x16_t __p0, uint8x16_t __p1) { | 433 | __ai uint8x16_t vabdq_u8(uint8x16_t __p0, uint8x16_t __p1) { |
| ... | @@ -7592,21 +7592,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -7592,21 +7592,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 7592 | }) | 7592 | }) |
| 7593 | #endif | 7593 | #endif |
| 7594 | 7594 | ||
| 7595 | #ifdef __LITTLE_ENDIAN__ | ||
| 7596 | #define vld1q_f16(__p0) __extension__ ({ \ | ||
| 7597 | float16x8_t __ret; \ | ||
| 7598 | __ret = (float16x8_t) __builtin_neon_vld1q_v(__p0, 40); \ | ||
| 7599 | __ret; \ | ||
| 7600 | }) | ||
| 7601 | #else | ||
| 7602 | #define vld1q_f16(__p0) __extension__ ({ \ | ||
| 7603 | float16x8_t __ret; \ | ||
| 7604 | __ret = (float16x8_t) __builtin_neon_vld1q_v(__p0, 40); \ | ||
| 7605 | __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 7606 | __ret; \ | ||
| 7607 | }) | ||
| 7608 | #endif | ||
| 7609 | |||
| 7610 | #ifdef __LITTLE_ENDIAN__ | 7595 | #ifdef __LITTLE_ENDIAN__ |
| 7611 | #define vld1q_s32(__p0) __extension__ ({ \ | 7596 | #define vld1q_s32(__p0) __extension__ ({ \ |
| 7612 | int32x4_t __ret; \ | 7597 | int32x4_t __ret; \ |
| ... | @@ -7741,21 +7726,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -7741,21 +7726,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 7741 | }) | 7726 | }) |
| 7742 | #endif | 7727 | #endif |
| 7743 | 7728 | ||
| 7744 | #ifdef __LITTLE_ENDIAN__ | ||
| 7745 | #define vld1_f16(__p0) __extension__ ({ \ | ||
| 7746 | float16x4_t __ret; \ | ||
| 7747 | __ret = (float16x4_t) __builtin_neon_vld1_v(__p0, 8); \ | ||
| 7748 | __ret; \ | ||
| 7749 | }) | ||
| 7750 | #else | ||
| 7751 | #define vld1_f16(__p0) __extension__ ({ \ | ||
| 7752 | float16x4_t __ret; \ | ||
| 7753 | __ret = (float16x4_t) __builtin_neon_vld1_v(__p0, 8); \ | ||
| 7754 | __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ | ||
| 7755 | __ret; \ | ||
| 7756 | }) | ||
| 7757 | #endif | ||
| 7758 | |||
| 7759 | #ifdef __LITTLE_ENDIAN__ | 7729 | #ifdef __LITTLE_ENDIAN__ |
| 7760 | #define vld1_s32(__p0) __extension__ ({ \ | 7730 | #define vld1_s32(__p0) __extension__ ({ \ |
| 7761 | int32x2_t __ret; \ | 7731 | int32x2_t __ret; \ |
| ... | @@ -7950,21 +7920,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -7950,21 +7920,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 7950 | }) | 7920 | }) |
| 7951 | #endif | 7921 | #endif |
| 7952 | 7922 | ||
| 7953 | #ifdef __LITTLE_ENDIAN__ | ||
| 7954 | #define vld1q_dup_f16(__p0) __extension__ ({ \ | ||
| 7955 | float16x8_t __ret; \ | ||
| 7956 | __ret = (float16x8_t) __builtin_neon_vld1q_dup_v(__p0, 40); \ | ||
| 7957 | __ret; \ | ||
| 7958 | }) | ||
| 7959 | #else | ||
| 7960 | #define vld1q_dup_f16(__p0) __extension__ ({ \ | ||
| 7961 | float16x8_t __ret; \ | ||
| 7962 | __ret = (float16x8_t) __builtin_neon_vld1q_dup_v(__p0, 40); \ | ||
| 7963 | __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 7964 | __ret; \ | ||
| 7965 | }) | ||
| 7966 | #endif | ||
| 7967 | |||
| 7968 | #ifdef __LITTLE_ENDIAN__ | 7923 | #ifdef __LITTLE_ENDIAN__ |
| 7969 | #define vld1q_dup_s32(__p0) __extension__ ({ \ | 7924 | #define vld1q_dup_s32(__p0) __extension__ ({ \ |
| 7970 | int32x4_t __ret; \ | 7925 | int32x4_t __ret; \ |
| ... | @@ -8099,21 +8054,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -8099,21 +8054,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 8099 | }) | 8054 | }) |
| 8100 | #endif | 8055 | #endif |
| 8101 | 8056 | ||
| 8102 | #ifdef __LITTLE_ENDIAN__ | ||
| 8103 | #define vld1_dup_f16(__p0) __extension__ ({ \ | ||
| 8104 | float16x4_t __ret; \ | ||
| 8105 | __ret = (float16x4_t) __builtin_neon_vld1_dup_v(__p0, 8); \ | ||
| 8106 | __ret; \ | ||
| 8107 | }) | ||
| 8108 | #else | ||
| 8109 | #define vld1_dup_f16(__p0) __extension__ ({ \ | ||
| 8110 | float16x4_t __ret; \ | ||
| 8111 | __ret = (float16x4_t) __builtin_neon_vld1_dup_v(__p0, 8); \ | ||
| 8112 | __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ | ||
| 8113 | __ret; \ | ||
| 8114 | }) | ||
| 8115 | #endif | ||
| 8116 | |||
| 8117 | #ifdef __LITTLE_ENDIAN__ | 8057 | #ifdef __LITTLE_ENDIAN__ |
| 8118 | #define vld1_dup_s32(__p0) __extension__ ({ \ | 8058 | #define vld1_dup_s32(__p0) __extension__ ({ \ |
| 8119 | int32x2_t __ret; \ | 8059 | int32x2_t __ret; \ |
| ... | @@ -8338,24 +8278,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -8338,24 +8278,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 8338 | }) | 8278 | }) |
| 8339 | #endif | 8279 | #endif |
| 8340 | 8280 | ||
| 8341 | #ifdef __LITTLE_ENDIAN__ | ||
| 8342 | #define vld1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 8343 | float16x8_t __s1 = __p1; \ | ||
| 8344 | float16x8_t __ret; \ | ||
| 8345 | __ret = (float16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 40); \ | ||
| 8346 | __ret; \ | ||
| 8347 | }) | ||
| 8348 | #else | ||
| 8349 | #define vld1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 8350 | float16x8_t __s1 = __p1; \ | ||
| 8351 | float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 8352 | float16x8_t __ret; \ | ||
| 8353 | __ret = (float16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 40); \ | ||
| 8354 | __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 8355 | __ret; \ | ||
| 8356 | }) | ||
| 8357 | #endif | ||
| 8358 | |||
| 8359 | #ifdef __LITTLE_ENDIAN__ | 8281 | #ifdef __LITTLE_ENDIAN__ |
| 8360 | #define vld1q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 8282 | #define vld1q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 8361 | int32x4_t __s1 = __p1; \ | 8283 | int32x4_t __s1 = __p1; \ |
| ... | @@ -8516,24 +8438,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -8516,24 +8438,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 8516 | }) | 8438 | }) |
| 8517 | #endif | 8439 | #endif |
| 8518 | 8440 | ||
| 8519 | #ifdef __LITTLE_ENDIAN__ | ||
| 8520 | #define vld1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 8521 | float16x4_t __s1 = __p1; \ | ||
| 8522 | float16x4_t __ret; \ | ||
| 8523 | __ret = (float16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 8); \ | ||
| 8524 | __ret; \ | ||
| 8525 | }) | ||
| 8526 | #else | ||
| 8527 | #define vld1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 8528 | float16x4_t __s1 = __p1; \ | ||
| 8529 | float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ | ||
| 8530 | float16x4_t __ret; \ | ||
| 8531 | __ret = (float16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 8); \ | ||
| 8532 | __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ | ||
| 8533 | __ret; \ | ||
| 8534 | }) | ||
| 8535 | #endif | ||
| 8536 | |||
| 8537 | #ifdef __LITTLE_ENDIAN__ | 8441 | #ifdef __LITTLE_ENDIAN__ |
| 8538 | #define vld1_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 8442 | #define vld1_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 8539 | int32x2_t __s1 = __p1; \ | 8443 | int32x2_t __s1 = __p1; \ |
| ... | @@ -8756,23 +8660,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -8756,23 +8660,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 8756 | }) | 8660 | }) |
| 8757 | #endif | 8661 | #endif |
| 8758 | 8662 | ||
| 8759 | #ifdef __LITTLE_ENDIAN__ | ||
| 8760 | #define vld1q_f16_x2(__p0) __extension__ ({ \ | ||
| 8761 | float16x8x2_t __ret; \ | ||
| 8762 | __builtin_neon_vld1q_x2_v(&__ret, __p0, 40); \ | ||
| 8763 | __ret; \ | ||
| 8764 | }) | ||
| 8765 | #else | ||
| 8766 | #define vld1q_f16_x2(__p0) __extension__ ({ \ | ||
| 8767 | float16x8x2_t __ret; \ | ||
| 8768 | __builtin_neon_vld1q_x2_v(&__ret, __p0, 40); \ | ||
| 8769 | \ | ||
| 8770 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 8771 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 8772 | __ret; \ | ||
| 8773 | }) | ||
| 8774 | #endif | ||
| 8775 | |||
| 8776 | #ifdef __LITTLE_ENDIAN__ | 8663 | #ifdef __LITTLE_ENDIAN__ |
| 8777 | #define vld1q_s32_x2(__p0) __extension__ ({ \ | 8664 | #define vld1q_s32_x2(__p0) __extension__ ({ \ |
| 8778 | int32x4x2_t __ret; \ | 8665 | int32x4x2_t __ret; \ |
| ... | @@ -8923,23 +8810,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -8923,23 +8810,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 8923 | }) | 8810 | }) |
| 8924 | #endif | 8811 | #endif |
| 8925 | 8812 | ||
| 8926 | #ifdef __LITTLE_ENDIAN__ | ||
| 8927 | #define vld1_f16_x2(__p0) __extension__ ({ \ | ||
| 8928 | float16x4x2_t __ret; \ | ||
| 8929 | __builtin_neon_vld1_x2_v(&__ret, __p0, 8); \ | ||
| 8930 | __ret; \ | ||
| 8931 | }) | ||
| 8932 | #else | ||
| 8933 | #define vld1_f16_x2(__p0) __extension__ ({ \ | ||
| 8934 | float16x4x2_t __ret; \ | ||
| 8935 | __builtin_neon_vld1_x2_v(&__ret, __p0, 8); \ | ||
| 8936 | \ | ||
| 8937 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 8938 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 8939 | __ret; \ | ||
| 8940 | }) | ||
| 8941 | #endif | ||
| 8942 | |||
| 8943 | #ifdef __LITTLE_ENDIAN__ | 8813 | #ifdef __LITTLE_ENDIAN__ |
| 8944 | #define vld1_s32_x2(__p0) __extension__ ({ \ | 8814 | #define vld1_s32_x2(__p0) __extension__ ({ \ |
| 8945 | int32x2x2_t __ret; \ | 8815 | int32x2x2_t __ret; \ |
| ... | @@ -9168,24 +9038,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -9168,24 +9038,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 9168 | }) | 9038 | }) |
| 9169 | #endif | 9039 | #endif |
| 9170 | 9040 | ||
| 9171 | #ifdef __LITTLE_ENDIAN__ | ||
| 9172 | #define vld1q_f16_x3(__p0) __extension__ ({ \ | ||
| 9173 | float16x8x3_t __ret; \ | ||
| 9174 | __builtin_neon_vld1q_x3_v(&__ret, __p0, 40); \ | ||
| 9175 | __ret; \ | ||
| 9176 | }) | ||
| 9177 | #else | ||
| 9178 | #define vld1q_f16_x3(__p0) __extension__ ({ \ | ||
| 9179 | float16x8x3_t __ret; \ | ||
| 9180 | __builtin_neon_vld1q_x3_v(&__ret, __p0, 40); \ | ||
| 9181 | \ | ||
| 9182 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 9183 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 9184 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 9185 | __ret; \ | ||
| 9186 | }) | ||
| 9187 | #endif | ||
| 9188 | |||
| 9189 | #ifdef __LITTLE_ENDIAN__ | 9041 | #ifdef __LITTLE_ENDIAN__ |
| 9190 | #define vld1q_s32_x3(__p0) __extension__ ({ \ | 9042 | #define vld1q_s32_x3(__p0) __extension__ ({ \ |
| 9191 | int32x4x3_t __ret; \ | 9043 | int32x4x3_t __ret; \ |
| ... | @@ -9344,24 +9196,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -9344,24 +9196,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 9344 | }) | 9196 | }) |
| 9345 | #endif | 9197 | #endif |
| 9346 | 9198 | ||
| 9347 | #ifdef __LITTLE_ENDIAN__ | ||
| 9348 | #define vld1_f16_x3(__p0) __extension__ ({ \ | ||
| 9349 | float16x4x3_t __ret; \ | ||
| 9350 | __builtin_neon_vld1_x3_v(&__ret, __p0, 8); \ | ||
| 9351 | __ret; \ | ||
| 9352 | }) | ||
| 9353 | #else | ||
| 9354 | #define vld1_f16_x3(__p0) __extension__ ({ \ | ||
| 9355 | float16x4x3_t __ret; \ | ||
| 9356 | __builtin_neon_vld1_x3_v(&__ret, __p0, 8); \ | ||
| 9357 | \ | ||
| 9358 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 9359 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 9360 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 9361 | __ret; \ | ||
| 9362 | }) | ||
| 9363 | #endif | ||
| 9364 | |||
| 9365 | #ifdef __LITTLE_ENDIAN__ | 9199 | #ifdef __LITTLE_ENDIAN__ |
| 9366 | #define vld1_s32_x3(__p0) __extension__ ({ \ | 9200 | #define vld1_s32_x3(__p0) __extension__ ({ \ |
| 9367 | int32x2x3_t __ret; \ | 9201 | int32x2x3_t __ret; \ |
| ... | @@ -9602,25 +9436,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -9602,25 +9436,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 9602 | }) | 9436 | }) |
| 9603 | #endif | 9437 | #endif |
| 9604 | 9438 | ||
| 9605 | #ifdef __LITTLE_ENDIAN__ | ||
| 9606 | #define vld1q_f16_x4(__p0) __extension__ ({ \ | ||
| 9607 | float16x8x4_t __ret; \ | ||
| 9608 | __builtin_neon_vld1q_x4_v(&__ret, __p0, 40); \ | ||
| 9609 | __ret; \ | ||
| 9610 | }) | ||
| 9611 | #else | ||
| 9612 | #define vld1q_f16_x4(__p0) __extension__ ({ \ | ||
| 9613 | float16x8x4_t __ret; \ | ||
| 9614 | __builtin_neon_vld1q_x4_v(&__ret, __p0, 40); \ | ||
| 9615 | \ | ||
| 9616 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 9617 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 9618 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 9619 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 9620 | __ret; \ | ||
| 9621 | }) | ||
| 9622 | #endif | ||
| 9623 | |||
| 9624 | #ifdef __LITTLE_ENDIAN__ | 9439 | #ifdef __LITTLE_ENDIAN__ |
| 9625 | #define vld1q_s32_x4(__p0) __extension__ ({ \ | 9440 | #define vld1q_s32_x4(__p0) __extension__ ({ \ |
| 9626 | int32x4x4_t __ret; \ | 9441 | int32x4x4_t __ret; \ |
| ... | @@ -9787,25 +9602,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -9787,25 +9602,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 9787 | }) | 9602 | }) |
| 9788 | #endif | 9603 | #endif |
| 9789 | 9604 | ||
| 9790 | #ifdef __LITTLE_ENDIAN__ | ||
| 9791 | #define vld1_f16_x4(__p0) __extension__ ({ \ | ||
| 9792 | float16x4x4_t __ret; \ | ||
| 9793 | __builtin_neon_vld1_x4_v(&__ret, __p0, 8); \ | ||
| 9794 | __ret; \ | ||
| 9795 | }) | ||
| 9796 | #else | ||
| 9797 | #define vld1_f16_x4(__p0) __extension__ ({ \ | ||
| 9798 | float16x4x4_t __ret; \ | ||
| 9799 | __builtin_neon_vld1_x4_v(&__ret, __p0, 8); \ | ||
| 9800 | \ | ||
| 9801 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 9802 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 9803 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 9804 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ | ||
| 9805 | __ret; \ | ||
| 9806 | }) | ||
| 9807 | #endif | ||
| 9808 | |||
| 9809 | #ifdef __LITTLE_ENDIAN__ | 9605 | #ifdef __LITTLE_ENDIAN__ |
| 9810 | #define vld1_s32_x4(__p0) __extension__ ({ \ | 9606 | #define vld1_s32_x4(__p0) __extension__ ({ \ |
| 9811 | int32x2x4_t __ret; \ | 9607 | int32x2x4_t __ret; \ |
| ... | @@ -10011,23 +9807,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -10011,23 +9807,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 10011 | }) | 9807 | }) |
| 10012 | #endif | 9808 | #endif |
| 10013 | 9809 | ||
| 10014 | #ifdef __LITTLE_ENDIAN__ | ||
| 10015 | #define vld2q_f16(__p0) __extension__ ({ \ | ||
| 10016 | float16x8x2_t __ret; \ | ||
| 10017 | __builtin_neon_vld2q_v(&__ret, __p0, 40); \ | ||
| 10018 | __ret; \ | ||
| 10019 | }) | ||
| 10020 | #else | ||
| 10021 | #define vld2q_f16(__p0) __extension__ ({ \ | ||
| 10022 | float16x8x2_t __ret; \ | ||
| 10023 | __builtin_neon_vld2q_v(&__ret, __p0, 40); \ | ||
| 10024 | \ | ||
| 10025 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 10026 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 10027 | __ret; \ | ||
| 10028 | }) | ||
| 10029 | #endif | ||
| 10030 | |||
| 10031 | #ifdef __LITTLE_ENDIAN__ | 9810 | #ifdef __LITTLE_ENDIAN__ |
| 10032 | #define vld2q_s32(__p0) __extension__ ({ \ | 9811 | #define vld2q_s32(__p0) __extension__ ({ \ |
| 10033 | int32x4x2_t __ret; \ | 9812 | int32x4x2_t __ret; \ |
| ... | @@ -10161,23 +9940,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -10161,23 +9940,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 10161 | }) | 9940 | }) |
| 10162 | #endif | 9941 | #endif |
| 10163 | 9942 | ||
| 10164 | #ifdef __LITTLE_ENDIAN__ | ||
| 10165 | #define vld2_f16(__p0) __extension__ ({ \ | ||
| 10166 | float16x4x2_t __ret; \ | ||
| 10167 | __builtin_neon_vld2_v(&__ret, __p0, 8); \ | ||
| 10168 | __ret; \ | ||
| 10169 | }) | ||
| 10170 | #else | ||
| 10171 | #define vld2_f16(__p0) __extension__ ({ \ | ||
| 10172 | float16x4x2_t __ret; \ | ||
| 10173 | __builtin_neon_vld2_v(&__ret, __p0, 8); \ | ||
| 10174 | \ | ||
| 10175 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 10176 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 10177 | __ret; \ | ||
| 10178 | }) | ||
| 10179 | #endif | ||
| 10180 | |||
| 10181 | #ifdef __LITTLE_ENDIAN__ | 9943 | #ifdef __LITTLE_ENDIAN__ |
| 10182 | #define vld2_s32(__p0) __extension__ ({ \ | 9944 | #define vld2_s32(__p0) __extension__ ({ \ |
| 10183 | int32x2x2_t __ret; \ | 9945 | int32x2x2_t __ret; \ |
| ... | @@ -10396,23 +10158,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -10396,23 +10158,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 10396 | }) | 10158 | }) |
| 10397 | #endif | 10159 | #endif |
| 10398 | 10160 | ||
| 10399 | #ifdef __LITTLE_ENDIAN__ | ||
| 10400 | #define vld2q_dup_f16(__p0) __extension__ ({ \ | ||
| 10401 | float16x8x2_t __ret; \ | ||
| 10402 | __builtin_neon_vld2q_dup_v(&__ret, __p0, 40); \ | ||
| 10403 | __ret; \ | ||
| 10404 | }) | ||
| 10405 | #else | ||
| 10406 | #define vld2q_dup_f16(__p0) __extension__ ({ \ | ||
| 10407 | float16x8x2_t __ret; \ | ||
| 10408 | __builtin_neon_vld2q_dup_v(&__ret, __p0, 40); \ | ||
| 10409 | \ | ||
| 10410 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 10411 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 10412 | __ret; \ | ||
| 10413 | }) | ||
| 10414 | #endif | ||
| 10415 | |||
| 10416 | #ifdef __LITTLE_ENDIAN__ | 10161 | #ifdef __LITTLE_ENDIAN__ |
| 10417 | #define vld2q_dup_s32(__p0) __extension__ ({ \ | 10162 | #define vld2q_dup_s32(__p0) __extension__ ({ \ |
| 10418 | int32x4x2_t __ret; \ | 10163 | int32x4x2_t __ret; \ |
| ... | @@ -10563,23 +10308,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -10563,23 +10308,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 10563 | }) | 10308 | }) |
| 10564 | #endif | 10309 | #endif |
| 10565 | 10310 | ||
| 10566 | #ifdef __LITTLE_ENDIAN__ | ||
| 10567 | #define vld2_dup_f16(__p0) __extension__ ({ \ | ||
| 10568 | float16x4x2_t __ret; \ | ||
| 10569 | __builtin_neon_vld2_dup_v(&__ret, __p0, 8); \ | ||
| 10570 | __ret; \ | ||
| 10571 | }) | ||
| 10572 | #else | ||
| 10573 | #define vld2_dup_f16(__p0) __extension__ ({ \ | ||
| 10574 | float16x4x2_t __ret; \ | ||
| 10575 | __builtin_neon_vld2_dup_v(&__ret, __p0, 8); \ | ||
| 10576 | \ | ||
| 10577 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 10578 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 10579 | __ret; \ | ||
| 10580 | }) | ||
| 10581 | #endif | ||
| 10582 | |||
| 10583 | #ifdef __LITTLE_ENDIAN__ | 10311 | #ifdef __LITTLE_ENDIAN__ |
| 10584 | #define vld2_dup_s32(__p0) __extension__ ({ \ | 10312 | #define vld2_dup_s32(__p0) __extension__ ({ \ |
| 10585 | int32x2x2_t __ret; \ | 10313 | int32x2x2_t __ret; \ |
| ... | @@ -10760,28 +10488,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -10760,28 +10488,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 10760 | }) | 10488 | }) |
| 10761 | #endif | 10489 | #endif |
| 10762 | 10490 | ||
| 10763 | #ifdef __LITTLE_ENDIAN__ | ||
| 10764 | #define vld2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 10765 | float16x8x2_t __s1 = __p1; \ | ||
| 10766 | float16x8x2_t __ret; \ | ||
| 10767 | __builtin_neon_vld2q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 40); \ | ||
| 10768 | __ret; \ | ||
| 10769 | }) | ||
| 10770 | #else | ||
| 10771 | #define vld2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 10772 | float16x8x2_t __s1 = __p1; \ | ||
| 10773 | float16x8x2_t __rev1; \ | ||
| 10774 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 10775 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 10776 | float16x8x2_t __ret; \ | ||
| 10777 | __builtin_neon_vld2q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 40); \ | ||
| 10778 | \ | ||
| 10779 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 10780 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 10781 | __ret; \ | ||
| 10782 | }) | ||
| 10783 | #endif | ||
| 10784 | |||
| 10785 | #ifdef __LITTLE_ENDIAN__ | 10491 | #ifdef __LITTLE_ENDIAN__ |
| 10786 | #define vld2q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 10492 | #define vld2q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 10787 | int32x4x2_t __s1 = __p1; \ | 10493 | int32x4x2_t __s1 = __p1; \ |
| ... | @@ -10936,28 +10642,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -10936,28 +10642,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 10936 | }) | 10642 | }) |
| 10937 | #endif | 10643 | #endif |
| 10938 | 10644 | ||
| 10939 | #ifdef __LITTLE_ENDIAN__ | ||
| 10940 | #define vld2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 10941 | float16x4x2_t __s1 = __p1; \ | ||
| 10942 | float16x4x2_t __ret; \ | ||
| 10943 | __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 8); \ | ||
| 10944 | __ret; \ | ||
| 10945 | }) | ||
| 10946 | #else | ||
| 10947 | #define vld2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 10948 | float16x4x2_t __s1 = __p1; \ | ||
| 10949 | float16x4x2_t __rev1; \ | ||
| 10950 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 10951 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 10952 | float16x4x2_t __ret; \ | ||
| 10953 | __builtin_neon_vld2_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 8); \ | ||
| 10954 | \ | ||
| 10955 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 10956 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 10957 | __ret; \ | ||
| 10958 | }) | ||
| 10959 | #endif | ||
| 10960 | |||
| 10961 | #ifdef __LITTLE_ENDIAN__ | 10645 | #ifdef __LITTLE_ENDIAN__ |
| 10962 | #define vld2_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 10646 | #define vld2_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 10963 | int32x2x2_t __s1 = __p1; \ | 10647 | int32x2x2_t __s1 = __p1; \ |
| ... | @@ -11164,24 +10848,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -11164,24 +10848,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 11164 | }) | 10848 | }) |
| 11165 | #endif | 10849 | #endif |
| 11166 | 10850 | ||
| 11167 | #ifdef __LITTLE_ENDIAN__ | ||
| 11168 | #define vld3q_f16(__p0) __extension__ ({ \ | ||
| 11169 | float16x8x3_t __ret; \ | ||
| 11170 | __builtin_neon_vld3q_v(&__ret, __p0, 40); \ | ||
| 11171 | __ret; \ | ||
| 11172 | }) | ||
| 11173 | #else | ||
| 11174 | #define vld3q_f16(__p0) __extension__ ({ \ | ||
| 11175 | float16x8x3_t __ret; \ | ||
| 11176 | __builtin_neon_vld3q_v(&__ret, __p0, 40); \ | ||
| 11177 | \ | ||
| 11178 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 11179 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 11180 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 11181 | __ret; \ | ||
| 11182 | }) | ||
| 11183 | #endif | ||
| 11184 | |||
| 11185 | #ifdef __LITTLE_ENDIAN__ | 10851 | #ifdef __LITTLE_ENDIAN__ |
| 11186 | #define vld3q_s32(__p0) __extension__ ({ \ | 10852 | #define vld3q_s32(__p0) __extension__ ({ \ |
| 11187 | int32x4x3_t __ret; \ | 10853 | int32x4x3_t __ret; \ |
| ... | @@ -11322,24 +10988,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -11322,24 +10988,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 11322 | }) | 10988 | }) |
| 11323 | #endif | 10989 | #endif |
| 11324 | 10990 | ||
| 11325 | #ifdef __LITTLE_ENDIAN__ | ||
| 11326 | #define vld3_f16(__p0) __extension__ ({ \ | ||
| 11327 | float16x4x3_t __ret; \ | ||
| 11328 | __builtin_neon_vld3_v(&__ret, __p0, 8); \ | ||
| 11329 | __ret; \ | ||
| 11330 | }) | ||
| 11331 | #else | ||
| 11332 | #define vld3_f16(__p0) __extension__ ({ \ | ||
| 11333 | float16x4x3_t __ret; \ | ||
| 11334 | __builtin_neon_vld3_v(&__ret, __p0, 8); \ | ||
| 11335 | \ | ||
| 11336 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 11337 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 11338 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 11339 | __ret; \ | ||
| 11340 | }) | ||
| 11341 | #endif | ||
| 11342 | |||
| 11343 | #ifdef __LITTLE_ENDIAN__ | 10991 | #ifdef __LITTLE_ENDIAN__ |
| 11344 | #define vld3_s32(__p0) __extension__ ({ \ | 10992 | #define vld3_s32(__p0) __extension__ ({ \ |
| 11345 | int32x2x3_t __ret; \ | 10993 | int32x2x3_t __ret; \ |
| ... | @@ -11570,24 +11218,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -11570,24 +11218,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 11570 | }) | 11218 | }) |
| 11571 | #endif | 11219 | #endif |
| 11572 | 11220 | ||
| 11573 | #ifdef __LITTLE_ENDIAN__ | ||
| 11574 | #define vld3q_dup_f16(__p0) __extension__ ({ \ | ||
| 11575 | float16x8x3_t __ret; \ | ||
| 11576 | __builtin_neon_vld3q_dup_v(&__ret, __p0, 40); \ | ||
| 11577 | __ret; \ | ||
| 11578 | }) | ||
| 11579 | #else | ||
| 11580 | #define vld3q_dup_f16(__p0) __extension__ ({ \ | ||
| 11581 | float16x8x3_t __ret; \ | ||
| 11582 | __builtin_neon_vld3q_dup_v(&__ret, __p0, 40); \ | ||
| 11583 | \ | ||
| 11584 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 11585 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 11586 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 11587 | __ret; \ | ||
| 11588 | }) | ||
| 11589 | #endif | ||
| 11590 | |||
| 11591 | #ifdef __LITTLE_ENDIAN__ | 11221 | #ifdef __LITTLE_ENDIAN__ |
| 11592 | #define vld3q_dup_s32(__p0) __extension__ ({ \ | 11222 | #define vld3q_dup_s32(__p0) __extension__ ({ \ |
| 11593 | int32x4x3_t __ret; \ | 11223 | int32x4x3_t __ret; \ |
| ... | @@ -11746,24 +11376,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -11746,24 +11376,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 11746 | }) | 11376 | }) |
| 11747 | #endif | 11377 | #endif |
| 11748 | 11378 | ||
| 11749 | #ifdef __LITTLE_ENDIAN__ | ||
| 11750 | #define vld3_dup_f16(__p0) __extension__ ({ \ | ||
| 11751 | float16x4x3_t __ret; \ | ||
| 11752 | __builtin_neon_vld3_dup_v(&__ret, __p0, 8); \ | ||
| 11753 | __ret; \ | ||
| 11754 | }) | ||
| 11755 | #else | ||
| 11756 | #define vld3_dup_f16(__p0) __extension__ ({ \ | ||
| 11757 | float16x4x3_t __ret; \ | ||
| 11758 | __builtin_neon_vld3_dup_v(&__ret, __p0, 8); \ | ||
| 11759 | \ | ||
| 11760 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 11761 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 11762 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 11763 | __ret; \ | ||
| 11764 | }) | ||
| 11765 | #endif | ||
| 11766 | |||
| 11767 | #ifdef __LITTLE_ENDIAN__ | 11379 | #ifdef __LITTLE_ENDIAN__ |
| 11768 | #define vld3_dup_s32(__p0) __extension__ ({ \ | 11380 | #define vld3_dup_s32(__p0) __extension__ ({ \ |
| 11769 | int32x2x3_t __ret; \ | 11381 | int32x2x3_t __ret; \ |
| ... | @@ -11958,30 +11570,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -11958,30 +11570,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 11958 | }) | 11570 | }) |
| 11959 | #endif | 11571 | #endif |
| 11960 | 11572 | ||
| 11961 | #ifdef __LITTLE_ENDIAN__ | ||
| 11962 | #define vld3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 11963 | float16x8x3_t __s1 = __p1; \ | ||
| 11964 | float16x8x3_t __ret; \ | ||
| 11965 | __builtin_neon_vld3q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 40); \ | ||
| 11966 | __ret; \ | ||
| 11967 | }) | ||
| 11968 | #else | ||
| 11969 | #define vld3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 11970 | float16x8x3_t __s1 = __p1; \ | ||
| 11971 | float16x8x3_t __rev1; \ | ||
| 11972 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 11973 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 11974 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 11975 | float16x8x3_t __ret; \ | ||
| 11976 | __builtin_neon_vld3q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 40); \ | ||
| 11977 | \ | ||
| 11978 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 11979 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 11980 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 11981 | __ret; \ | ||
| 11982 | }) | ||
| 11983 | #endif | ||
| 11984 | |||
| 11985 | #ifdef __LITTLE_ENDIAN__ | 11573 | #ifdef __LITTLE_ENDIAN__ |
| 11986 | #define vld3q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 11574 | #define vld3q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 11987 | int32x4x3_t __s1 = __p1; \ | 11575 | int32x4x3_t __s1 = __p1; \ |
| ... | @@ -12150,30 +11738,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -12150,30 +11738,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 12150 | }) | 11738 | }) |
| 12151 | #endif | 11739 | #endif |
| 12152 | 11740 | ||
| 12153 | #ifdef __LITTLE_ENDIAN__ | ||
| 12154 | #define vld3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 12155 | float16x4x3_t __s1 = __p1; \ | ||
| 12156 | float16x4x3_t __ret; \ | ||
| 12157 | __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 8); \ | ||
| 12158 | __ret; \ | ||
| 12159 | }) | ||
| 12160 | #else | ||
| 12161 | #define vld3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 12162 | float16x4x3_t __s1 = __p1; \ | ||
| 12163 | float16x4x3_t __rev1; \ | ||
| 12164 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 12165 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 12166 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 12167 | float16x4x3_t __ret; \ | ||
| 12168 | __builtin_neon_vld3_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 8); \ | ||
| 12169 | \ | ||
| 12170 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 12171 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 12172 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 12173 | __ret; \ | ||
| 12174 | }) | ||
| 12175 | #endif | ||
| 12176 | |||
| 12177 | #ifdef __LITTLE_ENDIAN__ | 11741 | #ifdef __LITTLE_ENDIAN__ |
| 12178 | #define vld3_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 11742 | #define vld3_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 12179 | int32x2x3_t __s1 = __p1; \ | 11743 | int32x2x3_t __s1 = __p1; \ |
| ... | @@ -12393,25 +11957,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -12393,25 +11957,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 12393 | }) | 11957 | }) |
| 12394 | #endif | 11958 | #endif |
| 12395 | 11959 | ||
| 12396 | #ifdef __LITTLE_ENDIAN__ | ||
| 12397 | #define vld4q_f16(__p0) __extension__ ({ \ | ||
| 12398 | float16x8x4_t __ret; \ | ||
| 12399 | __builtin_neon_vld4q_v(&__ret, __p0, 40); \ | ||
| 12400 | __ret; \ | ||
| 12401 | }) | ||
| 12402 | #else | ||
| 12403 | #define vld4q_f16(__p0) __extension__ ({ \ | ||
| 12404 | float16x8x4_t __ret; \ | ||
| 12405 | __builtin_neon_vld4q_v(&__ret, __p0, 40); \ | ||
| 12406 | \ | ||
| 12407 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 12408 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 12409 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 12410 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 12411 | __ret; \ | ||
| 12412 | }) | ||
| 12413 | #endif | ||
| 12414 | |||
| 12415 | #ifdef __LITTLE_ENDIAN__ | 11960 | #ifdef __LITTLE_ENDIAN__ |
| 12416 | #define vld4q_s32(__p0) __extension__ ({ \ | 11961 | #define vld4q_s32(__p0) __extension__ ({ \ |
| 12417 | int32x4x4_t __ret; \ | 11962 | int32x4x4_t __ret; \ |
| ... | @@ -12559,25 +12104,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -12559,25 +12104,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 12559 | }) | 12104 | }) |
| 12560 | #endif | 12105 | #endif |
| 12561 | 12106 | ||
| 12562 | #ifdef __LITTLE_ENDIAN__ | ||
| 12563 | #define vld4_f16(__p0) __extension__ ({ \ | ||
| 12564 | float16x4x4_t __ret; \ | ||
| 12565 | __builtin_neon_vld4_v(&__ret, __p0, 8); \ | ||
| 12566 | __ret; \ | ||
| 12567 | }) | ||
| 12568 | #else | ||
| 12569 | #define vld4_f16(__p0) __extension__ ({ \ | ||
| 12570 | float16x4x4_t __ret; \ | ||
| 12571 | __builtin_neon_vld4_v(&__ret, __p0, 8); \ | ||
| 12572 | \ | ||
| 12573 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 12574 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 12575 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 12576 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ | ||
| 12577 | __ret; \ | ||
| 12578 | }) | ||
| 12579 | #endif | ||
| 12580 | |||
| 12581 | #ifdef __LITTLE_ENDIAN__ | 12107 | #ifdef __LITTLE_ENDIAN__ |
| 12582 | #define vld4_s32(__p0) __extension__ ({ \ | 12108 | #define vld4_s32(__p0) __extension__ ({ \ |
| 12583 | int32x2x4_t __ret; \ | 12109 | int32x2x4_t __ret; \ |
| ... | @@ -12820,25 +12346,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -12820,25 +12346,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 12820 | }) | 12346 | }) |
| 12821 | #endif | 12347 | #endif |
| 12822 | 12348 | ||
| 12823 | #ifdef __LITTLE_ENDIAN__ | ||
| 12824 | #define vld4q_dup_f16(__p0) __extension__ ({ \ | ||
| 12825 | float16x8x4_t __ret; \ | ||
| 12826 | __builtin_neon_vld4q_dup_v(&__ret, __p0, 40); \ | ||
| 12827 | __ret; \ | ||
| 12828 | }) | ||
| 12829 | #else | ||
| 12830 | #define vld4q_dup_f16(__p0) __extension__ ({ \ | ||
| 12831 | float16x8x4_t __ret; \ | ||
| 12832 | __builtin_neon_vld4q_dup_v(&__ret, __p0, 40); \ | ||
| 12833 | \ | ||
| 12834 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 12835 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 12836 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 12837 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 12838 | __ret; \ | ||
| 12839 | }) | ||
| 12840 | #endif | ||
| 12841 | |||
| 12842 | #ifdef __LITTLE_ENDIAN__ | 12349 | #ifdef __LITTLE_ENDIAN__ |
| 12843 | #define vld4q_dup_s32(__p0) __extension__ ({ \ | 12350 | #define vld4q_dup_s32(__p0) __extension__ ({ \ |
| 12844 | int32x4x4_t __ret; \ | 12351 | int32x4x4_t __ret; \ |
| ... | @@ -13005,25 +12512,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -13005,25 +12512,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 13005 | }) | 12512 | }) |
| 13006 | #endif | 12513 | #endif |
| 13007 | 12514 | ||
| 13008 | #ifdef __LITTLE_ENDIAN__ | ||
| 13009 | #define vld4_dup_f16(__p0) __extension__ ({ \ | ||
| 13010 | float16x4x4_t __ret; \ | ||
| 13011 | __builtin_neon_vld4_dup_v(&__ret, __p0, 8); \ | ||
| 13012 | __ret; \ | ||
| 13013 | }) | ||
| 13014 | #else | ||
| 13015 | #define vld4_dup_f16(__p0) __extension__ ({ \ | ||
| 13016 | float16x4x4_t __ret; \ | ||
| 13017 | __builtin_neon_vld4_dup_v(&__ret, __p0, 8); \ | ||
| 13018 | \ | ||
| 13019 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 13020 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 13021 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 13022 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ | ||
| 13023 | __ret; \ | ||
| 13024 | }) | ||
| 13025 | #endif | ||
| 13026 | |||
| 13027 | #ifdef __LITTLE_ENDIAN__ | 12515 | #ifdef __LITTLE_ENDIAN__ |
| 13028 | #define vld4_dup_s32(__p0) __extension__ ({ \ | 12516 | #define vld4_dup_s32(__p0) __extension__ ({ \ |
| 13029 | int32x2x4_t __ret; \ | 12517 | int32x2x4_t __ret; \ |
| ... | @@ -13232,32 +12720,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -13232,32 +12720,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 13232 | }) | 12720 | }) |
| 13233 | #endif | 12721 | #endif |
| 13234 | 12722 | ||
| 13235 | #ifdef __LITTLE_ENDIAN__ | ||
| 13236 | #define vld4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 13237 | float16x8x4_t __s1 = __p1; \ | ||
| 13238 | float16x8x4_t __ret; \ | ||
| 13239 | __builtin_neon_vld4q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 40); \ | ||
| 13240 | __ret; \ | ||
| 13241 | }) | ||
| 13242 | #else | ||
| 13243 | #define vld4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 13244 | float16x8x4_t __s1 = __p1; \ | ||
| 13245 | float16x8x4_t __rev1; \ | ||
| 13246 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 13247 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 13248 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 13249 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 13250 | float16x8x4_t __ret; \ | ||
| 13251 | __builtin_neon_vld4q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 40); \ | ||
| 13252 | \ | ||
| 13253 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 13254 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 13255 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 13256 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 13257 | __ret; \ | ||
| 13258 | }) | ||
| 13259 | #endif | ||
| 13260 | |||
| 13261 | #ifdef __LITTLE_ENDIAN__ | 12723 | #ifdef __LITTLE_ENDIAN__ |
| 13262 | #define vld4q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 12724 | #define vld4q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 13263 | int32x4x4_t __s1 = __p1; \ | 12725 | int32x4x4_t __s1 = __p1; \ |
| ... | @@ -13440,32 +12902,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -13440,32 +12902,6 @@ __ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { |
| 13440 | }) | 12902 | }) |
| 13441 | #endif | 12903 | #endif |
| 13442 | 12904 | ||
| 13443 | #ifdef __LITTLE_ENDIAN__ | ||
| 13444 | #define vld4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 13445 | float16x4x4_t __s1 = __p1; \ | ||
| 13446 | float16x4x4_t __ret; \ | ||
| 13447 | __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 8); \ | ||
| 13448 | __ret; \ | ||
| 13449 | }) | ||
| 13450 | #else | ||
| 13451 | #define vld4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 13452 | float16x4x4_t __s1 = __p1; \ | ||
| 13453 | float16x4x4_t __rev1; \ | ||
| 13454 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 13455 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 13456 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 13457 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ | ||
| 13458 | float16x4x4_t __ret; \ | ||
| 13459 | __builtin_neon_vld4_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 8); \ | ||
| 13460 | \ | ||
| 13461 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 13462 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 13463 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 13464 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ | ||
| 13465 | __ret; \ | ||
| 13466 | }) | ||
| 13467 | #endif | ||
| 13468 | |||
| 13469 | #ifdef __LITTLE_ENDIAN__ | 12905 | #ifdef __LITTLE_ENDIAN__ |
| 13470 | #define vld4_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 12906 | #define vld4_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 13471 | int32x2x4_t __s1 = __p1; \ | 12907 | int32x2x4_t __s1 = __p1; \ |
| ... | @@ -27010,19 +26446,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -27010,19 +26446,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 27010 | }) | 26446 | }) |
| 27011 | #endif | 26447 | #endif |
| 27012 | 26448 | ||
| 27013 | #ifdef __LITTLE_ENDIAN__ | ||
| 27014 | #define vst1q_f16(__p0, __p1) __extension__ ({ \ | ||
| 27015 | float16x8_t __s1 = __p1; \ | ||
| 27016 | __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 40); \ | ||
| 27017 | }) | ||
| 27018 | #else | ||
| 27019 | #define vst1q_f16(__p0, __p1) __extension__ ({ \ | ||
| 27020 | float16x8_t __s1 = __p1; \ | ||
| 27021 | float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 27022 | __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 40); \ | ||
| 27023 | }) | ||
| 27024 | #endif | ||
| 27025 | |||
| 27026 | #ifdef __LITTLE_ENDIAN__ | 26449 | #ifdef __LITTLE_ENDIAN__ |
| 27027 | #define vst1q_s32(__p0, __p1) __extension__ ({ \ | 26450 | #define vst1q_s32(__p0, __p1) __extension__ ({ \ |
| 27028 | int32x4_t __s1 = __p1; \ | 26451 | int32x4_t __s1 = __p1; \ |
| ... | @@ -27139,19 +26562,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -27139,19 +26562,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 27139 | }) | 26562 | }) |
| 27140 | #endif | 26563 | #endif |
| 27141 | 26564 | ||
| 27142 | #ifdef __LITTLE_ENDIAN__ | ||
| 27143 | #define vst1_f16(__p0, __p1) __extension__ ({ \ | ||
| 27144 | float16x4_t __s1 = __p1; \ | ||
| 27145 | __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 8); \ | ||
| 27146 | }) | ||
| 27147 | #else | ||
| 27148 | #define vst1_f16(__p0, __p1) __extension__ ({ \ | ||
| 27149 | float16x4_t __s1 = __p1; \ | ||
| 27150 | float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ | ||
| 27151 | __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 8); \ | ||
| 27152 | }) | ||
| 27153 | #endif | ||
| 27154 | |||
| 27155 | #ifdef __LITTLE_ENDIAN__ | 26565 | #ifdef __LITTLE_ENDIAN__ |
| 27156 | #define vst1_s32(__p0, __p1) __extension__ ({ \ | 26566 | #define vst1_s32(__p0, __p1) __extension__ ({ \ |
| 27157 | int32x2_t __s1 = __p1; \ | 26567 | int32x2_t __s1 = __p1; \ |
| ... | @@ -27320,19 +26730,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -27320,19 +26730,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 27320 | }) | 26730 | }) |
| 27321 | #endif | 26731 | #endif |
| 27322 | 26732 | ||
| 27323 | #ifdef __LITTLE_ENDIAN__ | ||
| 27324 | #define vst1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 27325 | float16x8_t __s1 = __p1; \ | ||
| 27326 | __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 40); \ | ||
| 27327 | }) | ||
| 27328 | #else | ||
| 27329 | #define vst1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 27330 | float16x8_t __s1 = __p1; \ | ||
| 27331 | float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 27332 | __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 40); \ | ||
| 27333 | }) | ||
| 27334 | #endif | ||
| 27335 | |||
| 27336 | #ifdef __LITTLE_ENDIAN__ | 26733 | #ifdef __LITTLE_ENDIAN__ |
| 27337 | #define vst1q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 26734 | #define vst1q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 27338 | int32x4_t __s1 = __p1; \ | 26735 | int32x4_t __s1 = __p1; \ |
| ... | @@ -27449,19 +26846,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -27449,19 +26846,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 27449 | }) | 26846 | }) |
| 27450 | #endif | 26847 | #endif |
| 27451 | 26848 | ||
| 27452 | #ifdef __LITTLE_ENDIAN__ | ||
| 27453 | #define vst1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 27454 | float16x4_t __s1 = __p1; \ | ||
| 27455 | __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 8); \ | ||
| 27456 | }) | ||
| 27457 | #else | ||
| 27458 | #define vst1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 27459 | float16x4_t __s1 = __p1; \ | ||
| 27460 | float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ | ||
| 27461 | __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 8); \ | ||
| 27462 | }) | ||
| 27463 | #endif | ||
| 27464 | |||
| 27465 | #ifdef __LITTLE_ENDIAN__ | 26849 | #ifdef __LITTLE_ENDIAN__ |
| 27466 | #define vst1_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 26850 | #define vst1_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 27467 | int32x2_t __s1 = __p1; \ | 26851 | int32x2_t __s1 = __p1; \ |
| ... | @@ -27650,21 +27034,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -27650,21 +27034,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 27650 | }) | 27034 | }) |
| 27651 | #endif | 27035 | #endif |
| 27652 | 27036 | ||
| 27653 | #ifdef __LITTLE_ENDIAN__ | ||
| 27654 | #define vst1q_f16_x2(__p0, __p1) __extension__ ({ \ | ||
| 27655 | float16x8x2_t __s1 = __p1; \ | ||
| 27656 | __builtin_neon_vst1q_x2_v(__p0, __s1.val[0], __s1.val[1], 40); \ | ||
| 27657 | }) | ||
| 27658 | #else | ||
| 27659 | #define vst1q_f16_x2(__p0, __p1) __extension__ ({ \ | ||
| 27660 | float16x8x2_t __s1 = __p1; \ | ||
| 27661 | float16x8x2_t __rev1; \ | ||
| 27662 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 27663 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 27664 | __builtin_neon_vst1q_x2_v(__p0, __rev1.val[0], __rev1.val[1], 40); \ | ||
| 27665 | }) | ||
| 27666 | #endif | ||
| 27667 | |||
| 27668 | #ifdef __LITTLE_ENDIAN__ | 27037 | #ifdef __LITTLE_ENDIAN__ |
| 27669 | #define vst1q_s32_x2(__p0, __p1) __extension__ ({ \ | 27038 | #define vst1q_s32_x2(__p0, __p1) __extension__ ({ \ |
| 27670 | int32x4x2_t __s1 = __p1; \ | 27039 | int32x4x2_t __s1 = __p1; \ |
| ... | @@ -27797,21 +27166,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -27797,21 +27166,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 27797 | }) | 27166 | }) |
| 27798 | #endif | 27167 | #endif |
| 27799 | 27168 | ||
| 27800 | #ifdef __LITTLE_ENDIAN__ | ||
| 27801 | #define vst1_f16_x2(__p0, __p1) __extension__ ({ \ | ||
| 27802 | float16x4x2_t __s1 = __p1; \ | ||
| 27803 | __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 8); \ | ||
| 27804 | }) | ||
| 27805 | #else | ||
| 27806 | #define vst1_f16_x2(__p0, __p1) __extension__ ({ \ | ||
| 27807 | float16x4x2_t __s1 = __p1; \ | ||
| 27808 | float16x4x2_t __rev1; \ | ||
| 27809 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 27810 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 27811 | __builtin_neon_vst1_x2_v(__p0, __rev1.val[0], __rev1.val[1], 8); \ | ||
| 27812 | }) | ||
| 27813 | #endif | ||
| 27814 | |||
| 27815 | #ifdef __LITTLE_ENDIAN__ | 27169 | #ifdef __LITTLE_ENDIAN__ |
| 27816 | #define vst1_s32_x2(__p0, __p1) __extension__ ({ \ | 27170 | #define vst1_s32_x2(__p0, __p1) __extension__ ({ \ |
| 27817 | int32x2x2_t __s1 = __p1; \ | 27171 | int32x2x2_t __s1 = __p1; \ |
| ... | @@ -28014,22 +27368,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -28014,22 +27368,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 28014 | }) | 27368 | }) |
| 28015 | #endif | 27369 | #endif |
| 28016 | 27370 | ||
| 28017 | #ifdef __LITTLE_ENDIAN__ | ||
| 28018 | #define vst1q_f16_x3(__p0, __p1) __extension__ ({ \ | ||
| 28019 | float16x8x3_t __s1 = __p1; \ | ||
| 28020 | __builtin_neon_vst1q_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 40); \ | ||
| 28021 | }) | ||
| 28022 | #else | ||
| 28023 | #define vst1q_f16_x3(__p0, __p1) __extension__ ({ \ | ||
| 28024 | float16x8x3_t __s1 = __p1; \ | ||
| 28025 | float16x8x3_t __rev1; \ | ||
| 28026 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 28027 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 28028 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 28029 | __builtin_neon_vst1q_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 40); \ | ||
| 28030 | }) | ||
| 28031 | #endif | ||
| 28032 | |||
| 28033 | #ifdef __LITTLE_ENDIAN__ | 27371 | #ifdef __LITTLE_ENDIAN__ |
| 28034 | #define vst1q_s32_x3(__p0, __p1) __extension__ ({ \ | 27372 | #define vst1q_s32_x3(__p0, __p1) __extension__ ({ \ |
| 28035 | int32x4x3_t __s1 = __p1; \ | 27373 | int32x4x3_t __s1 = __p1; \ |
| ... | @@ -28170,22 +27508,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -28170,22 +27508,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 28170 | }) | 27508 | }) |
| 28171 | #endif | 27509 | #endif |
| 28172 | 27510 | ||
| 28173 | #ifdef __LITTLE_ENDIAN__ | ||
| 28174 | #define vst1_f16_x3(__p0, __p1) __extension__ ({ \ | ||
| 28175 | float16x4x3_t __s1 = __p1; \ | ||
| 28176 | __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 8); \ | ||
| 28177 | }) | ||
| 28178 | #else | ||
| 28179 | #define vst1_f16_x3(__p0, __p1) __extension__ ({ \ | ||
| 28180 | float16x4x3_t __s1 = __p1; \ | ||
| 28181 | float16x4x3_t __rev1; \ | ||
| 28182 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 28183 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 28184 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 28185 | __builtin_neon_vst1_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 8); \ | ||
| 28186 | }) | ||
| 28187 | #endif | ||
| 28188 | |||
| 28189 | #ifdef __LITTLE_ENDIAN__ | 27511 | #ifdef __LITTLE_ENDIAN__ |
| 28190 | #define vst1_s32_x3(__p0, __p1) __extension__ ({ \ | 27512 | #define vst1_s32_x3(__p0, __p1) __extension__ ({ \ |
| 28191 | int32x2x3_t __s1 = __p1; \ | 27513 | int32x2x3_t __s1 = __p1; \ |
| ... | @@ -28400,23 +27722,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -28400,23 +27722,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 28400 | }) | 27722 | }) |
| 28401 | #endif | 27723 | #endif |
| 28402 | 27724 | ||
| 28403 | #ifdef __LITTLE_ENDIAN__ | ||
| 28404 | #define vst1q_f16_x4(__p0, __p1) __extension__ ({ \ | ||
| 28405 | float16x8x4_t __s1 = __p1; \ | ||
| 28406 | __builtin_neon_vst1q_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 40); \ | ||
| 28407 | }) | ||
| 28408 | #else | ||
| 28409 | #define vst1q_f16_x4(__p0, __p1) __extension__ ({ \ | ||
| 28410 | float16x8x4_t __s1 = __p1; \ | ||
| 28411 | float16x8x4_t __rev1; \ | ||
| 28412 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 28413 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 28414 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 28415 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 28416 | __builtin_neon_vst1q_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 40); \ | ||
| 28417 | }) | ||
| 28418 | #endif | ||
| 28419 | |||
| 28420 | #ifdef __LITTLE_ENDIAN__ | 27725 | #ifdef __LITTLE_ENDIAN__ |
| 28421 | #define vst1q_s32_x4(__p0, __p1) __extension__ ({ \ | 27726 | #define vst1q_s32_x4(__p0, __p1) __extension__ ({ \ |
| 28422 | int32x4x4_t __s1 = __p1; \ | 27727 | int32x4x4_t __s1 = __p1; \ |
| ... | @@ -28565,23 +27870,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -28565,23 +27870,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 28565 | }) | 27870 | }) |
| 28566 | #endif | 27871 | #endif |
| 28567 | 27872 | ||
| 28568 | #ifdef __LITTLE_ENDIAN__ | ||
| 28569 | #define vst1_f16_x4(__p0, __p1) __extension__ ({ \ | ||
| 28570 | float16x4x4_t __s1 = __p1; \ | ||
| 28571 | __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 8); \ | ||
| 28572 | }) | ||
| 28573 | #else | ||
| 28574 | #define vst1_f16_x4(__p0, __p1) __extension__ ({ \ | ||
| 28575 | float16x4x4_t __s1 = __p1; \ | ||
| 28576 | float16x4x4_t __rev1; \ | ||
| 28577 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 28578 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 28579 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 28580 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ | ||
| 28581 | __builtin_neon_vst1_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 8); \ | ||
| 28582 | }) | ||
| 28583 | #endif | ||
| 28584 | |||
| 28585 | #ifdef __LITTLE_ENDIAN__ | 27873 | #ifdef __LITTLE_ENDIAN__ |
| 28586 | #define vst1_s32_x4(__p0, __p1) __extension__ ({ \ | 27874 | #define vst1_s32_x4(__p0, __p1) __extension__ ({ \ |
| 28587 | int32x2x4_t __s1 = __p1; \ | 27875 | int32x2x4_t __s1 = __p1; \ |
| ... | @@ -28763,21 +28051,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -28763,21 +28051,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 28763 | }) | 28051 | }) |
| 28764 | #endif | 28052 | #endif |
| 28765 | 28053 | ||
| 28766 | #ifdef __LITTLE_ENDIAN__ | ||
| 28767 | #define vst2q_f16(__p0, __p1) __extension__ ({ \ | ||
| 28768 | float16x8x2_t __s1 = __p1; \ | ||
| 28769 | __builtin_neon_vst2q_v(__p0, __s1.val[0], __s1.val[1], 40); \ | ||
| 28770 | }) | ||
| 28771 | #else | ||
| 28772 | #define vst2q_f16(__p0, __p1) __extension__ ({ \ | ||
| 28773 | float16x8x2_t __s1 = __p1; \ | ||
| 28774 | float16x8x2_t __rev1; \ | ||
| 28775 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 28776 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 28777 | __builtin_neon_vst2q_v(__p0, __rev1.val[0], __rev1.val[1], 40); \ | ||
| 28778 | }) | ||
| 28779 | #endif | ||
| 28780 | |||
| 28781 | #ifdef __LITTLE_ENDIAN__ | 28054 | #ifdef __LITTLE_ENDIAN__ |
| 28782 | #define vst2q_s32(__p0, __p1) __extension__ ({ \ | 28055 | #define vst2q_s32(__p0, __p1) __extension__ ({ \ |
| 28783 | int32x4x2_t __s1 = __p1; \ | 28056 | int32x4x2_t __s1 = __p1; \ |
| ... | @@ -28895,21 +28168,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -28895,21 +28168,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 28895 | }) | 28168 | }) |
| 28896 | #endif | 28169 | #endif |
| 28897 | 28170 | ||
| 28898 | #ifdef __LITTLE_ENDIAN__ | ||
| 28899 | #define vst2_f16(__p0, __p1) __extension__ ({ \ | ||
| 28900 | float16x4x2_t __s1 = __p1; \ | ||
| 28901 | __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 8); \ | ||
| 28902 | }) | ||
| 28903 | #else | ||
| 28904 | #define vst2_f16(__p0, __p1) __extension__ ({ \ | ||
| 28905 | float16x4x2_t __s1 = __p1; \ | ||
| 28906 | float16x4x2_t __rev1; \ | ||
| 28907 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 28908 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 28909 | __builtin_neon_vst2_v(__p0, __rev1.val[0], __rev1.val[1], 8); \ | ||
| 28910 | }) | ||
| 28911 | #endif | ||
| 28912 | |||
| 28913 | #ifdef __LITTLE_ENDIAN__ | 28171 | #ifdef __LITTLE_ENDIAN__ |
| 28914 | #define vst2_s32(__p0, __p1) __extension__ ({ \ | 28172 | #define vst2_s32(__p0, __p1) __extension__ ({ \ |
| 28915 | int32x2x2_t __s1 = __p1; \ | 28173 | int32x2x2_t __s1 = __p1; \ |
| ... | @@ -29042,21 +28300,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -29042,21 +28300,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 29042 | }) | 28300 | }) |
| 29043 | #endif | 28301 | #endif |
| 29044 | 28302 | ||
| 29045 | #ifdef __LITTLE_ENDIAN__ | ||
| 29046 | #define vst2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 29047 | float16x8x2_t __s1 = __p1; \ | ||
| 29048 | __builtin_neon_vst2q_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 40); \ | ||
| 29049 | }) | ||
| 29050 | #else | ||
| 29051 | #define vst2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 29052 | float16x8x2_t __s1 = __p1; \ | ||
| 29053 | float16x8x2_t __rev1; \ | ||
| 29054 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 29055 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 29056 | __builtin_neon_vst2q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 40); \ | ||
| 29057 | }) | ||
| 29058 | #endif | ||
| 29059 | |||
| 29060 | #ifdef __LITTLE_ENDIAN__ | 28303 | #ifdef __LITTLE_ENDIAN__ |
| 29061 | #define vst2q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 28304 | #define vst2q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 29062 | int32x4x2_t __s1 = __p1; \ | 28305 | int32x4x2_t __s1 = __p1; \ |
| ... | @@ -29162,21 +28405,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -29162,21 +28405,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 29162 | }) | 28405 | }) |
| 29163 | #endif | 28406 | #endif |
| 29164 | 28407 | ||
| 29165 | #ifdef __LITTLE_ENDIAN__ | ||
| 29166 | #define vst2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 29167 | float16x4x2_t __s1 = __p1; \ | ||
| 29168 | __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 8); \ | ||
| 29169 | }) | ||
| 29170 | #else | ||
| 29171 | #define vst2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 29172 | float16x4x2_t __s1 = __p1; \ | ||
| 29173 | float16x4x2_t __rev1; \ | ||
| 29174 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 29175 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 29176 | __builtin_neon_vst2_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 8); \ | ||
| 29177 | }) | ||
| 29178 | #endif | ||
| 29179 | |||
| 29180 | #ifdef __LITTLE_ENDIAN__ | 28408 | #ifdef __LITTLE_ENDIAN__ |
| 29181 | #define vst2_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 28409 | #define vst2_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 29182 | int32x2x2_t __s1 = __p1; \ | 28410 | int32x2x2_t __s1 = __p1; \ |
| ... | @@ -29351,22 +28579,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -29351,22 +28579,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 29351 | }) | 28579 | }) |
| 29352 | #endif | 28580 | #endif |
| 29353 | 28581 | ||
| 29354 | #ifdef __LITTLE_ENDIAN__ | ||
| 29355 | #define vst3q_f16(__p0, __p1) __extension__ ({ \ | ||
| 29356 | float16x8x3_t __s1 = __p1; \ | ||
| 29357 | __builtin_neon_vst3q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 40); \ | ||
| 29358 | }) | ||
| 29359 | #else | ||
| 29360 | #define vst3q_f16(__p0, __p1) __extension__ ({ \ | ||
| 29361 | float16x8x3_t __s1 = __p1; \ | ||
| 29362 | float16x8x3_t __rev1; \ | ||
| 29363 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 29364 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 29365 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 29366 | __builtin_neon_vst3q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 40); \ | ||
| 29367 | }) | ||
| 29368 | #endif | ||
| 29369 | |||
| 29370 | #ifdef __LITTLE_ENDIAN__ | 28582 | #ifdef __LITTLE_ENDIAN__ |
| 29371 | #define vst3q_s32(__p0, __p1) __extension__ ({ \ | 28583 | #define vst3q_s32(__p0, __p1) __extension__ ({ \ |
| 29372 | int32x4x3_t __s1 = __p1; \ | 28584 | int32x4x3_t __s1 = __p1; \ |
| ... | @@ -29491,22 +28703,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -29491,22 +28703,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 29491 | }) | 28703 | }) |
| 29492 | #endif | 28704 | #endif |
| 29493 | 28705 | ||
| 29494 | #ifdef __LITTLE_ENDIAN__ | ||
| 29495 | #define vst3_f16(__p0, __p1) __extension__ ({ \ | ||
| 29496 | float16x4x3_t __s1 = __p1; \ | ||
| 29497 | __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 8); \ | ||
| 29498 | }) | ||
| 29499 | #else | ||
| 29500 | #define vst3_f16(__p0, __p1) __extension__ ({ \ | ||
| 29501 | float16x4x3_t __s1 = __p1; \ | ||
| 29502 | float16x4x3_t __rev1; \ | ||
| 29503 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 29504 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 29505 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 29506 | __builtin_neon_vst3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 8); \ | ||
| 29507 | }) | ||
| 29508 | #endif | ||
| 29509 | |||
| 29510 | #ifdef __LITTLE_ENDIAN__ | 28706 | #ifdef __LITTLE_ENDIAN__ |
| 29511 | #define vst3_s32(__p0, __p1) __extension__ ({ \ | 28707 | #define vst3_s32(__p0, __p1) __extension__ ({ \ |
| 29512 | int32x2x3_t __s1 = __p1; \ | 28708 | int32x2x3_t __s1 = __p1; \ |
| ... | @@ -29647,22 +28843,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -29647,22 +28843,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 29647 | }) | 28843 | }) |
| 29648 | #endif | 28844 | #endif |
| 29649 | 28845 | ||
| 29650 | #ifdef __LITTLE_ENDIAN__ | ||
| 29651 | #define vst3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 29652 | float16x8x3_t __s1 = __p1; \ | ||
| 29653 | __builtin_neon_vst3q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 40); \ | ||
| 29654 | }) | ||
| 29655 | #else | ||
| 29656 | #define vst3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 29657 | float16x8x3_t __s1 = __p1; \ | ||
| 29658 | float16x8x3_t __rev1; \ | ||
| 29659 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 29660 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 29661 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 29662 | __builtin_neon_vst3q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 40); \ | ||
| 29663 | }) | ||
| 29664 | #endif | ||
| 29665 | |||
| 29666 | #ifdef __LITTLE_ENDIAN__ | 28846 | #ifdef __LITTLE_ENDIAN__ |
| 29667 | #define vst3q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 28847 | #define vst3q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 29668 | int32x4x3_t __s1 = __p1; \ | 28848 | int32x4x3_t __s1 = __p1; \ |
| ... | @@ -29775,22 +28955,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -29775,22 +28955,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 29775 | }) | 28955 | }) |
| 29776 | #endif | 28956 | #endif |
| 29777 | 28957 | ||
| 29778 | #ifdef __LITTLE_ENDIAN__ | ||
| 29779 | #define vst3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 29780 | float16x4x3_t __s1 = __p1; \ | ||
| 29781 | __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 8); \ | ||
| 29782 | }) | ||
| 29783 | #else | ||
| 29784 | #define vst3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 29785 | float16x4x3_t __s1 = __p1; \ | ||
| 29786 | float16x4x3_t __rev1; \ | ||
| 29787 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 29788 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 29789 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 29790 | __builtin_neon_vst3_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 8); \ | ||
| 29791 | }) | ||
| 29792 | #endif | ||
| 29793 | |||
| 29794 | #ifdef __LITTLE_ENDIAN__ | 28958 | #ifdef __LITTLE_ENDIAN__ |
| 29795 | #define vst3_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 28959 | #define vst3_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 29796 | int32x2x3_t __s1 = __p1; \ | 28960 | int32x2x3_t __s1 = __p1; \ |
| ... | @@ -29976,23 +29140,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -29976,23 +29140,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 29976 | }) | 29140 | }) |
| 29977 | #endif | 29141 | #endif |
| 29978 | 29142 | ||
| 29979 | #ifdef __LITTLE_ENDIAN__ | ||
| 29980 | #define vst4q_f16(__p0, __p1) __extension__ ({ \ | ||
| 29981 | float16x8x4_t __s1 = __p1; \ | ||
| 29982 | __builtin_neon_vst4q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 40); \ | ||
| 29983 | }) | ||
| 29984 | #else | ||
| 29985 | #define vst4q_f16(__p0, __p1) __extension__ ({ \ | ||
| 29986 | float16x8x4_t __s1 = __p1; \ | ||
| 29987 | float16x8x4_t __rev1; \ | ||
| 29988 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 29989 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 29990 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 29991 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 29992 | __builtin_neon_vst4q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 40); \ | ||
| 29993 | }) | ||
| 29994 | #endif | ||
| 29995 | |||
| 29996 | #ifdef __LITTLE_ENDIAN__ | 29143 | #ifdef __LITTLE_ENDIAN__ |
| 29997 | #define vst4q_s32(__p0, __p1) __extension__ ({ \ | 29144 | #define vst4q_s32(__p0, __p1) __extension__ ({ \ |
| 29998 | int32x4x4_t __s1 = __p1; \ | 29145 | int32x4x4_t __s1 = __p1; \ |
| ... | @@ -30124,23 +29271,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -30124,23 +29271,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 30124 | }) | 29271 | }) |
| 30125 | #endif | 29272 | #endif |
| 30126 | 29273 | ||
| 30127 | #ifdef __LITTLE_ENDIAN__ | ||
| 30128 | #define vst4_f16(__p0, __p1) __extension__ ({ \ | ||
| 30129 | float16x4x4_t __s1 = __p1; \ | ||
| 30130 | __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 8); \ | ||
| 30131 | }) | ||
| 30132 | #else | ||
| 30133 | #define vst4_f16(__p0, __p1) __extension__ ({ \ | ||
| 30134 | float16x4x4_t __s1 = __p1; \ | ||
| 30135 | float16x4x4_t __rev1; \ | ||
| 30136 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 30137 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 30138 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 30139 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ | ||
| 30140 | __builtin_neon_vst4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 8); \ | ||
| 30141 | }) | ||
| 30142 | #endif | ||
| 30143 | |||
| 30144 | #ifdef __LITTLE_ENDIAN__ | 29274 | #ifdef __LITTLE_ENDIAN__ |
| 30145 | #define vst4_s32(__p0, __p1) __extension__ ({ \ | 29275 | #define vst4_s32(__p0, __p1) __extension__ ({ \ |
| 30146 | int32x2x4_t __s1 = __p1; \ | 29276 | int32x2x4_t __s1 = __p1; \ |
| ... | @@ -30289,23 +29419,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -30289,23 +29419,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 30289 | }) | 29419 | }) |
| 30290 | #endif | 29420 | #endif |
| 30291 | 29421 | ||
| 30292 | #ifdef __LITTLE_ENDIAN__ | ||
| 30293 | #define vst4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 30294 | float16x8x4_t __s1 = __p1; \ | ||
| 30295 | __builtin_neon_vst4q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 40); \ | ||
| 30296 | }) | ||
| 30297 | #else | ||
| 30298 | #define vst4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 30299 | float16x8x4_t __s1 = __p1; \ | ||
| 30300 | float16x8x4_t __rev1; \ | ||
| 30301 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 30302 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 30303 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 30304 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 30305 | __builtin_neon_vst4q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 40); \ | ||
| 30306 | }) | ||
| 30307 | #endif | ||
| 30308 | |||
| 30309 | #ifdef __LITTLE_ENDIAN__ | 29422 | #ifdef __LITTLE_ENDIAN__ |
| 30310 | #define vst4q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 29423 | #define vst4q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 30311 | int32x4x4_t __s1 = __p1; \ | 29424 | int32x4x4_t __s1 = __p1; \ |
| ... | @@ -30425,23 +29538,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { | ... | @@ -30425,23 +29538,6 @@ __ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { |
| 30425 | }) | 29538 | }) |
| 30426 | #endif | 29539 | #endif |
| 30427 | 29540 | ||
| 30428 | #ifdef __LITTLE_ENDIAN__ | ||
| 30429 | #define vst4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 30430 | float16x4x4_t __s1 = __p1; \ | ||
| 30431 | __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 8); \ | ||
| 30432 | }) | ||
| 30433 | #else | ||
| 30434 | #define vst4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 30435 | float16x4x4_t __s1 = __p1; \ | ||
| 30436 | float16x4x4_t __rev1; \ | ||
| 30437 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 30438 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 30439 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 30440 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ | ||
| 30441 | __builtin_neon_vst4_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 8); \ | ||
| 30442 | }) | ||
| 30443 | #endif | ||
| 30444 | |||
| 30445 | #ifdef __LITTLE_ENDIAN__ | 29541 | #ifdef __LITTLE_ENDIAN__ |
| 30446 | #define vst4_lane_s32(__p0, __p1, __p2) __extension__ ({ \ | 29542 | #define vst4_lane_s32(__p0, __p1, __p2) __extension__ ({ \ |
| 30447 | int32x2x4_t __s1 = __p1; \ | 29543 | int32x2x4_t __s1 = __p1; \ |
| ... | @@ -36729,6 +35825,910 @@ __ai float32x4_t __noswap_vcvt_f32_f16(float16x4_t __p0) { | ... | @@ -36729,6 +35825,910 @@ __ai float32x4_t __noswap_vcvt_f32_f16(float16x4_t __p0) { |
| 36729 | } | 35825 | } |
| 36730 | #endif | 35826 | #endif |
| 36731 | 35827 | ||
| 35828 | #ifdef __LITTLE_ENDIAN__ | ||
| 35829 | #define vld1q_f16(__p0) __extension__ ({ \ | ||
| 35830 | float16x8_t __ret; \ | ||
| 35831 | __ret = (float16x8_t) __builtin_neon_vld1q_v(__p0, 40); \ | ||
| 35832 | __ret; \ | ||
| 35833 | }) | ||
| 35834 | #else | ||
| 35835 | #define vld1q_f16(__p0) __extension__ ({ \ | ||
| 35836 | float16x8_t __ret; \ | ||
| 35837 | __ret = (float16x8_t) __builtin_neon_vld1q_v(__p0, 40); \ | ||
| 35838 | __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 35839 | __ret; \ | ||
| 35840 | }) | ||
| 35841 | #endif | ||
| 35842 | |||
| 35843 | #ifdef __LITTLE_ENDIAN__ | ||
| 35844 | #define vld1_f16(__p0) __extension__ ({ \ | ||
| 35845 | float16x4_t __ret; \ | ||
| 35846 | __ret = (float16x4_t) __builtin_neon_vld1_v(__p0, 8); \ | ||
| 35847 | __ret; \ | ||
| 35848 | }) | ||
| 35849 | #else | ||
| 35850 | #define vld1_f16(__p0) __extension__ ({ \ | ||
| 35851 | float16x4_t __ret; \ | ||
| 35852 | __ret = (float16x4_t) __builtin_neon_vld1_v(__p0, 8); \ | ||
| 35853 | __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ | ||
| 35854 | __ret; \ | ||
| 35855 | }) | ||
| 35856 | #endif | ||
| 35857 | |||
| 35858 | #ifdef __LITTLE_ENDIAN__ | ||
| 35859 | #define vld1q_dup_f16(__p0) __extension__ ({ \ | ||
| 35860 | float16x8_t __ret; \ | ||
| 35861 | __ret = (float16x8_t) __builtin_neon_vld1q_dup_v(__p0, 40); \ | ||
| 35862 | __ret; \ | ||
| 35863 | }) | ||
| 35864 | #else | ||
| 35865 | #define vld1q_dup_f16(__p0) __extension__ ({ \ | ||
| 35866 | float16x8_t __ret; \ | ||
| 35867 | __ret = (float16x8_t) __builtin_neon_vld1q_dup_v(__p0, 40); \ | ||
| 35868 | __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 35869 | __ret; \ | ||
| 35870 | }) | ||
| 35871 | #endif | ||
| 35872 | |||
| 35873 | #ifdef __LITTLE_ENDIAN__ | ||
| 35874 | #define vld1_dup_f16(__p0) __extension__ ({ \ | ||
| 35875 | float16x4_t __ret; \ | ||
| 35876 | __ret = (float16x4_t) __builtin_neon_vld1_dup_v(__p0, 8); \ | ||
| 35877 | __ret; \ | ||
| 35878 | }) | ||
| 35879 | #else | ||
| 35880 | #define vld1_dup_f16(__p0) __extension__ ({ \ | ||
| 35881 | float16x4_t __ret; \ | ||
| 35882 | __ret = (float16x4_t) __builtin_neon_vld1_dup_v(__p0, 8); \ | ||
| 35883 | __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ | ||
| 35884 | __ret; \ | ||
| 35885 | }) | ||
| 35886 | #endif | ||
| 35887 | |||
| 35888 | #ifdef __LITTLE_ENDIAN__ | ||
| 35889 | #define vld1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 35890 | float16x8_t __s1 = __p1; \ | ||
| 35891 | float16x8_t __ret; \ | ||
| 35892 | __ret = (float16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 40); \ | ||
| 35893 | __ret; \ | ||
| 35894 | }) | ||
| 35895 | #else | ||
| 35896 | #define vld1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 35897 | float16x8_t __s1 = __p1; \ | ||
| 35898 | float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 35899 | float16x8_t __ret; \ | ||
| 35900 | __ret = (float16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 40); \ | ||
| 35901 | __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 35902 | __ret; \ | ||
| 35903 | }) | ||
| 35904 | #endif | ||
| 35905 | |||
| 35906 | #ifdef __LITTLE_ENDIAN__ | ||
| 35907 | #define vld1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 35908 | float16x4_t __s1 = __p1; \ | ||
| 35909 | float16x4_t __ret; \ | ||
| 35910 | __ret = (float16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 8); \ | ||
| 35911 | __ret; \ | ||
| 35912 | }) | ||
| 35913 | #else | ||
| 35914 | #define vld1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 35915 | float16x4_t __s1 = __p1; \ | ||
| 35916 | float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ | ||
| 35917 | float16x4_t __ret; \ | ||
| 35918 | __ret = (float16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 8); \ | ||
| 35919 | __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ | ||
| 35920 | __ret; \ | ||
| 35921 | }) | ||
| 35922 | #endif | ||
| 35923 | |||
| 35924 | #ifdef __LITTLE_ENDIAN__ | ||
| 35925 | #define vld1q_f16_x2(__p0) __extension__ ({ \ | ||
| 35926 | float16x8x2_t __ret; \ | ||
| 35927 | __builtin_neon_vld1q_x2_v(&__ret, __p0, 40); \ | ||
| 35928 | __ret; \ | ||
| 35929 | }) | ||
| 35930 | #else | ||
| 35931 | #define vld1q_f16_x2(__p0) __extension__ ({ \ | ||
| 35932 | float16x8x2_t __ret; \ | ||
| 35933 | __builtin_neon_vld1q_x2_v(&__ret, __p0, 40); \ | ||
| 35934 | \ | ||
| 35935 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 35936 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 35937 | __ret; \ | ||
| 35938 | }) | ||
| 35939 | #endif | ||
| 35940 | |||
| 35941 | #ifdef __LITTLE_ENDIAN__ | ||
| 35942 | #define vld1_f16_x2(__p0) __extension__ ({ \ | ||
| 35943 | float16x4x2_t __ret; \ | ||
| 35944 | __builtin_neon_vld1_x2_v(&__ret, __p0, 8); \ | ||
| 35945 | __ret; \ | ||
| 35946 | }) | ||
| 35947 | #else | ||
| 35948 | #define vld1_f16_x2(__p0) __extension__ ({ \ | ||
| 35949 | float16x4x2_t __ret; \ | ||
| 35950 | __builtin_neon_vld1_x2_v(&__ret, __p0, 8); \ | ||
| 35951 | \ | ||
| 35952 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 35953 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 35954 | __ret; \ | ||
| 35955 | }) | ||
| 35956 | #endif | ||
| 35957 | |||
| 35958 | #ifdef __LITTLE_ENDIAN__ | ||
| 35959 | #define vld1q_f16_x3(__p0) __extension__ ({ \ | ||
| 35960 | float16x8x3_t __ret; \ | ||
| 35961 | __builtin_neon_vld1q_x3_v(&__ret, __p0, 40); \ | ||
| 35962 | __ret; \ | ||
| 35963 | }) | ||
| 35964 | #else | ||
| 35965 | #define vld1q_f16_x3(__p0) __extension__ ({ \ | ||
| 35966 | float16x8x3_t __ret; \ | ||
| 35967 | __builtin_neon_vld1q_x3_v(&__ret, __p0, 40); \ | ||
| 35968 | \ | ||
| 35969 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 35970 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 35971 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 35972 | __ret; \ | ||
| 35973 | }) | ||
| 35974 | #endif | ||
| 35975 | |||
| 35976 | #ifdef __LITTLE_ENDIAN__ | ||
| 35977 | #define vld1_f16_x3(__p0) __extension__ ({ \ | ||
| 35978 | float16x4x3_t __ret; \ | ||
| 35979 | __builtin_neon_vld1_x3_v(&__ret, __p0, 8); \ | ||
| 35980 | __ret; \ | ||
| 35981 | }) | ||
| 35982 | #else | ||
| 35983 | #define vld1_f16_x3(__p0) __extension__ ({ \ | ||
| 35984 | float16x4x3_t __ret; \ | ||
| 35985 | __builtin_neon_vld1_x3_v(&__ret, __p0, 8); \ | ||
| 35986 | \ | ||
| 35987 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 35988 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 35989 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 35990 | __ret; \ | ||
| 35991 | }) | ||
| 35992 | #endif | ||
| 35993 | |||
| 35994 | #ifdef __LITTLE_ENDIAN__ | ||
| 35995 | #define vld1q_f16_x4(__p0) __extension__ ({ \ | ||
| 35996 | float16x8x4_t __ret; \ | ||
| 35997 | __builtin_neon_vld1q_x4_v(&__ret, __p0, 40); \ | ||
| 35998 | __ret; \ | ||
| 35999 | }) | ||
| 36000 | #else | ||
| 36001 | #define vld1q_f16_x4(__p0) __extension__ ({ \ | ||
| 36002 | float16x8x4_t __ret; \ | ||
| 36003 | __builtin_neon_vld1q_x4_v(&__ret, __p0, 40); \ | ||
| 36004 | \ | ||
| 36005 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36006 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36007 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36008 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36009 | __ret; \ | ||
| 36010 | }) | ||
| 36011 | #endif | ||
| 36012 | |||
| 36013 | #ifdef __LITTLE_ENDIAN__ | ||
| 36014 | #define vld1_f16_x4(__p0) __extension__ ({ \ | ||
| 36015 | float16x4x4_t __ret; \ | ||
| 36016 | __builtin_neon_vld1_x4_v(&__ret, __p0, 8); \ | ||
| 36017 | __ret; \ | ||
| 36018 | }) | ||
| 36019 | #else | ||
| 36020 | #define vld1_f16_x4(__p0) __extension__ ({ \ | ||
| 36021 | float16x4x4_t __ret; \ | ||
| 36022 | __builtin_neon_vld1_x4_v(&__ret, __p0, 8); \ | ||
| 36023 | \ | ||
| 36024 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 36025 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 36026 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 36027 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ | ||
| 36028 | __ret; \ | ||
| 36029 | }) | ||
| 36030 | #endif | ||
| 36031 | |||
| 36032 | #ifdef __LITTLE_ENDIAN__ | ||
| 36033 | #define vld2q_f16(__p0) __extension__ ({ \ | ||
| 36034 | float16x8x2_t __ret; \ | ||
| 36035 | __builtin_neon_vld2q_v(&__ret, __p0, 40); \ | ||
| 36036 | __ret; \ | ||
| 36037 | }) | ||
| 36038 | #else | ||
| 36039 | #define vld2q_f16(__p0) __extension__ ({ \ | ||
| 36040 | float16x8x2_t __ret; \ | ||
| 36041 | __builtin_neon_vld2q_v(&__ret, __p0, 40); \ | ||
| 36042 | \ | ||
| 36043 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36044 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36045 | __ret; \ | ||
| 36046 | }) | ||
| 36047 | #endif | ||
| 36048 | |||
| 36049 | #ifdef __LITTLE_ENDIAN__ | ||
| 36050 | #define vld2_f16(__p0) __extension__ ({ \ | ||
| 36051 | float16x4x2_t __ret; \ | ||
| 36052 | __builtin_neon_vld2_v(&__ret, __p0, 8); \ | ||
| 36053 | __ret; \ | ||
| 36054 | }) | ||
| 36055 | #else | ||
| 36056 | #define vld2_f16(__p0) __extension__ ({ \ | ||
| 36057 | float16x4x2_t __ret; \ | ||
| 36058 | __builtin_neon_vld2_v(&__ret, __p0, 8); \ | ||
| 36059 | \ | ||
| 36060 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 36061 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 36062 | __ret; \ | ||
| 36063 | }) | ||
| 36064 | #endif | ||
| 36065 | |||
| 36066 | #ifdef __LITTLE_ENDIAN__ | ||
| 36067 | #define vld2q_dup_f16(__p0) __extension__ ({ \ | ||
| 36068 | float16x8x2_t __ret; \ | ||
| 36069 | __builtin_neon_vld2q_dup_v(&__ret, __p0, 40); \ | ||
| 36070 | __ret; \ | ||
| 36071 | }) | ||
| 36072 | #else | ||
| 36073 | #define vld2q_dup_f16(__p0) __extension__ ({ \ | ||
| 36074 | float16x8x2_t __ret; \ | ||
| 36075 | __builtin_neon_vld2q_dup_v(&__ret, __p0, 40); \ | ||
| 36076 | \ | ||
| 36077 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36078 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36079 | __ret; \ | ||
| 36080 | }) | ||
| 36081 | #endif | ||
| 36082 | |||
| 36083 | #ifdef __LITTLE_ENDIAN__ | ||
| 36084 | #define vld2_dup_f16(__p0) __extension__ ({ \ | ||
| 36085 | float16x4x2_t __ret; \ | ||
| 36086 | __builtin_neon_vld2_dup_v(&__ret, __p0, 8); \ | ||
| 36087 | __ret; \ | ||
| 36088 | }) | ||
| 36089 | #else | ||
| 36090 | #define vld2_dup_f16(__p0) __extension__ ({ \ | ||
| 36091 | float16x4x2_t __ret; \ | ||
| 36092 | __builtin_neon_vld2_dup_v(&__ret, __p0, 8); \ | ||
| 36093 | \ | ||
| 36094 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 36095 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 36096 | __ret; \ | ||
| 36097 | }) | ||
| 36098 | #endif | ||
| 36099 | |||
| 36100 | #ifdef __LITTLE_ENDIAN__ | ||
| 36101 | #define vld2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36102 | float16x8x2_t __s1 = __p1; \ | ||
| 36103 | float16x8x2_t __ret; \ | ||
| 36104 | __builtin_neon_vld2q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 40); \ | ||
| 36105 | __ret; \ | ||
| 36106 | }) | ||
| 36107 | #else | ||
| 36108 | #define vld2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36109 | float16x8x2_t __s1 = __p1; \ | ||
| 36110 | float16x8x2_t __rev1; \ | ||
| 36111 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36112 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36113 | float16x8x2_t __ret; \ | ||
| 36114 | __builtin_neon_vld2q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 40); \ | ||
| 36115 | \ | ||
| 36116 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36117 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36118 | __ret; \ | ||
| 36119 | }) | ||
| 36120 | #endif | ||
| 36121 | |||
| 36122 | #ifdef __LITTLE_ENDIAN__ | ||
| 36123 | #define vld2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36124 | float16x4x2_t __s1 = __p1; \ | ||
| 36125 | float16x4x2_t __ret; \ | ||
| 36126 | __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 8); \ | ||
| 36127 | __ret; \ | ||
| 36128 | }) | ||
| 36129 | #else | ||
| 36130 | #define vld2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36131 | float16x4x2_t __s1 = __p1; \ | ||
| 36132 | float16x4x2_t __rev1; \ | ||
| 36133 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 36134 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 36135 | float16x4x2_t __ret; \ | ||
| 36136 | __builtin_neon_vld2_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 8); \ | ||
| 36137 | \ | ||
| 36138 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 36139 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 36140 | __ret; \ | ||
| 36141 | }) | ||
| 36142 | #endif | ||
| 36143 | |||
| 36144 | #ifdef __LITTLE_ENDIAN__ | ||
| 36145 | #define vld3q_f16(__p0) __extension__ ({ \ | ||
| 36146 | float16x8x3_t __ret; \ | ||
| 36147 | __builtin_neon_vld3q_v(&__ret, __p0, 40); \ | ||
| 36148 | __ret; \ | ||
| 36149 | }) | ||
| 36150 | #else | ||
| 36151 | #define vld3q_f16(__p0) __extension__ ({ \ | ||
| 36152 | float16x8x3_t __ret; \ | ||
| 36153 | __builtin_neon_vld3q_v(&__ret, __p0, 40); \ | ||
| 36154 | \ | ||
| 36155 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36156 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36157 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36158 | __ret; \ | ||
| 36159 | }) | ||
| 36160 | #endif | ||
| 36161 | |||
| 36162 | #ifdef __LITTLE_ENDIAN__ | ||
| 36163 | #define vld3_f16(__p0) __extension__ ({ \ | ||
| 36164 | float16x4x3_t __ret; \ | ||
| 36165 | __builtin_neon_vld3_v(&__ret, __p0, 8); \ | ||
| 36166 | __ret; \ | ||
| 36167 | }) | ||
| 36168 | #else | ||
| 36169 | #define vld3_f16(__p0) __extension__ ({ \ | ||
| 36170 | float16x4x3_t __ret; \ | ||
| 36171 | __builtin_neon_vld3_v(&__ret, __p0, 8); \ | ||
| 36172 | \ | ||
| 36173 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 36174 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 36175 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 36176 | __ret; \ | ||
| 36177 | }) | ||
| 36178 | #endif | ||
| 36179 | |||
| 36180 | #ifdef __LITTLE_ENDIAN__ | ||
| 36181 | #define vld3q_dup_f16(__p0) __extension__ ({ \ | ||
| 36182 | float16x8x3_t __ret; \ | ||
| 36183 | __builtin_neon_vld3q_dup_v(&__ret, __p0, 40); \ | ||
| 36184 | __ret; \ | ||
| 36185 | }) | ||
| 36186 | #else | ||
| 36187 | #define vld3q_dup_f16(__p0) __extension__ ({ \ | ||
| 36188 | float16x8x3_t __ret; \ | ||
| 36189 | __builtin_neon_vld3q_dup_v(&__ret, __p0, 40); \ | ||
| 36190 | \ | ||
| 36191 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36192 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36193 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36194 | __ret; \ | ||
| 36195 | }) | ||
| 36196 | #endif | ||
| 36197 | |||
| 36198 | #ifdef __LITTLE_ENDIAN__ | ||
| 36199 | #define vld3_dup_f16(__p0) __extension__ ({ \ | ||
| 36200 | float16x4x3_t __ret; \ | ||
| 36201 | __builtin_neon_vld3_dup_v(&__ret, __p0, 8); \ | ||
| 36202 | __ret; \ | ||
| 36203 | }) | ||
| 36204 | #else | ||
| 36205 | #define vld3_dup_f16(__p0) __extension__ ({ \ | ||
| 36206 | float16x4x3_t __ret; \ | ||
| 36207 | __builtin_neon_vld3_dup_v(&__ret, __p0, 8); \ | ||
| 36208 | \ | ||
| 36209 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 36210 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 36211 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 36212 | __ret; \ | ||
| 36213 | }) | ||
| 36214 | #endif | ||
| 36215 | |||
| 36216 | #ifdef __LITTLE_ENDIAN__ | ||
| 36217 | #define vld3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36218 | float16x8x3_t __s1 = __p1; \ | ||
| 36219 | float16x8x3_t __ret; \ | ||
| 36220 | __builtin_neon_vld3q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 40); \ | ||
| 36221 | __ret; \ | ||
| 36222 | }) | ||
| 36223 | #else | ||
| 36224 | #define vld3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36225 | float16x8x3_t __s1 = __p1; \ | ||
| 36226 | float16x8x3_t __rev1; \ | ||
| 36227 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36228 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36229 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36230 | float16x8x3_t __ret; \ | ||
| 36231 | __builtin_neon_vld3q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 40); \ | ||
| 36232 | \ | ||
| 36233 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36234 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36235 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36236 | __ret; \ | ||
| 36237 | }) | ||
| 36238 | #endif | ||
| 36239 | |||
| 36240 | #ifdef __LITTLE_ENDIAN__ | ||
| 36241 | #define vld3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36242 | float16x4x3_t __s1 = __p1; \ | ||
| 36243 | float16x4x3_t __ret; \ | ||
| 36244 | __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 8); \ | ||
| 36245 | __ret; \ | ||
| 36246 | }) | ||
| 36247 | #else | ||
| 36248 | #define vld3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36249 | float16x4x3_t __s1 = __p1; \ | ||
| 36250 | float16x4x3_t __rev1; \ | ||
| 36251 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 36252 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 36253 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 36254 | float16x4x3_t __ret; \ | ||
| 36255 | __builtin_neon_vld3_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 8); \ | ||
| 36256 | \ | ||
| 36257 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 36258 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 36259 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 36260 | __ret; \ | ||
| 36261 | }) | ||
| 36262 | #endif | ||
| 36263 | |||
| 36264 | #ifdef __LITTLE_ENDIAN__ | ||
| 36265 | #define vld4q_f16(__p0) __extension__ ({ \ | ||
| 36266 | float16x8x4_t __ret; \ | ||
| 36267 | __builtin_neon_vld4q_v(&__ret, __p0, 40); \ | ||
| 36268 | __ret; \ | ||
| 36269 | }) | ||
| 36270 | #else | ||
| 36271 | #define vld4q_f16(__p0) __extension__ ({ \ | ||
| 36272 | float16x8x4_t __ret; \ | ||
| 36273 | __builtin_neon_vld4q_v(&__ret, __p0, 40); \ | ||
| 36274 | \ | ||
| 36275 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36276 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36277 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36278 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36279 | __ret; \ | ||
| 36280 | }) | ||
| 36281 | #endif | ||
| 36282 | |||
| 36283 | #ifdef __LITTLE_ENDIAN__ | ||
| 36284 | #define vld4_f16(__p0) __extension__ ({ \ | ||
| 36285 | float16x4x4_t __ret; \ | ||
| 36286 | __builtin_neon_vld4_v(&__ret, __p0, 8); \ | ||
| 36287 | __ret; \ | ||
| 36288 | }) | ||
| 36289 | #else | ||
| 36290 | #define vld4_f16(__p0) __extension__ ({ \ | ||
| 36291 | float16x4x4_t __ret; \ | ||
| 36292 | __builtin_neon_vld4_v(&__ret, __p0, 8); \ | ||
| 36293 | \ | ||
| 36294 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 36295 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 36296 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 36297 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ | ||
| 36298 | __ret; \ | ||
| 36299 | }) | ||
| 36300 | #endif | ||
| 36301 | |||
| 36302 | #ifdef __LITTLE_ENDIAN__ | ||
| 36303 | #define vld4q_dup_f16(__p0) __extension__ ({ \ | ||
| 36304 | float16x8x4_t __ret; \ | ||
| 36305 | __builtin_neon_vld4q_dup_v(&__ret, __p0, 40); \ | ||
| 36306 | __ret; \ | ||
| 36307 | }) | ||
| 36308 | #else | ||
| 36309 | #define vld4q_dup_f16(__p0) __extension__ ({ \ | ||
| 36310 | float16x8x4_t __ret; \ | ||
| 36311 | __builtin_neon_vld4q_dup_v(&__ret, __p0, 40); \ | ||
| 36312 | \ | ||
| 36313 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36314 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36315 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36316 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36317 | __ret; \ | ||
| 36318 | }) | ||
| 36319 | #endif | ||
| 36320 | |||
| 36321 | #ifdef __LITTLE_ENDIAN__ | ||
| 36322 | #define vld4_dup_f16(__p0) __extension__ ({ \ | ||
| 36323 | float16x4x4_t __ret; \ | ||
| 36324 | __builtin_neon_vld4_dup_v(&__ret, __p0, 8); \ | ||
| 36325 | __ret; \ | ||
| 36326 | }) | ||
| 36327 | #else | ||
| 36328 | #define vld4_dup_f16(__p0) __extension__ ({ \ | ||
| 36329 | float16x4x4_t __ret; \ | ||
| 36330 | __builtin_neon_vld4_dup_v(&__ret, __p0, 8); \ | ||
| 36331 | \ | ||
| 36332 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 36333 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 36334 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 36335 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ | ||
| 36336 | __ret; \ | ||
| 36337 | }) | ||
| 36338 | #endif | ||
| 36339 | |||
| 36340 | #ifdef __LITTLE_ENDIAN__ | ||
| 36341 | #define vld4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36342 | float16x8x4_t __s1 = __p1; \ | ||
| 36343 | float16x8x4_t __ret; \ | ||
| 36344 | __builtin_neon_vld4q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 40); \ | ||
| 36345 | __ret; \ | ||
| 36346 | }) | ||
| 36347 | #else | ||
| 36348 | #define vld4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36349 | float16x8x4_t __s1 = __p1; \ | ||
| 36350 | float16x8x4_t __rev1; \ | ||
| 36351 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36352 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36353 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36354 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36355 | float16x8x4_t __ret; \ | ||
| 36356 | __builtin_neon_vld4q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 40); \ | ||
| 36357 | \ | ||
| 36358 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36359 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36360 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36361 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36362 | __ret; \ | ||
| 36363 | }) | ||
| 36364 | #endif | ||
| 36365 | |||
| 36366 | #ifdef __LITTLE_ENDIAN__ | ||
| 36367 | #define vld4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36368 | float16x4x4_t __s1 = __p1; \ | ||
| 36369 | float16x4x4_t __ret; \ | ||
| 36370 | __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 8); \ | ||
| 36371 | __ret; \ | ||
| 36372 | }) | ||
| 36373 | #else | ||
| 36374 | #define vld4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36375 | float16x4x4_t __s1 = __p1; \ | ||
| 36376 | float16x4x4_t __rev1; \ | ||
| 36377 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 36378 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 36379 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 36380 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ | ||
| 36381 | float16x4x4_t __ret; \ | ||
| 36382 | __builtin_neon_vld4_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 8); \ | ||
| 36383 | \ | ||
| 36384 | __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ | ||
| 36385 | __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ | ||
| 36386 | __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ | ||
| 36387 | __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ | ||
| 36388 | __ret; \ | ||
| 36389 | }) | ||
| 36390 | #endif | ||
| 36391 | |||
| 36392 | #ifdef __LITTLE_ENDIAN__ | ||
| 36393 | #define vst1q_f16(__p0, __p1) __extension__ ({ \ | ||
| 36394 | float16x8_t __s1 = __p1; \ | ||
| 36395 | __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 40); \ | ||
| 36396 | }) | ||
| 36397 | #else | ||
| 36398 | #define vst1q_f16(__p0, __p1) __extension__ ({ \ | ||
| 36399 | float16x8_t __s1 = __p1; \ | ||
| 36400 | float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36401 | __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 40); \ | ||
| 36402 | }) | ||
| 36403 | #endif | ||
| 36404 | |||
| 36405 | #ifdef __LITTLE_ENDIAN__ | ||
| 36406 | #define vst1_f16(__p0, __p1) __extension__ ({ \ | ||
| 36407 | float16x4_t __s1 = __p1; \ | ||
| 36408 | __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 8); \ | ||
| 36409 | }) | ||
| 36410 | #else | ||
| 36411 | #define vst1_f16(__p0, __p1) __extension__ ({ \ | ||
| 36412 | float16x4_t __s1 = __p1; \ | ||
| 36413 | float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ | ||
| 36414 | __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 8); \ | ||
| 36415 | }) | ||
| 36416 | #endif | ||
| 36417 | |||
| 36418 | #ifdef __LITTLE_ENDIAN__ | ||
| 36419 | #define vst1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36420 | float16x8_t __s1 = __p1; \ | ||
| 36421 | __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 40); \ | ||
| 36422 | }) | ||
| 36423 | #else | ||
| 36424 | #define vst1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36425 | float16x8_t __s1 = __p1; \ | ||
| 36426 | float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36427 | __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 40); \ | ||
| 36428 | }) | ||
| 36429 | #endif | ||
| 36430 | |||
| 36431 | #ifdef __LITTLE_ENDIAN__ | ||
| 36432 | #define vst1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36433 | float16x4_t __s1 = __p1; \ | ||
| 36434 | __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 8); \ | ||
| 36435 | }) | ||
| 36436 | #else | ||
| 36437 | #define vst1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36438 | float16x4_t __s1 = __p1; \ | ||
| 36439 | float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ | ||
| 36440 | __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 8); \ | ||
| 36441 | }) | ||
| 36442 | #endif | ||
| 36443 | |||
| 36444 | #ifdef __LITTLE_ENDIAN__ | ||
| 36445 | #define vst1q_f16_x2(__p0, __p1) __extension__ ({ \ | ||
| 36446 | float16x8x2_t __s1 = __p1; \ | ||
| 36447 | __builtin_neon_vst1q_x2_v(__p0, __s1.val[0], __s1.val[1], 40); \ | ||
| 36448 | }) | ||
| 36449 | #else | ||
| 36450 | #define vst1q_f16_x2(__p0, __p1) __extension__ ({ \ | ||
| 36451 | float16x8x2_t __s1 = __p1; \ | ||
| 36452 | float16x8x2_t __rev1; \ | ||
| 36453 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36454 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36455 | __builtin_neon_vst1q_x2_v(__p0, __rev1.val[0], __rev1.val[1], 40); \ | ||
| 36456 | }) | ||
| 36457 | #endif | ||
| 36458 | |||
| 36459 | #ifdef __LITTLE_ENDIAN__ | ||
| 36460 | #define vst1_f16_x2(__p0, __p1) __extension__ ({ \ | ||
| 36461 | float16x4x2_t __s1 = __p1; \ | ||
| 36462 | __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 8); \ | ||
| 36463 | }) | ||
| 36464 | #else | ||
| 36465 | #define vst1_f16_x2(__p0, __p1) __extension__ ({ \ | ||
| 36466 | float16x4x2_t __s1 = __p1; \ | ||
| 36467 | float16x4x2_t __rev1; \ | ||
| 36468 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 36469 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 36470 | __builtin_neon_vst1_x2_v(__p0, __rev1.val[0], __rev1.val[1], 8); \ | ||
| 36471 | }) | ||
| 36472 | #endif | ||
| 36473 | |||
| 36474 | #ifdef __LITTLE_ENDIAN__ | ||
| 36475 | #define vst1q_f16_x3(__p0, __p1) __extension__ ({ \ | ||
| 36476 | float16x8x3_t __s1 = __p1; \ | ||
| 36477 | __builtin_neon_vst1q_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 40); \ | ||
| 36478 | }) | ||
| 36479 | #else | ||
| 36480 | #define vst1q_f16_x3(__p0, __p1) __extension__ ({ \ | ||
| 36481 | float16x8x3_t __s1 = __p1; \ | ||
| 36482 | float16x8x3_t __rev1; \ | ||
| 36483 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36484 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36485 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36486 | __builtin_neon_vst1q_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 40); \ | ||
| 36487 | }) | ||
| 36488 | #endif | ||
| 36489 | |||
| 36490 | #ifdef __LITTLE_ENDIAN__ | ||
| 36491 | #define vst1_f16_x3(__p0, __p1) __extension__ ({ \ | ||
| 36492 | float16x4x3_t __s1 = __p1; \ | ||
| 36493 | __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 8); \ | ||
| 36494 | }) | ||
| 36495 | #else | ||
| 36496 | #define vst1_f16_x3(__p0, __p1) __extension__ ({ \ | ||
| 36497 | float16x4x3_t __s1 = __p1; \ | ||
| 36498 | float16x4x3_t __rev1; \ | ||
| 36499 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 36500 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 36501 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 36502 | __builtin_neon_vst1_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 8); \ | ||
| 36503 | }) | ||
| 36504 | #endif | ||
| 36505 | |||
| 36506 | #ifdef __LITTLE_ENDIAN__ | ||
| 36507 | #define vst1q_f16_x4(__p0, __p1) __extension__ ({ \ | ||
| 36508 | float16x8x4_t __s1 = __p1; \ | ||
| 36509 | __builtin_neon_vst1q_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 40); \ | ||
| 36510 | }) | ||
| 36511 | #else | ||
| 36512 | #define vst1q_f16_x4(__p0, __p1) __extension__ ({ \ | ||
| 36513 | float16x8x4_t __s1 = __p1; \ | ||
| 36514 | float16x8x4_t __rev1; \ | ||
| 36515 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36516 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36517 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36518 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36519 | __builtin_neon_vst1q_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 40); \ | ||
| 36520 | }) | ||
| 36521 | #endif | ||
| 36522 | |||
| 36523 | #ifdef __LITTLE_ENDIAN__ | ||
| 36524 | #define vst1_f16_x4(__p0, __p1) __extension__ ({ \ | ||
| 36525 | float16x4x4_t __s1 = __p1; \ | ||
| 36526 | __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 8); \ | ||
| 36527 | }) | ||
| 36528 | #else | ||
| 36529 | #define vst1_f16_x4(__p0, __p1) __extension__ ({ \ | ||
| 36530 | float16x4x4_t __s1 = __p1; \ | ||
| 36531 | float16x4x4_t __rev1; \ | ||
| 36532 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 36533 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 36534 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 36535 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ | ||
| 36536 | __builtin_neon_vst1_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 8); \ | ||
| 36537 | }) | ||
| 36538 | #endif | ||
| 36539 | |||
| 36540 | #ifdef __LITTLE_ENDIAN__ | ||
| 36541 | #define vst2q_f16(__p0, __p1) __extension__ ({ \ | ||
| 36542 | float16x8x2_t __s1 = __p1; \ | ||
| 36543 | __builtin_neon_vst2q_v(__p0, __s1.val[0], __s1.val[1], 40); \ | ||
| 36544 | }) | ||
| 36545 | #else | ||
| 36546 | #define vst2q_f16(__p0, __p1) __extension__ ({ \ | ||
| 36547 | float16x8x2_t __s1 = __p1; \ | ||
| 36548 | float16x8x2_t __rev1; \ | ||
| 36549 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36550 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36551 | __builtin_neon_vst2q_v(__p0, __rev1.val[0], __rev1.val[1], 40); \ | ||
| 36552 | }) | ||
| 36553 | #endif | ||
| 36554 | |||
| 36555 | #ifdef __LITTLE_ENDIAN__ | ||
| 36556 | #define vst2_f16(__p0, __p1) __extension__ ({ \ | ||
| 36557 | float16x4x2_t __s1 = __p1; \ | ||
| 36558 | __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 8); \ | ||
| 36559 | }) | ||
| 36560 | #else | ||
| 36561 | #define vst2_f16(__p0, __p1) __extension__ ({ \ | ||
| 36562 | float16x4x2_t __s1 = __p1; \ | ||
| 36563 | float16x4x2_t __rev1; \ | ||
| 36564 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 36565 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 36566 | __builtin_neon_vst2_v(__p0, __rev1.val[0], __rev1.val[1], 8); \ | ||
| 36567 | }) | ||
| 36568 | #endif | ||
| 36569 | |||
| 36570 | #ifdef __LITTLE_ENDIAN__ | ||
| 36571 | #define vst2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36572 | float16x8x2_t __s1 = __p1; \ | ||
| 36573 | __builtin_neon_vst2q_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 40); \ | ||
| 36574 | }) | ||
| 36575 | #else | ||
| 36576 | #define vst2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36577 | float16x8x2_t __s1 = __p1; \ | ||
| 36578 | float16x8x2_t __rev1; \ | ||
| 36579 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36580 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36581 | __builtin_neon_vst2q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 40); \ | ||
| 36582 | }) | ||
| 36583 | #endif | ||
| 36584 | |||
| 36585 | #ifdef __LITTLE_ENDIAN__ | ||
| 36586 | #define vst2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36587 | float16x4x2_t __s1 = __p1; \ | ||
| 36588 | __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 8); \ | ||
| 36589 | }) | ||
| 36590 | #else | ||
| 36591 | #define vst2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36592 | float16x4x2_t __s1 = __p1; \ | ||
| 36593 | float16x4x2_t __rev1; \ | ||
| 36594 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 36595 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 36596 | __builtin_neon_vst2_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 8); \ | ||
| 36597 | }) | ||
| 36598 | #endif | ||
| 36599 | |||
| 36600 | #ifdef __LITTLE_ENDIAN__ | ||
| 36601 | #define vst3q_f16(__p0, __p1) __extension__ ({ \ | ||
| 36602 | float16x8x3_t __s1 = __p1; \ | ||
| 36603 | __builtin_neon_vst3q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 40); \ | ||
| 36604 | }) | ||
| 36605 | #else | ||
| 36606 | #define vst3q_f16(__p0, __p1) __extension__ ({ \ | ||
| 36607 | float16x8x3_t __s1 = __p1; \ | ||
| 36608 | float16x8x3_t __rev1; \ | ||
| 36609 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36610 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36611 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36612 | __builtin_neon_vst3q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 40); \ | ||
| 36613 | }) | ||
| 36614 | #endif | ||
| 36615 | |||
| 36616 | #ifdef __LITTLE_ENDIAN__ | ||
| 36617 | #define vst3_f16(__p0, __p1) __extension__ ({ \ | ||
| 36618 | float16x4x3_t __s1 = __p1; \ | ||
| 36619 | __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 8); \ | ||
| 36620 | }) | ||
| 36621 | #else | ||
| 36622 | #define vst3_f16(__p0, __p1) __extension__ ({ \ | ||
| 36623 | float16x4x3_t __s1 = __p1; \ | ||
| 36624 | float16x4x3_t __rev1; \ | ||
| 36625 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 36626 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 36627 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 36628 | __builtin_neon_vst3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 8); \ | ||
| 36629 | }) | ||
| 36630 | #endif | ||
| 36631 | |||
| 36632 | #ifdef __LITTLE_ENDIAN__ | ||
| 36633 | #define vst3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36634 | float16x8x3_t __s1 = __p1; \ | ||
| 36635 | __builtin_neon_vst3q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 40); \ | ||
| 36636 | }) | ||
| 36637 | #else | ||
| 36638 | #define vst3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36639 | float16x8x3_t __s1 = __p1; \ | ||
| 36640 | float16x8x3_t __rev1; \ | ||
| 36641 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36642 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36643 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36644 | __builtin_neon_vst3q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 40); \ | ||
| 36645 | }) | ||
| 36646 | #endif | ||
| 36647 | |||
| 36648 | #ifdef __LITTLE_ENDIAN__ | ||
| 36649 | #define vst3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36650 | float16x4x3_t __s1 = __p1; \ | ||
| 36651 | __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 8); \ | ||
| 36652 | }) | ||
| 36653 | #else | ||
| 36654 | #define vst3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36655 | float16x4x3_t __s1 = __p1; \ | ||
| 36656 | float16x4x3_t __rev1; \ | ||
| 36657 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 36658 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 36659 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 36660 | __builtin_neon_vst3_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 8); \ | ||
| 36661 | }) | ||
| 36662 | #endif | ||
| 36663 | |||
| 36664 | #ifdef __LITTLE_ENDIAN__ | ||
| 36665 | #define vst4q_f16(__p0, __p1) __extension__ ({ \ | ||
| 36666 | float16x8x4_t __s1 = __p1; \ | ||
| 36667 | __builtin_neon_vst4q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 40); \ | ||
| 36668 | }) | ||
| 36669 | #else | ||
| 36670 | #define vst4q_f16(__p0, __p1) __extension__ ({ \ | ||
| 36671 | float16x8x4_t __s1 = __p1; \ | ||
| 36672 | float16x8x4_t __rev1; \ | ||
| 36673 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36674 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36675 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36676 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36677 | __builtin_neon_vst4q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 40); \ | ||
| 36678 | }) | ||
| 36679 | #endif | ||
| 36680 | |||
| 36681 | #ifdef __LITTLE_ENDIAN__ | ||
| 36682 | #define vst4_f16(__p0, __p1) __extension__ ({ \ | ||
| 36683 | float16x4x4_t __s1 = __p1; \ | ||
| 36684 | __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 8); \ | ||
| 36685 | }) | ||
| 36686 | #else | ||
| 36687 | #define vst4_f16(__p0, __p1) __extension__ ({ \ | ||
| 36688 | float16x4x4_t __s1 = __p1; \ | ||
| 36689 | float16x4x4_t __rev1; \ | ||
| 36690 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 36691 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 36692 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 36693 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ | ||
| 36694 | __builtin_neon_vst4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 8); \ | ||
| 36695 | }) | ||
| 36696 | #endif | ||
| 36697 | |||
| 36698 | #ifdef __LITTLE_ENDIAN__ | ||
| 36699 | #define vst4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36700 | float16x8x4_t __s1 = __p1; \ | ||
| 36701 | __builtin_neon_vst4q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 40); \ | ||
| 36702 | }) | ||
| 36703 | #else | ||
| 36704 | #define vst4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36705 | float16x8x4_t __s1 = __p1; \ | ||
| 36706 | float16x8x4_t __rev1; \ | ||
| 36707 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36708 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36709 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36710 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 36711 | __builtin_neon_vst4q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 40); \ | ||
| 36712 | }) | ||
| 36713 | #endif | ||
| 36714 | |||
| 36715 | #ifdef __LITTLE_ENDIAN__ | ||
| 36716 | #define vst4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36717 | float16x4x4_t __s1 = __p1; \ | ||
| 36718 | __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 8); \ | ||
| 36719 | }) | ||
| 36720 | #else | ||
| 36721 | #define vst4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ | ||
| 36722 | float16x4x4_t __s1 = __p1; \ | ||
| 36723 | float16x4x4_t __rev1; \ | ||
| 36724 | __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ | ||
| 36725 | __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ | ||
| 36726 | __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ | ||
| 36727 | __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ | ||
| 36728 | __builtin_neon_vst4_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 8); \ | ||
| 36729 | }) | ||
| 36730 | #endif | ||
| 36731 | |||
| 36732 | #endif | 36732 | #endif |
| 36733 | #if __ARM_ARCH >= 8 | 36733 | #if __ARM_ARCH >= 8 |
| 36734 | #ifdef __LITTLE_ENDIAN__ | 36734 | #ifdef __LITTLE_ENDIAN__ |
| ... | @@ -44244,6 +44244,192 @@ __ai float32x2_t vfms_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) | ... | @@ -44244,6 +44244,192 @@ __ai float32x2_t vfms_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) |
| 44244 | } | 44244 | } |
| 44245 | #endif | 44245 | #endif |
| 44246 | 44246 | ||
| 44247 | #endif | ||
| 44248 | #if defined(__ARM_FEATURE_FP16FML) && defined(__aarch64__) | ||
| 44249 | #ifdef __LITTLE_ENDIAN__ | ||
| 44250 | __ai float32x4_t vfmlalq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { | ||
| 44251 | float32x4_t __ret; | ||
| 44252 | __ret = (float32x4_t) __builtin_neon_vfmlalq_high_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41); | ||
| 44253 | return __ret; | ||
| 44254 | } | ||
| 44255 | #else | ||
| 44256 | __ai float32x4_t vfmlalq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { | ||
| 44257 | float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); | ||
| 44258 | float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); | ||
| 44259 | float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); | ||
| 44260 | float32x4_t __ret; | ||
| 44261 | __ret = (float32x4_t) __builtin_neon_vfmlalq_high_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 41); | ||
| 44262 | __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); | ||
| 44263 | return __ret; | ||
| 44264 | } | ||
| 44265 | __ai float32x4_t __noswap_vfmlalq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { | ||
| 44266 | float32x4_t __ret; | ||
| 44267 | __ret = (float32x4_t) __builtin_neon_vfmlalq_high_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41); | ||
| 44268 | return __ret; | ||
| 44269 | } | ||
| 44270 | #endif | ||
| 44271 | |||
| 44272 | #ifdef __LITTLE_ENDIAN__ | ||
| 44273 | __ai float32x2_t vfmlal_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) { | ||
| 44274 | float32x2_t __ret; | ||
| 44275 | __ret = (float32x2_t) __builtin_neon_vfmlal_high_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9); | ||
| 44276 | return __ret; | ||
| 44277 | } | ||
| 44278 | #else | ||
| 44279 | __ai float32x2_t vfmlal_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) { | ||
| 44280 | float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); | ||
| 44281 | float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); | ||
| 44282 | float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); | ||
| 44283 | float32x2_t __ret; | ||
| 44284 | __ret = (float32x2_t) __builtin_neon_vfmlal_high_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 9); | ||
| 44285 | __ret = __builtin_shufflevector(__ret, __ret, 1, 0); | ||
| 44286 | return __ret; | ||
| 44287 | } | ||
| 44288 | __ai float32x2_t __noswap_vfmlal_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) { | ||
| 44289 | float32x2_t __ret; | ||
| 44290 | __ret = (float32x2_t) __builtin_neon_vfmlal_high_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9); | ||
| 44291 | return __ret; | ||
| 44292 | } | ||
| 44293 | #endif | ||
| 44294 | |||
| 44295 | #ifdef __LITTLE_ENDIAN__ | ||
| 44296 | __ai float32x4_t vfmlalq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { | ||
| 44297 | float32x4_t __ret; | ||
| 44298 | __ret = (float32x4_t) __builtin_neon_vfmlalq_low_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41); | ||
| 44299 | return __ret; | ||
| 44300 | } | ||
| 44301 | #else | ||
| 44302 | __ai float32x4_t vfmlalq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { | ||
| 44303 | float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); | ||
| 44304 | float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); | ||
| 44305 | float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); | ||
| 44306 | float32x4_t __ret; | ||
| 44307 | __ret = (float32x4_t) __builtin_neon_vfmlalq_low_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 41); | ||
| 44308 | __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); | ||
| 44309 | return __ret; | ||
| 44310 | } | ||
| 44311 | __ai float32x4_t __noswap_vfmlalq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { | ||
| 44312 | float32x4_t __ret; | ||
| 44313 | __ret = (float32x4_t) __builtin_neon_vfmlalq_low_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41); | ||
| 44314 | return __ret; | ||
| 44315 | } | ||
| 44316 | #endif | ||
| 44317 | |||
| 44318 | #ifdef __LITTLE_ENDIAN__ | ||
| 44319 | __ai float32x2_t vfmlal_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) { | ||
| 44320 | float32x2_t __ret; | ||
| 44321 | __ret = (float32x2_t) __builtin_neon_vfmlal_low_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9); | ||
| 44322 | return __ret; | ||
| 44323 | } | ||
| 44324 | #else | ||
| 44325 | __ai float32x2_t vfmlal_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) { | ||
| 44326 | float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); | ||
| 44327 | float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); | ||
| 44328 | float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); | ||
| 44329 | float32x2_t __ret; | ||
| 44330 | __ret = (float32x2_t) __builtin_neon_vfmlal_low_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 9); | ||
| 44331 | __ret = __builtin_shufflevector(__ret, __ret, 1, 0); | ||
| 44332 | return __ret; | ||
| 44333 | } | ||
| 44334 | __ai float32x2_t __noswap_vfmlal_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) { | ||
| 44335 | float32x2_t __ret; | ||
| 44336 | __ret = (float32x2_t) __builtin_neon_vfmlal_low_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9); | ||
| 44337 | return __ret; | ||
| 44338 | } | ||
| 44339 | #endif | ||
| 44340 | |||
| 44341 | #ifdef __LITTLE_ENDIAN__ | ||
| 44342 | __ai float32x4_t vfmlslq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { | ||
| 44343 | float32x4_t __ret; | ||
| 44344 | __ret = (float32x4_t) __builtin_neon_vfmlslq_high_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41); | ||
| 44345 | return __ret; | ||
| 44346 | } | ||
| 44347 | #else | ||
| 44348 | __ai float32x4_t vfmlslq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { | ||
| 44349 | float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); | ||
| 44350 | float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); | ||
| 44351 | float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); | ||
| 44352 | float32x4_t __ret; | ||
| 44353 | __ret = (float32x4_t) __builtin_neon_vfmlslq_high_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 41); | ||
| 44354 | __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); | ||
| 44355 | return __ret; | ||
| 44356 | } | ||
| 44357 | __ai float32x4_t __noswap_vfmlslq_high_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { | ||
| 44358 | float32x4_t __ret; | ||
| 44359 | __ret = (float32x4_t) __builtin_neon_vfmlslq_high_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41); | ||
| 44360 | return __ret; | ||
| 44361 | } | ||
| 44362 | #endif | ||
| 44363 | |||
| 44364 | #ifdef __LITTLE_ENDIAN__ | ||
| 44365 | __ai float32x2_t vfmlsl_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) { | ||
| 44366 | float32x2_t __ret; | ||
| 44367 | __ret = (float32x2_t) __builtin_neon_vfmlsl_high_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9); | ||
| 44368 | return __ret; | ||
| 44369 | } | ||
| 44370 | #else | ||
| 44371 | __ai float32x2_t vfmlsl_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) { | ||
| 44372 | float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); | ||
| 44373 | float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); | ||
| 44374 | float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); | ||
| 44375 | float32x2_t __ret; | ||
| 44376 | __ret = (float32x2_t) __builtin_neon_vfmlsl_high_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 9); | ||
| 44377 | __ret = __builtin_shufflevector(__ret, __ret, 1, 0); | ||
| 44378 | return __ret; | ||
| 44379 | } | ||
| 44380 | __ai float32x2_t __noswap_vfmlsl_high_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) { | ||
| 44381 | float32x2_t __ret; | ||
| 44382 | __ret = (float32x2_t) __builtin_neon_vfmlsl_high_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9); | ||
| 44383 | return __ret; | ||
| 44384 | } | ||
| 44385 | #endif | ||
| 44386 | |||
| 44387 | #ifdef __LITTLE_ENDIAN__ | ||
| 44388 | __ai float32x4_t vfmlslq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { | ||
| 44389 | float32x4_t __ret; | ||
| 44390 | __ret = (float32x4_t) __builtin_neon_vfmlslq_low_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41); | ||
| 44391 | return __ret; | ||
| 44392 | } | ||
| 44393 | #else | ||
| 44394 | __ai float32x4_t vfmlslq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { | ||
| 44395 | float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); | ||
| 44396 | float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); | ||
| 44397 | float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); | ||
| 44398 | float32x4_t __ret; | ||
| 44399 | __ret = (float32x4_t) __builtin_neon_vfmlslq_low_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 41); | ||
| 44400 | __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); | ||
| 44401 | return __ret; | ||
| 44402 | } | ||
| 44403 | __ai float32x4_t __noswap_vfmlslq_low_u32(float32x4_t __p0, float16x8_t __p1, float16x8_t __p2) { | ||
| 44404 | float32x4_t __ret; | ||
| 44405 | __ret = (float32x4_t) __builtin_neon_vfmlslq_low_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41); | ||
| 44406 | return __ret; | ||
| 44407 | } | ||
| 44408 | #endif | ||
| 44409 | |||
| 44410 | #ifdef __LITTLE_ENDIAN__ | ||
| 44411 | __ai float32x2_t vfmlsl_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) { | ||
| 44412 | float32x2_t __ret; | ||
| 44413 | __ret = (float32x2_t) __builtin_neon_vfmlsl_low_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9); | ||
| 44414 | return __ret; | ||
| 44415 | } | ||
| 44416 | #else | ||
| 44417 | __ai float32x2_t vfmlsl_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) { | ||
| 44418 | float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); | ||
| 44419 | float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); | ||
| 44420 | float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); | ||
| 44421 | float32x2_t __ret; | ||
| 44422 | __ret = (float32x2_t) __builtin_neon_vfmlsl_low_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 9); | ||
| 44423 | __ret = __builtin_shufflevector(__ret, __ret, 1, 0); | ||
| 44424 | return __ret; | ||
| 44425 | } | ||
| 44426 | __ai float32x2_t __noswap_vfmlsl_low_u32(float32x2_t __p0, float16x4_t __p1, float16x4_t __p2) { | ||
| 44427 | float32x2_t __ret; | ||
| 44428 | __ret = (float32x2_t) __builtin_neon_vfmlsl_low_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9); | ||
| 44429 | return __ret; | ||
| 44430 | } | ||
| 44431 | #endif | ||
| 44432 | |||
| 44247 | #endif | 44433 | #endif |
| 44248 | #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) | 44434 | #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) |
| 44249 | #ifdef __LITTLE_ENDIAN__ | 44435 | #ifdef __LITTLE_ENDIAN__ |
| ... | @@ -71724,45 +71910,431 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in | ... | @@ -71724,45 +71910,431 @@ int16x8_t __reint2_263 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_263, *(in |
| 71724 | }) | 71910 | }) |
| 71725 | #endif | 71911 | #endif |
| 71726 | 71912 | ||
| 71727 | #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(__aarch64__) | 71913 | #if defined(__ARM_FEATURE_FP16FML) && defined(__aarch64__) |
| 71728 | #ifdef __LITTLE_ENDIAN__ | 71914 | #ifdef __LITTLE_ENDIAN__ |
| 71729 | #define vmulh_lane_f16(__p0_264, __p1_264, __p2_264) __extension__ ({ \ | 71915 | #define vfmlalq_lane_high_u32(__p0_264, __p1_264, __p2_264, __p3_264) __extension__ ({ \ |
| 71730 | float16_t __s0_264 = __p0_264; \ | 71916 | float32x4_t __s0_264 = __p0_264; \ |
| 71731 | float16x4_t __s1_264 = __p1_264; \ | 71917 | float16x8_t __s1_264 = __p1_264; \ |
| 71732 | float16_t __ret_264; \ | 71918 | float16x4_t __s2_264 = __p2_264; \ |
| 71733 | __ret_264 = __s0_264 * vget_lane_f16(__s1_264, __p2_264); \ | 71919 | float32x4_t __ret_264; \ |
| 71920 | __ret_264 = vfmlalq_high_u32(__s0_264, __s1_264, (float16x8_t) {vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264), vget_lane_f16(__s2_264, __p3_264)}); \ | ||
| 71734 | __ret_264; \ | 71921 | __ret_264; \ |
| 71735 | }) | 71922 | }) |
| 71736 | #else | 71923 | #else |
| 71737 | #define vmulh_lane_f16(__p0_265, __p1_265, __p2_265) __extension__ ({ \ | 71924 | #define vfmlalq_lane_high_u32(__p0_265, __p1_265, __p2_265, __p3_265) __extension__ ({ \ |
| 71738 | float16_t __s0_265 = __p0_265; \ | 71925 | float32x4_t __s0_265 = __p0_265; \ |
| 71739 | float16x4_t __s1_265 = __p1_265; \ | 71926 | float16x8_t __s1_265 = __p1_265; \ |
| 71740 | float16x4_t __rev1_265; __rev1_265 = __builtin_shufflevector(__s1_265, __s1_265, 3, 2, 1, 0); \ | 71927 | float16x4_t __s2_265 = __p2_265; \ |
| 71741 | float16_t __ret_265; \ | 71928 | float32x4_t __rev0_265; __rev0_265 = __builtin_shufflevector(__s0_265, __s0_265, 3, 2, 1, 0); \ |
| 71742 | __ret_265 = __s0_265 * __noswap_vget_lane_f16(__rev1_265, __p2_265); \ | 71929 | float16x8_t __rev1_265; __rev1_265 = __builtin_shufflevector(__s1_265, __s1_265, 7, 6, 5, 4, 3, 2, 1, 0); \ |
| 71930 | float16x4_t __rev2_265; __rev2_265 = __builtin_shufflevector(__s2_265, __s2_265, 3, 2, 1, 0); \ | ||
| 71931 | float32x4_t __ret_265; \ | ||
| 71932 | __ret_265 = __noswap_vfmlalq_high_u32(__rev0_265, __rev1_265, (float16x8_t) {__noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265), __noswap_vget_lane_f16(__rev2_265, __p3_265)}); \ | ||
| 71933 | __ret_265 = __builtin_shufflevector(__ret_265, __ret_265, 3, 2, 1, 0); \ | ||
| 71743 | __ret_265; \ | 71934 | __ret_265; \ |
| 71744 | }) | 71935 | }) |
| 71745 | #endif | 71936 | #endif |
| 71746 | 71937 | ||
| 71747 | #ifdef __LITTLE_ENDIAN__ | 71938 | #ifdef __LITTLE_ENDIAN__ |
| 71748 | #define vmulh_laneq_f16(__p0_266, __p1_266, __p2_266) __extension__ ({ \ | 71939 | #define vfmlal_lane_high_u32(__p0_266, __p1_266, __p2_266, __p3_266) __extension__ ({ \ |
| 71749 | float16_t __s0_266 = __p0_266; \ | 71940 | float32x2_t __s0_266 = __p0_266; \ |
| 71750 | float16x8_t __s1_266 = __p1_266; \ | 71941 | float16x4_t __s1_266 = __p1_266; \ |
| 71751 | float16_t __ret_266; \ | 71942 | float16x4_t __s2_266 = __p2_266; \ |
| 71752 | __ret_266 = __s0_266 * vgetq_lane_f16(__s1_266, __p2_266); \ | 71943 | float32x2_t __ret_266; \ |
| 71944 | __ret_266 = vfmlal_high_u32(__s0_266, __s1_266, (float16x4_t) {vget_lane_f16(__s2_266, __p3_266), vget_lane_f16(__s2_266, __p3_266), vget_lane_f16(__s2_266, __p3_266), vget_lane_f16(__s2_266, __p3_266)}); \ | ||
| 71753 | __ret_266; \ | 71945 | __ret_266; \ |
| 71754 | }) | 71946 | }) |
| 71755 | #else | 71947 | #else |
| 71756 | #define vmulh_laneq_f16(__p0_267, __p1_267, __p2_267) __extension__ ({ \ | 71948 | #define vfmlal_lane_high_u32(__p0_267, __p1_267, __p2_267, __p3_267) __extension__ ({ \ |
| 71757 | float16_t __s0_267 = __p0_267; \ | 71949 | float32x2_t __s0_267 = __p0_267; \ |
| 71758 | float16x8_t __s1_267 = __p1_267; \ | 71950 | float16x4_t __s1_267 = __p1_267; \ |
| 71759 | float16x8_t __rev1_267; __rev1_267 = __builtin_shufflevector(__s1_267, __s1_267, 7, 6, 5, 4, 3, 2, 1, 0); \ | 71951 | float16x4_t __s2_267 = __p2_267; \ |
| 71760 | float16_t __ret_267; \ | 71952 | float32x2_t __rev0_267; __rev0_267 = __builtin_shufflevector(__s0_267, __s0_267, 1, 0); \ |
| 71761 | __ret_267 = __s0_267 * __noswap_vgetq_lane_f16(__rev1_267, __p2_267); \ | 71953 | float16x4_t __rev1_267; __rev1_267 = __builtin_shufflevector(__s1_267, __s1_267, 3, 2, 1, 0); \ |
| 71954 | float16x4_t __rev2_267; __rev2_267 = __builtin_shufflevector(__s2_267, __s2_267, 3, 2, 1, 0); \ | ||
| 71955 | float32x2_t __ret_267; \ | ||
| 71956 | __ret_267 = __noswap_vfmlal_high_u32(__rev0_267, __rev1_267, (float16x4_t) {__noswap_vget_lane_f16(__rev2_267, __p3_267), __noswap_vget_lane_f16(__rev2_267, __p3_267), __noswap_vget_lane_f16(__rev2_267, __p3_267), __noswap_vget_lane_f16(__rev2_267, __p3_267)}); \ | ||
| 71957 | __ret_267 = __builtin_shufflevector(__ret_267, __ret_267, 1, 0); \ | ||
| 71762 | __ret_267; \ | 71958 | __ret_267; \ |
| 71763 | }) | 71959 | }) |
| 71764 | #endif | 71960 | #endif |
| 71765 | 71961 | ||
| 71962 | #ifdef __LITTLE_ENDIAN__ | ||
| 71963 | #define vfmlalq_lane_low_u32(__p0_268, __p1_268, __p2_268, __p3_268) __extension__ ({ \ | ||
| 71964 | float32x4_t __s0_268 = __p0_268; \ | ||
| 71965 | float16x8_t __s1_268 = __p1_268; \ | ||
| 71966 | float16x4_t __s2_268 = __p2_268; \ | ||
| 71967 | float32x4_t __ret_268; \ | ||
| 71968 | __ret_268 = vfmlalq_low_u32(__s0_268, __s1_268, (float16x8_t) {vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268), vget_lane_f16(__s2_268, __p3_268)}); \ | ||
| 71969 | __ret_268; \ | ||
| 71970 | }) | ||
| 71971 | #else | ||
| 71972 | #define vfmlalq_lane_low_u32(__p0_269, __p1_269, __p2_269, __p3_269) __extension__ ({ \ | ||
| 71973 | float32x4_t __s0_269 = __p0_269; \ | ||
| 71974 | float16x8_t __s1_269 = __p1_269; \ | ||
| 71975 | float16x4_t __s2_269 = __p2_269; \ | ||
| 71976 | float32x4_t __rev0_269; __rev0_269 = __builtin_shufflevector(__s0_269, __s0_269, 3, 2, 1, 0); \ | ||
| 71977 | float16x8_t __rev1_269; __rev1_269 = __builtin_shufflevector(__s1_269, __s1_269, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 71978 | float16x4_t __rev2_269; __rev2_269 = __builtin_shufflevector(__s2_269, __s2_269, 3, 2, 1, 0); \ | ||
| 71979 | float32x4_t __ret_269; \ | ||
| 71980 | __ret_269 = __noswap_vfmlalq_low_u32(__rev0_269, __rev1_269, (float16x8_t) {__noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269), __noswap_vget_lane_f16(__rev2_269, __p3_269)}); \ | ||
| 71981 | __ret_269 = __builtin_shufflevector(__ret_269, __ret_269, 3, 2, 1, 0); \ | ||
| 71982 | __ret_269; \ | ||
| 71983 | }) | ||
| 71984 | #endif | ||
| 71985 | |||
| 71986 | #ifdef __LITTLE_ENDIAN__ | ||
| 71987 | #define vfmlal_lane_low_u32(__p0_270, __p1_270, __p2_270, __p3_270) __extension__ ({ \ | ||
| 71988 | float32x2_t __s0_270 = __p0_270; \ | ||
| 71989 | float16x4_t __s1_270 = __p1_270; \ | ||
| 71990 | float16x4_t __s2_270 = __p2_270; \ | ||
| 71991 | float32x2_t __ret_270; \ | ||
| 71992 | __ret_270 = vfmlal_low_u32(__s0_270, __s1_270, (float16x4_t) {vget_lane_f16(__s2_270, __p3_270), vget_lane_f16(__s2_270, __p3_270), vget_lane_f16(__s2_270, __p3_270), vget_lane_f16(__s2_270, __p3_270)}); \ | ||
| 71993 | __ret_270; \ | ||
| 71994 | }) | ||
| 71995 | #else | ||
| 71996 | #define vfmlal_lane_low_u32(__p0_271, __p1_271, __p2_271, __p3_271) __extension__ ({ \ | ||
| 71997 | float32x2_t __s0_271 = __p0_271; \ | ||
| 71998 | float16x4_t __s1_271 = __p1_271; \ | ||
| 71999 | float16x4_t __s2_271 = __p2_271; \ | ||
| 72000 | float32x2_t __rev0_271; __rev0_271 = __builtin_shufflevector(__s0_271, __s0_271, 1, 0); \ | ||
| 72001 | float16x4_t __rev1_271; __rev1_271 = __builtin_shufflevector(__s1_271, __s1_271, 3, 2, 1, 0); \ | ||
| 72002 | float16x4_t __rev2_271; __rev2_271 = __builtin_shufflevector(__s2_271, __s2_271, 3, 2, 1, 0); \ | ||
| 72003 | float32x2_t __ret_271; \ | ||
| 72004 | __ret_271 = __noswap_vfmlal_low_u32(__rev0_271, __rev1_271, (float16x4_t) {__noswap_vget_lane_f16(__rev2_271, __p3_271), __noswap_vget_lane_f16(__rev2_271, __p3_271), __noswap_vget_lane_f16(__rev2_271, __p3_271), __noswap_vget_lane_f16(__rev2_271, __p3_271)}); \ | ||
| 72005 | __ret_271 = __builtin_shufflevector(__ret_271, __ret_271, 1, 0); \ | ||
| 72006 | __ret_271; \ | ||
| 72007 | }) | ||
| 72008 | #endif | ||
| 72009 | |||
| 72010 | #ifdef __LITTLE_ENDIAN__ | ||
| 72011 | #define vfmlalq_laneq_high_u32(__p0_272, __p1_272, __p2_272, __p3_272) __extension__ ({ \ | ||
| 72012 | float32x4_t __s0_272 = __p0_272; \ | ||
| 72013 | float16x8_t __s1_272 = __p1_272; \ | ||
| 72014 | float16x8_t __s2_272 = __p2_272; \ | ||
| 72015 | float32x4_t __ret_272; \ | ||
| 72016 | __ret_272 = vfmlalq_high_u32(__s0_272, __s1_272, (float16x8_t) {vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272), vgetq_lane_f16(__s2_272, __p3_272)}); \ | ||
| 72017 | __ret_272; \ | ||
| 72018 | }) | ||
| 72019 | #else | ||
| 72020 | #define vfmlalq_laneq_high_u32(__p0_273, __p1_273, __p2_273, __p3_273) __extension__ ({ \ | ||
| 72021 | float32x4_t __s0_273 = __p0_273; \ | ||
| 72022 | float16x8_t __s1_273 = __p1_273; \ | ||
| 72023 | float16x8_t __s2_273 = __p2_273; \ | ||
| 72024 | float32x4_t __rev0_273; __rev0_273 = __builtin_shufflevector(__s0_273, __s0_273, 3, 2, 1, 0); \ | ||
| 72025 | float16x8_t __rev1_273; __rev1_273 = __builtin_shufflevector(__s1_273, __s1_273, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72026 | float16x8_t __rev2_273; __rev2_273 = __builtin_shufflevector(__s2_273, __s2_273, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72027 | float32x4_t __ret_273; \ | ||
| 72028 | __ret_273 = __noswap_vfmlalq_high_u32(__rev0_273, __rev1_273, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273), __noswap_vgetq_lane_f16(__rev2_273, __p3_273)}); \ | ||
| 72029 | __ret_273 = __builtin_shufflevector(__ret_273, __ret_273, 3, 2, 1, 0); \ | ||
| 72030 | __ret_273; \ | ||
| 72031 | }) | ||
| 72032 | #endif | ||
| 72033 | |||
| 72034 | #ifdef __LITTLE_ENDIAN__ | ||
| 72035 | #define vfmlal_laneq_high_u32(__p0_274, __p1_274, __p2_274, __p3_274) __extension__ ({ \ | ||
| 72036 | float32x2_t __s0_274 = __p0_274; \ | ||
| 72037 | float16x4_t __s1_274 = __p1_274; \ | ||
| 72038 | float16x8_t __s2_274 = __p2_274; \ | ||
| 72039 | float32x2_t __ret_274; \ | ||
| 72040 | __ret_274 = vfmlal_high_u32(__s0_274, __s1_274, (float16x4_t) {vgetq_lane_f16(__s2_274, __p3_274), vgetq_lane_f16(__s2_274, __p3_274), vgetq_lane_f16(__s2_274, __p3_274), vgetq_lane_f16(__s2_274, __p3_274)}); \ | ||
| 72041 | __ret_274; \ | ||
| 72042 | }) | ||
| 72043 | #else | ||
| 72044 | #define vfmlal_laneq_high_u32(__p0_275, __p1_275, __p2_275, __p3_275) __extension__ ({ \ | ||
| 72045 | float32x2_t __s0_275 = __p0_275; \ | ||
| 72046 | float16x4_t __s1_275 = __p1_275; \ | ||
| 72047 | float16x8_t __s2_275 = __p2_275; \ | ||
| 72048 | float32x2_t __rev0_275; __rev0_275 = __builtin_shufflevector(__s0_275, __s0_275, 1, 0); \ | ||
| 72049 | float16x4_t __rev1_275; __rev1_275 = __builtin_shufflevector(__s1_275, __s1_275, 3, 2, 1, 0); \ | ||
| 72050 | float16x8_t __rev2_275; __rev2_275 = __builtin_shufflevector(__s2_275, __s2_275, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72051 | float32x2_t __ret_275; \ | ||
| 72052 | __ret_275 = __noswap_vfmlal_high_u32(__rev0_275, __rev1_275, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_275, __p3_275), __noswap_vgetq_lane_f16(__rev2_275, __p3_275), __noswap_vgetq_lane_f16(__rev2_275, __p3_275), __noswap_vgetq_lane_f16(__rev2_275, __p3_275)}); \ | ||
| 72053 | __ret_275 = __builtin_shufflevector(__ret_275, __ret_275, 1, 0); \ | ||
| 72054 | __ret_275; \ | ||
| 72055 | }) | ||
| 72056 | #endif | ||
| 72057 | |||
| 72058 | #ifdef __LITTLE_ENDIAN__ | ||
| 72059 | #define vfmlalq_laneq_low_u32(__p0_276, __p1_276, __p2_276, __p3_276) __extension__ ({ \ | ||
| 72060 | float32x4_t __s0_276 = __p0_276; \ | ||
| 72061 | float16x8_t __s1_276 = __p1_276; \ | ||
| 72062 | float16x8_t __s2_276 = __p2_276; \ | ||
| 72063 | float32x4_t __ret_276; \ | ||
| 72064 | __ret_276 = vfmlalq_low_u32(__s0_276, __s1_276, (float16x8_t) {vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276), vgetq_lane_f16(__s2_276, __p3_276)}); \ | ||
| 72065 | __ret_276; \ | ||
| 72066 | }) | ||
| 72067 | #else | ||
| 72068 | #define vfmlalq_laneq_low_u32(__p0_277, __p1_277, __p2_277, __p3_277) __extension__ ({ \ | ||
| 72069 | float32x4_t __s0_277 = __p0_277; \ | ||
| 72070 | float16x8_t __s1_277 = __p1_277; \ | ||
| 72071 | float16x8_t __s2_277 = __p2_277; \ | ||
| 72072 | float32x4_t __rev0_277; __rev0_277 = __builtin_shufflevector(__s0_277, __s0_277, 3, 2, 1, 0); \ | ||
| 72073 | float16x8_t __rev1_277; __rev1_277 = __builtin_shufflevector(__s1_277, __s1_277, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72074 | float16x8_t __rev2_277; __rev2_277 = __builtin_shufflevector(__s2_277, __s2_277, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72075 | float32x4_t __ret_277; \ | ||
| 72076 | __ret_277 = __noswap_vfmlalq_low_u32(__rev0_277, __rev1_277, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277), __noswap_vgetq_lane_f16(__rev2_277, __p3_277)}); \ | ||
| 72077 | __ret_277 = __builtin_shufflevector(__ret_277, __ret_277, 3, 2, 1, 0); \ | ||
| 72078 | __ret_277; \ | ||
| 72079 | }) | ||
| 72080 | #endif | ||
| 72081 | |||
| 72082 | #ifdef __LITTLE_ENDIAN__ | ||
| 72083 | #define vfmlal_laneq_low_u32(__p0_278, __p1_278, __p2_278, __p3_278) __extension__ ({ \ | ||
| 72084 | float32x2_t __s0_278 = __p0_278; \ | ||
| 72085 | float16x4_t __s1_278 = __p1_278; \ | ||
| 72086 | float16x8_t __s2_278 = __p2_278; \ | ||
| 72087 | float32x2_t __ret_278; \ | ||
| 72088 | __ret_278 = vfmlal_low_u32(__s0_278, __s1_278, (float16x4_t) {vgetq_lane_f16(__s2_278, __p3_278), vgetq_lane_f16(__s2_278, __p3_278), vgetq_lane_f16(__s2_278, __p3_278), vgetq_lane_f16(__s2_278, __p3_278)}); \ | ||
| 72089 | __ret_278; \ | ||
| 72090 | }) | ||
| 72091 | #else | ||
| 72092 | #define vfmlal_laneq_low_u32(__p0_279, __p1_279, __p2_279, __p3_279) __extension__ ({ \ | ||
| 72093 | float32x2_t __s0_279 = __p0_279; \ | ||
| 72094 | float16x4_t __s1_279 = __p1_279; \ | ||
| 72095 | float16x8_t __s2_279 = __p2_279; \ | ||
| 72096 | float32x2_t __rev0_279; __rev0_279 = __builtin_shufflevector(__s0_279, __s0_279, 1, 0); \ | ||
| 72097 | float16x4_t __rev1_279; __rev1_279 = __builtin_shufflevector(__s1_279, __s1_279, 3, 2, 1, 0); \ | ||
| 72098 | float16x8_t __rev2_279; __rev2_279 = __builtin_shufflevector(__s2_279, __s2_279, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72099 | float32x2_t __ret_279; \ | ||
| 72100 | __ret_279 = __noswap_vfmlal_low_u32(__rev0_279, __rev1_279, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_279, __p3_279), __noswap_vgetq_lane_f16(__rev2_279, __p3_279), __noswap_vgetq_lane_f16(__rev2_279, __p3_279), __noswap_vgetq_lane_f16(__rev2_279, __p3_279)}); \ | ||
| 72101 | __ret_279 = __builtin_shufflevector(__ret_279, __ret_279, 1, 0); \ | ||
| 72102 | __ret_279; \ | ||
| 72103 | }) | ||
| 72104 | #endif | ||
| 72105 | |||
| 72106 | #ifdef __LITTLE_ENDIAN__ | ||
| 72107 | #define vfmlslq_lane_high_u32(__p0_280, __p1_280, __p2_280, __p3_280) __extension__ ({ \ | ||
| 72108 | float32x4_t __s0_280 = __p0_280; \ | ||
| 72109 | float16x8_t __s1_280 = __p1_280; \ | ||
| 72110 | float16x4_t __s2_280 = __p2_280; \ | ||
| 72111 | float32x4_t __ret_280; \ | ||
| 72112 | __ret_280 = vfmlslq_high_u32(__s0_280, __s1_280, (float16x8_t) {vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280), vget_lane_f16(__s2_280, __p3_280)}); \ | ||
| 72113 | __ret_280; \ | ||
| 72114 | }) | ||
| 72115 | #else | ||
| 72116 | #define vfmlslq_lane_high_u32(__p0_281, __p1_281, __p2_281, __p3_281) __extension__ ({ \ | ||
| 72117 | float32x4_t __s0_281 = __p0_281; \ | ||
| 72118 | float16x8_t __s1_281 = __p1_281; \ | ||
| 72119 | float16x4_t __s2_281 = __p2_281; \ | ||
| 72120 | float32x4_t __rev0_281; __rev0_281 = __builtin_shufflevector(__s0_281, __s0_281, 3, 2, 1, 0); \ | ||
| 72121 | float16x8_t __rev1_281; __rev1_281 = __builtin_shufflevector(__s1_281, __s1_281, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72122 | float16x4_t __rev2_281; __rev2_281 = __builtin_shufflevector(__s2_281, __s2_281, 3, 2, 1, 0); \ | ||
| 72123 | float32x4_t __ret_281; \ | ||
| 72124 | __ret_281 = __noswap_vfmlslq_high_u32(__rev0_281, __rev1_281, (float16x8_t) {__noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281), __noswap_vget_lane_f16(__rev2_281, __p3_281)}); \ | ||
| 72125 | __ret_281 = __builtin_shufflevector(__ret_281, __ret_281, 3, 2, 1, 0); \ | ||
| 72126 | __ret_281; \ | ||
| 72127 | }) | ||
| 72128 | #endif | ||
| 72129 | |||
| 72130 | #ifdef __LITTLE_ENDIAN__ | ||
| 72131 | #define vfmlsl_lane_high_u32(__p0_282, __p1_282, __p2_282, __p3_282) __extension__ ({ \ | ||
| 72132 | float32x2_t __s0_282 = __p0_282; \ | ||
| 72133 | float16x4_t __s1_282 = __p1_282; \ | ||
| 72134 | float16x4_t __s2_282 = __p2_282; \ | ||
| 72135 | float32x2_t __ret_282; \ | ||
| 72136 | __ret_282 = vfmlsl_high_u32(__s0_282, __s1_282, (float16x4_t) {vget_lane_f16(__s2_282, __p3_282), vget_lane_f16(__s2_282, __p3_282), vget_lane_f16(__s2_282, __p3_282), vget_lane_f16(__s2_282, __p3_282)}); \ | ||
| 72137 | __ret_282; \ | ||
| 72138 | }) | ||
| 72139 | #else | ||
| 72140 | #define vfmlsl_lane_high_u32(__p0_283, __p1_283, __p2_283, __p3_283) __extension__ ({ \ | ||
| 72141 | float32x2_t __s0_283 = __p0_283; \ | ||
| 72142 | float16x4_t __s1_283 = __p1_283; \ | ||
| 72143 | float16x4_t __s2_283 = __p2_283; \ | ||
| 72144 | float32x2_t __rev0_283; __rev0_283 = __builtin_shufflevector(__s0_283, __s0_283, 1, 0); \ | ||
| 72145 | float16x4_t __rev1_283; __rev1_283 = __builtin_shufflevector(__s1_283, __s1_283, 3, 2, 1, 0); \ | ||
| 72146 | float16x4_t __rev2_283; __rev2_283 = __builtin_shufflevector(__s2_283, __s2_283, 3, 2, 1, 0); \ | ||
| 72147 | float32x2_t __ret_283; \ | ||
| 72148 | __ret_283 = __noswap_vfmlsl_high_u32(__rev0_283, __rev1_283, (float16x4_t) {__noswap_vget_lane_f16(__rev2_283, __p3_283), __noswap_vget_lane_f16(__rev2_283, __p3_283), __noswap_vget_lane_f16(__rev2_283, __p3_283), __noswap_vget_lane_f16(__rev2_283, __p3_283)}); \ | ||
| 72149 | __ret_283 = __builtin_shufflevector(__ret_283, __ret_283, 1, 0); \ | ||
| 72150 | __ret_283; \ | ||
| 72151 | }) | ||
| 72152 | #endif | ||
| 72153 | |||
| 72154 | #ifdef __LITTLE_ENDIAN__ | ||
| 72155 | #define vfmlslq_lane_low_u32(__p0_284, __p1_284, __p2_284, __p3_284) __extension__ ({ \ | ||
| 72156 | float32x4_t __s0_284 = __p0_284; \ | ||
| 72157 | float16x8_t __s1_284 = __p1_284; \ | ||
| 72158 | float16x4_t __s2_284 = __p2_284; \ | ||
| 72159 | float32x4_t __ret_284; \ | ||
| 72160 | __ret_284 = vfmlslq_low_u32(__s0_284, __s1_284, (float16x8_t) {vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284), vget_lane_f16(__s2_284, __p3_284)}); \ | ||
| 72161 | __ret_284; \ | ||
| 72162 | }) | ||
| 72163 | #else | ||
| 72164 | #define vfmlslq_lane_low_u32(__p0_285, __p1_285, __p2_285, __p3_285) __extension__ ({ \ | ||
| 72165 | float32x4_t __s0_285 = __p0_285; \ | ||
| 72166 | float16x8_t __s1_285 = __p1_285; \ | ||
| 72167 | float16x4_t __s2_285 = __p2_285; \ | ||
| 72168 | float32x4_t __rev0_285; __rev0_285 = __builtin_shufflevector(__s0_285, __s0_285, 3, 2, 1, 0); \ | ||
| 72169 | float16x8_t __rev1_285; __rev1_285 = __builtin_shufflevector(__s1_285, __s1_285, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72170 | float16x4_t __rev2_285; __rev2_285 = __builtin_shufflevector(__s2_285, __s2_285, 3, 2, 1, 0); \ | ||
| 72171 | float32x4_t __ret_285; \ | ||
| 72172 | __ret_285 = __noswap_vfmlslq_low_u32(__rev0_285, __rev1_285, (float16x8_t) {__noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285), __noswap_vget_lane_f16(__rev2_285, __p3_285)}); \ | ||
| 72173 | __ret_285 = __builtin_shufflevector(__ret_285, __ret_285, 3, 2, 1, 0); \ | ||
| 72174 | __ret_285; \ | ||
| 72175 | }) | ||
| 72176 | #endif | ||
| 72177 | |||
| 72178 | #ifdef __LITTLE_ENDIAN__ | ||
| 72179 | #define vfmlsl_lane_low_u32(__p0_286, __p1_286, __p2_286, __p3_286) __extension__ ({ \ | ||
| 72180 | float32x2_t __s0_286 = __p0_286; \ | ||
| 72181 | float16x4_t __s1_286 = __p1_286; \ | ||
| 72182 | float16x4_t __s2_286 = __p2_286; \ | ||
| 72183 | float32x2_t __ret_286; \ | ||
| 72184 | __ret_286 = vfmlsl_low_u32(__s0_286, __s1_286, (float16x4_t) {vget_lane_f16(__s2_286, __p3_286), vget_lane_f16(__s2_286, __p3_286), vget_lane_f16(__s2_286, __p3_286), vget_lane_f16(__s2_286, __p3_286)}); \ | ||
| 72185 | __ret_286; \ | ||
| 72186 | }) | ||
| 72187 | #else | ||
| 72188 | #define vfmlsl_lane_low_u32(__p0_287, __p1_287, __p2_287, __p3_287) __extension__ ({ \ | ||
| 72189 | float32x2_t __s0_287 = __p0_287; \ | ||
| 72190 | float16x4_t __s1_287 = __p1_287; \ | ||
| 72191 | float16x4_t __s2_287 = __p2_287; \ | ||
| 72192 | float32x2_t __rev0_287; __rev0_287 = __builtin_shufflevector(__s0_287, __s0_287, 1, 0); \ | ||
| 72193 | float16x4_t __rev1_287; __rev1_287 = __builtin_shufflevector(__s1_287, __s1_287, 3, 2, 1, 0); \ | ||
| 72194 | float16x4_t __rev2_287; __rev2_287 = __builtin_shufflevector(__s2_287, __s2_287, 3, 2, 1, 0); \ | ||
| 72195 | float32x2_t __ret_287; \ | ||
| 72196 | __ret_287 = __noswap_vfmlsl_low_u32(__rev0_287, __rev1_287, (float16x4_t) {__noswap_vget_lane_f16(__rev2_287, __p3_287), __noswap_vget_lane_f16(__rev2_287, __p3_287), __noswap_vget_lane_f16(__rev2_287, __p3_287), __noswap_vget_lane_f16(__rev2_287, __p3_287)}); \ | ||
| 72197 | __ret_287 = __builtin_shufflevector(__ret_287, __ret_287, 1, 0); \ | ||
| 72198 | __ret_287; \ | ||
| 72199 | }) | ||
| 72200 | #endif | ||
| 72201 | |||
| 72202 | #ifdef __LITTLE_ENDIAN__ | ||
| 72203 | #define vfmlslq_laneq_high_u32(__p0_288, __p1_288, __p2_288, __p3_288) __extension__ ({ \ | ||
| 72204 | float32x4_t __s0_288 = __p0_288; \ | ||
| 72205 | float16x8_t __s1_288 = __p1_288; \ | ||
| 72206 | float16x8_t __s2_288 = __p2_288; \ | ||
| 72207 | float32x4_t __ret_288; \ | ||
| 72208 | __ret_288 = vfmlslq_high_u32(__s0_288, __s1_288, (float16x8_t) {vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288), vgetq_lane_f16(__s2_288, __p3_288)}); \ | ||
| 72209 | __ret_288; \ | ||
| 72210 | }) | ||
| 72211 | #else | ||
| 72212 | #define vfmlslq_laneq_high_u32(__p0_289, __p1_289, __p2_289, __p3_289) __extension__ ({ \ | ||
| 72213 | float32x4_t __s0_289 = __p0_289; \ | ||
| 72214 | float16x8_t __s1_289 = __p1_289; \ | ||
| 72215 | float16x8_t __s2_289 = __p2_289; \ | ||
| 72216 | float32x4_t __rev0_289; __rev0_289 = __builtin_shufflevector(__s0_289, __s0_289, 3, 2, 1, 0); \ | ||
| 72217 | float16x8_t __rev1_289; __rev1_289 = __builtin_shufflevector(__s1_289, __s1_289, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72218 | float16x8_t __rev2_289; __rev2_289 = __builtin_shufflevector(__s2_289, __s2_289, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72219 | float32x4_t __ret_289; \ | ||
| 72220 | __ret_289 = __noswap_vfmlslq_high_u32(__rev0_289, __rev1_289, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289), __noswap_vgetq_lane_f16(__rev2_289, __p3_289)}); \ | ||
| 72221 | __ret_289 = __builtin_shufflevector(__ret_289, __ret_289, 3, 2, 1, 0); \ | ||
| 72222 | __ret_289; \ | ||
| 72223 | }) | ||
| 72224 | #endif | ||
| 72225 | |||
| 72226 | #ifdef __LITTLE_ENDIAN__ | ||
| 72227 | #define vfmlsl_laneq_high_u32(__p0_290, __p1_290, __p2_290, __p3_290) __extension__ ({ \ | ||
| 72228 | float32x2_t __s0_290 = __p0_290; \ | ||
| 72229 | float16x4_t __s1_290 = __p1_290; \ | ||
| 72230 | float16x8_t __s2_290 = __p2_290; \ | ||
| 72231 | float32x2_t __ret_290; \ | ||
| 72232 | __ret_290 = vfmlsl_high_u32(__s0_290, __s1_290, (float16x4_t) {vgetq_lane_f16(__s2_290, __p3_290), vgetq_lane_f16(__s2_290, __p3_290), vgetq_lane_f16(__s2_290, __p3_290), vgetq_lane_f16(__s2_290, __p3_290)}); \ | ||
| 72233 | __ret_290; \ | ||
| 72234 | }) | ||
| 72235 | #else | ||
| 72236 | #define vfmlsl_laneq_high_u32(__p0_291, __p1_291, __p2_291, __p3_291) __extension__ ({ \ | ||
| 72237 | float32x2_t __s0_291 = __p0_291; \ | ||
| 72238 | float16x4_t __s1_291 = __p1_291; \ | ||
| 72239 | float16x8_t __s2_291 = __p2_291; \ | ||
| 72240 | float32x2_t __rev0_291; __rev0_291 = __builtin_shufflevector(__s0_291, __s0_291, 1, 0); \ | ||
| 72241 | float16x4_t __rev1_291; __rev1_291 = __builtin_shufflevector(__s1_291, __s1_291, 3, 2, 1, 0); \ | ||
| 72242 | float16x8_t __rev2_291; __rev2_291 = __builtin_shufflevector(__s2_291, __s2_291, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72243 | float32x2_t __ret_291; \ | ||
| 72244 | __ret_291 = __noswap_vfmlsl_high_u32(__rev0_291, __rev1_291, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_291, __p3_291), __noswap_vgetq_lane_f16(__rev2_291, __p3_291), __noswap_vgetq_lane_f16(__rev2_291, __p3_291), __noswap_vgetq_lane_f16(__rev2_291, __p3_291)}); \ | ||
| 72245 | __ret_291 = __builtin_shufflevector(__ret_291, __ret_291, 1, 0); \ | ||
| 72246 | __ret_291; \ | ||
| 72247 | }) | ||
| 72248 | #endif | ||
| 72249 | |||
| 72250 | #ifdef __LITTLE_ENDIAN__ | ||
| 72251 | #define vfmlslq_laneq_low_u32(__p0_292, __p1_292, __p2_292, __p3_292) __extension__ ({ \ | ||
| 72252 | float32x4_t __s0_292 = __p0_292; \ | ||
| 72253 | float16x8_t __s1_292 = __p1_292; \ | ||
| 72254 | float16x8_t __s2_292 = __p2_292; \ | ||
| 72255 | float32x4_t __ret_292; \ | ||
| 72256 | __ret_292 = vfmlslq_low_u32(__s0_292, __s1_292, (float16x8_t) {vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292), vgetq_lane_f16(__s2_292, __p3_292)}); \ | ||
| 72257 | __ret_292; \ | ||
| 72258 | }) | ||
| 72259 | #else | ||
| 72260 | #define vfmlslq_laneq_low_u32(__p0_293, __p1_293, __p2_293, __p3_293) __extension__ ({ \ | ||
| 72261 | float32x4_t __s0_293 = __p0_293; \ | ||
| 72262 | float16x8_t __s1_293 = __p1_293; \ | ||
| 72263 | float16x8_t __s2_293 = __p2_293; \ | ||
| 72264 | float32x4_t __rev0_293; __rev0_293 = __builtin_shufflevector(__s0_293, __s0_293, 3, 2, 1, 0); \ | ||
| 72265 | float16x8_t __rev1_293; __rev1_293 = __builtin_shufflevector(__s1_293, __s1_293, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72266 | float16x8_t __rev2_293; __rev2_293 = __builtin_shufflevector(__s2_293, __s2_293, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72267 | float32x4_t __ret_293; \ | ||
| 72268 | __ret_293 = __noswap_vfmlslq_low_u32(__rev0_293, __rev1_293, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293), __noswap_vgetq_lane_f16(__rev2_293, __p3_293)}); \ | ||
| 72269 | __ret_293 = __builtin_shufflevector(__ret_293, __ret_293, 3, 2, 1, 0); \ | ||
| 72270 | __ret_293; \ | ||
| 72271 | }) | ||
| 72272 | #endif | ||
| 72273 | |||
| 72274 | #ifdef __LITTLE_ENDIAN__ | ||
| 72275 | #define vfmlsl_laneq_low_u32(__p0_294, __p1_294, __p2_294, __p3_294) __extension__ ({ \ | ||
| 72276 | float32x2_t __s0_294 = __p0_294; \ | ||
| 72277 | float16x4_t __s1_294 = __p1_294; \ | ||
| 72278 | float16x8_t __s2_294 = __p2_294; \ | ||
| 72279 | float32x2_t __ret_294; \ | ||
| 72280 | __ret_294 = vfmlsl_low_u32(__s0_294, __s1_294, (float16x4_t) {vgetq_lane_f16(__s2_294, __p3_294), vgetq_lane_f16(__s2_294, __p3_294), vgetq_lane_f16(__s2_294, __p3_294), vgetq_lane_f16(__s2_294, __p3_294)}); \ | ||
| 72281 | __ret_294; \ | ||
| 72282 | }) | ||
| 72283 | #else | ||
| 72284 | #define vfmlsl_laneq_low_u32(__p0_295, __p1_295, __p2_295, __p3_295) __extension__ ({ \ | ||
| 72285 | float32x2_t __s0_295 = __p0_295; \ | ||
| 72286 | float16x4_t __s1_295 = __p1_295; \ | ||
| 72287 | float16x8_t __s2_295 = __p2_295; \ | ||
| 72288 | float32x2_t __rev0_295; __rev0_295 = __builtin_shufflevector(__s0_295, __s0_295, 1, 0); \ | ||
| 72289 | float16x4_t __rev1_295; __rev1_295 = __builtin_shufflevector(__s1_295, __s1_295, 3, 2, 1, 0); \ | ||
| 72290 | float16x8_t __rev2_295; __rev2_295 = __builtin_shufflevector(__s2_295, __s2_295, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72291 | float32x2_t __ret_295; \ | ||
| 72292 | __ret_295 = __noswap_vfmlsl_low_u32(__rev0_295, __rev1_295, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_295, __p3_295), __noswap_vgetq_lane_f16(__rev2_295, __p3_295), __noswap_vgetq_lane_f16(__rev2_295, __p3_295), __noswap_vgetq_lane_f16(__rev2_295, __p3_295)}); \ | ||
| 72293 | __ret_295 = __builtin_shufflevector(__ret_295, __ret_295, 1, 0); \ | ||
| 72294 | __ret_295; \ | ||
| 72295 | }) | ||
| 72296 | #endif | ||
| 72297 | |||
| 72298 | #endif | ||
| 72299 | #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(__aarch64__) | ||
| 72300 | #ifdef __LITTLE_ENDIAN__ | ||
| 72301 | #define vmulh_lane_f16(__p0_296, __p1_296, __p2_296) __extension__ ({ \ | ||
| 72302 | float16_t __s0_296 = __p0_296; \ | ||
| 72303 | float16x4_t __s1_296 = __p1_296; \ | ||
| 72304 | float16_t __ret_296; \ | ||
| 72305 | __ret_296 = __s0_296 * vget_lane_f16(__s1_296, __p2_296); \ | ||
| 72306 | __ret_296; \ | ||
| 72307 | }) | ||
| 72308 | #else | ||
| 72309 | #define vmulh_lane_f16(__p0_297, __p1_297, __p2_297) __extension__ ({ \ | ||
| 72310 | float16_t __s0_297 = __p0_297; \ | ||
| 72311 | float16x4_t __s1_297 = __p1_297; \ | ||
| 72312 | float16x4_t __rev1_297; __rev1_297 = __builtin_shufflevector(__s1_297, __s1_297, 3, 2, 1, 0); \ | ||
| 72313 | float16_t __ret_297; \ | ||
| 72314 | __ret_297 = __s0_297 * __noswap_vget_lane_f16(__rev1_297, __p2_297); \ | ||
| 72315 | __ret_297; \ | ||
| 72316 | }) | ||
| 72317 | #endif | ||
| 72318 | |||
| 72319 | #ifdef __LITTLE_ENDIAN__ | ||
| 72320 | #define vmulh_laneq_f16(__p0_298, __p1_298, __p2_298) __extension__ ({ \ | ||
| 72321 | float16_t __s0_298 = __p0_298; \ | ||
| 72322 | float16x8_t __s1_298 = __p1_298; \ | ||
| 72323 | float16_t __ret_298; \ | ||
| 72324 | __ret_298 = __s0_298 * vgetq_lane_f16(__s1_298, __p2_298); \ | ||
| 72325 | __ret_298; \ | ||
| 72326 | }) | ||
| 72327 | #else | ||
| 72328 | #define vmulh_laneq_f16(__p0_299, __p1_299, __p2_299) __extension__ ({ \ | ||
| 72329 | float16_t __s0_299 = __p0_299; \ | ||
| 72330 | float16x8_t __s1_299 = __p1_299; \ | ||
| 72331 | float16x8_t __rev1_299; __rev1_299 = __builtin_shufflevector(__s1_299, __s1_299, 7, 6, 5, 4, 3, 2, 1, 0); \ | ||
| 72332 | float16_t __ret_299; \ | ||
| 72333 | __ret_299 = __s0_299 * __noswap_vgetq_lane_f16(__rev1_299, __p2_299); \ | ||
| 72334 | __ret_299; \ | ||
| 72335 | }) | ||
| 72336 | #endif | ||
| 72337 | |||
| 71766 | #endif | 72338 | #endif |
| 71767 | #if defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__) | 72339 | #if defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__) |
| 71768 | #ifdef __LITTLE_ENDIAN__ | 72340 | #ifdef __LITTLE_ENDIAN__ |
| ... | @@ -71794,86 +72366,86 @@ __ai int16_t vqrdmlahh_s16(int16_t __p0, int16_t __p1, int16_t __p2) { | ... | @@ -71794,86 +72366,86 @@ __ai int16_t vqrdmlahh_s16(int16_t __p0, int16_t __p1, int16_t __p2) { |
| 71794 | #endif | 72366 | #endif |
| 71795 | 72367 | ||
| 71796 | #ifdef __LITTLE_ENDIAN__ | 72368 | #ifdef __LITTLE_ENDIAN__ |
| 71797 | #define vqrdmlahs_lane_s32(__p0_268, __p1_268, __p2_268, __p3_268) __extension__ ({ \ | 72369 | #define vqrdmlahs_lane_s32(__p0_300, __p1_300, __p2_300, __p3_300) __extension__ ({ \ |
| 71798 | int32_t __s0_268 = __p0_268; \ | 72370 | int32_t __s0_300 = __p0_300; \ |
| 71799 | int32_t __s1_268 = __p1_268; \ | 72371 | int32_t __s1_300 = __p1_300; \ |
| 71800 | int32x2_t __s2_268 = __p2_268; \ | 72372 | int32x2_t __s2_300 = __p2_300; \ |
| 71801 | int32_t __ret_268; \ | 72373 | int32_t __ret_300; \ |
| 71802 | __ret_268 = vqadds_s32(__s0_268, vqrdmulhs_s32(__s1_268, vget_lane_s32(__s2_268, __p3_268))); \ | 72374 | __ret_300 = vqadds_s32(__s0_300, vqrdmulhs_s32(__s1_300, vget_lane_s32(__s2_300, __p3_300))); \ |
| 71803 | __ret_268; \ | 72375 | __ret_300; \ |
| 71804 | }) | 72376 | }) |
| 71805 | #else | 72377 | #else |
| 71806 | #define vqrdmlahs_lane_s32(__p0_269, __p1_269, __p2_269, __p3_269) __extension__ ({ \ | 72378 | #define vqrdmlahs_lane_s32(__p0_301, __p1_301, __p2_301, __p3_301) __extension__ ({ \ |
| 71807 | int32_t __s0_269 = __p0_269; \ | 72379 | int32_t __s0_301 = __p0_301; \ |
| 71808 | int32_t __s1_269 = __p1_269; \ | 72380 | int32_t __s1_301 = __p1_301; \ |
| 71809 | int32x2_t __s2_269 = __p2_269; \ | 72381 | int32x2_t __s2_301 = __p2_301; \ |
| 71810 | int32x2_t __rev2_269; __rev2_269 = __builtin_shufflevector(__s2_269, __s2_269, 1, 0); \ | 72382 | int32x2_t __rev2_301; __rev2_301 = __builtin_shufflevector(__s2_301, __s2_301, 1, 0); \ |
| 71811 | int32_t __ret_269; \ | 72383 | int32_t __ret_301; \ |
| 71812 | __ret_269 = __noswap_vqadds_s32(__s0_269, __noswap_vqrdmulhs_s32(__s1_269, __noswap_vget_lane_s32(__rev2_269, __p3_269))); \ | 72384 | __ret_301 = __noswap_vqadds_s32(__s0_301, __noswap_vqrdmulhs_s32(__s1_301, __noswap_vget_lane_s32(__rev2_301, __p3_301))); \ |
| 71813 | __ret_269; \ | 72385 | __ret_301; \ |
| 71814 | }) | 72386 | }) |
| 71815 | #endif | 72387 | #endif |
| 71816 | 72388 | ||
| 71817 | #ifdef __LITTLE_ENDIAN__ | 72389 | #ifdef __LITTLE_ENDIAN__ |
| 71818 | #define vqrdmlahh_lane_s16(__p0_270, __p1_270, __p2_270, __p3_270) __extension__ ({ \ | 72390 | #define vqrdmlahh_lane_s16(__p0_302, __p1_302, __p2_302, __p3_302) __extension__ ({ \ |
| 71819 | int16_t __s0_270 = __p0_270; \ | 72391 | int16_t __s0_302 = __p0_302; \ |
| 71820 | int16_t __s1_270 = __p1_270; \ | 72392 | int16_t __s1_302 = __p1_302; \ |
| 71821 | int16x4_t __s2_270 = __p2_270; \ | 72393 | int16x4_t __s2_302 = __p2_302; \ |
| 71822 | int16_t __ret_270; \ | 72394 | int16_t __ret_302; \ |
| 71823 | __ret_270 = vqaddh_s16(__s0_270, vqrdmulhh_s16(__s1_270, vget_lane_s16(__s2_270, __p3_270))); \ | 72395 | __ret_302 = vqaddh_s16(__s0_302, vqrdmulhh_s16(__s1_302, vget_lane_s16(__s2_302, __p3_302))); \ |
| 71824 | __ret_270; \ | 72396 | __ret_302; \ |
| 71825 | }) | 72397 | }) |
| 71826 | #else | 72398 | #else |
| 71827 | #define vqrdmlahh_lane_s16(__p0_271, __p1_271, __p2_271, __p3_271) __extension__ ({ \ | 72399 | #define vqrdmlahh_lane_s16(__p0_303, __p1_303, __p2_303, __p3_303) __extension__ ({ \ |
| 71828 | int16_t __s0_271 = __p0_271; \ | 72400 | int16_t __s0_303 = __p0_303; \ |
| 71829 | int16_t __s1_271 = __p1_271; \ | 72401 | int16_t __s1_303 = __p1_303; \ |
| 71830 | int16x4_t __s2_271 = __p2_271; \ | 72402 | int16x4_t __s2_303 = __p2_303; \ |
| 71831 | int16x4_t __rev2_271; __rev2_271 = __builtin_shufflevector(__s2_271, __s2_271, 3, 2, 1, 0); \ | 72403 | int16x4_t __rev2_303; __rev2_303 = __builtin_shufflevector(__s2_303, __s2_303, 3, 2, 1, 0); \ |
| 71832 | int16_t __ret_271; \ | 72404 | int16_t __ret_303; \ |
| 71833 | __ret_271 = __noswap_vqaddh_s16(__s0_271, __noswap_vqrdmulhh_s16(__s1_271, __noswap_vget_lane_s16(__rev2_271, __p3_271))); \ | 72405 | __ret_303 = __noswap_vqaddh_s16(__s0_303, __noswap_vqrdmulhh_s16(__s1_303, __noswap_vget_lane_s16(__rev2_303, __p3_303))); \ |
| 71834 | __ret_271; \ | 72406 | __ret_303; \ |
| 71835 | }) | 72407 | }) |
| 71836 | #endif | 72408 | #endif |
| 71837 | 72409 | ||
| 71838 | #ifdef __LITTLE_ENDIAN__ | 72410 | #ifdef __LITTLE_ENDIAN__ |
| 71839 | #define vqrdmlahs_laneq_s32(__p0_272, __p1_272, __p2_272, __p3_272) __extension__ ({ \ | 72411 | #define vqrdmlahs_laneq_s32(__p0_304, __p1_304, __p2_304, __p3_304) __extension__ ({ \ |
| 71840 | int32_t __s0_272 = __p0_272; \ | 72412 | int32_t __s0_304 = __p0_304; \ |
| 71841 | int32_t __s1_272 = __p1_272; \ | 72413 | int32_t __s1_304 = __p1_304; \ |
| 71842 | int32x4_t __s2_272 = __p2_272; \ | 72414 | int32x4_t __s2_304 = __p2_304; \ |
| 71843 | int32_t __ret_272; \ | 72415 | int32_t __ret_304; \ |
| 71844 | __ret_272 = vqadds_s32(__s0_272, vqrdmulhs_s32(__s1_272, vgetq_lane_s32(__s2_272, __p3_272))); \ | 72416 | __ret_304 = vqadds_s32(__s0_304, vqrdmulhs_s32(__s1_304, vgetq_lane_s32(__s2_304, __p3_304))); \ |
| 71845 | __ret_272; \ | 72417 | __ret_304; \ |
| 71846 | }) | 72418 | }) |
| 71847 | #else | 72419 | #else |
| 71848 | #define vqrdmlahs_laneq_s32(__p0_273, __p1_273, __p2_273, __p3_273) __extension__ ({ \ | 72420 | #define vqrdmlahs_laneq_s32(__p0_305, __p1_305, __p2_305, __p3_305) __extension__ ({ \ |
| 71849 | int32_t __s0_273 = __p0_273; \ | 72421 | int32_t __s0_305 = __p0_305; \ |
| 71850 | int32_t __s1_273 = __p1_273; \ | 72422 | int32_t __s1_305 = __p1_305; \ |
| 71851 | int32x4_t __s2_273 = __p2_273; \ | 72423 | int32x4_t __s2_305 = __p2_305; \ |
| 71852 | int32x4_t __rev2_273; __rev2_273 = __builtin_shufflevector(__s2_273, __s2_273, 3, 2, 1, 0); \ | 72424 | int32x4_t __rev2_305; __rev2_305 = __builtin_shufflevector(__s2_305, __s2_305, 3, 2, 1, 0); \ |
| 71853 | int32_t __ret_273; \ | 72425 | int32_t __ret_305; \ |
| 71854 | __ret_273 = __noswap_vqadds_s32(__s0_273, __noswap_vqrdmulhs_s32(__s1_273, __noswap_vgetq_lane_s32(__rev2_273, __p3_273))); \ | 72426 | __ret_305 = __noswap_vqadds_s32(__s0_305, __noswap_vqrdmulhs_s32(__s1_305, __noswap_vgetq_lane_s32(__rev2_305, __p3_305))); \ |
| 71855 | __ret_273; \ | 72427 | __ret_305; \ |
| 71856 | }) | 72428 | }) |
| 71857 | #endif | 72429 | #endif |
| 71858 | 72430 | ||
| 71859 | #ifdef __LITTLE_ENDIAN__ | 72431 | #ifdef __LITTLE_ENDIAN__ |
| 71860 | #define vqrdmlahh_laneq_s16(__p0_274, __p1_274, __p2_274, __p3_274) __extension__ ({ \ | 72432 | #define vqrdmlahh_laneq_s16(__p0_306, __p1_306, __p2_306, __p3_306) __extension__ ({ \ |
| 71861 | int16_t __s0_274 = __p0_274; \ | 72433 | int16_t __s0_306 = __p0_306; \ |
| 71862 | int16_t __s1_274 = __p1_274; \ | 72434 | int16_t __s1_306 = __p1_306; \ |
| 71863 | int16x8_t __s2_274 = __p2_274; \ | 72435 | int16x8_t __s2_306 = __p2_306; \ |
| 71864 | int16_t __ret_274; \ | 72436 | int16_t __ret_306; \ |
| 71865 | __ret_274 = vqaddh_s16(__s0_274, vqrdmulhh_s16(__s1_274, vgetq_lane_s16(__s2_274, __p3_274))); \ | 72437 | __ret_306 = vqaddh_s16(__s0_306, vqrdmulhh_s16(__s1_306, vgetq_lane_s16(__s2_306, __p3_306))); \ |
| 71866 | __ret_274; \ | 72438 | __ret_306; \ |
| 71867 | }) | 72439 | }) |
| 71868 | #else | 72440 | #else |
| 71869 | #define vqrdmlahh_laneq_s16(__p0_275, __p1_275, __p2_275, __p3_275) __extension__ ({ \ | 72441 | #define vqrdmlahh_laneq_s16(__p0_307, __p1_307, __p2_307, __p3_307) __extension__ ({ \ |
| 71870 | int16_t __s0_275 = __p0_275; \ | 72442 | int16_t __s0_307 = __p0_307; \ |
| 71871 | int16_t __s1_275 = __p1_275; \ | 72443 | int16_t __s1_307 = __p1_307; \ |
| 71872 | int16x8_t __s2_275 = __p2_275; \ | 72444 | int16x8_t __s2_307 = __p2_307; \ |
| 71873 | int16x8_t __rev2_275; __rev2_275 = __builtin_shufflevector(__s2_275, __s2_275, 7, 6, 5, 4, 3, 2, 1, 0); \ | 72445 | int16x8_t __rev2_307; __rev2_307 = __builtin_shufflevector(__s2_307, __s2_307, 7, 6, 5, 4, 3, 2, 1, 0); \ |
| 71874 | int16_t __ret_275; \ | 72446 | int16_t __ret_307; \ |
| 71875 | __ret_275 = __noswap_vqaddh_s16(__s0_275, __noswap_vqrdmulhh_s16(__s1_275, __noswap_vgetq_lane_s16(__rev2_275, __p3_275))); \ | 72447 | __ret_307 = __noswap_vqaddh_s16(__s0_307, __noswap_vqrdmulhh_s16(__s1_307, __noswap_vgetq_lane_s16(__rev2_307, __p3_307))); \ |
| 71876 | __ret_275; \ | 72448 | __ret_307; \ |
| 71877 | }) | 72449 | }) |
| 71878 | #endif | 72450 | #endif |
| 71879 | 72451 | ||
| ... | @@ -71906,86 +72478,86 @@ __ai int16_t vqrdmlshh_s16(int16_t __p0, int16_t __p1, int16_t __p2) { | ... | @@ -71906,86 +72478,86 @@ __ai int16_t vqrdmlshh_s16(int16_t __p0, int16_t __p1, int16_t __p2) { |
| 71906 | #endif | 72478 | #endif |
| 71907 | 72479 | ||
| 71908 | #ifdef __LITTLE_ENDIAN__ | 72480 | #ifdef __LITTLE_ENDIAN__ |
| 71909 | #define vqrdmlshs_lane_s32(__p0_276, __p1_276, __p2_276, __p3_276) __extension__ ({ \ | 72481 | #define vqrdmlshs_lane_s32(__p0_308, __p1_308, __p2_308, __p3_308) __extension__ ({ \ |
| 71910 | int32_t __s0_276 = __p0_276; \ | 72482 | int32_t __s0_308 = __p0_308; \ |
| 71911 | int32_t __s1_276 = __p1_276; \ | 72483 | int32_t __s1_308 = __p1_308; \ |
| 71912 | int32x2_t __s2_276 = __p2_276; \ | 72484 | int32x2_t __s2_308 = __p2_308; \ |
| 71913 | int32_t __ret_276; \ | 72485 | int32_t __ret_308; \ |
| 71914 | __ret_276 = vqsubs_s32(__s0_276, vqrdmulhs_s32(__s1_276, vget_lane_s32(__s2_276, __p3_276))); \ | 72486 | __ret_308 = vqsubs_s32(__s0_308, vqrdmulhs_s32(__s1_308, vget_lane_s32(__s2_308, __p3_308))); \ |
| 71915 | __ret_276; \ | 72487 | __ret_308; \ |
| 71916 | }) | 72488 | }) |
| 71917 | #else | 72489 | #else |
| 71918 | #define vqrdmlshs_lane_s32(__p0_277, __p1_277, __p2_277, __p3_277) __extension__ ({ \ | 72490 | #define vqrdmlshs_lane_s32(__p0_309, __p1_309, __p2_309, __p3_309) __extension__ ({ \ |
| 71919 | int32_t __s0_277 = __p0_277; \ | 72491 | int32_t __s0_309 = __p0_309; \ |
| 71920 | int32_t __s1_277 = __p1_277; \ | 72492 | int32_t __s1_309 = __p1_309; \ |
| 71921 | int32x2_t __s2_277 = __p2_277; \ | 72493 | int32x2_t __s2_309 = __p2_309; \ |
| 71922 | int32x2_t __rev2_277; __rev2_277 = __builtin_shufflevector(__s2_277, __s2_277, 1, 0); \ | 72494 | int32x2_t __rev2_309; __rev2_309 = __builtin_shufflevector(__s2_309, __s2_309, 1, 0); \ |
| 71923 | int32_t __ret_277; \ | 72495 | int32_t __ret_309; \ |
| 71924 | __ret_277 = __noswap_vqsubs_s32(__s0_277, __noswap_vqrdmulhs_s32(__s1_277, __noswap_vget_lane_s32(__rev2_277, __p3_277))); \ | 72496 | __ret_309 = __noswap_vqsubs_s32(__s0_309, __noswap_vqrdmulhs_s32(__s1_309, __noswap_vget_lane_s32(__rev2_309, __p3_309))); \ |
| 71925 | __ret_277; \ | 72497 | __ret_309; \ |
| 71926 | }) | 72498 | }) |
| 71927 | #endif | 72499 | #endif |
| 71928 | 72500 | ||
| 71929 | #ifdef __LITTLE_ENDIAN__ | 72501 | #ifdef __LITTLE_ENDIAN__ |
| 71930 | #define vqrdmlshh_lane_s16(__p0_278, __p1_278, __p2_278, __p3_278) __extension__ ({ \ | 72502 | #define vqrdmlshh_lane_s16(__p0_310, __p1_310, __p2_310, __p3_310) __extension__ ({ \ |
| 71931 | int16_t __s0_278 = __p0_278; \ | 72503 | int16_t __s0_310 = __p0_310; \ |
| 71932 | int16_t __s1_278 = __p1_278; \ | 72504 | int16_t __s1_310 = __p1_310; \ |
| 71933 | int16x4_t __s2_278 = __p2_278; \ | 72505 | int16x4_t __s2_310 = __p2_310; \ |
| 71934 | int16_t __ret_278; \ | 72506 | int16_t __ret_310; \ |
| 71935 | __ret_278 = vqsubh_s16(__s0_278, vqrdmulhh_s16(__s1_278, vget_lane_s16(__s2_278, __p3_278))); \ | 72507 | __ret_310 = vqsubh_s16(__s0_310, vqrdmulhh_s16(__s1_310, vget_lane_s16(__s2_310, __p3_310))); \ |
| 71936 | __ret_278; \ | 72508 | __ret_310; \ |
| 71937 | }) | 72509 | }) |
| 71938 | #else | 72510 | #else |
| 71939 | #define vqrdmlshh_lane_s16(__p0_279, __p1_279, __p2_279, __p3_279) __extension__ ({ \ | 72511 | #define vqrdmlshh_lane_s16(__p0_311, __p1_311, __p2_311, __p3_311) __extension__ ({ \ |
| 71940 | int16_t __s0_279 = __p0_279; \ | 72512 | int16_t __s0_311 = __p0_311; \ |
| 71941 | int16_t __s1_279 = __p1_279; \ | 72513 | int16_t __s1_311 = __p1_311; \ |
| 71942 | int16x4_t __s2_279 = __p2_279; \ | 72514 | int16x4_t __s2_311 = __p2_311; \ |
| 71943 | int16x4_t __rev2_279; __rev2_279 = __builtin_shufflevector(__s2_279, __s2_279, 3, 2, 1, 0); \ | 72515 | int16x4_t __rev2_311; __rev2_311 = __builtin_shufflevector(__s2_311, __s2_311, 3, 2, 1, 0); \ |
| 71944 | int16_t __ret_279; \ | 72516 | int16_t __ret_311; \ |
| 71945 | __ret_279 = __noswap_vqsubh_s16(__s0_279, __noswap_vqrdmulhh_s16(__s1_279, __noswap_vget_lane_s16(__rev2_279, __p3_279))); \ | 72517 | __ret_311 = __noswap_vqsubh_s16(__s0_311, __noswap_vqrdmulhh_s16(__s1_311, __noswap_vget_lane_s16(__rev2_311, __p3_311))); \ |
| 71946 | __ret_279; \ | 72518 | __ret_311; \ |
| 71947 | }) | 72519 | }) |
| 71948 | #endif | 72520 | #endif |
| 71949 | 72521 | ||
| 71950 | #ifdef __LITTLE_ENDIAN__ | 72522 | #ifdef __LITTLE_ENDIAN__ |
| 71951 | #define vqrdmlshs_laneq_s32(__p0_280, __p1_280, __p2_280, __p3_280) __extension__ ({ \ | 72523 | #define vqrdmlshs_laneq_s32(__p0_312, __p1_312, __p2_312, __p3_312) __extension__ ({ \ |
| 71952 | int32_t __s0_280 = __p0_280; \ | 72524 | int32_t __s0_312 = __p0_312; \ |
| 71953 | int32_t __s1_280 = __p1_280; \ | 72525 | int32_t __s1_312 = __p1_312; \ |
| 71954 | int32x4_t __s2_280 = __p2_280; \ | 72526 | int32x4_t __s2_312 = __p2_312; \ |
| 71955 | int32_t __ret_280; \ | 72527 | int32_t __ret_312; \ |
| 71956 | __ret_280 = vqsubs_s32(__s0_280, vqrdmulhs_s32(__s1_280, vgetq_lane_s32(__s2_280, __p3_280))); \ | 72528 | __ret_312 = vqsubs_s32(__s0_312, vqrdmulhs_s32(__s1_312, vgetq_lane_s32(__s2_312, __p3_312))); \ |
| 71957 | __ret_280; \ | 72529 | __ret_312; \ |
| 71958 | }) | 72530 | }) |
| 71959 | #else | 72531 | #else |
| 71960 | #define vqrdmlshs_laneq_s32(__p0_281, __p1_281, __p2_281, __p3_281) __extension__ ({ \ | 72532 | #define vqrdmlshs_laneq_s32(__p0_313, __p1_313, __p2_313, __p3_313) __extension__ ({ \ |
| 71961 | int32_t __s0_281 = __p0_281; \ | 72533 | int32_t __s0_313 = __p0_313; \ |
| 71962 | int32_t __s1_281 = __p1_281; \ | 72534 | int32_t __s1_313 = __p1_313; \ |
| 71963 | int32x4_t __s2_281 = __p2_281; \ | 72535 | int32x4_t __s2_313 = __p2_313; \ |
| 71964 | int32x4_t __rev2_281; __rev2_281 = __builtin_shufflevector(__s2_281, __s2_281, 3, 2, 1, 0); \ | 72536 | int32x4_t __rev2_313; __rev2_313 = __builtin_shufflevector(__s2_313, __s2_313, 3, 2, 1, 0); \ |
| 71965 | int32_t __ret_281; \ | 72537 | int32_t __ret_313; \ |
| 71966 | __ret_281 = __noswap_vqsubs_s32(__s0_281, __noswap_vqrdmulhs_s32(__s1_281, __noswap_vgetq_lane_s32(__rev2_281, __p3_281))); \ | 72538 | __ret_313 = __noswap_vqsubs_s32(__s0_313, __noswap_vqrdmulhs_s32(__s1_313, __noswap_vgetq_lane_s32(__rev2_313, __p3_313))); \ |
| 71967 | __ret_281; \ | 72539 | __ret_313; \ |
| 71968 | }) | 72540 | }) |
| 71969 | #endif | 72541 | #endif |
| 71970 | 72542 | ||
| 71971 | #ifdef __LITTLE_ENDIAN__ | 72543 | #ifdef __LITTLE_ENDIAN__ |
| 71972 | #define vqrdmlshh_laneq_s16(__p0_282, __p1_282, __p2_282, __p3_282) __extension__ ({ \ | 72544 | #define vqrdmlshh_laneq_s16(__p0_314, __p1_314, __p2_314, __p3_314) __extension__ ({ \ |
| 71973 | int16_t __s0_282 = __p0_282; \ | 72545 | int16_t __s0_314 = __p0_314; \ |
| 71974 | int16_t __s1_282 = __p1_282; \ | 72546 | int16_t __s1_314 = __p1_314; \ |
| 71975 | int16x8_t __s2_282 = __p2_282; \ | 72547 | int16x8_t __s2_314 = __p2_314; \ |
| 71976 | int16_t __ret_282; \ | 72548 | int16_t __ret_314; \ |
| 71977 | __ret_282 = vqsubh_s16(__s0_282, vqrdmulhh_s16(__s1_282, vgetq_lane_s16(__s2_282, __p3_282))); \ | 72549 | __ret_314 = vqsubh_s16(__s0_314, vqrdmulhh_s16(__s1_314, vgetq_lane_s16(__s2_314, __p3_314))); \ |
| 71978 | __ret_282; \ | 72550 | __ret_314; \ |
| 71979 | }) | 72551 | }) |
| 71980 | #else | 72552 | #else |
| 71981 | #define vqrdmlshh_laneq_s16(__p0_283, __p1_283, __p2_283, __p3_283) __extension__ ({ \ | 72553 | #define vqrdmlshh_laneq_s16(__p0_315, __p1_315, __p2_315, __p3_315) __extension__ ({ \ |
| 71982 | int16_t __s0_283 = __p0_283; \ | 72554 | int16_t __s0_315 = __p0_315; \ |
| 71983 | int16_t __s1_283 = __p1_283; \ | 72555 | int16_t __s1_315 = __p1_315; \ |
| 71984 | int16x8_t __s2_283 = __p2_283; \ | 72556 | int16x8_t __s2_315 = __p2_315; \ |
| 71985 | int16x8_t __rev2_283; __rev2_283 = __builtin_shufflevector(__s2_283, __s2_283, 7, 6, 5, 4, 3, 2, 1, 0); \ | 72557 | int16x8_t __rev2_315; __rev2_315 = __builtin_shufflevector(__s2_315, __s2_315, 7, 6, 5, 4, 3, 2, 1, 0); \ |
| 71986 | int16_t __ret_283; \ | 72558 | int16_t __ret_315; \ |
| 71987 | __ret_283 = __noswap_vqsubh_s16(__s0_283, __noswap_vqrdmulhh_s16(__s1_283, __noswap_vgetq_lane_s16(__rev2_283, __p3_283))); \ | 72559 | __ret_315 = __noswap_vqsubh_s16(__s0_315, __noswap_vqrdmulhh_s16(__s1_315, __noswap_vgetq_lane_s16(__rev2_315, __p3_315))); \ |
| 71988 | __ret_283; \ | 72560 | __ret_315; \ |
| 71989 | }) | 72561 | }) |
| 71990 | #endif | 72562 | #endif |
| 71991 | 72563 | ||
| ... | @@ -72298,158 +72870,158 @@ __ai int32x4_t vaddw_high_s16(int32x4_t __p0, int16x8_t __p1) { | ... | @@ -72298,158 +72870,158 @@ __ai int32x4_t vaddw_high_s16(int32x4_t __p0, int16x8_t __p1) { |
| 72298 | #endif | 72870 | #endif |
| 72299 | 72871 | ||
| 72300 | #ifdef __LITTLE_ENDIAN__ | 72872 | #ifdef __LITTLE_ENDIAN__ |
| 72301 | #define vcopyq_lane_p64(__p0_284, __p1_284, __p2_284, __p3_284) __extension__ ({ \ | 72873 | #define vcopyq_lane_p64(__p0_316, __p1_316, __p2_316, __p3_316) __extension__ ({ \ |
| 72302 | poly64x2_t __s0_284 = __p0_284; \ | 72874 | poly64x2_t __s0_316 = __p0_316; \ |
| 72303 | poly64x1_t __s2_284 = __p2_284; \ | 72875 | poly64x1_t __s2_316 = __p2_316; \ |
| 72304 | poly64x2_t __ret_284; \ | 72876 | poly64x2_t __ret_316; \ |
| 72305 | __ret_284 = vsetq_lane_p64(vget_lane_p64(__s2_284, __p3_284), __s0_284, __p1_284); \ | 72877 | __ret_316 = vsetq_lane_p64(vget_lane_p64(__s2_316, __p3_316), __s0_316, __p1_316); \ |
| 72306 | __ret_284; \ | 72878 | __ret_316; \ |
| 72307 | }) | 72879 | }) |
| 72308 | #else | 72880 | #else |
| 72309 | #define vcopyq_lane_p64(__p0_285, __p1_285, __p2_285, __p3_285) __extension__ ({ \ | 72881 | #define vcopyq_lane_p64(__p0_317, __p1_317, __p2_317, __p3_317) __extension__ ({ \ |
| 72310 | poly64x2_t __s0_285 = __p0_285; \ | 72882 | poly64x2_t __s0_317 = __p0_317; \ |
| 72311 | poly64x1_t __s2_285 = __p2_285; \ | 72883 | poly64x1_t __s2_317 = __p2_317; \ |
| 72312 | poly64x2_t __rev0_285; __rev0_285 = __builtin_shufflevector(__s0_285, __s0_285, 1, 0); \ | 72884 | poly64x2_t __rev0_317; __rev0_317 = __builtin_shufflevector(__s0_317, __s0_317, 1, 0); \ |
| 72313 | poly64x2_t __ret_285; \ | 72885 | poly64x2_t __ret_317; \ |
| 72314 | __ret_285 = __noswap_vsetq_lane_p64(__noswap_vget_lane_p64(__s2_285, __p3_285), __rev0_285, __p1_285); \ | 72886 | __ret_317 = __noswap_vsetq_lane_p64(__noswap_vget_lane_p64(__s2_317, __p3_317), __rev0_317, __p1_317); \ |
| 72315 | __ret_285 = __builtin_shufflevector(__ret_285, __ret_285, 1, 0); \ | 72887 | __ret_317 = __builtin_shufflevector(__ret_317, __ret_317, 1, 0); \ |
| 72316 | __ret_285; \ | 72888 | __ret_317; \ |
| 72317 | }) | 72889 | }) |
| 72318 | #endif | 72890 | #endif |
| 72319 | 72891 | ||
| 72320 | #ifdef __LITTLE_ENDIAN__ | 72892 | #ifdef __LITTLE_ENDIAN__ |
| 72321 | #define vcopyq_lane_f64(__p0_286, __p1_286, __p2_286, __p3_286) __extension__ ({ \ | 72893 | #define vcopyq_lane_f64(__p0_318, __p1_318, __p2_318, __p3_318) __extension__ ({ \ |
| 72322 | float64x2_t __s0_286 = __p0_286; \ | 72894 | float64x2_t __s0_318 = __p0_318; \ |
| 72323 | float64x1_t __s2_286 = __p2_286; \ | 72895 | float64x1_t __s2_318 = __p2_318; \ |
| 72324 | float64x2_t __ret_286; \ | 72896 | float64x2_t __ret_318; \ |
| 72325 | __ret_286 = vsetq_lane_f64(vget_lane_f64(__s2_286, __p3_286), __s0_286, __p1_286); \ | 72897 | __ret_318 = vsetq_lane_f64(vget_lane_f64(__s2_318, __p3_318), __s0_318, __p1_318); \ |
| 72326 | __ret_286; \ | 72898 | __ret_318; \ |
| 72327 | }) | 72899 | }) |
| 72328 | #else | 72900 | #else |
| 72329 | #define vcopyq_lane_f64(__p0_287, __p1_287, __p2_287, __p3_287) __extension__ ({ \ | 72901 | #define vcopyq_lane_f64(__p0_319, __p1_319, __p2_319, __p3_319) __extension__ ({ \ |
| 72330 | float64x2_t __s0_287 = __p0_287; \ | 72902 | float64x2_t __s0_319 = __p0_319; \ |
| 72331 | float64x1_t __s2_287 = __p2_287; \ | 72903 | float64x1_t __s2_319 = __p2_319; \ |
| 72332 | float64x2_t __rev0_287; __rev0_287 = __builtin_shufflevector(__s0_287, __s0_287, 1, 0); \ | 72904 | float64x2_t __rev0_319; __rev0_319 = __builtin_shufflevector(__s0_319, __s0_319, 1, 0); \ |
| 72333 | float64x2_t __ret_287; \ | 72905 | float64x2_t __ret_319; \ |
| 72334 | __ret_287 = __noswap_vsetq_lane_f64(__noswap_vget_lane_f64(__s2_287, __p3_287), __rev0_287, __p1_287); \ | 72906 | __ret_319 = __noswap_vsetq_lane_f64(__noswap_vget_lane_f64(__s2_319, __p3_319), __rev0_319, __p1_319); \ |
| 72335 | __ret_287 = __builtin_shufflevector(__ret_287, __ret_287, 1, 0); \ | 72907 | __ret_319 = __builtin_shufflevector(__ret_319, __ret_319, 1, 0); \ |
| 72336 | __ret_287; \ | 72908 | __ret_319; \ |
| 72337 | }) | 72909 | }) |
| 72338 | #endif | 72910 | #endif |
| 72339 | 72911 | ||
| 72340 | #ifdef __LITTLE_ENDIAN__ | 72912 | #ifdef __LITTLE_ENDIAN__ |
| 72341 | #define vcopy_lane_p64(__p0_288, __p1_288, __p2_288, __p3_288) __extension__ ({ \ | 72913 | #define vcopy_lane_p64(__p0_320, __p1_320, __p2_320, __p3_320) __extension__ ({ \ |
| 72342 | poly64x1_t __s0_288 = __p0_288; \ | 72914 | poly64x1_t __s0_320 = __p0_320; \ |
| 72343 | poly64x1_t __s2_288 = __p2_288; \ | 72915 | poly64x1_t __s2_320 = __p2_320; \ |
| 72344 | poly64x1_t __ret_288; \ | 72916 | poly64x1_t __ret_320; \ |
| 72345 | __ret_288 = vset_lane_p64(vget_lane_p64(__s2_288, __p3_288), __s0_288, __p1_288); \ | 72917 | __ret_320 = vset_lane_p64(vget_lane_p64(__s2_320, __p3_320), __s0_320, __p1_320); \ |
| 72346 | __ret_288; \ | 72918 | __ret_320; \ |
| 72347 | }) | 72919 | }) |
| 72348 | #else | 72920 | #else |
| 72349 | #define vcopy_lane_p64(__p0_289, __p1_289, __p2_289, __p3_289) __extension__ ({ \ | 72921 | #define vcopy_lane_p64(__p0_321, __p1_321, __p2_321, __p3_321) __extension__ ({ \ |
| 72350 | poly64x1_t __s0_289 = __p0_289; \ | 72922 | poly64x1_t __s0_321 = __p0_321; \ |
| 72351 | poly64x1_t __s2_289 = __p2_289; \ | 72923 | poly64x1_t __s2_321 = __p2_321; \ |
| 72352 | poly64x1_t __ret_289; \ | 72924 | poly64x1_t __ret_321; \ |
| 72353 | __ret_289 = __noswap_vset_lane_p64(__noswap_vget_lane_p64(__s2_289, __p3_289), __s0_289, __p1_289); \ | 72925 | __ret_321 = __noswap_vset_lane_p64(__noswap_vget_lane_p64(__s2_321, __p3_321), __s0_321, __p1_321); \ |
| 72354 | __ret_289; \ | 72926 | __ret_321; \ |
| 72355 | }) | 72927 | }) |
| 72356 | #endif | 72928 | #endif |
| 72357 | 72929 | ||
| 72358 | #ifdef __LITTLE_ENDIAN__ | 72930 | #ifdef __LITTLE_ENDIAN__ |
| 72359 | #define vcopy_lane_f64(__p0_290, __p1_290, __p2_290, __p3_290) __extension__ ({ \ | 72931 | #define vcopy_lane_f64(__p0_322, __p1_322, __p2_322, __p3_322) __extension__ ({ \ |
| 72360 | float64x1_t __s0_290 = __p0_290; \ | 72932 | float64x1_t __s0_322 = __p0_322; \ |
| 72361 | float64x1_t __s2_290 = __p2_290; \ | 72933 | float64x1_t __s2_322 = __p2_322; \ |
| 72362 | float64x1_t __ret_290; \ | 72934 | float64x1_t __ret_322; \ |
| 72363 | __ret_290 = vset_lane_f64(vget_lane_f64(__s2_290, __p3_290), __s0_290, __p1_290); \ | 72935 | __ret_322 = vset_lane_f64(vget_lane_f64(__s2_322, __p3_322), __s0_322, __p1_322); \ |
| 72364 | __ret_290; \ | 72936 | __ret_322; \ |
| 72365 | }) | 72937 | }) |
| 72366 | #else | 72938 | #else |
| 72367 | #define vcopy_lane_f64(__p0_291, __p1_291, __p2_291, __p3_291) __extension__ ({ \ | 72939 | #define vcopy_lane_f64(__p0_323, __p1_323, __p2_323, __p3_323) __extension__ ({ \ |
| 72368 | float64x1_t __s0_291 = __p0_291; \ | 72940 | float64x1_t __s0_323 = __p0_323; \ |
| 72369 | float64x1_t __s2_291 = __p2_291; \ | 72941 | float64x1_t __s2_323 = __p2_323; \ |
| 72370 | float64x1_t __ret_291; \ | 72942 | float64x1_t __ret_323; \ |
| 72371 | __ret_291 = __noswap_vset_lane_f64(__noswap_vget_lane_f64(__s2_291, __p3_291), __s0_291, __p1_291); \ | 72943 | __ret_323 = __noswap_vset_lane_f64(__noswap_vget_lane_f64(__s2_323, __p3_323), __s0_323, __p1_323); \ |
| 72372 | __ret_291; \ | 72944 | __ret_323; \ |
| 72373 | }) | 72945 | }) |
| 72374 | #endif | 72946 | #endif |
| 72375 | 72947 | ||
| 72376 | #ifdef __LITTLE_ENDIAN__ | 72948 | #ifdef __LITTLE_ENDIAN__ |
| 72377 | #define vcopyq_laneq_p64(__p0_292, __p1_292, __p2_292, __p3_292) __extension__ ({ \ | 72949 | #define vcopyq_laneq_p64(__p0_324, __p1_324, __p2_324, __p3_324) __extension__ ({ \ |
| 72378 | poly64x2_t __s0_292 = __p0_292; \ | 72950 | poly64x2_t __s0_324 = __p0_324; \ |
| 72379 | poly64x2_t __s2_292 = __p2_292; \ | 72951 | poly64x2_t __s2_324 = __p2_324; \ |
| 72380 | poly64x2_t __ret_292; \ | 72952 | poly64x2_t __ret_324; \ |
| 72381 | __ret_292 = vsetq_lane_p64(vgetq_lane_p64(__s2_292, __p3_292), __s0_292, __p1_292); \ | 72953 | __ret_324 = vsetq_lane_p64(vgetq_lane_p64(__s2_324, __p3_324), __s0_324, __p1_324); \ |
| 72382 | __ret_292; \ | 72954 | __ret_324; \ |
| 72383 | }) | 72955 | }) |
| 72384 | #else | 72956 | #else |
| 72385 | #define vcopyq_laneq_p64(__p0_293, __p1_293, __p2_293, __p3_293) __extension__ ({ \ | 72957 | #define vcopyq_laneq_p64(__p0_325, __p1_325, __p2_325, __p3_325) __extension__ ({ \ |
| 72386 | poly64x2_t __s0_293 = __p0_293; \ | 72958 | poly64x2_t __s0_325 = __p0_325; \ |
| 72387 | poly64x2_t __s2_293 = __p2_293; \ | 72959 | poly64x2_t __s2_325 = __p2_325; \ |
| 72388 | poly64x2_t __rev0_293; __rev0_293 = __builtin_shufflevector(__s0_293, __s0_293, 1, 0); \ | 72960 | poly64x2_t __rev0_325; __rev0_325 = __builtin_shufflevector(__s0_325, __s0_325, 1, 0); \ |
| 72389 | poly64x2_t __rev2_293; __rev2_293 = __builtin_shufflevector(__s2_293, __s2_293, 1, 0); \ | 72961 | poly64x2_t __rev2_325; __rev2_325 = __builtin_shufflevector(__s2_325, __s2_325, 1, 0); \ |
| 72390 | poly64x2_t __ret_293; \ | 72962 | poly64x2_t __ret_325; \ |
| 72391 | __ret_293 = __noswap_vsetq_lane_p64(__noswap_vgetq_lane_p64(__rev2_293, __p3_293), __rev0_293, __p1_293); \ | 72963 | __ret_325 = __noswap_vsetq_lane_p64(__noswap_vgetq_lane_p64(__rev2_325, __p3_325), __rev0_325, __p1_325); \ |
| 72392 | __ret_293 = __builtin_shufflevector(__ret_293, __ret_293, 1, 0); \ | 72964 | __ret_325 = __builtin_shufflevector(__ret_325, __ret_325, 1, 0); \ |
| 72393 | __ret_293; \ | 72965 | __ret_325; \ |
| 72394 | }) | 72966 | }) |
| 72395 | #endif | 72967 | #endif |
| 72396 | 72968 | ||
| 72397 | #ifdef __LITTLE_ENDIAN__ | 72969 | #ifdef __LITTLE_ENDIAN__ |
| 72398 | #define vcopyq_laneq_f64(__p0_294, __p1_294, __p2_294, __p3_294) __extension__ ({ \ | 72970 | #define vcopyq_laneq_f64(__p0_326, __p1_326, __p2_326, __p3_326) __extension__ ({ \ |
| 72399 | float64x2_t __s0_294 = __p0_294; \ | 72971 | float64x2_t __s0_326 = __p0_326; \ |
| 72400 | float64x2_t __s2_294 = __p2_294; \ | 72972 | float64x2_t __s2_326 = __p2_326; \ |
| 72401 | float64x2_t __ret_294; \ | 72973 | float64x2_t __ret_326; \ |
| 72402 | __ret_294 = vsetq_lane_f64(vgetq_lane_f64(__s2_294, __p3_294), __s0_294, __p1_294); \ | 72974 | __ret_326 = vsetq_lane_f64(vgetq_lane_f64(__s2_326, __p3_326), __s0_326, __p1_326); \ |
| 72403 | __ret_294; \ | 72975 | __ret_326; \ |
| 72404 | }) | 72976 | }) |
| 72405 | #else | 72977 | #else |
| 72406 | #define vcopyq_laneq_f64(__p0_295, __p1_295, __p2_295, __p3_295) __extension__ ({ \ | 72978 | #define vcopyq_laneq_f64(__p0_327, __p1_327, __p2_327, __p3_327) __extension__ ({ \ |
| 72407 | float64x2_t __s0_295 = __p0_295; \ | 72979 | float64x2_t __s0_327 = __p0_327; \ |
| 72408 | float64x2_t __s2_295 = __p2_295; \ | 72980 | float64x2_t __s2_327 = __p2_327; \ |
| 72409 | float64x2_t __rev0_295; __rev0_295 = __builtin_shufflevector(__s0_295, __s0_295, 1, 0); \ | 72981 | float64x2_t __rev0_327; __rev0_327 = __builtin_shufflevector(__s0_327, __s0_327, 1, 0); \ |
| 72410 | float64x2_t __rev2_295; __rev2_295 = __builtin_shufflevector(__s2_295, __s2_295, 1, 0); \ | 72982 | float64x2_t __rev2_327; __rev2_327 = __builtin_shufflevector(__s2_327, __s2_327, 1, 0); \ |
| 72411 | float64x2_t __ret_295; \ | 72983 | float64x2_t __ret_327; \ |
| 72412 | __ret_295 = __noswap_vsetq_lane_f64(__noswap_vgetq_lane_f64(__rev2_295, __p3_295), __rev0_295, __p1_295); \ | 72984 | __ret_327 = __noswap_vsetq_lane_f64(__noswap_vgetq_lane_f64(__rev2_327, __p3_327), __rev0_327, __p1_327); \ |
| 72413 | __ret_295 = __builtin_shufflevector(__ret_295, __ret_295, 1, 0); \ | 72985 | __ret_327 = __builtin_shufflevector(__ret_327, __ret_327, 1, 0); \ |
| 72414 | __ret_295; \ | 72986 | __ret_327; \ |
| 72415 | }) | 72987 | }) |
| 72416 | #endif | 72988 | #endif |
| 72417 | 72989 | ||
| 72418 | #ifdef __LITTLE_ENDIAN__ | 72990 | #ifdef __LITTLE_ENDIAN__ |
| 72419 | #define vcopy_laneq_p64(__p0_296, __p1_296, __p2_296, __p3_296) __extension__ ({ \ | 72991 | #define vcopy_laneq_p64(__p0_328, __p1_328, __p2_328, __p3_328) __extension__ ({ \ |
| 72420 | poly64x1_t __s0_296 = __p0_296; \ | 72992 | poly64x1_t __s0_328 = __p0_328; \ |
| 72421 | poly64x2_t __s2_296 = __p2_296; \ | 72993 | poly64x2_t __s2_328 = __p2_328; \ |
| 72422 | poly64x1_t __ret_296; \ | 72994 | poly64x1_t __ret_328; \ |
| 72423 | __ret_296 = vset_lane_p64(vgetq_lane_p64(__s2_296, __p3_296), __s0_296, __p1_296); \ | 72995 | __ret_328 = vset_lane_p64(vgetq_lane_p64(__s2_328, __p3_328), __s0_328, __p1_328); \ |
| 72424 | __ret_296; \ | 72996 | __ret_328; \ |
| 72425 | }) | 72997 | }) |
| 72426 | #else | 72998 | #else |
| 72427 | #define vcopy_laneq_p64(__p0_297, __p1_297, __p2_297, __p3_297) __extension__ ({ \ | 72999 | #define vcopy_laneq_p64(__p0_329, __p1_329, __p2_329, __p3_329) __extension__ ({ \ |
| 72428 | poly64x1_t __s0_297 = __p0_297; \ | 73000 | poly64x1_t __s0_329 = __p0_329; \ |
| 72429 | poly64x2_t __s2_297 = __p2_297; \ | 73001 | poly64x2_t __s2_329 = __p2_329; \ |
| 72430 | poly64x2_t __rev2_297; __rev2_297 = __builtin_shufflevector(__s2_297, __s2_297, 1, 0); \ | 73002 | poly64x2_t __rev2_329; __rev2_329 = __builtin_shufflevector(__s2_329, __s2_329, 1, 0); \ |
| 72431 | poly64x1_t __ret_297; \ | 73003 | poly64x1_t __ret_329; \ |
| 72432 | __ret_297 = __noswap_vset_lane_p64(__noswap_vgetq_lane_p64(__rev2_297, __p3_297), __s0_297, __p1_297); \ | 73004 | __ret_329 = __noswap_vset_lane_p64(__noswap_vgetq_lane_p64(__rev2_329, __p3_329), __s0_329, __p1_329); \ |
| 72433 | __ret_297; \ | 73005 | __ret_329; \ |
| 72434 | }) | 73006 | }) |
| 72435 | #endif | 73007 | #endif |
| 72436 | 73008 | ||
| 72437 | #ifdef __LITTLE_ENDIAN__ | 73009 | #ifdef __LITTLE_ENDIAN__ |
| 72438 | #define vcopy_laneq_f64(__p0_298, __p1_298, __p2_298, __p3_298) __extension__ ({ \ | 73010 | #define vcopy_laneq_f64(__p0_330, __p1_330, __p2_330, __p3_330) __extension__ ({ \ |
| 72439 | float64x1_t __s0_298 = __p0_298; \ | 73011 | float64x1_t __s0_330 = __p0_330; \ |
| 72440 | float64x2_t __s2_298 = __p2_298; \ | 73012 | float64x2_t __s2_330 = __p2_330; \ |
| 72441 | float64x1_t __ret_298; \ | 73013 | float64x1_t __ret_330; \ |
| 72442 | __ret_298 = vset_lane_f64(vgetq_lane_f64(__s2_298, __p3_298), __s0_298, __p1_298); \ | 73014 | __ret_330 = vset_lane_f64(vgetq_lane_f64(__s2_330, __p3_330), __s0_330, __p1_330); \ |
| 72443 | __ret_298; \ | 73015 | __ret_330; \ |
| 72444 | }) | 73016 | }) |
| 72445 | #else | 73017 | #else |
| 72446 | #define vcopy_laneq_f64(__p0_299, __p1_299, __p2_299, __p3_299) __extension__ ({ \ | 73018 | #define vcopy_laneq_f64(__p0_331, __p1_331, __p2_331, __p3_331) __extension__ ({ \ |
| 72447 | float64x1_t __s0_299 = __p0_299; \ | 73019 | float64x1_t __s0_331 = __p0_331; \ |
| 72448 | float64x2_t __s2_299 = __p2_299; \ | 73020 | float64x2_t __s2_331 = __p2_331; \ |
| 72449 | float64x2_t __rev2_299; __rev2_299 = __builtin_shufflevector(__s2_299, __s2_299, 1, 0); \ | 73021 | float64x2_t __rev2_331; __rev2_331 = __builtin_shufflevector(__s2_331, __s2_331, 1, 0); \ |
| 72450 | float64x1_t __ret_299; \ | 73022 | float64x1_t __ret_331; \ |
| 72451 | __ret_299 = __noswap_vset_lane_f64(__noswap_vgetq_lane_f64(__rev2_299, __p3_299), __s0_299, __p1_299); \ | 73023 | __ret_331 = __noswap_vset_lane_f64(__noswap_vgetq_lane_f64(__rev2_331, __p3_331), __s0_331, __p1_331); \ |
| 72452 | __ret_299; \ | 73024 | __ret_331; \ |
| 72453 | }) | 73025 | }) |
| 72454 | #endif | 73026 | #endif |
| 72455 | 73027 | ||
| ... | @@ -72806,51 +73378,51 @@ __ai int32x4_t vmlsl_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { | ... | @@ -72806,51 +73378,51 @@ __ai int32x4_t vmlsl_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { |
| 72806 | #endif | 73378 | #endif |
| 72807 | 73379 | ||
| 72808 | #ifdef __LITTLE_ENDIAN__ | 73380 | #ifdef __LITTLE_ENDIAN__ |
| 72809 | #define vmulx_lane_f64(__p0_300, __p1_300, __p2_300) __extension__ ({ \ | 73381 | #define vmulx_lane_f64(__p0_332, __p1_332, __p2_332) __extension__ ({ \ |
| 72810 | float64x1_t __s0_300 = __p0_300; \ | 73382 | float64x1_t __s0_332 = __p0_332; \ |
| 72811 | float64x1_t __s1_300 = __p1_300; \ | 73383 | float64x1_t __s1_332 = __p1_332; \ |
| 72812 | float64x1_t __ret_300; \ | 73384 | float64x1_t __ret_332; \ |
| 72813 | float64_t __x_300 = vget_lane_f64(__s0_300, 0); \ | 73385 | float64_t __x_332 = vget_lane_f64(__s0_332, 0); \ |
| 72814 | float64_t __y_300 = vget_lane_f64(__s1_300, __p2_300); \ | 73386 | float64_t __y_332 = vget_lane_f64(__s1_332, __p2_332); \ |
| 72815 | float64_t __z_300 = vmulxd_f64(__x_300, __y_300); \ | 73387 | float64_t __z_332 = vmulxd_f64(__x_332, __y_332); \ |
| 72816 | __ret_300 = vset_lane_f64(__z_300, __s0_300, __p2_300); \ | 73388 | __ret_332 = vset_lane_f64(__z_332, __s0_332, __p2_332); \ |
| 72817 | __ret_300; \ | 73389 | __ret_332; \ |
| 72818 | }) | 73390 | }) |
| 72819 | #else | 73391 | #else |
| 72820 | #define vmulx_lane_f64(__p0_301, __p1_301, __p2_301) __extension__ ({ \ | 73392 | #define vmulx_lane_f64(__p0_333, __p1_333, __p2_333) __extension__ ({ \ |
| 72821 | float64x1_t __s0_301 = __p0_301; \ | 73393 | float64x1_t __s0_333 = __p0_333; \ |
| 72822 | float64x1_t __s1_301 = __p1_301; \ | 73394 | float64x1_t __s1_333 = __p1_333; \ |
| 72823 | float64x1_t __ret_301; \ | 73395 | float64x1_t __ret_333; \ |
| 72824 | float64_t __x_301 = __noswap_vget_lane_f64(__s0_301, 0); \ | 73396 | float64_t __x_333 = __noswap_vget_lane_f64(__s0_333, 0); \ |
| 72825 | float64_t __y_301 = __noswap_vget_lane_f64(__s1_301, __p2_301); \ | 73397 | float64_t __y_333 = __noswap_vget_lane_f64(__s1_333, __p2_333); \ |
| 72826 | float64_t __z_301 = __noswap_vmulxd_f64(__x_301, __y_301); \ | 73398 | float64_t __z_333 = __noswap_vmulxd_f64(__x_333, __y_333); \ |
| 72827 | __ret_301 = __noswap_vset_lane_f64(__z_301, __s0_301, __p2_301); \ | 73399 | __ret_333 = __noswap_vset_lane_f64(__z_333, __s0_333, __p2_333); \ |
| 72828 | __ret_301; \ | 73400 | __ret_333; \ |
| 72829 | }) | 73401 | }) |
| 72830 | #endif | 73402 | #endif |
| 72831 | 73403 | ||
| 72832 | #ifdef __LITTLE_ENDIAN__ | 73404 | #ifdef __LITTLE_ENDIAN__ |
| 72833 | #define vmulx_laneq_f64(__p0_302, __p1_302, __p2_302) __extension__ ({ \ | 73405 | #define vmulx_laneq_f64(__p0_334, __p1_334, __p2_334) __extension__ ({ \ |
| 72834 | float64x1_t __s0_302 = __p0_302; \ | 73406 | float64x1_t __s0_334 = __p0_334; \ |
| 72835 | float64x2_t __s1_302 = __p1_302; \ | 73407 | float64x2_t __s1_334 = __p1_334; \ |
| 72836 | float64x1_t __ret_302; \ | 73408 | float64x1_t __ret_334; \ |
| 72837 | float64_t __x_302 = vget_lane_f64(__s0_302, 0); \ | 73409 | float64_t __x_334 = vget_lane_f64(__s0_334, 0); \ |
| 72838 | float64_t __y_302 = vgetq_lane_f64(__s1_302, __p2_302); \ | 73410 | float64_t __y_334 = vgetq_lane_f64(__s1_334, __p2_334); \ |
| 72839 | float64_t __z_302 = vmulxd_f64(__x_302, __y_302); \ | 73411 | float64_t __z_334 = vmulxd_f64(__x_334, __y_334); \ |
| 72840 | __ret_302 = vset_lane_f64(__z_302, __s0_302, 0); \ | 73412 | __ret_334 = vset_lane_f64(__z_334, __s0_334, 0); \ |
| 72841 | __ret_302; \ | 73413 | __ret_334; \ |
| 72842 | }) | 73414 | }) |
| 72843 | #else | 73415 | #else |
| 72844 | #define vmulx_laneq_f64(__p0_303, __p1_303, __p2_303) __extension__ ({ \ | 73416 | #define vmulx_laneq_f64(__p0_335, __p1_335, __p2_335) __extension__ ({ \ |
| 72845 | float64x1_t __s0_303 = __p0_303; \ | 73417 | float64x1_t __s0_335 = __p0_335; \ |
| 72846 | float64x2_t __s1_303 = __p1_303; \ | 73418 | float64x2_t __s1_335 = __p1_335; \ |
| 72847 | float64x2_t __rev1_303; __rev1_303 = __builtin_shufflevector(__s1_303, __s1_303, 1, 0); \ | 73419 | float64x2_t __rev1_335; __rev1_335 = __builtin_shufflevector(__s1_335, __s1_335, 1, 0); \ |
| 72848 | float64x1_t __ret_303; \ | 73420 | float64x1_t __ret_335; \ |
| 72849 | float64_t __x_303 = __noswap_vget_lane_f64(__s0_303, 0); \ | 73421 | float64_t __x_335 = __noswap_vget_lane_f64(__s0_335, 0); \ |
| 72850 | float64_t __y_303 = __noswap_vgetq_lane_f64(__rev1_303, __p2_303); \ | 73422 | float64_t __y_335 = __noswap_vgetq_lane_f64(__rev1_335, __p2_335); \ |
| 72851 | float64_t __z_303 = __noswap_vmulxd_f64(__x_303, __y_303); \ | 73423 | float64_t __z_335 = __noswap_vmulxd_f64(__x_335, __y_335); \ |
| 72852 | __ret_303 = __noswap_vset_lane_f64(__z_303, __s0_303, 0); \ | 73424 | __ret_335 = __noswap_vset_lane_f64(__z_335, __s0_335, 0); \ |
| 72853 | __ret_303; \ | 73425 | __ret_335; \ |
| 72854 | }) | 73426 | }) |
| 72855 | #endif | 73427 | #endif |
| 72856 | 73428 |
c_headers/avx512bwintrin.h+518-321
| ... | @@ -32,7 +32,216 @@ typedef unsigned int __mmask32; | ... | @@ -32,7 +32,216 @@ typedef unsigned int __mmask32; |
| 32 | typedef unsigned long long __mmask64; | 32 | typedef unsigned long long __mmask64; |
| 33 | 33 | ||
| 34 | /* Define the default attributes for the functions in this file. */ | 34 | /* Define the default attributes for the functions in this file. */ |
| 35 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("avx512bw"), __min_vector_width__(512))) | 35 | #define __DEFAULT_FN_ATTRS512 __attribute__((__always_inline__, __nodebug__, __target__("avx512bw"), __min_vector_width__(512))) |
| 36 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("avx512bw"))) | ||
| 37 | |||
| 38 | static __inline __mmask32 __DEFAULT_FN_ATTRS | ||
| 39 | _knot_mask32(__mmask32 __M) | ||
| 40 | { | ||
| 41 | return __builtin_ia32_knotsi(__M); | ||
| 42 | } | ||
| 43 | |||
| 44 | static __inline __mmask64 __DEFAULT_FN_ATTRS | ||
| 45 | _knot_mask64(__mmask64 __M) | ||
| 46 | { | ||
| 47 | return __builtin_ia32_knotdi(__M); | ||
| 48 | } | ||
| 49 | |||
| 50 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS | ||
| 51 | _kand_mask32(__mmask32 __A, __mmask32 __B) | ||
| 52 | { | ||
| 53 | return (__mmask32)__builtin_ia32_kandsi((__mmask32)__A, (__mmask32)__B); | ||
| 54 | } | ||
| 55 | |||
| 56 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS | ||
| 57 | _kand_mask64(__mmask64 __A, __mmask64 __B) | ||
| 58 | { | ||
| 59 | return (__mmask64)__builtin_ia32_kanddi((__mmask64)__A, (__mmask64)__B); | ||
| 60 | } | ||
| 61 | |||
| 62 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS | ||
| 63 | _kandn_mask32(__mmask32 __A, __mmask32 __B) | ||
| 64 | { | ||
| 65 | return (__mmask32)__builtin_ia32_kandnsi((__mmask32)__A, (__mmask32)__B); | ||
| 66 | } | ||
| 67 | |||
| 68 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS | ||
| 69 | _kandn_mask64(__mmask64 __A, __mmask64 __B) | ||
| 70 | { | ||
| 71 | return (__mmask64)__builtin_ia32_kandndi((__mmask64)__A, (__mmask64)__B); | ||
| 72 | } | ||
| 73 | |||
| 74 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS | ||
| 75 | _kor_mask32(__mmask32 __A, __mmask32 __B) | ||
| 76 | { | ||
| 77 | return (__mmask32)__builtin_ia32_korsi((__mmask32)__A, (__mmask32)__B); | ||
| 78 | } | ||
| 79 | |||
| 80 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS | ||
| 81 | _kor_mask64(__mmask64 __A, __mmask64 __B) | ||
| 82 | { | ||
| 83 | return (__mmask64)__builtin_ia32_kordi((__mmask64)__A, (__mmask64)__B); | ||
| 84 | } | ||
| 85 | |||
| 86 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS | ||
| 87 | _kxnor_mask32(__mmask32 __A, __mmask32 __B) | ||
| 88 | { | ||
| 89 | return (__mmask32)__builtin_ia32_kxnorsi((__mmask32)__A, (__mmask32)__B); | ||
| 90 | } | ||
| 91 | |||
| 92 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS | ||
| 93 | _kxnor_mask64(__mmask64 __A, __mmask64 __B) | ||
| 94 | { | ||
| 95 | return (__mmask64)__builtin_ia32_kxnordi((__mmask64)__A, (__mmask64)__B); | ||
| 96 | } | ||
| 97 | |||
| 98 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS | ||
| 99 | _kxor_mask32(__mmask32 __A, __mmask32 __B) | ||
| 100 | { | ||
| 101 | return (__mmask32)__builtin_ia32_kxorsi((__mmask32)__A, (__mmask32)__B); | ||
| 102 | } | ||
| 103 | |||
| 104 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS | ||
| 105 | _kxor_mask64(__mmask64 __A, __mmask64 __B) | ||
| 106 | { | ||
| 107 | return (__mmask64)__builtin_ia32_kxordi((__mmask64)__A, (__mmask64)__B); | ||
| 108 | } | ||
| 109 | |||
| 110 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 111 | _kortestc_mask32_u8(__mmask32 __A, __mmask32 __B) | ||
| 112 | { | ||
| 113 | return (unsigned char)__builtin_ia32_kortestcsi(__A, __B); | ||
| 114 | } | ||
| 115 | |||
| 116 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 117 | _kortestz_mask32_u8(__mmask32 __A, __mmask32 __B) | ||
| 118 | { | ||
| 119 | return (unsigned char)__builtin_ia32_kortestzsi(__A, __B); | ||
| 120 | } | ||
| 121 | |||
| 122 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 123 | _kortest_mask32_u8(__mmask32 __A, __mmask32 __B, unsigned char *__C) { | ||
| 124 | *__C = (unsigned char)__builtin_ia32_kortestcsi(__A, __B); | ||
| 125 | return (unsigned char)__builtin_ia32_kortestzsi(__A, __B); | ||
| 126 | } | ||
| 127 | |||
| 128 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 129 | _kortestc_mask64_u8(__mmask64 __A, __mmask64 __B) | ||
| 130 | { | ||
| 131 | return (unsigned char)__builtin_ia32_kortestcdi(__A, __B); | ||
| 132 | } | ||
| 133 | |||
| 134 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 135 | _kortestz_mask64_u8(__mmask64 __A, __mmask64 __B) | ||
| 136 | { | ||
| 137 | return (unsigned char)__builtin_ia32_kortestzdi(__A, __B); | ||
| 138 | } | ||
| 139 | |||
| 140 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 141 | _kortest_mask64_u8(__mmask64 __A, __mmask64 __B, unsigned char *__C) { | ||
| 142 | *__C = (unsigned char)__builtin_ia32_kortestcdi(__A, __B); | ||
| 143 | return (unsigned char)__builtin_ia32_kortestzdi(__A, __B); | ||
| 144 | } | ||
| 145 | |||
| 146 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 147 | _ktestc_mask32_u8(__mmask32 __A, __mmask32 __B) | ||
| 148 | { | ||
| 149 | return (unsigned char)__builtin_ia32_ktestcsi(__A, __B); | ||
| 150 | } | ||
| 151 | |||
| 152 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 153 | _ktestz_mask32_u8(__mmask32 __A, __mmask32 __B) | ||
| 154 | { | ||
| 155 | return (unsigned char)__builtin_ia32_ktestzsi(__A, __B); | ||
| 156 | } | ||
| 157 | |||
| 158 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 159 | _ktest_mask32_u8(__mmask32 __A, __mmask32 __B, unsigned char *__C) { | ||
| 160 | *__C = (unsigned char)__builtin_ia32_ktestcsi(__A, __B); | ||
| 161 | return (unsigned char)__builtin_ia32_ktestzsi(__A, __B); | ||
| 162 | } | ||
| 163 | |||
| 164 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 165 | _ktestc_mask64_u8(__mmask64 __A, __mmask64 __B) | ||
| 166 | { | ||
| 167 | return (unsigned char)__builtin_ia32_ktestcdi(__A, __B); | ||
| 168 | } | ||
| 169 | |||
| 170 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 171 | _ktestz_mask64_u8(__mmask64 __A, __mmask64 __B) | ||
| 172 | { | ||
| 173 | return (unsigned char)__builtin_ia32_ktestzdi(__A, __B); | ||
| 174 | } | ||
| 175 | |||
| 176 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 177 | _ktest_mask64_u8(__mmask64 __A, __mmask64 __B, unsigned char *__C) { | ||
| 178 | *__C = (unsigned char)__builtin_ia32_ktestcdi(__A, __B); | ||
| 179 | return (unsigned char)__builtin_ia32_ktestzdi(__A, __B); | ||
| 180 | } | ||
| 181 | |||
| 182 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS | ||
| 183 | _kadd_mask32(__mmask32 __A, __mmask32 __B) | ||
| 184 | { | ||
| 185 | return (__mmask32)__builtin_ia32_kaddsi((__mmask32)__A, (__mmask32)__B); | ||
| 186 | } | ||
| 187 | |||
| 188 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS | ||
| 189 | _kadd_mask64(__mmask64 __A, __mmask64 __B) | ||
| 190 | { | ||
| 191 | return (__mmask64)__builtin_ia32_kadddi((__mmask64)__A, (__mmask64)__B); | ||
| 192 | } | ||
| 193 | |||
| 194 | #define _kshiftli_mask32(A, I) \ | ||
| 195 | (__mmask32)__builtin_ia32_kshiftlisi((__mmask32)(A), (unsigned int)(I)) | ||
| 196 | |||
| 197 | #define _kshiftri_mask32(A, I) \ | ||
| 198 | (__mmask32)__builtin_ia32_kshiftrisi((__mmask32)(A), (unsigned int)(I)) | ||
| 199 | |||
| 200 | #define _kshiftli_mask64(A, I) \ | ||
| 201 | (__mmask64)__builtin_ia32_kshiftlidi((__mmask64)(A), (unsigned int)(I)) | ||
| 202 | |||
| 203 | #define _kshiftri_mask64(A, I) \ | ||
| 204 | (__mmask64)__builtin_ia32_kshiftridi((__mmask64)(A), (unsigned int)(I)) | ||
| 205 | |||
| 206 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | ||
| 207 | _cvtmask32_u32(__mmask32 __A) { | ||
| 208 | return (unsigned int)__builtin_ia32_kmovd((__mmask32)__A); | ||
| 209 | } | ||
| 210 | |||
| 211 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | ||
| 212 | _cvtmask64_u64(__mmask64 __A) { | ||
| 213 | return (unsigned long long)__builtin_ia32_kmovq((__mmask64)__A); | ||
| 214 | } | ||
| 215 | |||
| 216 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS | ||
| 217 | _cvtu32_mask32(unsigned int __A) { | ||
| 218 | return (__mmask32)__builtin_ia32_kmovd((__mmask32)__A); | ||
| 219 | } | ||
| 220 | |||
| 221 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS | ||
| 222 | _cvtu64_mask64(unsigned long long __A) { | ||
| 223 | return (__mmask64)__builtin_ia32_kmovq((__mmask64)__A); | ||
| 224 | } | ||
| 225 | |||
| 226 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS | ||
| 227 | _load_mask32(__mmask32 *__A) { | ||
| 228 | return (__mmask32)__builtin_ia32_kmovd(*(__mmask32 *)__A); | ||
| 229 | } | ||
| 230 | |||
| 231 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS | ||
| 232 | _load_mask64(__mmask64 *__A) { | ||
| 233 | return (__mmask64)__builtin_ia32_kmovq(*(__mmask64 *)__A); | ||
| 234 | } | ||
| 235 | |||
| 236 | static __inline__ void __DEFAULT_FN_ATTRS | ||
| 237 | _store_mask32(__mmask32 *__A, __mmask32 __B) { | ||
| 238 | *(__mmask32 *)__A = __builtin_ia32_kmovd((__mmask32)__B); | ||
| 239 | } | ||
| 240 | |||
| 241 | static __inline__ void __DEFAULT_FN_ATTRS | ||
| 242 | _store_mask64(__mmask64 *__A, __mmask64 __B) { | ||
| 243 | *(__mmask64 *)__A = __builtin_ia32_kmovq((__mmask64)__B); | ||
| 244 | } | ||
| 36 | 245 | ||
| 37 | /* Integer compare */ | 246 | /* Integer compare */ |
| 38 | 247 | ||
| ... | @@ -176,102 +385,102 @@ typedef unsigned long long __mmask64; | ... | @@ -176,102 +385,102 @@ typedef unsigned long long __mmask64; |
| 176 | #define _mm512_mask_cmpneq_epu16_mask(k, A, B) \ | 385 | #define _mm512_mask_cmpneq_epu16_mask(k, A, B) \ |
| 177 | _mm512_mask_cmp_epu16_mask((k), (A), (B), _MM_CMPINT_NE) | 386 | _mm512_mask_cmp_epu16_mask((k), (A), (B), _MM_CMPINT_NE) |
| 178 | 387 | ||
| 179 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 388 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 180 | _mm512_add_epi8 (__m512i __A, __m512i __B) { | 389 | _mm512_add_epi8 (__m512i __A, __m512i __B) { |
| 181 | return (__m512i) ((__v64qu) __A + (__v64qu) __B); | 390 | return (__m512i) ((__v64qu) __A + (__v64qu) __B); |
| 182 | } | 391 | } |
| 183 | 392 | ||
| 184 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 393 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 185 | _mm512_mask_add_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) { | 394 | _mm512_mask_add_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) { |
| 186 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, | 395 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 187 | (__v64qi)_mm512_add_epi8(__A, __B), | 396 | (__v64qi)_mm512_add_epi8(__A, __B), |
| 188 | (__v64qi)__W); | 397 | (__v64qi)__W); |
| 189 | } | 398 | } |
| 190 | 399 | ||
| 191 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 400 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 192 | _mm512_maskz_add_epi8(__mmask64 __U, __m512i __A, __m512i __B) { | 401 | _mm512_maskz_add_epi8(__mmask64 __U, __m512i __A, __m512i __B) { |
| 193 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, | 402 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 194 | (__v64qi)_mm512_add_epi8(__A, __B), | 403 | (__v64qi)_mm512_add_epi8(__A, __B), |
| 195 | (__v64qi)_mm512_setzero_si512()); | 404 | (__v64qi)_mm512_setzero_si512()); |
| 196 | } | 405 | } |
| 197 | 406 | ||
| 198 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 407 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 199 | _mm512_sub_epi8 (__m512i __A, __m512i __B) { | 408 | _mm512_sub_epi8 (__m512i __A, __m512i __B) { |
| 200 | return (__m512i) ((__v64qu) __A - (__v64qu) __B); | 409 | return (__m512i) ((__v64qu) __A - (__v64qu) __B); |
| 201 | } | 410 | } |
| 202 | 411 | ||
| 203 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 412 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 204 | _mm512_mask_sub_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) { | 413 | _mm512_mask_sub_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) { |
| 205 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, | 414 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 206 | (__v64qi)_mm512_sub_epi8(__A, __B), | 415 | (__v64qi)_mm512_sub_epi8(__A, __B), |
| 207 | (__v64qi)__W); | 416 | (__v64qi)__W); |
| 208 | } | 417 | } |
| 209 | 418 | ||
| 210 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 419 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 211 | _mm512_maskz_sub_epi8(__mmask64 __U, __m512i __A, __m512i __B) { | 420 | _mm512_maskz_sub_epi8(__mmask64 __U, __m512i __A, __m512i __B) { |
| 212 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, | 421 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 213 | (__v64qi)_mm512_sub_epi8(__A, __B), | 422 | (__v64qi)_mm512_sub_epi8(__A, __B), |
| 214 | (__v64qi)_mm512_setzero_si512()); | 423 | (__v64qi)_mm512_setzero_si512()); |
| 215 | } | 424 | } |
| 216 | 425 | ||
| 217 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 426 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 218 | _mm512_add_epi16 (__m512i __A, __m512i __B) { | 427 | _mm512_add_epi16 (__m512i __A, __m512i __B) { |
| 219 | return (__m512i) ((__v32hu) __A + (__v32hu) __B); | 428 | return (__m512i) ((__v32hu) __A + (__v32hu) __B); |
| 220 | } | 429 | } |
| 221 | 430 | ||
| 222 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 431 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 223 | _mm512_mask_add_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) { | 432 | _mm512_mask_add_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) { |
| 224 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 433 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 225 | (__v32hi)_mm512_add_epi16(__A, __B), | 434 | (__v32hi)_mm512_add_epi16(__A, __B), |
| 226 | (__v32hi)__W); | 435 | (__v32hi)__W); |
| 227 | } | 436 | } |
| 228 | 437 | ||
| 229 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 438 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 230 | _mm512_maskz_add_epi16(__mmask32 __U, __m512i __A, __m512i __B) { | 439 | _mm512_maskz_add_epi16(__mmask32 __U, __m512i __A, __m512i __B) { |
| 231 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 440 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 232 | (__v32hi)_mm512_add_epi16(__A, __B), | 441 | (__v32hi)_mm512_add_epi16(__A, __B), |
| 233 | (__v32hi)_mm512_setzero_si512()); | 442 | (__v32hi)_mm512_setzero_si512()); |
| 234 | } | 443 | } |
| 235 | 444 | ||
| 236 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 445 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 237 | _mm512_sub_epi16 (__m512i __A, __m512i __B) { | 446 | _mm512_sub_epi16 (__m512i __A, __m512i __B) { |
| 238 | return (__m512i) ((__v32hu) __A - (__v32hu) __B); | 447 | return (__m512i) ((__v32hu) __A - (__v32hu) __B); |
| 239 | } | 448 | } |
| 240 | 449 | ||
| 241 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 450 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 242 | _mm512_mask_sub_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) { | 451 | _mm512_mask_sub_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) { |
| 243 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 452 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 244 | (__v32hi)_mm512_sub_epi16(__A, __B), | 453 | (__v32hi)_mm512_sub_epi16(__A, __B), |
| 245 | (__v32hi)__W); | 454 | (__v32hi)__W); |
| 246 | } | 455 | } |
| 247 | 456 | ||
| 248 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 457 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 249 | _mm512_maskz_sub_epi16(__mmask32 __U, __m512i __A, __m512i __B) { | 458 | _mm512_maskz_sub_epi16(__mmask32 __U, __m512i __A, __m512i __B) { |
| 250 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 459 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 251 | (__v32hi)_mm512_sub_epi16(__A, __B), | 460 | (__v32hi)_mm512_sub_epi16(__A, __B), |
| 252 | (__v32hi)_mm512_setzero_si512()); | 461 | (__v32hi)_mm512_setzero_si512()); |
| 253 | } | 462 | } |
| 254 | 463 | ||
| 255 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 464 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 256 | _mm512_mullo_epi16 (__m512i __A, __m512i __B) { | 465 | _mm512_mullo_epi16 (__m512i __A, __m512i __B) { |
| 257 | return (__m512i) ((__v32hu) __A * (__v32hu) __B); | 466 | return (__m512i) ((__v32hu) __A * (__v32hu) __B); |
| 258 | } | 467 | } |
| 259 | 468 | ||
| 260 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 469 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 261 | _mm512_mask_mullo_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) { | 470 | _mm512_mask_mullo_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) { |
| 262 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 471 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 263 | (__v32hi)_mm512_mullo_epi16(__A, __B), | 472 | (__v32hi)_mm512_mullo_epi16(__A, __B), |
| 264 | (__v32hi)__W); | 473 | (__v32hi)__W); |
| 265 | } | 474 | } |
| 266 | 475 | ||
| 267 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 476 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 268 | _mm512_maskz_mullo_epi16(__mmask32 __U, __m512i __A, __m512i __B) { | 477 | _mm512_maskz_mullo_epi16(__mmask32 __U, __m512i __A, __m512i __B) { |
| 269 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 478 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 270 | (__v32hi)_mm512_mullo_epi16(__A, __B), | 479 | (__v32hi)_mm512_mullo_epi16(__A, __B), |
| 271 | (__v32hi)_mm512_setzero_si512()); | 480 | (__v32hi)_mm512_setzero_si512()); |
| 272 | } | 481 | } |
| 273 | 482 | ||
| 274 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 483 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 275 | _mm512_mask_blend_epi8 (__mmask64 __U, __m512i __A, __m512i __W) | 484 | _mm512_mask_blend_epi8 (__mmask64 __U, __m512i __A, __m512i __W) |
| 276 | { | 485 | { |
| 277 | return (__m512i) __builtin_ia32_selectb_512 ((__mmask64) __U, | 486 | return (__m512i) __builtin_ia32_selectb_512 ((__mmask64) __U, |
| ... | @@ -279,7 +488,7 @@ _mm512_mask_blend_epi8 (__mmask64 __U, __m512i __A, __m512i __W) | ... | @@ -279,7 +488,7 @@ _mm512_mask_blend_epi8 (__mmask64 __U, __m512i __A, __m512i __W) |
| 279 | (__v64qi) __A); | 488 | (__v64qi) __A); |
| 280 | } | 489 | } |
| 281 | 490 | ||
| 282 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 491 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 283 | _mm512_mask_blend_epi16 (__mmask32 __U, __m512i __A, __m512i __W) | 492 | _mm512_mask_blend_epi16 (__mmask32 __U, __m512i __A, __m512i __W) |
| 284 | { | 493 | { |
| 285 | return (__m512i) __builtin_ia32_selectw_512 ((__mmask32) __U, | 494 | return (__m512i) __builtin_ia32_selectw_512 ((__mmask32) __U, |
| ... | @@ -287,13 +496,13 @@ _mm512_mask_blend_epi16 (__mmask32 __U, __m512i __A, __m512i __W) | ... | @@ -287,13 +496,13 @@ _mm512_mask_blend_epi16 (__mmask32 __U, __m512i __A, __m512i __W) |
| 287 | (__v32hi) __A); | 496 | (__v32hi) __A); |
| 288 | } | 497 | } |
| 289 | 498 | ||
| 290 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 499 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 291 | _mm512_abs_epi8 (__m512i __A) | 500 | _mm512_abs_epi8 (__m512i __A) |
| 292 | { | 501 | { |
| 293 | return (__m512i)__builtin_ia32_pabsb512((__v64qi)__A); | 502 | return (__m512i)__builtin_ia32_pabsb512((__v64qi)__A); |
| 294 | } | 503 | } |
| 295 | 504 | ||
| 296 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 505 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 297 | _mm512_mask_abs_epi8 (__m512i __W, __mmask64 __U, __m512i __A) | 506 | _mm512_mask_abs_epi8 (__m512i __W, __mmask64 __U, __m512i __A) |
| 298 | { | 507 | { |
| 299 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, | 508 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| ... | @@ -301,7 +510,7 @@ _mm512_mask_abs_epi8 (__m512i __W, __mmask64 __U, __m512i __A) | ... | @@ -301,7 +510,7 @@ _mm512_mask_abs_epi8 (__m512i __W, __mmask64 __U, __m512i __A) |
| 301 | (__v64qi)__W); | 510 | (__v64qi)__W); |
| 302 | } | 511 | } |
| 303 | 512 | ||
| 304 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 513 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 305 | _mm512_maskz_abs_epi8 (__mmask64 __U, __m512i __A) | 514 | _mm512_maskz_abs_epi8 (__mmask64 __U, __m512i __A) |
| 306 | { | 515 | { |
| 307 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, | 516 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| ... | @@ -309,13 +518,13 @@ _mm512_maskz_abs_epi8 (__mmask64 __U, __m512i __A) | ... | @@ -309,13 +518,13 @@ _mm512_maskz_abs_epi8 (__mmask64 __U, __m512i __A) |
| 309 | (__v64qi)_mm512_setzero_si512()); | 518 | (__v64qi)_mm512_setzero_si512()); |
| 310 | } | 519 | } |
| 311 | 520 | ||
| 312 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 521 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 313 | _mm512_abs_epi16 (__m512i __A) | 522 | _mm512_abs_epi16 (__m512i __A) |
| 314 | { | 523 | { |
| 315 | return (__m512i)__builtin_ia32_pabsw512((__v32hi)__A); | 524 | return (__m512i)__builtin_ia32_pabsw512((__v32hi)__A); |
| 316 | } | 525 | } |
| 317 | 526 | ||
| 318 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 527 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 319 | _mm512_mask_abs_epi16 (__m512i __W, __mmask32 __U, __m512i __A) | 528 | _mm512_mask_abs_epi16 (__m512i __W, __mmask32 __U, __m512i __A) |
| 320 | { | 529 | { |
| 321 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 530 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -323,7 +532,7 @@ _mm512_mask_abs_epi16 (__m512i __W, __mmask32 __U, __m512i __A) | ... | @@ -323,7 +532,7 @@ _mm512_mask_abs_epi16 (__m512i __W, __mmask32 __U, __m512i __A) |
| 323 | (__v32hi)__W); | 532 | (__v32hi)__W); |
| 324 | } | 533 | } |
| 325 | 534 | ||
| 326 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 535 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 327 | _mm512_maskz_abs_epi16 (__mmask32 __U, __m512i __A) | 536 | _mm512_maskz_abs_epi16 (__mmask32 __U, __m512i __A) |
| 328 | { | 537 | { |
| 329 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 538 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -331,13 +540,13 @@ _mm512_maskz_abs_epi16 (__mmask32 __U, __m512i __A) | ... | @@ -331,13 +540,13 @@ _mm512_maskz_abs_epi16 (__mmask32 __U, __m512i __A) |
| 331 | (__v32hi)_mm512_setzero_si512()); | 540 | (__v32hi)_mm512_setzero_si512()); |
| 332 | } | 541 | } |
| 333 | 542 | ||
| 334 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 543 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 335 | _mm512_packs_epi32(__m512i __A, __m512i __B) | 544 | _mm512_packs_epi32(__m512i __A, __m512i __B) |
| 336 | { | 545 | { |
| 337 | return (__m512i)__builtin_ia32_packssdw512((__v16si)__A, (__v16si)__B); | 546 | return (__m512i)__builtin_ia32_packssdw512((__v16si)__A, (__v16si)__B); |
| 338 | } | 547 | } |
| 339 | 548 | ||
| 340 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 549 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 341 | _mm512_maskz_packs_epi32(__mmask32 __M, __m512i __A, __m512i __B) | 550 | _mm512_maskz_packs_epi32(__mmask32 __M, __m512i __A, __m512i __B) |
| 342 | { | 551 | { |
| 343 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, | 552 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, |
| ... | @@ -345,7 +554,7 @@ _mm512_maskz_packs_epi32(__mmask32 __M, __m512i __A, __m512i __B) | ... | @@ -345,7 +554,7 @@ _mm512_maskz_packs_epi32(__mmask32 __M, __m512i __A, __m512i __B) |
| 345 | (__v32hi)_mm512_setzero_si512()); | 554 | (__v32hi)_mm512_setzero_si512()); |
| 346 | } | 555 | } |
| 347 | 556 | ||
| 348 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 557 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 349 | _mm512_mask_packs_epi32(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) | 558 | _mm512_mask_packs_epi32(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) |
| 350 | { | 559 | { |
| 351 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, | 560 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, |
| ... | @@ -353,13 +562,13 @@ _mm512_mask_packs_epi32(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) | ... | @@ -353,13 +562,13 @@ _mm512_mask_packs_epi32(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) |
| 353 | (__v32hi)__W); | 562 | (__v32hi)__W); |
| 354 | } | 563 | } |
| 355 | 564 | ||
| 356 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 565 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 357 | _mm512_packs_epi16(__m512i __A, __m512i __B) | 566 | _mm512_packs_epi16(__m512i __A, __m512i __B) |
| 358 | { | 567 | { |
| 359 | return (__m512i)__builtin_ia32_packsswb512((__v32hi)__A, (__v32hi) __B); | 568 | return (__m512i)__builtin_ia32_packsswb512((__v32hi)__A, (__v32hi) __B); |
| 360 | } | 569 | } |
| 361 | 570 | ||
| 362 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 571 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 363 | _mm512_mask_packs_epi16(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) | 572 | _mm512_mask_packs_epi16(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) |
| 364 | { | 573 | { |
| 365 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, | 574 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, |
| ... | @@ -367,7 +576,7 @@ _mm512_mask_packs_epi16(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) | ... | @@ -367,7 +576,7 @@ _mm512_mask_packs_epi16(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) |
| 367 | (__v64qi)__W); | 576 | (__v64qi)__W); |
| 368 | } | 577 | } |
| 369 | 578 | ||
| 370 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 579 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 371 | _mm512_maskz_packs_epi16(__mmask64 __M, __m512i __A, __m512i __B) | 580 | _mm512_maskz_packs_epi16(__mmask64 __M, __m512i __A, __m512i __B) |
| 372 | { | 581 | { |
| 373 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, | 582 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, |
| ... | @@ -375,13 +584,13 @@ _mm512_maskz_packs_epi16(__mmask64 __M, __m512i __A, __m512i __B) | ... | @@ -375,13 +584,13 @@ _mm512_maskz_packs_epi16(__mmask64 __M, __m512i __A, __m512i __B) |
| 375 | (__v64qi)_mm512_setzero_si512()); | 584 | (__v64qi)_mm512_setzero_si512()); |
| 376 | } | 585 | } |
| 377 | 586 | ||
| 378 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 587 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 379 | _mm512_packus_epi32(__m512i __A, __m512i __B) | 588 | _mm512_packus_epi32(__m512i __A, __m512i __B) |
| 380 | { | 589 | { |
| 381 | return (__m512i)__builtin_ia32_packusdw512((__v16si) __A, (__v16si) __B); | 590 | return (__m512i)__builtin_ia32_packusdw512((__v16si) __A, (__v16si) __B); |
| 382 | } | 591 | } |
| 383 | 592 | ||
| 384 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 593 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 385 | _mm512_maskz_packus_epi32(__mmask32 __M, __m512i __A, __m512i __B) | 594 | _mm512_maskz_packus_epi32(__mmask32 __M, __m512i __A, __m512i __B) |
| 386 | { | 595 | { |
| 387 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, | 596 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, |
| ... | @@ -389,7 +598,7 @@ _mm512_maskz_packus_epi32(__mmask32 __M, __m512i __A, __m512i __B) | ... | @@ -389,7 +598,7 @@ _mm512_maskz_packus_epi32(__mmask32 __M, __m512i __A, __m512i __B) |
| 389 | (__v32hi)_mm512_setzero_si512()); | 598 | (__v32hi)_mm512_setzero_si512()); |
| 390 | } | 599 | } |
| 391 | 600 | ||
| 392 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 601 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 393 | _mm512_mask_packus_epi32(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) | 602 | _mm512_mask_packus_epi32(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) |
| 394 | { | 603 | { |
| 395 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, | 604 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, |
| ... | @@ -397,13 +606,13 @@ _mm512_mask_packus_epi32(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) | ... | @@ -397,13 +606,13 @@ _mm512_mask_packus_epi32(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) |
| 397 | (__v32hi)__W); | 606 | (__v32hi)__W); |
| 398 | } | 607 | } |
| 399 | 608 | ||
| 400 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 609 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 401 | _mm512_packus_epi16(__m512i __A, __m512i __B) | 610 | _mm512_packus_epi16(__m512i __A, __m512i __B) |
| 402 | { | 611 | { |
| 403 | return (__m512i)__builtin_ia32_packuswb512((__v32hi) __A, (__v32hi) __B); | 612 | return (__m512i)__builtin_ia32_packuswb512((__v32hi) __A, (__v32hi) __B); |
| 404 | } | 613 | } |
| 405 | 614 | ||
| 406 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 615 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 407 | _mm512_mask_packus_epi16(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) | 616 | _mm512_mask_packus_epi16(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) |
| 408 | { | 617 | { |
| 409 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, | 618 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, |
| ... | @@ -411,7 +620,7 @@ _mm512_mask_packus_epi16(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) | ... | @@ -411,7 +620,7 @@ _mm512_mask_packus_epi16(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) |
| 411 | (__v64qi)__W); | 620 | (__v64qi)__W); |
| 412 | } | 621 | } |
| 413 | 622 | ||
| 414 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 623 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 415 | _mm512_maskz_packus_epi16(__mmask64 __M, __m512i __A, __m512i __B) | 624 | _mm512_maskz_packus_epi16(__mmask64 __M, __m512i __A, __m512i __B) |
| 416 | { | 625 | { |
| 417 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, | 626 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, |
| ... | @@ -419,119 +628,95 @@ _mm512_maskz_packus_epi16(__mmask64 __M, __m512i __A, __m512i __B) | ... | @@ -419,119 +628,95 @@ _mm512_maskz_packus_epi16(__mmask64 __M, __m512i __A, __m512i __B) |
| 419 | (__v64qi)_mm512_setzero_si512()); | 628 | (__v64qi)_mm512_setzero_si512()); |
| 420 | } | 629 | } |
| 421 | 630 | ||
| 422 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 631 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 423 | _mm512_adds_epi8 (__m512i __A, __m512i __B) | 632 | _mm512_adds_epi8 (__m512i __A, __m512i __B) |
| 424 | { | 633 | { |
| 425 | return (__m512i) __builtin_ia32_paddsb512_mask ((__v64qi) __A, | 634 | return (__m512i)__builtin_ia32_paddsb512((__v64qi)__A, (__v64qi)__B); |
| 426 | (__v64qi) __B, | ||
| 427 | (__v64qi) _mm512_setzero_si512(), | ||
| 428 | (__mmask64) -1); | ||
| 429 | } | 635 | } |
| 430 | 636 | ||
| 431 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 637 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 432 | _mm512_mask_adds_epi8 (__m512i __W, __mmask64 __U, __m512i __A, | 638 | _mm512_mask_adds_epi8 (__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) |
| 433 | __m512i __B) | ||
| 434 | { | 639 | { |
| 435 | return (__m512i) __builtin_ia32_paddsb512_mask ((__v64qi) __A, | 640 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 436 | (__v64qi) __B, | 641 | (__v64qi)_mm512_adds_epi8(__A, __B), |
| 437 | (__v64qi) __W, | 642 | (__v64qi)__W); |
| 438 | (__mmask64) __U); | ||
| 439 | } | 643 | } |
| 440 | 644 | ||
| 441 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 645 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 442 | _mm512_maskz_adds_epi8 (__mmask64 __U, __m512i __A, __m512i __B) | 646 | _mm512_maskz_adds_epi8 (__mmask64 __U, __m512i __A, __m512i __B) |
| 443 | { | 647 | { |
| 444 | return (__m512i) __builtin_ia32_paddsb512_mask ((__v64qi) __A, | 648 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 445 | (__v64qi) __B, | 649 | (__v64qi)_mm512_adds_epi8(__A, __B), |
| 446 | (__v64qi) _mm512_setzero_si512(), | 650 | (__v64qi)_mm512_setzero_si512()); |
| 447 | (__mmask64) __U); | ||
| 448 | } | 651 | } |
| 449 | 652 | ||
| 450 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 653 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 451 | _mm512_adds_epi16 (__m512i __A, __m512i __B) | 654 | _mm512_adds_epi16 (__m512i __A, __m512i __B) |
| 452 | { | 655 | { |
| 453 | return (__m512i) __builtin_ia32_paddsw512_mask ((__v32hi) __A, | 656 | return (__m512i)__builtin_ia32_paddsw512((__v32hi)__A, (__v32hi)__B); |
| 454 | (__v32hi) __B, | ||
| 455 | (__v32hi) _mm512_setzero_si512(), | ||
| 456 | (__mmask32) -1); | ||
| 457 | } | 657 | } |
| 458 | 658 | ||
| 459 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 659 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 460 | _mm512_mask_adds_epi16 (__m512i __W, __mmask32 __U, __m512i __A, | 660 | _mm512_mask_adds_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) |
| 461 | __m512i __B) | ||
| 462 | { | 661 | { |
| 463 | return (__m512i) __builtin_ia32_paddsw512_mask ((__v32hi) __A, | 662 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 464 | (__v32hi) __B, | 663 | (__v32hi)_mm512_adds_epi16(__A, __B), |
| 465 | (__v32hi) __W, | 664 | (__v32hi)__W); |
| 466 | (__mmask32) __U); | ||
| 467 | } | 665 | } |
| 468 | 666 | ||
| 469 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 667 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 470 | _mm512_maskz_adds_epi16 (__mmask32 __U, __m512i __A, __m512i __B) | 668 | _mm512_maskz_adds_epi16 (__mmask32 __U, __m512i __A, __m512i __B) |
| 471 | { | 669 | { |
| 472 | return (__m512i) __builtin_ia32_paddsw512_mask ((__v32hi) __A, | 670 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 473 | (__v32hi) __B, | 671 | (__v32hi)_mm512_adds_epi16(__A, __B), |
| 474 | (__v32hi) _mm512_setzero_si512(), | 672 | (__v32hi)_mm512_setzero_si512()); |
| 475 | (__mmask32) __U); | ||
| 476 | } | 673 | } |
| 477 | 674 | ||
| 478 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 675 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 479 | _mm512_adds_epu8 (__m512i __A, __m512i __B) | 676 | _mm512_adds_epu8 (__m512i __A, __m512i __B) |
| 480 | { | 677 | { |
| 481 | return (__m512i) __builtin_ia32_paddusb512_mask ((__v64qi) __A, | 678 | return (__m512i)__builtin_ia32_paddusb512((__v64qi) __A, (__v64qi) __B); |
| 482 | (__v64qi) __B, | ||
| 483 | (__v64qi) _mm512_setzero_si512(), | ||
| 484 | (__mmask64) -1); | ||
| 485 | } | 679 | } |
| 486 | 680 | ||
| 487 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 681 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 488 | _mm512_mask_adds_epu8 (__m512i __W, __mmask64 __U, __m512i __A, | 682 | _mm512_mask_adds_epu8 (__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) |
| 489 | __m512i __B) | ||
| 490 | { | 683 | { |
| 491 | return (__m512i) __builtin_ia32_paddusb512_mask ((__v64qi) __A, | 684 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 492 | (__v64qi) __B, | 685 | (__v64qi)_mm512_adds_epu8(__A, __B), |
| 493 | (__v64qi) __W, | 686 | (__v64qi)__W); |
| 494 | (__mmask64) __U); | ||
| 495 | } | 687 | } |
| 496 | 688 | ||
| 497 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 689 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 498 | _mm512_maskz_adds_epu8 (__mmask64 __U, __m512i __A, __m512i __B) | 690 | _mm512_maskz_adds_epu8 (__mmask64 __U, __m512i __A, __m512i __B) |
| 499 | { | 691 | { |
| 500 | return (__m512i) __builtin_ia32_paddusb512_mask ((__v64qi) __A, | 692 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 501 | (__v64qi) __B, | 693 | (__v64qi)_mm512_adds_epu8(__A, __B), |
| 502 | (__v64qi) _mm512_setzero_si512(), | 694 | (__v64qi)_mm512_setzero_si512()); |
| 503 | (__mmask64) __U); | ||
| 504 | } | 695 | } |
| 505 | 696 | ||
| 506 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 697 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 507 | _mm512_adds_epu16 (__m512i __A, __m512i __B) | 698 | _mm512_adds_epu16 (__m512i __A, __m512i __B) |
| 508 | { | 699 | { |
| 509 | return (__m512i) __builtin_ia32_paddusw512_mask ((__v32hi) __A, | 700 | return (__m512i)__builtin_ia32_paddusw512((__v32hi) __A, (__v32hi) __B); |
| 510 | (__v32hi) __B, | ||
| 511 | (__v32hi) _mm512_setzero_si512(), | ||
| 512 | (__mmask32) -1); | ||
| 513 | } | 701 | } |
| 514 | 702 | ||
| 515 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 703 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 516 | _mm512_mask_adds_epu16 (__m512i __W, __mmask32 __U, __m512i __A, | 704 | _mm512_mask_adds_epu16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) |
| 517 | __m512i __B) | ||
| 518 | { | 705 | { |
| 519 | return (__m512i) __builtin_ia32_paddusw512_mask ((__v32hi) __A, | 706 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 520 | (__v32hi) __B, | 707 | (__v32hi)_mm512_adds_epu16(__A, __B), |
| 521 | (__v32hi) __W, | 708 | (__v32hi)__W); |
| 522 | (__mmask32) __U); | ||
| 523 | } | 709 | } |
| 524 | 710 | ||
| 525 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 711 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 526 | _mm512_maskz_adds_epu16 (__mmask32 __U, __m512i __A, __m512i __B) | 712 | _mm512_maskz_adds_epu16 (__mmask32 __U, __m512i __A, __m512i __B) |
| 527 | { | 713 | { |
| 528 | return (__m512i) __builtin_ia32_paddusw512_mask ((__v32hi) __A, | 714 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 529 | (__v32hi) __B, | 715 | (__v32hi)_mm512_adds_epu16(__A, __B), |
| 530 | (__v32hi) _mm512_setzero_si512(), | 716 | (__v32hi)_mm512_setzero_si512()); |
| 531 | (__mmask32) __U); | ||
| 532 | } | 717 | } |
| 533 | 718 | ||
| 534 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 719 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 535 | _mm512_avg_epu8 (__m512i __A, __m512i __B) | 720 | _mm512_avg_epu8 (__m512i __A, __m512i __B) |
| 536 | { | 721 | { |
| 537 | typedef unsigned short __v64hu __attribute__((__vector_size__(128))); | 722 | typedef unsigned short __v64hu __attribute__((__vector_size__(128))); |
| ... | @@ -541,7 +726,7 @@ _mm512_avg_epu8 (__m512i __A, __m512i __B) | ... | @@ -541,7 +726,7 @@ _mm512_avg_epu8 (__m512i __A, __m512i __B) |
| 541 | >> 1, __v64qu); | 726 | >> 1, __v64qu); |
| 542 | } | 727 | } |
| 543 | 728 | ||
| 544 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 729 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 545 | _mm512_mask_avg_epu8 (__m512i __W, __mmask64 __U, __m512i __A, | 730 | _mm512_mask_avg_epu8 (__m512i __W, __mmask64 __U, __m512i __A, |
| 546 | __m512i __B) | 731 | __m512i __B) |
| 547 | { | 732 | { |
| ... | @@ -550,7 +735,7 @@ _mm512_mask_avg_epu8 (__m512i __W, __mmask64 __U, __m512i __A, | ... | @@ -550,7 +735,7 @@ _mm512_mask_avg_epu8 (__m512i __W, __mmask64 __U, __m512i __A, |
| 550 | (__v64qi)__W); | 735 | (__v64qi)__W); |
| 551 | } | 736 | } |
| 552 | 737 | ||
| 553 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 738 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 554 | _mm512_maskz_avg_epu8 (__mmask64 __U, __m512i __A, __m512i __B) | 739 | _mm512_maskz_avg_epu8 (__mmask64 __U, __m512i __A, __m512i __B) |
| 555 | { | 740 | { |
| 556 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, | 741 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| ... | @@ -558,7 +743,7 @@ _mm512_maskz_avg_epu8 (__mmask64 __U, __m512i __A, __m512i __B) | ... | @@ -558,7 +743,7 @@ _mm512_maskz_avg_epu8 (__mmask64 __U, __m512i __A, __m512i __B) |
| 558 | (__v64qi)_mm512_setzero_si512()); | 743 | (__v64qi)_mm512_setzero_si512()); |
| 559 | } | 744 | } |
| 560 | 745 | ||
| 561 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 746 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 562 | _mm512_avg_epu16 (__m512i __A, __m512i __B) | 747 | _mm512_avg_epu16 (__m512i __A, __m512i __B) |
| 563 | { | 748 | { |
| 564 | typedef unsigned int __v32su __attribute__((__vector_size__(128))); | 749 | typedef unsigned int __v32su __attribute__((__vector_size__(128))); |
| ... | @@ -568,7 +753,7 @@ _mm512_avg_epu16 (__m512i __A, __m512i __B) | ... | @@ -568,7 +753,7 @@ _mm512_avg_epu16 (__m512i __A, __m512i __B) |
| 568 | >> 1, __v32hu); | 753 | >> 1, __v32hu); |
| 569 | } | 754 | } |
| 570 | 755 | ||
| 571 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 756 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 572 | _mm512_mask_avg_epu16 (__m512i __W, __mmask32 __U, __m512i __A, | 757 | _mm512_mask_avg_epu16 (__m512i __W, __mmask32 __U, __m512i __A, |
| 573 | __m512i __B) | 758 | __m512i __B) |
| 574 | { | 759 | { |
| ... | @@ -577,7 +762,7 @@ _mm512_mask_avg_epu16 (__m512i __W, __mmask32 __U, __m512i __A, | ... | @@ -577,7 +762,7 @@ _mm512_mask_avg_epu16 (__m512i __W, __mmask32 __U, __m512i __A, |
| 577 | (__v32hi)__W); | 762 | (__v32hi)__W); |
| 578 | } | 763 | } |
| 579 | 764 | ||
| 580 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 765 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 581 | _mm512_maskz_avg_epu16 (__mmask32 __U, __m512i __A, __m512i __B) | 766 | _mm512_maskz_avg_epu16 (__mmask32 __U, __m512i __A, __m512i __B) |
| 582 | { | 767 | { |
| 583 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 768 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -585,13 +770,13 @@ _mm512_maskz_avg_epu16 (__mmask32 __U, __m512i __A, __m512i __B) | ... | @@ -585,13 +770,13 @@ _mm512_maskz_avg_epu16 (__mmask32 __U, __m512i __A, __m512i __B) |
| 585 | (__v32hi) _mm512_setzero_si512()); | 770 | (__v32hi) _mm512_setzero_si512()); |
| 586 | } | 771 | } |
| 587 | 772 | ||
| 588 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 773 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 589 | _mm512_max_epi8 (__m512i __A, __m512i __B) | 774 | _mm512_max_epi8 (__m512i __A, __m512i __B) |
| 590 | { | 775 | { |
| 591 | return (__m512i)__builtin_ia32_pmaxsb512((__v64qi) __A, (__v64qi) __B); | 776 | return (__m512i)__builtin_ia32_pmaxsb512((__v64qi) __A, (__v64qi) __B); |
| 592 | } | 777 | } |
| 593 | 778 | ||
| 594 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 779 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 595 | _mm512_maskz_max_epi8 (__mmask64 __M, __m512i __A, __m512i __B) | 780 | _mm512_maskz_max_epi8 (__mmask64 __M, __m512i __A, __m512i __B) |
| 596 | { | 781 | { |
| 597 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, | 782 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, |
| ... | @@ -599,7 +784,7 @@ _mm512_maskz_max_epi8 (__mmask64 __M, __m512i __A, __m512i __B) | ... | @@ -599,7 +784,7 @@ _mm512_maskz_max_epi8 (__mmask64 __M, __m512i __A, __m512i __B) |
| 599 | (__v64qi)_mm512_setzero_si512()); | 784 | (__v64qi)_mm512_setzero_si512()); |
| 600 | } | 785 | } |
| 601 | 786 | ||
| 602 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 787 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 603 | _mm512_mask_max_epi8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) | 788 | _mm512_mask_max_epi8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) |
| 604 | { | 789 | { |
| 605 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, | 790 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, |
| ... | @@ -607,13 +792,13 @@ _mm512_mask_max_epi8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) | ... | @@ -607,13 +792,13 @@ _mm512_mask_max_epi8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) |
| 607 | (__v64qi)__W); | 792 | (__v64qi)__W); |
| 608 | } | 793 | } |
| 609 | 794 | ||
| 610 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 795 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 611 | _mm512_max_epi16 (__m512i __A, __m512i __B) | 796 | _mm512_max_epi16 (__m512i __A, __m512i __B) |
| 612 | { | 797 | { |
| 613 | return (__m512i)__builtin_ia32_pmaxsw512((__v32hi) __A, (__v32hi) __B); | 798 | return (__m512i)__builtin_ia32_pmaxsw512((__v32hi) __A, (__v32hi) __B); |
| 614 | } | 799 | } |
| 615 | 800 | ||
| 616 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 801 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 617 | _mm512_maskz_max_epi16 (__mmask32 __M, __m512i __A, __m512i __B) | 802 | _mm512_maskz_max_epi16 (__mmask32 __M, __m512i __A, __m512i __B) |
| 618 | { | 803 | { |
| 619 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, | 804 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, |
| ... | @@ -621,7 +806,7 @@ _mm512_maskz_max_epi16 (__mmask32 __M, __m512i __A, __m512i __B) | ... | @@ -621,7 +806,7 @@ _mm512_maskz_max_epi16 (__mmask32 __M, __m512i __A, __m512i __B) |
| 621 | (__v32hi)_mm512_setzero_si512()); | 806 | (__v32hi)_mm512_setzero_si512()); |
| 622 | } | 807 | } |
| 623 | 808 | ||
| 624 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 809 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 625 | _mm512_mask_max_epi16 (__m512i __W, __mmask32 __M, __m512i __A, | 810 | _mm512_mask_max_epi16 (__m512i __W, __mmask32 __M, __m512i __A, |
| 626 | __m512i __B) | 811 | __m512i __B) |
| 627 | { | 812 | { |
| ... | @@ -630,13 +815,13 @@ _mm512_mask_max_epi16 (__m512i __W, __mmask32 __M, __m512i __A, | ... | @@ -630,13 +815,13 @@ _mm512_mask_max_epi16 (__m512i __W, __mmask32 __M, __m512i __A, |
| 630 | (__v32hi)__W); | 815 | (__v32hi)__W); |
| 631 | } | 816 | } |
| 632 | 817 | ||
| 633 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 818 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 634 | _mm512_max_epu8 (__m512i __A, __m512i __B) | 819 | _mm512_max_epu8 (__m512i __A, __m512i __B) |
| 635 | { | 820 | { |
| 636 | return (__m512i)__builtin_ia32_pmaxub512((__v64qi)__A, (__v64qi)__B); | 821 | return (__m512i)__builtin_ia32_pmaxub512((__v64qi)__A, (__v64qi)__B); |
| 637 | } | 822 | } |
| 638 | 823 | ||
| 639 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 824 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 640 | _mm512_maskz_max_epu8 (__mmask64 __M, __m512i __A, __m512i __B) | 825 | _mm512_maskz_max_epu8 (__mmask64 __M, __m512i __A, __m512i __B) |
| 641 | { | 826 | { |
| 642 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, | 827 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, |
| ... | @@ -644,7 +829,7 @@ _mm512_maskz_max_epu8 (__mmask64 __M, __m512i __A, __m512i __B) | ... | @@ -644,7 +829,7 @@ _mm512_maskz_max_epu8 (__mmask64 __M, __m512i __A, __m512i __B) |
| 644 | (__v64qi)_mm512_setzero_si512()); | 829 | (__v64qi)_mm512_setzero_si512()); |
| 645 | } | 830 | } |
| 646 | 831 | ||
| 647 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 832 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 648 | _mm512_mask_max_epu8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) | 833 | _mm512_mask_max_epu8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) |
| 649 | { | 834 | { |
| 650 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, | 835 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, |
| ... | @@ -652,13 +837,13 @@ _mm512_mask_max_epu8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) | ... | @@ -652,13 +837,13 @@ _mm512_mask_max_epu8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) |
| 652 | (__v64qi)__W); | 837 | (__v64qi)__W); |
| 653 | } | 838 | } |
| 654 | 839 | ||
| 655 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 840 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 656 | _mm512_max_epu16 (__m512i __A, __m512i __B) | 841 | _mm512_max_epu16 (__m512i __A, __m512i __B) |
| 657 | { | 842 | { |
| 658 | return (__m512i)__builtin_ia32_pmaxuw512((__v32hi)__A, (__v32hi)__B); | 843 | return (__m512i)__builtin_ia32_pmaxuw512((__v32hi)__A, (__v32hi)__B); |
| 659 | } | 844 | } |
| 660 | 845 | ||
| 661 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 846 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 662 | _mm512_maskz_max_epu16 (__mmask32 __M, __m512i __A, __m512i __B) | 847 | _mm512_maskz_max_epu16 (__mmask32 __M, __m512i __A, __m512i __B) |
| 663 | { | 848 | { |
| 664 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, | 849 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, |
| ... | @@ -666,7 +851,7 @@ _mm512_maskz_max_epu16 (__mmask32 __M, __m512i __A, __m512i __B) | ... | @@ -666,7 +851,7 @@ _mm512_maskz_max_epu16 (__mmask32 __M, __m512i __A, __m512i __B) |
| 666 | (__v32hi)_mm512_setzero_si512()); | 851 | (__v32hi)_mm512_setzero_si512()); |
| 667 | } | 852 | } |
| 668 | 853 | ||
| 669 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 854 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 670 | _mm512_mask_max_epu16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) | 855 | _mm512_mask_max_epu16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) |
| 671 | { | 856 | { |
| 672 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, | 857 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, |
| ... | @@ -674,13 +859,13 @@ _mm512_mask_max_epu16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) | ... | @@ -674,13 +859,13 @@ _mm512_mask_max_epu16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) |
| 674 | (__v32hi)__W); | 859 | (__v32hi)__W); |
| 675 | } | 860 | } |
| 676 | 861 | ||
| 677 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 862 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 678 | _mm512_min_epi8 (__m512i __A, __m512i __B) | 863 | _mm512_min_epi8 (__m512i __A, __m512i __B) |
| 679 | { | 864 | { |
| 680 | return (__m512i)__builtin_ia32_pminsb512((__v64qi) __A, (__v64qi) __B); | 865 | return (__m512i)__builtin_ia32_pminsb512((__v64qi) __A, (__v64qi) __B); |
| 681 | } | 866 | } |
| 682 | 867 | ||
| 683 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 868 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 684 | _mm512_maskz_min_epi8 (__mmask64 __M, __m512i __A, __m512i __B) | 869 | _mm512_maskz_min_epi8 (__mmask64 __M, __m512i __A, __m512i __B) |
| 685 | { | 870 | { |
| 686 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, | 871 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, |
| ... | @@ -688,7 +873,7 @@ _mm512_maskz_min_epi8 (__mmask64 __M, __m512i __A, __m512i __B) | ... | @@ -688,7 +873,7 @@ _mm512_maskz_min_epi8 (__mmask64 __M, __m512i __A, __m512i __B) |
| 688 | (__v64qi)_mm512_setzero_si512()); | 873 | (__v64qi)_mm512_setzero_si512()); |
| 689 | } | 874 | } |
| 690 | 875 | ||
| 691 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 876 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 692 | _mm512_mask_min_epi8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) | 877 | _mm512_mask_min_epi8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) |
| 693 | { | 878 | { |
| 694 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, | 879 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, |
| ... | @@ -696,13 +881,13 @@ _mm512_mask_min_epi8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) | ... | @@ -696,13 +881,13 @@ _mm512_mask_min_epi8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) |
| 696 | (__v64qi)__W); | 881 | (__v64qi)__W); |
| 697 | } | 882 | } |
| 698 | 883 | ||
| 699 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 884 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 700 | _mm512_min_epi16 (__m512i __A, __m512i __B) | 885 | _mm512_min_epi16 (__m512i __A, __m512i __B) |
| 701 | { | 886 | { |
| 702 | return (__m512i)__builtin_ia32_pminsw512((__v32hi) __A, (__v32hi) __B); | 887 | return (__m512i)__builtin_ia32_pminsw512((__v32hi) __A, (__v32hi) __B); |
| 703 | } | 888 | } |
| 704 | 889 | ||
| 705 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 890 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 706 | _mm512_maskz_min_epi16 (__mmask32 __M, __m512i __A, __m512i __B) | 891 | _mm512_maskz_min_epi16 (__mmask32 __M, __m512i __A, __m512i __B) |
| 707 | { | 892 | { |
| 708 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, | 893 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, |
| ... | @@ -710,7 +895,7 @@ _mm512_maskz_min_epi16 (__mmask32 __M, __m512i __A, __m512i __B) | ... | @@ -710,7 +895,7 @@ _mm512_maskz_min_epi16 (__mmask32 __M, __m512i __A, __m512i __B) |
| 710 | (__v32hi)_mm512_setzero_si512()); | 895 | (__v32hi)_mm512_setzero_si512()); |
| 711 | } | 896 | } |
| 712 | 897 | ||
| 713 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 898 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 714 | _mm512_mask_min_epi16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) | 899 | _mm512_mask_min_epi16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) |
| 715 | { | 900 | { |
| 716 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, | 901 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, |
| ... | @@ -718,13 +903,13 @@ _mm512_mask_min_epi16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) | ... | @@ -718,13 +903,13 @@ _mm512_mask_min_epi16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) |
| 718 | (__v32hi)__W); | 903 | (__v32hi)__W); |
| 719 | } | 904 | } |
| 720 | 905 | ||
| 721 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 906 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 722 | _mm512_min_epu8 (__m512i __A, __m512i __B) | 907 | _mm512_min_epu8 (__m512i __A, __m512i __B) |
| 723 | { | 908 | { |
| 724 | return (__m512i)__builtin_ia32_pminub512((__v64qi)__A, (__v64qi)__B); | 909 | return (__m512i)__builtin_ia32_pminub512((__v64qi)__A, (__v64qi)__B); |
| 725 | } | 910 | } |
| 726 | 911 | ||
| 727 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 912 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 728 | _mm512_maskz_min_epu8 (__mmask64 __M, __m512i __A, __m512i __B) | 913 | _mm512_maskz_min_epu8 (__mmask64 __M, __m512i __A, __m512i __B) |
| 729 | { | 914 | { |
| 730 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, | 915 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, |
| ... | @@ -732,7 +917,7 @@ _mm512_maskz_min_epu8 (__mmask64 __M, __m512i __A, __m512i __B) | ... | @@ -732,7 +917,7 @@ _mm512_maskz_min_epu8 (__mmask64 __M, __m512i __A, __m512i __B) |
| 732 | (__v64qi)_mm512_setzero_si512()); | 917 | (__v64qi)_mm512_setzero_si512()); |
| 733 | } | 918 | } |
| 734 | 919 | ||
| 735 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 920 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 736 | _mm512_mask_min_epu8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) | 921 | _mm512_mask_min_epu8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) |
| 737 | { | 922 | { |
| 738 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, | 923 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, |
| ... | @@ -740,13 +925,13 @@ _mm512_mask_min_epu8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) | ... | @@ -740,13 +925,13 @@ _mm512_mask_min_epu8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) |
| 740 | (__v64qi)__W); | 925 | (__v64qi)__W); |
| 741 | } | 926 | } |
| 742 | 927 | ||
| 743 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 928 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 744 | _mm512_min_epu16 (__m512i __A, __m512i __B) | 929 | _mm512_min_epu16 (__m512i __A, __m512i __B) |
| 745 | { | 930 | { |
| 746 | return (__m512i)__builtin_ia32_pminuw512((__v32hi)__A, (__v32hi)__B); | 931 | return (__m512i)__builtin_ia32_pminuw512((__v32hi)__A, (__v32hi)__B); |
| 747 | } | 932 | } |
| 748 | 933 | ||
| 749 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 934 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 750 | _mm512_maskz_min_epu16 (__mmask32 __M, __m512i __A, __m512i __B) | 935 | _mm512_maskz_min_epu16 (__mmask32 __M, __m512i __A, __m512i __B) |
| 751 | { | 936 | { |
| 752 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, | 937 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, |
| ... | @@ -754,7 +939,7 @@ _mm512_maskz_min_epu16 (__mmask32 __M, __m512i __A, __m512i __B) | ... | @@ -754,7 +939,7 @@ _mm512_maskz_min_epu16 (__mmask32 __M, __m512i __A, __m512i __B) |
| 754 | (__v32hi)_mm512_setzero_si512()); | 939 | (__v32hi)_mm512_setzero_si512()); |
| 755 | } | 940 | } |
| 756 | 941 | ||
| 757 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 942 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 758 | _mm512_mask_min_epu16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) | 943 | _mm512_mask_min_epu16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) |
| 759 | { | 944 | { |
| 760 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, | 945 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M, |
| ... | @@ -762,13 +947,13 @@ _mm512_mask_min_epu16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) | ... | @@ -762,13 +947,13 @@ _mm512_mask_min_epu16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) |
| 762 | (__v32hi)__W); | 947 | (__v32hi)__W); |
| 763 | } | 948 | } |
| 764 | 949 | ||
| 765 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 950 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 766 | _mm512_shuffle_epi8(__m512i __A, __m512i __B) | 951 | _mm512_shuffle_epi8(__m512i __A, __m512i __B) |
| 767 | { | 952 | { |
| 768 | return (__m512i)__builtin_ia32_pshufb512((__v64qi)__A,(__v64qi)__B); | 953 | return (__m512i)__builtin_ia32_pshufb512((__v64qi)__A,(__v64qi)__B); |
| 769 | } | 954 | } |
| 770 | 955 | ||
| 771 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 956 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 772 | _mm512_mask_shuffle_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) | 957 | _mm512_mask_shuffle_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) |
| 773 | { | 958 | { |
| 774 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, | 959 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| ... | @@ -776,7 +961,7 @@ _mm512_mask_shuffle_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) | ... | @@ -776,7 +961,7 @@ _mm512_mask_shuffle_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) |
| 776 | (__v64qi)__W); | 961 | (__v64qi)__W); |
| 777 | } | 962 | } |
| 778 | 963 | ||
| 779 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 964 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 780 | _mm512_maskz_shuffle_epi8(__mmask64 __U, __m512i __A, __m512i __B) | 965 | _mm512_maskz_shuffle_epi8(__mmask64 __U, __m512i __A, __m512i __B) |
| 781 | { | 966 | { |
| 782 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, | 967 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| ... | @@ -784,126 +969,102 @@ _mm512_maskz_shuffle_epi8(__mmask64 __U, __m512i __A, __m512i __B) | ... | @@ -784,126 +969,102 @@ _mm512_maskz_shuffle_epi8(__mmask64 __U, __m512i __A, __m512i __B) |
| 784 | (__v64qi)_mm512_setzero_si512()); | 969 | (__v64qi)_mm512_setzero_si512()); |
| 785 | } | 970 | } |
| 786 | 971 | ||
| 787 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 972 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 788 | _mm512_subs_epi8 (__m512i __A, __m512i __B) | 973 | _mm512_subs_epi8 (__m512i __A, __m512i __B) |
| 789 | { | 974 | { |
| 790 | return (__m512i) __builtin_ia32_psubsb512_mask ((__v64qi) __A, | 975 | return (__m512i)__builtin_ia32_psubsb512((__v64qi)__A, (__v64qi)__B); |
| 791 | (__v64qi) __B, | ||
| 792 | (__v64qi) _mm512_setzero_si512(), | ||
| 793 | (__mmask64) -1); | ||
| 794 | } | 976 | } |
| 795 | 977 | ||
| 796 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 978 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 797 | _mm512_mask_subs_epi8 (__m512i __W, __mmask64 __U, __m512i __A, | 979 | _mm512_mask_subs_epi8 (__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) |
| 798 | __m512i __B) | ||
| 799 | { | 980 | { |
| 800 | return (__m512i) __builtin_ia32_psubsb512_mask ((__v64qi) __A, | 981 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 801 | (__v64qi) __B, | 982 | (__v64qi)_mm512_subs_epi8(__A, __B), |
| 802 | (__v64qi) __W, | 983 | (__v64qi)__W); |
| 803 | (__mmask64) __U); | ||
| 804 | } | 984 | } |
| 805 | 985 | ||
| 806 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 986 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 807 | _mm512_maskz_subs_epi8 (__mmask64 __U, __m512i __A, __m512i __B) | 987 | _mm512_maskz_subs_epi8 (__mmask64 __U, __m512i __A, __m512i __B) |
| 808 | { | 988 | { |
| 809 | return (__m512i) __builtin_ia32_psubsb512_mask ((__v64qi) __A, | 989 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 810 | (__v64qi) __B, | 990 | (__v64qi)_mm512_subs_epi8(__A, __B), |
| 811 | (__v64qi) _mm512_setzero_si512(), | 991 | (__v64qi)_mm512_setzero_si512()); |
| 812 | (__mmask64) __U); | ||
| 813 | } | 992 | } |
| 814 | 993 | ||
| 815 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 994 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 816 | _mm512_subs_epi16 (__m512i __A, __m512i __B) | 995 | _mm512_subs_epi16 (__m512i __A, __m512i __B) |
| 817 | { | 996 | { |
| 818 | return (__m512i) __builtin_ia32_psubsw512_mask ((__v32hi) __A, | 997 | return (__m512i)__builtin_ia32_psubsw512((__v32hi)__A, (__v32hi)__B); |
| 819 | (__v32hi) __B, | ||
| 820 | (__v32hi) _mm512_setzero_si512(), | ||
| 821 | (__mmask32) -1); | ||
| 822 | } | 998 | } |
| 823 | 999 | ||
| 824 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1000 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 825 | _mm512_mask_subs_epi16 (__m512i __W, __mmask32 __U, __m512i __A, | 1001 | _mm512_mask_subs_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) |
| 826 | __m512i __B) | ||
| 827 | { | 1002 | { |
| 828 | return (__m512i) __builtin_ia32_psubsw512_mask ((__v32hi) __A, | 1003 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 829 | (__v32hi) __B, | 1004 | (__v32hi)_mm512_subs_epi16(__A, __B), |
| 830 | (__v32hi) __W, | 1005 | (__v32hi)__W); |
| 831 | (__mmask32) __U); | ||
| 832 | } | 1006 | } |
| 833 | 1007 | ||
| 834 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1008 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 835 | _mm512_maskz_subs_epi16 (__mmask32 __U, __m512i __A, __m512i __B) | 1009 | _mm512_maskz_subs_epi16 (__mmask32 __U, __m512i __A, __m512i __B) |
| 836 | { | 1010 | { |
| 837 | return (__m512i) __builtin_ia32_psubsw512_mask ((__v32hi) __A, | 1011 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 838 | (__v32hi) __B, | 1012 | (__v32hi)_mm512_subs_epi16(__A, __B), |
| 839 | (__v32hi) _mm512_setzero_si512(), | 1013 | (__v32hi)_mm512_setzero_si512()); |
| 840 | (__mmask32) __U); | ||
| 841 | } | 1014 | } |
| 842 | 1015 | ||
| 843 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1016 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 844 | _mm512_subs_epu8 (__m512i __A, __m512i __B) | 1017 | _mm512_subs_epu8 (__m512i __A, __m512i __B) |
| 845 | { | 1018 | { |
| 846 | return (__m512i) __builtin_ia32_psubusb512_mask ((__v64qi) __A, | 1019 | return (__m512i)__builtin_ia32_psubusb512((__v64qi) __A, (__v64qi) __B); |
| 847 | (__v64qi) __B, | ||
| 848 | (__v64qi) _mm512_setzero_si512(), | ||
| 849 | (__mmask64) -1); | ||
| 850 | } | 1020 | } |
| 851 | 1021 | ||
| 852 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1022 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 853 | _mm512_mask_subs_epu8 (__m512i __W, __mmask64 __U, __m512i __A, | 1023 | _mm512_mask_subs_epu8 (__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) |
| 854 | __m512i __B) | ||
| 855 | { | 1024 | { |
| 856 | return (__m512i) __builtin_ia32_psubusb512_mask ((__v64qi) __A, | 1025 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 857 | (__v64qi) __B, | 1026 | (__v64qi)_mm512_subs_epu8(__A, __B), |
| 858 | (__v64qi) __W, | 1027 | (__v64qi)__W); |
| 859 | (__mmask64) __U); | ||
| 860 | } | 1028 | } |
| 861 | 1029 | ||
| 862 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1030 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 863 | _mm512_maskz_subs_epu8 (__mmask64 __U, __m512i __A, __m512i __B) | 1031 | _mm512_maskz_subs_epu8 (__mmask64 __U, __m512i __A, __m512i __B) |
| 864 | { | 1032 | { |
| 865 | return (__m512i) __builtin_ia32_psubusb512_mask ((__v64qi) __A, | 1033 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 866 | (__v64qi) __B, | 1034 | (__v64qi)_mm512_subs_epu8(__A, __B), |
| 867 | (__v64qi) _mm512_setzero_si512(), | 1035 | (__v64qi)_mm512_setzero_si512()); |
| 868 | (__mmask64) __U); | ||
| 869 | } | 1036 | } |
| 870 | 1037 | ||
| 871 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1038 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 872 | _mm512_subs_epu16 (__m512i __A, __m512i __B) | 1039 | _mm512_subs_epu16 (__m512i __A, __m512i __B) |
| 873 | { | 1040 | { |
| 874 | return (__m512i) __builtin_ia32_psubusw512_mask ((__v32hi) __A, | 1041 | return (__m512i)__builtin_ia32_psubusw512((__v32hi) __A, (__v32hi) __B); |
| 875 | (__v32hi) __B, | ||
| 876 | (__v32hi) _mm512_setzero_si512(), | ||
| 877 | (__mmask32) -1); | ||
| 878 | } | 1042 | } |
| 879 | 1043 | ||
| 880 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1044 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 881 | _mm512_mask_subs_epu16 (__m512i __W, __mmask32 __U, __m512i __A, | 1045 | _mm512_mask_subs_epu16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) |
| 882 | __m512i __B) | ||
| 883 | { | 1046 | { |
| 884 | return (__m512i) __builtin_ia32_psubusw512_mask ((__v32hi) __A, | 1047 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 885 | (__v32hi) __B, | 1048 | (__v32hi)_mm512_subs_epu16(__A, __B), |
| 886 | (__v32hi) __W, | 1049 | (__v32hi)__W); |
| 887 | (__mmask32) __U); | ||
| 888 | } | 1050 | } |
| 889 | 1051 | ||
| 890 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1052 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 891 | _mm512_maskz_subs_epu16 (__mmask32 __U, __m512i __A, __m512i __B) | 1053 | _mm512_maskz_subs_epu16 (__mmask32 __U, __m512i __A, __m512i __B) |
| 892 | { | 1054 | { |
| 893 | return (__m512i) __builtin_ia32_psubusw512_mask ((__v32hi) __A, | 1055 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 894 | (__v32hi) __B, | 1056 | (__v32hi)_mm512_subs_epu16(__A, __B), |
| 895 | (__v32hi) _mm512_setzero_si512(), | 1057 | (__v32hi)_mm512_setzero_si512()); |
| 896 | (__mmask32) __U); | ||
| 897 | } | 1058 | } |
| 898 | 1059 | ||
| 899 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1060 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 900 | _mm512_permutex2var_epi16(__m512i __A, __m512i __I, __m512i __B) | 1061 | _mm512_permutex2var_epi16(__m512i __A, __m512i __I, __m512i __B) |
| 901 | { | 1062 | { |
| 902 | return (__m512i)__builtin_ia32_vpermi2varhi512((__v32hi)__A, (__v32hi)__I, | 1063 | return (__m512i)__builtin_ia32_vpermi2varhi512((__v32hi)__A, (__v32hi)__I, |
| 903 | (__v32hi)__B); | 1064 | (__v32hi)__B); |
| 904 | } | 1065 | } |
| 905 | 1066 | ||
| 906 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1067 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 907 | _mm512_mask_permutex2var_epi16(__m512i __A, __mmask32 __U, __m512i __I, | 1068 | _mm512_mask_permutex2var_epi16(__m512i __A, __mmask32 __U, __m512i __I, |
| 908 | __m512i __B) | 1069 | __m512i __B) |
| 909 | { | 1070 | { |
| ... | @@ -912,7 +1073,7 @@ _mm512_mask_permutex2var_epi16(__m512i __A, __mmask32 __U, __m512i __I, | ... | @@ -912,7 +1073,7 @@ _mm512_mask_permutex2var_epi16(__m512i __A, __mmask32 __U, __m512i __I, |
| 912 | (__v32hi)__A); | 1073 | (__v32hi)__A); |
| 913 | } | 1074 | } |
| 914 | 1075 | ||
| 915 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1076 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 916 | _mm512_mask2_permutex2var_epi16(__m512i __A, __m512i __I, __mmask32 __U, | 1077 | _mm512_mask2_permutex2var_epi16(__m512i __A, __m512i __I, __mmask32 __U, |
| 917 | __m512i __B) | 1078 | __m512i __B) |
| 918 | { | 1079 | { |
| ... | @@ -921,7 +1082,7 @@ _mm512_mask2_permutex2var_epi16(__m512i __A, __m512i __I, __mmask32 __U, | ... | @@ -921,7 +1082,7 @@ _mm512_mask2_permutex2var_epi16(__m512i __A, __m512i __I, __mmask32 __U, |
| 921 | (__v32hi)__I); | 1082 | (__v32hi)__I); |
| 922 | } | 1083 | } |
| 923 | 1084 | ||
| 924 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1085 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 925 | _mm512_maskz_permutex2var_epi16(__mmask32 __U, __m512i __A, __m512i __I, | 1086 | _mm512_maskz_permutex2var_epi16(__mmask32 __U, __m512i __A, __m512i __I, |
| 926 | __m512i __B) | 1087 | __m512i __B) |
| 927 | { | 1088 | { |
| ... | @@ -930,13 +1091,13 @@ _mm512_maskz_permutex2var_epi16(__mmask32 __U, __m512i __A, __m512i __I, | ... | @@ -930,13 +1091,13 @@ _mm512_maskz_permutex2var_epi16(__mmask32 __U, __m512i __A, __m512i __I, |
| 930 | (__v32hi)_mm512_setzero_si512()); | 1091 | (__v32hi)_mm512_setzero_si512()); |
| 931 | } | 1092 | } |
| 932 | 1093 | ||
| 933 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1094 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 934 | _mm512_mulhrs_epi16(__m512i __A, __m512i __B) | 1095 | _mm512_mulhrs_epi16(__m512i __A, __m512i __B) |
| 935 | { | 1096 | { |
| 936 | return (__m512i)__builtin_ia32_pmulhrsw512((__v32hi)__A, (__v32hi)__B); | 1097 | return (__m512i)__builtin_ia32_pmulhrsw512((__v32hi)__A, (__v32hi)__B); |
| 937 | } | 1098 | } |
| 938 | 1099 | ||
| 939 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1100 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 940 | _mm512_mask_mulhrs_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) | 1101 | _mm512_mask_mulhrs_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) |
| 941 | { | 1102 | { |
| 942 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1103 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -944,7 +1105,7 @@ _mm512_mask_mulhrs_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) | ... | @@ -944,7 +1105,7 @@ _mm512_mask_mulhrs_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) |
| 944 | (__v32hi)__W); | 1105 | (__v32hi)__W); |
| 945 | } | 1106 | } |
| 946 | 1107 | ||
| 947 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1108 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 948 | _mm512_maskz_mulhrs_epi16(__mmask32 __U, __m512i __A, __m512i __B) | 1109 | _mm512_maskz_mulhrs_epi16(__mmask32 __U, __m512i __A, __m512i __B) |
| 949 | { | 1110 | { |
| 950 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1111 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -952,13 +1113,13 @@ _mm512_maskz_mulhrs_epi16(__mmask32 __U, __m512i __A, __m512i __B) | ... | @@ -952,13 +1113,13 @@ _mm512_maskz_mulhrs_epi16(__mmask32 __U, __m512i __A, __m512i __B) |
| 952 | (__v32hi)_mm512_setzero_si512()); | 1113 | (__v32hi)_mm512_setzero_si512()); |
| 953 | } | 1114 | } |
| 954 | 1115 | ||
| 955 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1116 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 956 | _mm512_mulhi_epi16(__m512i __A, __m512i __B) | 1117 | _mm512_mulhi_epi16(__m512i __A, __m512i __B) |
| 957 | { | 1118 | { |
| 958 | return (__m512i)__builtin_ia32_pmulhw512((__v32hi) __A, (__v32hi) __B); | 1119 | return (__m512i)__builtin_ia32_pmulhw512((__v32hi) __A, (__v32hi) __B); |
| 959 | } | 1120 | } |
| 960 | 1121 | ||
| 961 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1122 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 962 | _mm512_mask_mulhi_epi16(__m512i __W, __mmask32 __U, __m512i __A, | 1123 | _mm512_mask_mulhi_epi16(__m512i __W, __mmask32 __U, __m512i __A, |
| 963 | __m512i __B) | 1124 | __m512i __B) |
| 964 | { | 1125 | { |
| ... | @@ -967,7 +1128,7 @@ _mm512_mask_mulhi_epi16(__m512i __W, __mmask32 __U, __m512i __A, | ... | @@ -967,7 +1128,7 @@ _mm512_mask_mulhi_epi16(__m512i __W, __mmask32 __U, __m512i __A, |
| 967 | (__v32hi)__W); | 1128 | (__v32hi)__W); |
| 968 | } | 1129 | } |
| 969 | 1130 | ||
| 970 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1131 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 971 | _mm512_maskz_mulhi_epi16(__mmask32 __U, __m512i __A, __m512i __B) | 1132 | _mm512_maskz_mulhi_epi16(__mmask32 __U, __m512i __A, __m512i __B) |
| 972 | { | 1133 | { |
| 973 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1134 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -975,13 +1136,13 @@ _mm512_maskz_mulhi_epi16(__mmask32 __U, __m512i __A, __m512i __B) | ... | @@ -975,13 +1136,13 @@ _mm512_maskz_mulhi_epi16(__mmask32 __U, __m512i __A, __m512i __B) |
| 975 | (__v32hi)_mm512_setzero_si512()); | 1136 | (__v32hi)_mm512_setzero_si512()); |
| 976 | } | 1137 | } |
| 977 | 1138 | ||
| 978 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1139 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 979 | _mm512_mulhi_epu16(__m512i __A, __m512i __B) | 1140 | _mm512_mulhi_epu16(__m512i __A, __m512i __B) |
| 980 | { | 1141 | { |
| 981 | return (__m512i)__builtin_ia32_pmulhuw512((__v32hi) __A, (__v32hi) __B); | 1142 | return (__m512i)__builtin_ia32_pmulhuw512((__v32hi) __A, (__v32hi) __B); |
| 982 | } | 1143 | } |
| 983 | 1144 | ||
| 984 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1145 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 985 | _mm512_mask_mulhi_epu16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) | 1146 | _mm512_mask_mulhi_epu16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) |
| 986 | { | 1147 | { |
| 987 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1148 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -989,7 +1150,7 @@ _mm512_mask_mulhi_epu16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) | ... | @@ -989,7 +1150,7 @@ _mm512_mask_mulhi_epu16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) |
| 989 | (__v32hi)__W); | 1150 | (__v32hi)__W); |
| 990 | } | 1151 | } |
| 991 | 1152 | ||
| 992 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1153 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 993 | _mm512_maskz_mulhi_epu16 (__mmask32 __U, __m512i __A, __m512i __B) | 1154 | _mm512_maskz_mulhi_epu16 (__mmask32 __U, __m512i __A, __m512i __B) |
| 994 | { | 1155 | { |
| 995 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1156 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -997,12 +1158,12 @@ _mm512_maskz_mulhi_epu16 (__mmask32 __U, __m512i __A, __m512i __B) | ... | @@ -997,12 +1158,12 @@ _mm512_maskz_mulhi_epu16 (__mmask32 __U, __m512i __A, __m512i __B) |
| 997 | (__v32hi)_mm512_setzero_si512()); | 1158 | (__v32hi)_mm512_setzero_si512()); |
| 998 | } | 1159 | } |
| 999 | 1160 | ||
| 1000 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1161 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1001 | _mm512_maddubs_epi16(__m512i __X, __m512i __Y) { | 1162 | _mm512_maddubs_epi16(__m512i __X, __m512i __Y) { |
| 1002 | return (__m512i)__builtin_ia32_pmaddubsw512((__v64qi)__X, (__v64qi)__Y); | 1163 | return (__m512i)__builtin_ia32_pmaddubsw512((__v64qi)__X, (__v64qi)__Y); |
| 1003 | } | 1164 | } |
| 1004 | 1165 | ||
| 1005 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1166 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1006 | _mm512_mask_maddubs_epi16(__m512i __W, __mmask32 __U, __m512i __X, | 1167 | _mm512_mask_maddubs_epi16(__m512i __W, __mmask32 __U, __m512i __X, |
| 1007 | __m512i __Y) { | 1168 | __m512i __Y) { |
| 1008 | return (__m512i)__builtin_ia32_selectw_512((__mmask32) __U, | 1169 | return (__m512i)__builtin_ia32_selectw_512((__mmask32) __U, |
| ... | @@ -1010,114 +1171,114 @@ _mm512_mask_maddubs_epi16(__m512i __W, __mmask32 __U, __m512i __X, | ... | @@ -1010,114 +1171,114 @@ _mm512_mask_maddubs_epi16(__m512i __W, __mmask32 __U, __m512i __X, |
| 1010 | (__v32hi)__W); | 1171 | (__v32hi)__W); |
| 1011 | } | 1172 | } |
| 1012 | 1173 | ||
| 1013 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1174 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1014 | _mm512_maskz_maddubs_epi16(__mmask32 __U, __m512i __X, __m512i __Y) { | 1175 | _mm512_maskz_maddubs_epi16(__mmask32 __U, __m512i __X, __m512i __Y) { |
| 1015 | return (__m512i)__builtin_ia32_selectw_512((__mmask32) __U, | 1176 | return (__m512i)__builtin_ia32_selectw_512((__mmask32) __U, |
| 1016 | (__v32hi)_mm512_maddubs_epi16(__X, __Y), | 1177 | (__v32hi)_mm512_maddubs_epi16(__X, __Y), |
| 1017 | (__v32hi)_mm512_setzero_si512()); | 1178 | (__v32hi)_mm512_setzero_si512()); |
| 1018 | } | 1179 | } |
| 1019 | 1180 | ||
| 1020 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1181 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1021 | _mm512_madd_epi16(__m512i __A, __m512i __B) { | 1182 | _mm512_madd_epi16(__m512i __A, __m512i __B) { |
| 1022 | return (__m512i)__builtin_ia32_pmaddwd512((__v32hi)__A, (__v32hi)__B); | 1183 | return (__m512i)__builtin_ia32_pmaddwd512((__v32hi)__A, (__v32hi)__B); |
| 1023 | } | 1184 | } |
| 1024 | 1185 | ||
| 1025 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1186 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1026 | _mm512_mask_madd_epi16(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) { | 1187 | _mm512_mask_madd_epi16(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) { |
| 1027 | return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U, | 1188 | return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U, |
| 1028 | (__v16si)_mm512_madd_epi16(__A, __B), | 1189 | (__v16si)_mm512_madd_epi16(__A, __B), |
| 1029 | (__v16si)__W); | 1190 | (__v16si)__W); |
| 1030 | } | 1191 | } |
| 1031 | 1192 | ||
| 1032 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1193 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1033 | _mm512_maskz_madd_epi16(__mmask16 __U, __m512i __A, __m512i __B) { | 1194 | _mm512_maskz_madd_epi16(__mmask16 __U, __m512i __A, __m512i __B) { |
| 1034 | return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U, | 1195 | return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U, |
| 1035 | (__v16si)_mm512_madd_epi16(__A, __B), | 1196 | (__v16si)_mm512_madd_epi16(__A, __B), |
| 1036 | (__v16si)_mm512_setzero_si512()); | 1197 | (__v16si)_mm512_setzero_si512()); |
| 1037 | } | 1198 | } |
| 1038 | 1199 | ||
| 1039 | static __inline__ __m256i __DEFAULT_FN_ATTRS | 1200 | static __inline__ __m256i __DEFAULT_FN_ATTRS512 |
| 1040 | _mm512_cvtsepi16_epi8 (__m512i __A) { | 1201 | _mm512_cvtsepi16_epi8 (__m512i __A) { |
| 1041 | return (__m256i) __builtin_ia32_pmovswb512_mask ((__v32hi) __A, | 1202 | return (__m256i) __builtin_ia32_pmovswb512_mask ((__v32hi) __A, |
| 1042 | (__v32qi)_mm256_setzero_si256(), | 1203 | (__v32qi)_mm256_setzero_si256(), |
| 1043 | (__mmask32) -1); | 1204 | (__mmask32) -1); |
| 1044 | } | 1205 | } |
| 1045 | 1206 | ||
| 1046 | static __inline__ __m256i __DEFAULT_FN_ATTRS | 1207 | static __inline__ __m256i __DEFAULT_FN_ATTRS512 |
| 1047 | _mm512_mask_cvtsepi16_epi8 (__m256i __O, __mmask32 __M, __m512i __A) { | 1208 | _mm512_mask_cvtsepi16_epi8 (__m256i __O, __mmask32 __M, __m512i __A) { |
| 1048 | return (__m256i) __builtin_ia32_pmovswb512_mask ((__v32hi) __A, | 1209 | return (__m256i) __builtin_ia32_pmovswb512_mask ((__v32hi) __A, |
| 1049 | (__v32qi)__O, | 1210 | (__v32qi)__O, |
| 1050 | __M); | 1211 | __M); |
| 1051 | } | 1212 | } |
| 1052 | 1213 | ||
| 1053 | static __inline__ __m256i __DEFAULT_FN_ATTRS | 1214 | static __inline__ __m256i __DEFAULT_FN_ATTRS512 |
| 1054 | _mm512_maskz_cvtsepi16_epi8 (__mmask32 __M, __m512i __A) { | 1215 | _mm512_maskz_cvtsepi16_epi8 (__mmask32 __M, __m512i __A) { |
| 1055 | return (__m256i) __builtin_ia32_pmovswb512_mask ((__v32hi) __A, | 1216 | return (__m256i) __builtin_ia32_pmovswb512_mask ((__v32hi) __A, |
| 1056 | (__v32qi) _mm256_setzero_si256(), | 1217 | (__v32qi) _mm256_setzero_si256(), |
| 1057 | __M); | 1218 | __M); |
| 1058 | } | 1219 | } |
| 1059 | 1220 | ||
| 1060 | static __inline__ __m256i __DEFAULT_FN_ATTRS | 1221 | static __inline__ __m256i __DEFAULT_FN_ATTRS512 |
| 1061 | _mm512_cvtusepi16_epi8 (__m512i __A) { | 1222 | _mm512_cvtusepi16_epi8 (__m512i __A) { |
| 1062 | return (__m256i) __builtin_ia32_pmovuswb512_mask ((__v32hi) __A, | 1223 | return (__m256i) __builtin_ia32_pmovuswb512_mask ((__v32hi) __A, |
| 1063 | (__v32qi) _mm256_setzero_si256(), | 1224 | (__v32qi) _mm256_setzero_si256(), |
| 1064 | (__mmask32) -1); | 1225 | (__mmask32) -1); |
| 1065 | } | 1226 | } |
| 1066 | 1227 | ||
| 1067 | static __inline__ __m256i __DEFAULT_FN_ATTRS | 1228 | static __inline__ __m256i __DEFAULT_FN_ATTRS512 |
| 1068 | _mm512_mask_cvtusepi16_epi8 (__m256i __O, __mmask32 __M, __m512i __A) { | 1229 | _mm512_mask_cvtusepi16_epi8 (__m256i __O, __mmask32 __M, __m512i __A) { |
| 1069 | return (__m256i) __builtin_ia32_pmovuswb512_mask ((__v32hi) __A, | 1230 | return (__m256i) __builtin_ia32_pmovuswb512_mask ((__v32hi) __A, |
| 1070 | (__v32qi) __O, | 1231 | (__v32qi) __O, |
| 1071 | __M); | 1232 | __M); |
| 1072 | } | 1233 | } |
| 1073 | 1234 | ||
| 1074 | static __inline__ __m256i __DEFAULT_FN_ATTRS | 1235 | static __inline__ __m256i __DEFAULT_FN_ATTRS512 |
| 1075 | _mm512_maskz_cvtusepi16_epi8 (__mmask32 __M, __m512i __A) { | 1236 | _mm512_maskz_cvtusepi16_epi8 (__mmask32 __M, __m512i __A) { |
| 1076 | return (__m256i) __builtin_ia32_pmovuswb512_mask ((__v32hi) __A, | 1237 | return (__m256i) __builtin_ia32_pmovuswb512_mask ((__v32hi) __A, |
| 1077 | (__v32qi) _mm256_setzero_si256(), | 1238 | (__v32qi) _mm256_setzero_si256(), |
| 1078 | __M); | 1239 | __M); |
| 1079 | } | 1240 | } |
| 1080 | 1241 | ||
| 1081 | static __inline__ __m256i __DEFAULT_FN_ATTRS | 1242 | static __inline__ __m256i __DEFAULT_FN_ATTRS512 |
| 1082 | _mm512_cvtepi16_epi8 (__m512i __A) { | 1243 | _mm512_cvtepi16_epi8 (__m512i __A) { |
| 1083 | return (__m256i) __builtin_ia32_pmovwb512_mask ((__v32hi) __A, | 1244 | return (__m256i) __builtin_ia32_pmovwb512_mask ((__v32hi) __A, |
| 1084 | (__v32qi) _mm256_undefined_si256(), | 1245 | (__v32qi) _mm256_undefined_si256(), |
| 1085 | (__mmask32) -1); | 1246 | (__mmask32) -1); |
| 1086 | } | 1247 | } |
| 1087 | 1248 | ||
| 1088 | static __inline__ __m256i __DEFAULT_FN_ATTRS | 1249 | static __inline__ __m256i __DEFAULT_FN_ATTRS512 |
| 1089 | _mm512_mask_cvtepi16_epi8 (__m256i __O, __mmask32 __M, __m512i __A) { | 1250 | _mm512_mask_cvtepi16_epi8 (__m256i __O, __mmask32 __M, __m512i __A) { |
| 1090 | return (__m256i) __builtin_ia32_pmovwb512_mask ((__v32hi) __A, | 1251 | return (__m256i) __builtin_ia32_pmovwb512_mask ((__v32hi) __A, |
| 1091 | (__v32qi) __O, | 1252 | (__v32qi) __O, |
| 1092 | __M); | 1253 | __M); |
| 1093 | } | 1254 | } |
| 1094 | 1255 | ||
| 1095 | static __inline__ __m256i __DEFAULT_FN_ATTRS | 1256 | static __inline__ __m256i __DEFAULT_FN_ATTRS512 |
| 1096 | _mm512_maskz_cvtepi16_epi8 (__mmask32 __M, __m512i __A) { | 1257 | _mm512_maskz_cvtepi16_epi8 (__mmask32 __M, __m512i __A) { |
| 1097 | return (__m256i) __builtin_ia32_pmovwb512_mask ((__v32hi) __A, | 1258 | return (__m256i) __builtin_ia32_pmovwb512_mask ((__v32hi) __A, |
| 1098 | (__v32qi) _mm256_setzero_si256(), | 1259 | (__v32qi) _mm256_setzero_si256(), |
| 1099 | __M); | 1260 | __M); |
| 1100 | } | 1261 | } |
| 1101 | 1262 | ||
| 1102 | static __inline__ void __DEFAULT_FN_ATTRS | 1263 | static __inline__ void __DEFAULT_FN_ATTRS512 |
| 1103 | _mm512_mask_cvtepi16_storeu_epi8 (void * __P, __mmask32 __M, __m512i __A) | 1264 | _mm512_mask_cvtepi16_storeu_epi8 (void * __P, __mmask32 __M, __m512i __A) |
| 1104 | { | 1265 | { |
| 1105 | __builtin_ia32_pmovwb512mem_mask ((__v32qi *) __P, (__v32hi) __A, __M); | 1266 | __builtin_ia32_pmovwb512mem_mask ((__v32qi *) __P, (__v32hi) __A, __M); |
| 1106 | } | 1267 | } |
| 1107 | 1268 | ||
| 1108 | static __inline__ void __DEFAULT_FN_ATTRS | 1269 | static __inline__ void __DEFAULT_FN_ATTRS512 |
| 1109 | _mm512_mask_cvtsepi16_storeu_epi8 (void * __P, __mmask32 __M, __m512i __A) | 1270 | _mm512_mask_cvtsepi16_storeu_epi8 (void * __P, __mmask32 __M, __m512i __A) |
| 1110 | { | 1271 | { |
| 1111 | __builtin_ia32_pmovswb512mem_mask ((__v32qi *) __P, (__v32hi) __A, __M); | 1272 | __builtin_ia32_pmovswb512mem_mask ((__v32qi *) __P, (__v32hi) __A, __M); |
| 1112 | } | 1273 | } |
| 1113 | 1274 | ||
| 1114 | static __inline__ void __DEFAULT_FN_ATTRS | 1275 | static __inline__ void __DEFAULT_FN_ATTRS512 |
| 1115 | _mm512_mask_cvtusepi16_storeu_epi8 (void * __P, __mmask32 __M, __m512i __A) | 1276 | _mm512_mask_cvtusepi16_storeu_epi8 (void * __P, __mmask32 __M, __m512i __A) |
| 1116 | { | 1277 | { |
| 1117 | __builtin_ia32_pmovuswb512mem_mask ((__v32qi *) __P, (__v32hi) __A, __M); | 1278 | __builtin_ia32_pmovuswb512mem_mask ((__v32qi *) __P, (__v32hi) __A, __M); |
| 1118 | } | 1279 | } |
| 1119 | 1280 | ||
| 1120 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1281 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1121 | _mm512_unpackhi_epi8(__m512i __A, __m512i __B) { | 1282 | _mm512_unpackhi_epi8(__m512i __A, __m512i __B) { |
| 1122 | return (__m512i)__builtin_shufflevector((__v64qi)__A, (__v64qi)__B, | 1283 | return (__m512i)__builtin_shufflevector((__v64qi)__A, (__v64qi)__B, |
| 1123 | 8, 64+8, 9, 64+9, | 1284 | 8, 64+8, 9, 64+9, |
| ... | @@ -1138,21 +1299,21 @@ _mm512_unpackhi_epi8(__m512i __A, __m512i __B) { | ... | @@ -1138,21 +1299,21 @@ _mm512_unpackhi_epi8(__m512i __A, __m512i __B) { |
| 1138 | 62, 64+62, 63, 64+63); | 1299 | 62, 64+62, 63, 64+63); |
| 1139 | } | 1300 | } |
| 1140 | 1301 | ||
| 1141 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1302 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1142 | _mm512_mask_unpackhi_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) { | 1303 | _mm512_mask_unpackhi_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) { |
| 1143 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, | 1304 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 1144 | (__v64qi)_mm512_unpackhi_epi8(__A, __B), | 1305 | (__v64qi)_mm512_unpackhi_epi8(__A, __B), |
| 1145 | (__v64qi)__W); | 1306 | (__v64qi)__W); |
| 1146 | } | 1307 | } |
| 1147 | 1308 | ||
| 1148 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1309 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1149 | _mm512_maskz_unpackhi_epi8(__mmask64 __U, __m512i __A, __m512i __B) { | 1310 | _mm512_maskz_unpackhi_epi8(__mmask64 __U, __m512i __A, __m512i __B) { |
| 1150 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, | 1311 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 1151 | (__v64qi)_mm512_unpackhi_epi8(__A, __B), | 1312 | (__v64qi)_mm512_unpackhi_epi8(__A, __B), |
| 1152 | (__v64qi)_mm512_setzero_si512()); | 1313 | (__v64qi)_mm512_setzero_si512()); |
| 1153 | } | 1314 | } |
| 1154 | 1315 | ||
| 1155 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1316 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1156 | _mm512_unpackhi_epi16(__m512i __A, __m512i __B) { | 1317 | _mm512_unpackhi_epi16(__m512i __A, __m512i __B) { |
| 1157 | return (__m512i)__builtin_shufflevector((__v32hi)__A, (__v32hi)__B, | 1318 | return (__m512i)__builtin_shufflevector((__v32hi)__A, (__v32hi)__B, |
| 1158 | 4, 32+4, 5, 32+5, | 1319 | 4, 32+4, 5, 32+5, |
| ... | @@ -1165,21 +1326,21 @@ _mm512_unpackhi_epi16(__m512i __A, __m512i __B) { | ... | @@ -1165,21 +1326,21 @@ _mm512_unpackhi_epi16(__m512i __A, __m512i __B) { |
| 1165 | 30, 32+30, 31, 32+31); | 1326 | 30, 32+30, 31, 32+31); |
| 1166 | } | 1327 | } |
| 1167 | 1328 | ||
| 1168 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1329 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1169 | _mm512_mask_unpackhi_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) { | 1330 | _mm512_mask_unpackhi_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) { |
| 1170 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1331 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 1171 | (__v32hi)_mm512_unpackhi_epi16(__A, __B), | 1332 | (__v32hi)_mm512_unpackhi_epi16(__A, __B), |
| 1172 | (__v32hi)__W); | 1333 | (__v32hi)__W); |
| 1173 | } | 1334 | } |
| 1174 | 1335 | ||
| 1175 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1336 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1176 | _mm512_maskz_unpackhi_epi16(__mmask32 __U, __m512i __A, __m512i __B) { | 1337 | _mm512_maskz_unpackhi_epi16(__mmask32 __U, __m512i __A, __m512i __B) { |
| 1177 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1338 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 1178 | (__v32hi)_mm512_unpackhi_epi16(__A, __B), | 1339 | (__v32hi)_mm512_unpackhi_epi16(__A, __B), |
| 1179 | (__v32hi)_mm512_setzero_si512()); | 1340 | (__v32hi)_mm512_setzero_si512()); |
| 1180 | } | 1341 | } |
| 1181 | 1342 | ||
| 1182 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1343 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1183 | _mm512_unpacklo_epi8(__m512i __A, __m512i __B) { | 1344 | _mm512_unpacklo_epi8(__m512i __A, __m512i __B) { |
| 1184 | return (__m512i)__builtin_shufflevector((__v64qi)__A, (__v64qi)__B, | 1345 | return (__m512i)__builtin_shufflevector((__v64qi)__A, (__v64qi)__B, |
| 1185 | 0, 64+0, 1, 64+1, | 1346 | 0, 64+0, 1, 64+1, |
| ... | @@ -1200,21 +1361,21 @@ _mm512_unpacklo_epi8(__m512i __A, __m512i __B) { | ... | @@ -1200,21 +1361,21 @@ _mm512_unpacklo_epi8(__m512i __A, __m512i __B) { |
| 1200 | 54, 64+54, 55, 64+55); | 1361 | 54, 64+54, 55, 64+55); |
| 1201 | } | 1362 | } |
| 1202 | 1363 | ||
| 1203 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1364 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1204 | _mm512_mask_unpacklo_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) { | 1365 | _mm512_mask_unpacklo_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) { |
| 1205 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, | 1366 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 1206 | (__v64qi)_mm512_unpacklo_epi8(__A, __B), | 1367 | (__v64qi)_mm512_unpacklo_epi8(__A, __B), |
| 1207 | (__v64qi)__W); | 1368 | (__v64qi)__W); |
| 1208 | } | 1369 | } |
| 1209 | 1370 | ||
| 1210 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1371 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1211 | _mm512_maskz_unpacklo_epi8(__mmask64 __U, __m512i __A, __m512i __B) { | 1372 | _mm512_maskz_unpacklo_epi8(__mmask64 __U, __m512i __A, __m512i __B) { |
| 1212 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, | 1373 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U, |
| 1213 | (__v64qi)_mm512_unpacklo_epi8(__A, __B), | 1374 | (__v64qi)_mm512_unpacklo_epi8(__A, __B), |
| 1214 | (__v64qi)_mm512_setzero_si512()); | 1375 | (__v64qi)_mm512_setzero_si512()); |
| 1215 | } | 1376 | } |
| 1216 | 1377 | ||
| 1217 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1378 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1218 | _mm512_unpacklo_epi16(__m512i __A, __m512i __B) { | 1379 | _mm512_unpacklo_epi16(__m512i __A, __m512i __B) { |
| 1219 | return (__m512i)__builtin_shufflevector((__v32hi)__A, (__v32hi)__B, | 1380 | return (__m512i)__builtin_shufflevector((__v32hi)__A, (__v32hi)__B, |
| 1220 | 0, 32+0, 1, 32+1, | 1381 | 0, 32+0, 1, 32+1, |
| ... | @@ -1227,21 +1388,21 @@ _mm512_unpacklo_epi16(__m512i __A, __m512i __B) { | ... | @@ -1227,21 +1388,21 @@ _mm512_unpacklo_epi16(__m512i __A, __m512i __B) { |
| 1227 | 26, 32+26, 27, 32+27); | 1388 | 26, 32+26, 27, 32+27); |
| 1228 | } | 1389 | } |
| 1229 | 1390 | ||
| 1230 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1391 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1231 | _mm512_mask_unpacklo_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) { | 1392 | _mm512_mask_unpacklo_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) { |
| 1232 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1393 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 1233 | (__v32hi)_mm512_unpacklo_epi16(__A, __B), | 1394 | (__v32hi)_mm512_unpacklo_epi16(__A, __B), |
| 1234 | (__v32hi)__W); | 1395 | (__v32hi)__W); |
| 1235 | } | 1396 | } |
| 1236 | 1397 | ||
| 1237 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1398 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1238 | _mm512_maskz_unpacklo_epi16(__mmask32 __U, __m512i __A, __m512i __B) { | 1399 | _mm512_maskz_unpacklo_epi16(__mmask32 __U, __m512i __A, __m512i __B) { |
| 1239 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1400 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| 1240 | (__v32hi)_mm512_unpacklo_epi16(__A, __B), | 1401 | (__v32hi)_mm512_unpacklo_epi16(__A, __B), |
| 1241 | (__v32hi)_mm512_setzero_si512()); | 1402 | (__v32hi)_mm512_setzero_si512()); |
| 1242 | } | 1403 | } |
| 1243 | 1404 | ||
| 1244 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1405 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1245 | _mm512_cvtepi8_epi16(__m256i __A) | 1406 | _mm512_cvtepi8_epi16(__m256i __A) |
| 1246 | { | 1407 | { |
| 1247 | /* This function always performs a signed extension, but __v32qi is a char | 1408 | /* This function always performs a signed extension, but __v32qi is a char |
| ... | @@ -1249,7 +1410,7 @@ _mm512_cvtepi8_epi16(__m256i __A) | ... | @@ -1249,7 +1410,7 @@ _mm512_cvtepi8_epi16(__m256i __A) |
| 1249 | return (__m512i)__builtin_convertvector((__v32qs)__A, __v32hi); | 1410 | return (__m512i)__builtin_convertvector((__v32qs)__A, __v32hi); |
| 1250 | } | 1411 | } |
| 1251 | 1412 | ||
| 1252 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1413 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1253 | _mm512_mask_cvtepi8_epi16(__m512i __W, __mmask32 __U, __m256i __A) | 1414 | _mm512_mask_cvtepi8_epi16(__m512i __W, __mmask32 __U, __m256i __A) |
| 1254 | { | 1415 | { |
| 1255 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1416 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1257,7 +1418,7 @@ _mm512_mask_cvtepi8_epi16(__m512i __W, __mmask32 __U, __m256i __A) | ... | @@ -1257,7 +1418,7 @@ _mm512_mask_cvtepi8_epi16(__m512i __W, __mmask32 __U, __m256i __A) |
| 1257 | (__v32hi)__W); | 1418 | (__v32hi)__W); |
| 1258 | } | 1419 | } |
| 1259 | 1420 | ||
| 1260 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1421 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1261 | _mm512_maskz_cvtepi8_epi16(__mmask32 __U, __m256i __A) | 1422 | _mm512_maskz_cvtepi8_epi16(__mmask32 __U, __m256i __A) |
| 1262 | { | 1423 | { |
| 1263 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1424 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1265,13 +1426,13 @@ _mm512_maskz_cvtepi8_epi16(__mmask32 __U, __m256i __A) | ... | @@ -1265,13 +1426,13 @@ _mm512_maskz_cvtepi8_epi16(__mmask32 __U, __m256i __A) |
| 1265 | (__v32hi)_mm512_setzero_si512()); | 1426 | (__v32hi)_mm512_setzero_si512()); |
| 1266 | } | 1427 | } |
| 1267 | 1428 | ||
| 1268 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1429 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1269 | _mm512_cvtepu8_epi16(__m256i __A) | 1430 | _mm512_cvtepu8_epi16(__m256i __A) |
| 1270 | { | 1431 | { |
| 1271 | return (__m512i)__builtin_convertvector((__v32qu)__A, __v32hi); | 1432 | return (__m512i)__builtin_convertvector((__v32qu)__A, __v32hi); |
| 1272 | } | 1433 | } |
| 1273 | 1434 | ||
| 1274 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1435 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1275 | _mm512_mask_cvtepu8_epi16(__m512i __W, __mmask32 __U, __m256i __A) | 1436 | _mm512_mask_cvtepu8_epi16(__m512i __W, __mmask32 __U, __m256i __A) |
| 1276 | { | 1437 | { |
| 1277 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1438 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1279,7 +1440,7 @@ _mm512_mask_cvtepu8_epi16(__m512i __W, __mmask32 __U, __m256i __A) | ... | @@ -1279,7 +1440,7 @@ _mm512_mask_cvtepu8_epi16(__m512i __W, __mmask32 __U, __m256i __A) |
| 1279 | (__v32hi)__W); | 1440 | (__v32hi)__W); |
| 1280 | } | 1441 | } |
| 1281 | 1442 | ||
| 1282 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1443 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1283 | _mm512_maskz_cvtepu8_epi16(__mmask32 __U, __m256i __A) | 1444 | _mm512_maskz_cvtepu8_epi16(__mmask32 __U, __m256i __A) |
| 1284 | { | 1445 | { |
| 1285 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1446 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1320,13 +1481,13 @@ _mm512_maskz_cvtepu8_epi16(__mmask32 __U, __m256i __A) | ... | @@ -1320,13 +1481,13 @@ _mm512_maskz_cvtepu8_epi16(__mmask32 __U, __m256i __A) |
| 1320 | (imm)), \ | 1481 | (imm)), \ |
| 1321 | (__v32hi)_mm512_setzero_si512()) | 1482 | (__v32hi)_mm512_setzero_si512()) |
| 1322 | 1483 | ||
| 1323 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1484 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1324 | _mm512_sllv_epi16(__m512i __A, __m512i __B) | 1485 | _mm512_sllv_epi16(__m512i __A, __m512i __B) |
| 1325 | { | 1486 | { |
| 1326 | return (__m512i)__builtin_ia32_psllv32hi((__v32hi) __A, (__v32hi) __B); | 1487 | return (__m512i)__builtin_ia32_psllv32hi((__v32hi) __A, (__v32hi) __B); |
| 1327 | } | 1488 | } |
| 1328 | 1489 | ||
| 1329 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1490 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1330 | _mm512_mask_sllv_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) | 1491 | _mm512_mask_sllv_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) |
| 1331 | { | 1492 | { |
| 1332 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1493 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1334,7 +1495,7 @@ _mm512_mask_sllv_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) | ... | @@ -1334,7 +1495,7 @@ _mm512_mask_sllv_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) |
| 1334 | (__v32hi)__W); | 1495 | (__v32hi)__W); |
| 1335 | } | 1496 | } |
| 1336 | 1497 | ||
| 1337 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1498 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1338 | _mm512_maskz_sllv_epi16(__mmask32 __U, __m512i __A, __m512i __B) | 1499 | _mm512_maskz_sllv_epi16(__mmask32 __U, __m512i __A, __m512i __B) |
| 1339 | { | 1500 | { |
| 1340 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1501 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1342,13 +1503,13 @@ _mm512_maskz_sllv_epi16(__mmask32 __U, __m512i __A, __m512i __B) | ... | @@ -1342,13 +1503,13 @@ _mm512_maskz_sllv_epi16(__mmask32 __U, __m512i __A, __m512i __B) |
| 1342 | (__v32hi)_mm512_setzero_si512()); | 1503 | (__v32hi)_mm512_setzero_si512()); |
| 1343 | } | 1504 | } |
| 1344 | 1505 | ||
| 1345 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1506 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1346 | _mm512_sll_epi16(__m512i __A, __m128i __B) | 1507 | _mm512_sll_epi16(__m512i __A, __m128i __B) |
| 1347 | { | 1508 | { |
| 1348 | return (__m512i)__builtin_ia32_psllw512((__v32hi) __A, (__v8hi) __B); | 1509 | return (__m512i)__builtin_ia32_psllw512((__v32hi) __A, (__v8hi) __B); |
| 1349 | } | 1510 | } |
| 1350 | 1511 | ||
| 1351 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1512 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1352 | _mm512_mask_sll_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) | 1513 | _mm512_mask_sll_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) |
| 1353 | { | 1514 | { |
| 1354 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1515 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1356,7 +1517,7 @@ _mm512_mask_sll_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) | ... | @@ -1356,7 +1517,7 @@ _mm512_mask_sll_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) |
| 1356 | (__v32hi)__W); | 1517 | (__v32hi)__W); |
| 1357 | } | 1518 | } |
| 1358 | 1519 | ||
| 1359 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1520 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1360 | _mm512_maskz_sll_epi16(__mmask32 __U, __m512i __A, __m128i __B) | 1521 | _mm512_maskz_sll_epi16(__mmask32 __U, __m512i __A, __m128i __B) |
| 1361 | { | 1522 | { |
| 1362 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1523 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1364,13 +1525,13 @@ _mm512_maskz_sll_epi16(__mmask32 __U, __m512i __A, __m128i __B) | ... | @@ -1364,13 +1525,13 @@ _mm512_maskz_sll_epi16(__mmask32 __U, __m512i __A, __m128i __B) |
| 1364 | (__v32hi)_mm512_setzero_si512()); | 1525 | (__v32hi)_mm512_setzero_si512()); |
| 1365 | } | 1526 | } |
| 1366 | 1527 | ||
| 1367 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1528 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1368 | _mm512_slli_epi16(__m512i __A, int __B) | 1529 | _mm512_slli_epi16(__m512i __A, int __B) |
| 1369 | { | 1530 | { |
| 1370 | return (__m512i)__builtin_ia32_psllwi512((__v32hi)__A, __B); | 1531 | return (__m512i)__builtin_ia32_psllwi512((__v32hi)__A, __B); |
| 1371 | } | 1532 | } |
| 1372 | 1533 | ||
| 1373 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1534 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1374 | _mm512_mask_slli_epi16(__m512i __W, __mmask32 __U, __m512i __A, int __B) | 1535 | _mm512_mask_slli_epi16(__m512i __W, __mmask32 __U, __m512i __A, int __B) |
| 1375 | { | 1536 | { |
| 1376 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1537 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1378,7 +1539,7 @@ _mm512_mask_slli_epi16(__m512i __W, __mmask32 __U, __m512i __A, int __B) | ... | @@ -1378,7 +1539,7 @@ _mm512_mask_slli_epi16(__m512i __W, __mmask32 __U, __m512i __A, int __B) |
| 1378 | (__v32hi)__W); | 1539 | (__v32hi)__W); |
| 1379 | } | 1540 | } |
| 1380 | 1541 | ||
| 1381 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1542 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1382 | _mm512_maskz_slli_epi16(__mmask32 __U, __m512i __A, int __B) | 1543 | _mm512_maskz_slli_epi16(__mmask32 __U, __m512i __A, int __B) |
| 1383 | { | 1544 | { |
| 1384 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1545 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1389,13 +1550,13 @@ _mm512_maskz_slli_epi16(__mmask32 __U, __m512i __A, int __B) | ... | @@ -1389,13 +1550,13 @@ _mm512_maskz_slli_epi16(__mmask32 __U, __m512i __A, int __B) |
| 1389 | #define _mm512_bslli_epi128(a, imm) \ | 1550 | #define _mm512_bslli_epi128(a, imm) \ |
| 1390 | (__m512i)__builtin_ia32_pslldqi512_byteshift((__v8di)(__m512i)(a), (int)(imm)) | 1551 | (__m512i)__builtin_ia32_pslldqi512_byteshift((__v8di)(__m512i)(a), (int)(imm)) |
| 1391 | 1552 | ||
| 1392 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1553 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1393 | _mm512_srlv_epi16(__m512i __A, __m512i __B) | 1554 | _mm512_srlv_epi16(__m512i __A, __m512i __B) |
| 1394 | { | 1555 | { |
| 1395 | return (__m512i)__builtin_ia32_psrlv32hi((__v32hi)__A, (__v32hi)__B); | 1556 | return (__m512i)__builtin_ia32_psrlv32hi((__v32hi)__A, (__v32hi)__B); |
| 1396 | } | 1557 | } |
| 1397 | 1558 | ||
| 1398 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1559 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1399 | _mm512_mask_srlv_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) | 1560 | _mm512_mask_srlv_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) |
| 1400 | { | 1561 | { |
| 1401 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1562 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1403,7 +1564,7 @@ _mm512_mask_srlv_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) | ... | @@ -1403,7 +1564,7 @@ _mm512_mask_srlv_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) |
| 1403 | (__v32hi)__W); | 1564 | (__v32hi)__W); |
| 1404 | } | 1565 | } |
| 1405 | 1566 | ||
| 1406 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1567 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1407 | _mm512_maskz_srlv_epi16(__mmask32 __U, __m512i __A, __m512i __B) | 1568 | _mm512_maskz_srlv_epi16(__mmask32 __U, __m512i __A, __m512i __B) |
| 1408 | { | 1569 | { |
| 1409 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1570 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1411,13 +1572,13 @@ _mm512_maskz_srlv_epi16(__mmask32 __U, __m512i __A, __m512i __B) | ... | @@ -1411,13 +1572,13 @@ _mm512_maskz_srlv_epi16(__mmask32 __U, __m512i __A, __m512i __B) |
| 1411 | (__v32hi)_mm512_setzero_si512()); | 1572 | (__v32hi)_mm512_setzero_si512()); |
| 1412 | } | 1573 | } |
| 1413 | 1574 | ||
| 1414 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1575 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1415 | _mm512_srav_epi16(__m512i __A, __m512i __B) | 1576 | _mm512_srav_epi16(__m512i __A, __m512i __B) |
| 1416 | { | 1577 | { |
| 1417 | return (__m512i)__builtin_ia32_psrav32hi((__v32hi)__A, (__v32hi)__B); | 1578 | return (__m512i)__builtin_ia32_psrav32hi((__v32hi)__A, (__v32hi)__B); |
| 1418 | } | 1579 | } |
| 1419 | 1580 | ||
| 1420 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1581 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1421 | _mm512_mask_srav_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) | 1582 | _mm512_mask_srav_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) |
| 1422 | { | 1583 | { |
| 1423 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1584 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1425,7 +1586,7 @@ _mm512_mask_srav_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) | ... | @@ -1425,7 +1586,7 @@ _mm512_mask_srav_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) |
| 1425 | (__v32hi)__W); | 1586 | (__v32hi)__W); |
| 1426 | } | 1587 | } |
| 1427 | 1588 | ||
| 1428 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1589 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1429 | _mm512_maskz_srav_epi16(__mmask32 __U, __m512i __A, __m512i __B) | 1590 | _mm512_maskz_srav_epi16(__mmask32 __U, __m512i __A, __m512i __B) |
| 1430 | { | 1591 | { |
| 1431 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1592 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1433,13 +1594,13 @@ _mm512_maskz_srav_epi16(__mmask32 __U, __m512i __A, __m512i __B) | ... | @@ -1433,13 +1594,13 @@ _mm512_maskz_srav_epi16(__mmask32 __U, __m512i __A, __m512i __B) |
| 1433 | (__v32hi)_mm512_setzero_si512()); | 1594 | (__v32hi)_mm512_setzero_si512()); |
| 1434 | } | 1595 | } |
| 1435 | 1596 | ||
| 1436 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1597 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1437 | _mm512_sra_epi16(__m512i __A, __m128i __B) | 1598 | _mm512_sra_epi16(__m512i __A, __m128i __B) |
| 1438 | { | 1599 | { |
| 1439 | return (__m512i)__builtin_ia32_psraw512((__v32hi) __A, (__v8hi) __B); | 1600 | return (__m512i)__builtin_ia32_psraw512((__v32hi) __A, (__v8hi) __B); |
| 1440 | } | 1601 | } |
| 1441 | 1602 | ||
| 1442 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1603 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1443 | _mm512_mask_sra_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) | 1604 | _mm512_mask_sra_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) |
| 1444 | { | 1605 | { |
| 1445 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1606 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1447,7 +1608,7 @@ _mm512_mask_sra_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) | ... | @@ -1447,7 +1608,7 @@ _mm512_mask_sra_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) |
| 1447 | (__v32hi)__W); | 1608 | (__v32hi)__W); |
| 1448 | } | 1609 | } |
| 1449 | 1610 | ||
| 1450 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1611 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1451 | _mm512_maskz_sra_epi16(__mmask32 __U, __m512i __A, __m128i __B) | 1612 | _mm512_maskz_sra_epi16(__mmask32 __U, __m512i __A, __m128i __B) |
| 1452 | { | 1613 | { |
| 1453 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1614 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1455,13 +1616,13 @@ _mm512_maskz_sra_epi16(__mmask32 __U, __m512i __A, __m128i __B) | ... | @@ -1455,13 +1616,13 @@ _mm512_maskz_sra_epi16(__mmask32 __U, __m512i __A, __m128i __B) |
| 1455 | (__v32hi)_mm512_setzero_si512()); | 1616 | (__v32hi)_mm512_setzero_si512()); |
| 1456 | } | 1617 | } |
| 1457 | 1618 | ||
| 1458 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1619 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1459 | _mm512_srai_epi16(__m512i __A, int __B) | 1620 | _mm512_srai_epi16(__m512i __A, int __B) |
| 1460 | { | 1621 | { |
| 1461 | return (__m512i)__builtin_ia32_psrawi512((__v32hi)__A, __B); | 1622 | return (__m512i)__builtin_ia32_psrawi512((__v32hi)__A, __B); |
| 1462 | } | 1623 | } |
| 1463 | 1624 | ||
| 1464 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1625 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1465 | _mm512_mask_srai_epi16(__m512i __W, __mmask32 __U, __m512i __A, int __B) | 1626 | _mm512_mask_srai_epi16(__m512i __W, __mmask32 __U, __m512i __A, int __B) |
| 1466 | { | 1627 | { |
| 1467 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1628 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1469,7 +1630,7 @@ _mm512_mask_srai_epi16(__m512i __W, __mmask32 __U, __m512i __A, int __B) | ... | @@ -1469,7 +1630,7 @@ _mm512_mask_srai_epi16(__m512i __W, __mmask32 __U, __m512i __A, int __B) |
| 1469 | (__v32hi)__W); | 1630 | (__v32hi)__W); |
| 1470 | } | 1631 | } |
| 1471 | 1632 | ||
| 1472 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1633 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1473 | _mm512_maskz_srai_epi16(__mmask32 __U, __m512i __A, int __B) | 1634 | _mm512_maskz_srai_epi16(__mmask32 __U, __m512i __A, int __B) |
| 1474 | { | 1635 | { |
| 1475 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1636 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1477,13 +1638,13 @@ _mm512_maskz_srai_epi16(__mmask32 __U, __m512i __A, int __B) | ... | @@ -1477,13 +1638,13 @@ _mm512_maskz_srai_epi16(__mmask32 __U, __m512i __A, int __B) |
| 1477 | (__v32hi)_mm512_setzero_si512()); | 1638 | (__v32hi)_mm512_setzero_si512()); |
| 1478 | } | 1639 | } |
| 1479 | 1640 | ||
| 1480 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1641 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1481 | _mm512_srl_epi16(__m512i __A, __m128i __B) | 1642 | _mm512_srl_epi16(__m512i __A, __m128i __B) |
| 1482 | { | 1643 | { |
| 1483 | return (__m512i)__builtin_ia32_psrlw512((__v32hi) __A, (__v8hi) __B); | 1644 | return (__m512i)__builtin_ia32_psrlw512((__v32hi) __A, (__v8hi) __B); |
| 1484 | } | 1645 | } |
| 1485 | 1646 | ||
| 1486 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1647 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1487 | _mm512_mask_srl_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) | 1648 | _mm512_mask_srl_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) |
| 1488 | { | 1649 | { |
| 1489 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1650 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1491,7 +1652,7 @@ _mm512_mask_srl_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) | ... | @@ -1491,7 +1652,7 @@ _mm512_mask_srl_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) |
| 1491 | (__v32hi)__W); | 1652 | (__v32hi)__W); |
| 1492 | } | 1653 | } |
| 1493 | 1654 | ||
| 1494 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1655 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1495 | _mm512_maskz_srl_epi16(__mmask32 __U, __m512i __A, __m128i __B) | 1656 | _mm512_maskz_srl_epi16(__mmask32 __U, __m512i __A, __m128i __B) |
| 1496 | { | 1657 | { |
| 1497 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1658 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1499,13 +1660,13 @@ _mm512_maskz_srl_epi16(__mmask32 __U, __m512i __A, __m128i __B) | ... | @@ -1499,13 +1660,13 @@ _mm512_maskz_srl_epi16(__mmask32 __U, __m512i __A, __m128i __B) |
| 1499 | (__v32hi)_mm512_setzero_si512()); | 1660 | (__v32hi)_mm512_setzero_si512()); |
| 1500 | } | 1661 | } |
| 1501 | 1662 | ||
| 1502 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1663 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1503 | _mm512_srli_epi16(__m512i __A, int __B) | 1664 | _mm512_srli_epi16(__m512i __A, int __B) |
| 1504 | { | 1665 | { |
| 1505 | return (__m512i)__builtin_ia32_psrlwi512((__v32hi)__A, __B); | 1666 | return (__m512i)__builtin_ia32_psrlwi512((__v32hi)__A, __B); |
| 1506 | } | 1667 | } |
| 1507 | 1668 | ||
| 1508 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1669 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1509 | _mm512_mask_srli_epi16(__m512i __W, __mmask32 __U, __m512i __A, int __B) | 1670 | _mm512_mask_srli_epi16(__m512i __W, __mmask32 __U, __m512i __A, int __B) |
| 1510 | { | 1671 | { |
| 1511 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1672 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1513,7 +1674,7 @@ _mm512_mask_srli_epi16(__m512i __W, __mmask32 __U, __m512i __A, int __B) | ... | @@ -1513,7 +1674,7 @@ _mm512_mask_srli_epi16(__m512i __W, __mmask32 __U, __m512i __A, int __B) |
| 1513 | (__v32hi)__W); | 1674 | (__v32hi)__W); |
| 1514 | } | 1675 | } |
| 1515 | 1676 | ||
| 1516 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1677 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1517 | _mm512_maskz_srli_epi16(__mmask32 __U, __m512i __A, int __B) | 1678 | _mm512_maskz_srli_epi16(__mmask32 __U, __m512i __A, int __B) |
| 1518 | { | 1679 | { |
| 1519 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, | 1680 | return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U, |
| ... | @@ -1524,7 +1685,7 @@ _mm512_maskz_srli_epi16(__mmask32 __U, __m512i __A, int __B) | ... | @@ -1524,7 +1685,7 @@ _mm512_maskz_srli_epi16(__mmask32 __U, __m512i __A, int __B) |
| 1524 | #define _mm512_bsrli_epi128(a, imm) \ | 1685 | #define _mm512_bsrli_epi128(a, imm) \ |
| 1525 | (__m512i)__builtin_ia32_psrldqi512_byteshift((__v8di)(__m512i)(a), (int)(imm)) | 1686 | (__m512i)__builtin_ia32_psrldqi512_byteshift((__v8di)(__m512i)(a), (int)(imm)) |
| 1526 | 1687 | ||
| 1527 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1688 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1528 | _mm512_mask_mov_epi16 (__m512i __W, __mmask32 __U, __m512i __A) | 1689 | _mm512_mask_mov_epi16 (__m512i __W, __mmask32 __U, __m512i __A) |
| 1529 | { | 1690 | { |
| 1530 | return (__m512i) __builtin_ia32_selectw_512 ((__mmask32) __U, | 1691 | return (__m512i) __builtin_ia32_selectw_512 ((__mmask32) __U, |
| ... | @@ -1532,7 +1693,7 @@ _mm512_mask_mov_epi16 (__m512i __W, __mmask32 __U, __m512i __A) | ... | @@ -1532,7 +1693,7 @@ _mm512_mask_mov_epi16 (__m512i __W, __mmask32 __U, __m512i __A) |
| 1532 | (__v32hi) __W); | 1693 | (__v32hi) __W); |
| 1533 | } | 1694 | } |
| 1534 | 1695 | ||
| 1535 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1696 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1536 | _mm512_maskz_mov_epi16 (__mmask32 __U, __m512i __A) | 1697 | _mm512_maskz_mov_epi16 (__mmask32 __U, __m512i __A) |
| 1537 | { | 1698 | { |
| 1538 | return (__m512i) __builtin_ia32_selectw_512 ((__mmask32) __U, | 1699 | return (__m512i) __builtin_ia32_selectw_512 ((__mmask32) __U, |
| ... | @@ -1540,7 +1701,7 @@ _mm512_maskz_mov_epi16 (__mmask32 __U, __m512i __A) | ... | @@ -1540,7 +1701,7 @@ _mm512_maskz_mov_epi16 (__mmask32 __U, __m512i __A) |
| 1540 | (__v32hi) _mm512_setzero_si512 ()); | 1701 | (__v32hi) _mm512_setzero_si512 ()); |
| 1541 | } | 1702 | } |
| 1542 | 1703 | ||
| 1543 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1704 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1544 | _mm512_mask_mov_epi8 (__m512i __W, __mmask64 __U, __m512i __A) | 1705 | _mm512_mask_mov_epi8 (__m512i __W, __mmask64 __U, __m512i __A) |
| 1545 | { | 1706 | { |
| 1546 | return (__m512i) __builtin_ia32_selectb_512 ((__mmask64) __U, | 1707 | return (__m512i) __builtin_ia32_selectb_512 ((__mmask64) __U, |
| ... | @@ -1548,7 +1709,7 @@ _mm512_mask_mov_epi8 (__m512i __W, __mmask64 __U, __m512i __A) | ... | @@ -1548,7 +1709,7 @@ _mm512_mask_mov_epi8 (__m512i __W, __mmask64 __U, __m512i __A) |
| 1548 | (__v64qi) __W); | 1709 | (__v64qi) __W); |
| 1549 | } | 1710 | } |
| 1550 | 1711 | ||
| 1551 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1712 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1552 | _mm512_maskz_mov_epi8 (__mmask64 __U, __m512i __A) | 1713 | _mm512_maskz_mov_epi8 (__mmask64 __U, __m512i __A) |
| 1553 | { | 1714 | { |
| 1554 | return (__m512i) __builtin_ia32_selectb_512 ((__mmask64) __U, | 1715 | return (__m512i) __builtin_ia32_selectb_512 ((__mmask64) __U, |
| ... | @@ -1556,7 +1717,7 @@ _mm512_maskz_mov_epi8 (__mmask64 __U, __m512i __A) | ... | @@ -1556,7 +1717,7 @@ _mm512_maskz_mov_epi8 (__mmask64 __U, __m512i __A) |
| 1556 | (__v64qi) _mm512_setzero_si512 ()); | 1717 | (__v64qi) _mm512_setzero_si512 ()); |
| 1557 | } | 1718 | } |
| 1558 | 1719 | ||
| 1559 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1720 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1560 | _mm512_mask_set1_epi8 (__m512i __O, __mmask64 __M, char __A) | 1721 | _mm512_mask_set1_epi8 (__m512i __O, __mmask64 __M, char __A) |
| 1561 | { | 1722 | { |
| 1562 | return (__m512i) __builtin_ia32_selectb_512(__M, | 1723 | return (__m512i) __builtin_ia32_selectb_512(__M, |
| ... | @@ -1564,7 +1725,7 @@ _mm512_mask_set1_epi8 (__m512i __O, __mmask64 __M, char __A) | ... | @@ -1564,7 +1725,7 @@ _mm512_mask_set1_epi8 (__m512i __O, __mmask64 __M, char __A) |
| 1564 | (__v64qi) __O); | 1725 | (__v64qi) __O); |
| 1565 | } | 1726 | } |
| 1566 | 1727 | ||
| 1567 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1728 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1568 | _mm512_maskz_set1_epi8 (__mmask64 __M, char __A) | 1729 | _mm512_maskz_set1_epi8 (__mmask64 __M, char __A) |
| 1569 | { | 1730 | { |
| 1570 | return (__m512i) __builtin_ia32_selectb_512(__M, | 1731 | return (__m512i) __builtin_ia32_selectb_512(__M, |
| ... | @@ -1572,21 +1733,30 @@ _mm512_maskz_set1_epi8 (__mmask64 __M, char __A) | ... | @@ -1572,21 +1733,30 @@ _mm512_maskz_set1_epi8 (__mmask64 __M, char __A) |
| 1572 | (__v64qi) _mm512_setzero_si512()); | 1733 | (__v64qi) _mm512_setzero_si512()); |
| 1573 | } | 1734 | } |
| 1574 | 1735 | ||
| 1575 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS | 1736 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS512 |
| 1576 | _mm512_kunpackd (__mmask64 __A, __mmask64 __B) | 1737 | _mm512_kunpackd (__mmask64 __A, __mmask64 __B) |
| 1577 | { | 1738 | { |
| 1578 | return (__mmask64) __builtin_ia32_kunpckdi ((__mmask64) __A, | 1739 | return (__mmask64) __builtin_ia32_kunpckdi ((__mmask64) __A, |
| 1579 | (__mmask64) __B); | 1740 | (__mmask64) __B); |
| 1580 | } | 1741 | } |
| 1581 | 1742 | ||
| 1582 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS | 1743 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS512 |
| 1583 | _mm512_kunpackw (__mmask32 __A, __mmask32 __B) | 1744 | _mm512_kunpackw (__mmask32 __A, __mmask32 __B) |
| 1584 | { | 1745 | { |
| 1585 | return (__mmask32) __builtin_ia32_kunpcksi ((__mmask32) __A, | 1746 | return (__mmask32) __builtin_ia32_kunpcksi ((__mmask32) __A, |
| 1586 | (__mmask32) __B); | 1747 | (__mmask32) __B); |
| 1587 | } | 1748 | } |
| 1588 | 1749 | ||
| 1589 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1750 | static __inline __m512i __DEFAULT_FN_ATTRS512 |
| 1751 | _mm512_loadu_epi16 (void const *__P) | ||
| 1752 | { | ||
| 1753 | struct __loadu_epi16 { | ||
| 1754 | __m512i __v; | ||
| 1755 | } __attribute__((__packed__, __may_alias__)); | ||
| 1756 | return ((struct __loadu_epi16*)__P)->__v; | ||
| 1757 | } | ||
| 1758 | |||
| 1759 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 | ||
| 1590 | _mm512_mask_loadu_epi16 (__m512i __W, __mmask32 __U, void const *__P) | 1760 | _mm512_mask_loadu_epi16 (__m512i __W, __mmask32 __U, void const *__P) |
| 1591 | { | 1761 | { |
| 1592 | return (__m512i) __builtin_ia32_loaddquhi512_mask ((__v32hi *) __P, | 1762 | return (__m512i) __builtin_ia32_loaddquhi512_mask ((__v32hi *) __P, |
| ... | @@ -1594,7 +1764,7 @@ _mm512_mask_loadu_epi16 (__m512i __W, __mmask32 __U, void const *__P) | ... | @@ -1594,7 +1764,7 @@ _mm512_mask_loadu_epi16 (__m512i __W, __mmask32 __U, void const *__P) |
| 1594 | (__mmask32) __U); | 1764 | (__mmask32) __U); |
| 1595 | } | 1765 | } |
| 1596 | 1766 | ||
| 1597 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1767 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1598 | _mm512_maskz_loadu_epi16 (__mmask32 __U, void const *__P) | 1768 | _mm512_maskz_loadu_epi16 (__mmask32 __U, void const *__P) |
| 1599 | { | 1769 | { |
| 1600 | return (__m512i) __builtin_ia32_loaddquhi512_mask ((__v32hi *) __P, | 1770 | return (__m512i) __builtin_ia32_loaddquhi512_mask ((__v32hi *) __P, |
| ... | @@ -1603,7 +1773,16 @@ _mm512_maskz_loadu_epi16 (__mmask32 __U, void const *__P) | ... | @@ -1603,7 +1773,16 @@ _mm512_maskz_loadu_epi16 (__mmask32 __U, void const *__P) |
| 1603 | (__mmask32) __U); | 1773 | (__mmask32) __U); |
| 1604 | } | 1774 | } |
| 1605 | 1775 | ||
| 1606 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1776 | static __inline __m512i __DEFAULT_FN_ATTRS512 |
| 1777 | _mm512_loadu_epi8 (void const *__P) | ||
| 1778 | { | ||
| 1779 | struct __loadu_epi8 { | ||
| 1780 | __m512i __v; | ||
| 1781 | } __attribute__((__packed__, __may_alias__)); | ||
| 1782 | return ((struct __loadu_epi8*)__P)->__v; | ||
| 1783 | } | ||
| 1784 | |||
| 1785 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 | ||
| 1607 | _mm512_mask_loadu_epi8 (__m512i __W, __mmask64 __U, void const *__P) | 1786 | _mm512_mask_loadu_epi8 (__m512i __W, __mmask64 __U, void const *__P) |
| 1608 | { | 1787 | { |
| 1609 | return (__m512i) __builtin_ia32_loaddquqi512_mask ((__v64qi *) __P, | 1788 | return (__m512i) __builtin_ia32_loaddquqi512_mask ((__v64qi *) __P, |
| ... | @@ -1611,7 +1790,7 @@ _mm512_mask_loadu_epi8 (__m512i __W, __mmask64 __U, void const *__P) | ... | @@ -1611,7 +1790,7 @@ _mm512_mask_loadu_epi8 (__m512i __W, __mmask64 __U, void const *__P) |
| 1611 | (__mmask64) __U); | 1790 | (__mmask64) __U); |
| 1612 | } | 1791 | } |
| 1613 | 1792 | ||
| 1614 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1793 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1615 | _mm512_maskz_loadu_epi8 (__mmask64 __U, void const *__P) | 1794 | _mm512_maskz_loadu_epi8 (__mmask64 __U, void const *__P) |
| 1616 | { | 1795 | { |
| 1617 | return (__m512i) __builtin_ia32_loaddquqi512_mask ((__v64qi *) __P, | 1796 | return (__m512i) __builtin_ia32_loaddquqi512_mask ((__v64qi *) __P, |
| ... | @@ -1619,7 +1798,17 @@ _mm512_maskz_loadu_epi8 (__mmask64 __U, void const *__P) | ... | @@ -1619,7 +1798,17 @@ _mm512_maskz_loadu_epi8 (__mmask64 __U, void const *__P) |
| 1619 | _mm512_setzero_si512 (), | 1798 | _mm512_setzero_si512 (), |
| 1620 | (__mmask64) __U); | 1799 | (__mmask64) __U); |
| 1621 | } | 1800 | } |
| 1622 | static __inline__ void __DEFAULT_FN_ATTRS | 1801 | |
| 1802 | static __inline void __DEFAULT_FN_ATTRS512 | ||
| 1803 | _mm512_storeu_epi16 (void *__P, __m512i __A) | ||
| 1804 | { | ||
| 1805 | struct __storeu_epi16 { | ||
| 1806 | __m512i __v; | ||
| 1807 | } __attribute__((__packed__, __may_alias__)); | ||
| 1808 | ((struct __storeu_epi16*)__P)->__v = __A; | ||
| 1809 | } | ||
| 1810 | |||
| 1811 | static __inline__ void __DEFAULT_FN_ATTRS512 | ||
| 1623 | _mm512_mask_storeu_epi16 (void *__P, __mmask32 __U, __m512i __A) | 1812 | _mm512_mask_storeu_epi16 (void *__P, __mmask32 __U, __m512i __A) |
| 1624 | { | 1813 | { |
| 1625 | __builtin_ia32_storedquhi512_mask ((__v32hi *) __P, | 1814 | __builtin_ia32_storedquhi512_mask ((__v32hi *) __P, |
| ... | @@ -1627,7 +1816,16 @@ _mm512_mask_storeu_epi16 (void *__P, __mmask32 __U, __m512i __A) | ... | @@ -1627,7 +1816,16 @@ _mm512_mask_storeu_epi16 (void *__P, __mmask32 __U, __m512i __A) |
| 1627 | (__mmask32) __U); | 1816 | (__mmask32) __U); |
| 1628 | } | 1817 | } |
| 1629 | 1818 | ||
| 1630 | static __inline__ void __DEFAULT_FN_ATTRS | 1819 | static __inline void __DEFAULT_FN_ATTRS512 |
| 1820 | _mm512_storeu_epi8 (void *__P, __m512i __A) | ||
| 1821 | { | ||
| 1822 | struct __storeu_epi8 { | ||
| 1823 | __m512i __v; | ||
| 1824 | } __attribute__((__packed__, __may_alias__)); | ||
| 1825 | ((struct __storeu_epi8*)__P)->__v = __A; | ||
| 1826 | } | ||
| 1827 | |||
| 1828 | static __inline__ void __DEFAULT_FN_ATTRS512 | ||
| 1631 | _mm512_mask_storeu_epi8 (void *__P, __mmask64 __U, __m512i __A) | 1829 | _mm512_mask_storeu_epi8 (void *__P, __mmask64 __U, __m512i __A) |
| 1632 | { | 1830 | { |
| 1633 | __builtin_ia32_storedquqi512_mask ((__v64qi *) __P, | 1831 | __builtin_ia32_storedquqi512_mask ((__v64qi *) __P, |
| ... | @@ -1635,86 +1833,86 @@ _mm512_mask_storeu_epi8 (void *__P, __mmask64 __U, __m512i __A) | ... | @@ -1635,86 +1833,86 @@ _mm512_mask_storeu_epi8 (void *__P, __mmask64 __U, __m512i __A) |
| 1635 | (__mmask64) __U); | 1833 | (__mmask64) __U); |
| 1636 | } | 1834 | } |
| 1637 | 1835 | ||
| 1638 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS | 1836 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS512 |
| 1639 | _mm512_test_epi8_mask (__m512i __A, __m512i __B) | 1837 | _mm512_test_epi8_mask (__m512i __A, __m512i __B) |
| 1640 | { | 1838 | { |
| 1641 | return _mm512_cmpneq_epi8_mask (_mm512_and_epi32 (__A, __B), | 1839 | return _mm512_cmpneq_epi8_mask (_mm512_and_epi32 (__A, __B), |
| 1642 | _mm512_setzero_si512()); | 1840 | _mm512_setzero_si512()); |
| 1643 | } | 1841 | } |
| 1644 | 1842 | ||
| 1645 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS | 1843 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS512 |
| 1646 | _mm512_mask_test_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B) | 1844 | _mm512_mask_test_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B) |
| 1647 | { | 1845 | { |
| 1648 | return _mm512_mask_cmpneq_epi8_mask (__U, _mm512_and_epi32 (__A, __B), | 1846 | return _mm512_mask_cmpneq_epi8_mask (__U, _mm512_and_epi32 (__A, __B), |
| 1649 | _mm512_setzero_si512()); | 1847 | _mm512_setzero_si512()); |
| 1650 | } | 1848 | } |
| 1651 | 1849 | ||
| 1652 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS | 1850 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS512 |
| 1653 | _mm512_test_epi16_mask (__m512i __A, __m512i __B) | 1851 | _mm512_test_epi16_mask (__m512i __A, __m512i __B) |
| 1654 | { | 1852 | { |
| 1655 | return _mm512_cmpneq_epi16_mask (_mm512_and_epi32 (__A, __B), | 1853 | return _mm512_cmpneq_epi16_mask (_mm512_and_epi32 (__A, __B), |
| 1656 | _mm512_setzero_si512()); | 1854 | _mm512_setzero_si512()); |
| 1657 | } | 1855 | } |
| 1658 | 1856 | ||
| 1659 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS | 1857 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS512 |
| 1660 | _mm512_mask_test_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B) | 1858 | _mm512_mask_test_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B) |
| 1661 | { | 1859 | { |
| 1662 | return _mm512_mask_cmpneq_epi16_mask (__U, _mm512_and_epi32 (__A, __B), | 1860 | return _mm512_mask_cmpneq_epi16_mask (__U, _mm512_and_epi32 (__A, __B), |
| 1663 | _mm512_setzero_si512()); | 1861 | _mm512_setzero_si512()); |
| 1664 | } | 1862 | } |
| 1665 | 1863 | ||
| 1666 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS | 1864 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS512 |
| 1667 | _mm512_testn_epi8_mask (__m512i __A, __m512i __B) | 1865 | _mm512_testn_epi8_mask (__m512i __A, __m512i __B) |
| 1668 | { | 1866 | { |
| 1669 | return _mm512_cmpeq_epi8_mask (_mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); | 1867 | return _mm512_cmpeq_epi8_mask (_mm512_and_epi32 (__A, __B), _mm512_setzero_si512()); |
| 1670 | } | 1868 | } |
| 1671 | 1869 | ||
| 1672 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS | 1870 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS512 |
| 1673 | _mm512_mask_testn_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B) | 1871 | _mm512_mask_testn_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B) |
| 1674 | { | 1872 | { |
| 1675 | return _mm512_mask_cmpeq_epi8_mask (__U, _mm512_and_epi32 (__A, __B), | 1873 | return _mm512_mask_cmpeq_epi8_mask (__U, _mm512_and_epi32 (__A, __B), |
| 1676 | _mm512_setzero_si512()); | 1874 | _mm512_setzero_si512()); |
| 1677 | } | 1875 | } |
| 1678 | 1876 | ||
| 1679 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS | 1877 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS512 |
| 1680 | _mm512_testn_epi16_mask (__m512i __A, __m512i __B) | 1878 | _mm512_testn_epi16_mask (__m512i __A, __m512i __B) |
| 1681 | { | 1879 | { |
| 1682 | return _mm512_cmpeq_epi16_mask (_mm512_and_epi32 (__A, __B), | 1880 | return _mm512_cmpeq_epi16_mask (_mm512_and_epi32 (__A, __B), |
| 1683 | _mm512_setzero_si512()); | 1881 | _mm512_setzero_si512()); |
| 1684 | } | 1882 | } |
| 1685 | 1883 | ||
| 1686 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS | 1884 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS512 |
| 1687 | _mm512_mask_testn_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B) | 1885 | _mm512_mask_testn_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B) |
| 1688 | { | 1886 | { |
| 1689 | return _mm512_mask_cmpeq_epi16_mask (__U, _mm512_and_epi32 (__A, __B), | 1887 | return _mm512_mask_cmpeq_epi16_mask (__U, _mm512_and_epi32 (__A, __B), |
| 1690 | _mm512_setzero_si512()); | 1888 | _mm512_setzero_si512()); |
| 1691 | } | 1889 | } |
| 1692 | 1890 | ||
| 1693 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS | 1891 | static __inline__ __mmask64 __DEFAULT_FN_ATTRS512 |
| 1694 | _mm512_movepi8_mask (__m512i __A) | 1892 | _mm512_movepi8_mask (__m512i __A) |
| 1695 | { | 1893 | { |
| 1696 | return (__mmask64) __builtin_ia32_cvtb2mask512 ((__v64qi) __A); | 1894 | return (__mmask64) __builtin_ia32_cvtb2mask512 ((__v64qi) __A); |
| 1697 | } | 1895 | } |
| 1698 | 1896 | ||
| 1699 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS | 1897 | static __inline__ __mmask32 __DEFAULT_FN_ATTRS512 |
| 1700 | _mm512_movepi16_mask (__m512i __A) | 1898 | _mm512_movepi16_mask (__m512i __A) |
| 1701 | { | 1899 | { |
| 1702 | return (__mmask32) __builtin_ia32_cvtw2mask512 ((__v32hi) __A); | 1900 | return (__mmask32) __builtin_ia32_cvtw2mask512 ((__v32hi) __A); |
| 1703 | } | 1901 | } |
| 1704 | 1902 | ||
| 1705 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1903 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1706 | _mm512_movm_epi8 (__mmask64 __A) | 1904 | _mm512_movm_epi8 (__mmask64 __A) |
| 1707 | { | 1905 | { |
| 1708 | return (__m512i) __builtin_ia32_cvtmask2b512 (__A); | 1906 | return (__m512i) __builtin_ia32_cvtmask2b512 (__A); |
| 1709 | } | 1907 | } |
| 1710 | 1908 | ||
| 1711 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1909 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1712 | _mm512_movm_epi16 (__mmask32 __A) | 1910 | _mm512_movm_epi16 (__mmask32 __A) |
| 1713 | { | 1911 | { |
| 1714 | return (__m512i) __builtin_ia32_cvtmask2w512 (__A); | 1912 | return (__m512i) __builtin_ia32_cvtmask2w512 (__A); |
| 1715 | } | 1913 | } |
| 1716 | 1914 | ||
| 1717 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1915 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1718 | _mm512_broadcastb_epi8 (__m128i __A) | 1916 | _mm512_broadcastb_epi8 (__m128i __A) |
| 1719 | { | 1917 | { |
| 1720 | return (__m512i)__builtin_shufflevector((__v16qi) __A, (__v16qi) __A, | 1918 | return (__m512i)__builtin_shufflevector((__v16qi) __A, (__v16qi) __A, |
| ... | @@ -1724,7 +1922,7 @@ _mm512_broadcastb_epi8 (__m128i __A) | ... | @@ -1724,7 +1922,7 @@ _mm512_broadcastb_epi8 (__m128i __A) |
| 1724 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); | 1922 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); |
| 1725 | } | 1923 | } |
| 1726 | 1924 | ||
| 1727 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1925 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1728 | _mm512_mask_broadcastb_epi8 (__m512i __O, __mmask64 __M, __m128i __A) | 1926 | _mm512_mask_broadcastb_epi8 (__m512i __O, __mmask64 __M, __m128i __A) |
| 1729 | { | 1927 | { |
| 1730 | return (__m512i)__builtin_ia32_selectb_512(__M, | 1928 | return (__m512i)__builtin_ia32_selectb_512(__M, |
| ... | @@ -1732,7 +1930,7 @@ _mm512_mask_broadcastb_epi8 (__m512i __O, __mmask64 __M, __m128i __A) | ... | @@ -1732,7 +1930,7 @@ _mm512_mask_broadcastb_epi8 (__m512i __O, __mmask64 __M, __m128i __A) |
| 1732 | (__v64qi) __O); | 1930 | (__v64qi) __O); |
| 1733 | } | 1931 | } |
| 1734 | 1932 | ||
| 1735 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1933 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1736 | _mm512_maskz_broadcastb_epi8 (__mmask64 __M, __m128i __A) | 1934 | _mm512_maskz_broadcastb_epi8 (__mmask64 __M, __m128i __A) |
| 1737 | { | 1935 | { |
| 1738 | return (__m512i)__builtin_ia32_selectb_512(__M, | 1936 | return (__m512i)__builtin_ia32_selectb_512(__M, |
| ... | @@ -1740,7 +1938,7 @@ _mm512_maskz_broadcastb_epi8 (__mmask64 __M, __m128i __A) | ... | @@ -1740,7 +1938,7 @@ _mm512_maskz_broadcastb_epi8 (__mmask64 __M, __m128i __A) |
| 1740 | (__v64qi) _mm512_setzero_si512()); | 1938 | (__v64qi) _mm512_setzero_si512()); |
| 1741 | } | 1939 | } |
| 1742 | 1940 | ||
| 1743 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1941 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1744 | _mm512_mask_set1_epi16 (__m512i __O, __mmask32 __M, short __A) | 1942 | _mm512_mask_set1_epi16 (__m512i __O, __mmask32 __M, short __A) |
| 1745 | { | 1943 | { |
| 1746 | return (__m512i) __builtin_ia32_selectw_512(__M, | 1944 | return (__m512i) __builtin_ia32_selectw_512(__M, |
| ... | @@ -1748,7 +1946,7 @@ _mm512_mask_set1_epi16 (__m512i __O, __mmask32 __M, short __A) | ... | @@ -1748,7 +1946,7 @@ _mm512_mask_set1_epi16 (__m512i __O, __mmask32 __M, short __A) |
| 1748 | (__v32hi) __O); | 1946 | (__v32hi) __O); |
| 1749 | } | 1947 | } |
| 1750 | 1948 | ||
| 1751 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1949 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1752 | _mm512_maskz_set1_epi16 (__mmask32 __M, short __A) | 1950 | _mm512_maskz_set1_epi16 (__mmask32 __M, short __A) |
| 1753 | { | 1951 | { |
| 1754 | return (__m512i) __builtin_ia32_selectw_512(__M, | 1952 | return (__m512i) __builtin_ia32_selectw_512(__M, |
| ... | @@ -1756,7 +1954,7 @@ _mm512_maskz_set1_epi16 (__mmask32 __M, short __A) | ... | @@ -1756,7 +1954,7 @@ _mm512_maskz_set1_epi16 (__mmask32 __M, short __A) |
| 1756 | (__v32hi) _mm512_setzero_si512()); | 1954 | (__v32hi) _mm512_setzero_si512()); |
| 1757 | } | 1955 | } |
| 1758 | 1956 | ||
| 1759 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1957 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1760 | _mm512_broadcastw_epi16 (__m128i __A) | 1958 | _mm512_broadcastw_epi16 (__m128i __A) |
| 1761 | { | 1959 | { |
| 1762 | return (__m512i)__builtin_shufflevector((__v8hi) __A, (__v8hi) __A, | 1960 | return (__m512i)__builtin_shufflevector((__v8hi) __A, (__v8hi) __A, |
| ... | @@ -1764,7 +1962,7 @@ _mm512_broadcastw_epi16 (__m128i __A) | ... | @@ -1764,7 +1962,7 @@ _mm512_broadcastw_epi16 (__m128i __A) |
| 1764 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); | 1962 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); |
| 1765 | } | 1963 | } |
| 1766 | 1964 | ||
| 1767 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1965 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1768 | _mm512_mask_broadcastw_epi16 (__m512i __O, __mmask32 __M, __m128i __A) | 1966 | _mm512_mask_broadcastw_epi16 (__m512i __O, __mmask32 __M, __m128i __A) |
| 1769 | { | 1967 | { |
| 1770 | return (__m512i)__builtin_ia32_selectw_512(__M, | 1968 | return (__m512i)__builtin_ia32_selectw_512(__M, |
| ... | @@ -1772,7 +1970,7 @@ _mm512_mask_broadcastw_epi16 (__m512i __O, __mmask32 __M, __m128i __A) | ... | @@ -1772,7 +1970,7 @@ _mm512_mask_broadcastw_epi16 (__m512i __O, __mmask32 __M, __m128i __A) |
| 1772 | (__v32hi) __O); | 1970 | (__v32hi) __O); |
| 1773 | } | 1971 | } |
| 1774 | 1972 | ||
| 1775 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1973 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1776 | _mm512_maskz_broadcastw_epi16 (__mmask32 __M, __m128i __A) | 1974 | _mm512_maskz_broadcastw_epi16 (__mmask32 __M, __m128i __A) |
| 1777 | { | 1975 | { |
| 1778 | return (__m512i)__builtin_ia32_selectw_512(__M, | 1976 | return (__m512i)__builtin_ia32_selectw_512(__M, |
| ... | @@ -1780,13 +1978,13 @@ _mm512_maskz_broadcastw_epi16 (__mmask32 __M, __m128i __A) | ... | @@ -1780,13 +1978,13 @@ _mm512_maskz_broadcastw_epi16 (__mmask32 __M, __m128i __A) |
| 1780 | (__v32hi) _mm512_setzero_si512()); | 1978 | (__v32hi) _mm512_setzero_si512()); |
| 1781 | } | 1979 | } |
| 1782 | 1980 | ||
| 1783 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1981 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1784 | _mm512_permutexvar_epi16 (__m512i __A, __m512i __B) | 1982 | _mm512_permutexvar_epi16 (__m512i __A, __m512i __B) |
| 1785 | { | 1983 | { |
| 1786 | return (__m512i)__builtin_ia32_permvarhi512((__v32hi)__B, (__v32hi)__A); | 1984 | return (__m512i)__builtin_ia32_permvarhi512((__v32hi)__B, (__v32hi)__A); |
| 1787 | } | 1985 | } |
| 1788 | 1986 | ||
| 1789 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1987 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1790 | _mm512_maskz_permutexvar_epi16 (__mmask32 __M, __m512i __A, | 1988 | _mm512_maskz_permutexvar_epi16 (__mmask32 __M, __m512i __A, |
| 1791 | __m512i __B) | 1989 | __m512i __B) |
| 1792 | { | 1990 | { |
| ... | @@ -1795,7 +1993,7 @@ _mm512_maskz_permutexvar_epi16 (__mmask32 __M, __m512i __A, | ... | @@ -1795,7 +1993,7 @@ _mm512_maskz_permutexvar_epi16 (__mmask32 __M, __m512i __A, |
| 1795 | (__v32hi)_mm512_setzero_si512()); | 1993 | (__v32hi)_mm512_setzero_si512()); |
| 1796 | } | 1994 | } |
| 1797 | 1995 | ||
| 1798 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1996 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1799 | _mm512_mask_permutexvar_epi16 (__m512i __W, __mmask32 __M, __m512i __A, | 1997 | _mm512_mask_permutexvar_epi16 (__m512i __W, __mmask32 __M, __m512i __A, |
| 1800 | __m512i __B) | 1998 | __m512i __B) |
| 1801 | { | 1999 | { |
| ... | @@ -1832,15 +2030,14 @@ _mm512_mask_permutexvar_epi16 (__m512i __W, __mmask32 __M, __m512i __A, | ... | @@ -1832,15 +2030,14 @@ _mm512_mask_permutexvar_epi16 (__m512i __W, __mmask32 __M, __m512i __A, |
| 1832 | (__v32hi)_mm512_dbsad_epu8((A), (B), (imm)), \ | 2030 | (__v32hi)_mm512_dbsad_epu8((A), (B), (imm)), \ |
| 1833 | (__v32hi)_mm512_setzero_si512()) | 2031 | (__v32hi)_mm512_setzero_si512()) |
| 1834 | 2032 | ||
| 1835 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 2033 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1836 | _mm512_sad_epu8 (__m512i __A, __m512i __B) | 2034 | _mm512_sad_epu8 (__m512i __A, __m512i __B) |
| 1837 | { | 2035 | { |
| 1838 | return (__m512i) __builtin_ia32_psadbw512 ((__v64qi) __A, | 2036 | return (__m512i) __builtin_ia32_psadbw512 ((__v64qi) __A, |
| 1839 | (__v64qi) __B); | 2037 | (__v64qi) __B); |
| 1840 | } | 2038 | } |
| 1841 | 2039 | ||
| 1842 | 2040 | #undef __DEFAULT_FN_ATTRS512 | |
| 1843 | |||
| 1844 | #undef __DEFAULT_FN_ATTRS | 2041 | #undef __DEFAULT_FN_ATTRS |
| 1845 | 2042 | ||
| 1846 | #endif | 2043 | #endif |
c_headers/avx512dqintrin.h+216-86
| ... | @@ -29,180 +29,309 @@ | ... | @@ -29,180 +29,309 @@ |
| 29 | #define __AVX512DQINTRIN_H | 29 | #define __AVX512DQINTRIN_H |
| 30 | 30 | ||
| 31 | /* Define the default attributes for the functions in this file. */ | 31 | /* Define the default attributes for the functions in this file. */ |
| 32 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("avx512dq"), __min_vector_width__(512))) | 32 | #define __DEFAULT_FN_ATTRS512 __attribute__((__always_inline__, __nodebug__, __target__("avx512dq"), __min_vector_width__(512))) |
| 33 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("avx512dq"))) | ||
| 33 | 34 | ||
| 34 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 35 | static __inline __mmask8 __DEFAULT_FN_ATTRS |
| 36 | _knot_mask8(__mmask8 __M) | ||
| 37 | { | ||
| 38 | return __builtin_ia32_knotqi(__M); | ||
| 39 | } | ||
| 40 | |||
| 41 | static __inline__ __mmask8 __DEFAULT_FN_ATTRS | ||
| 42 | _kand_mask8(__mmask8 __A, __mmask8 __B) | ||
| 43 | { | ||
| 44 | return (__mmask8)__builtin_ia32_kandqi((__mmask8)__A, (__mmask8)__B); | ||
| 45 | } | ||
| 46 | |||
| 47 | static __inline__ __mmask8 __DEFAULT_FN_ATTRS | ||
| 48 | _kandn_mask8(__mmask8 __A, __mmask8 __B) | ||
| 49 | { | ||
| 50 | return (__mmask8)__builtin_ia32_kandnqi((__mmask8)__A, (__mmask8)__B); | ||
| 51 | } | ||
| 52 | |||
| 53 | static __inline__ __mmask8 __DEFAULT_FN_ATTRS | ||
| 54 | _kor_mask8(__mmask8 __A, __mmask8 __B) | ||
| 55 | { | ||
| 56 | return (__mmask8)__builtin_ia32_korqi((__mmask8)__A, (__mmask8)__B); | ||
| 57 | } | ||
| 58 | |||
| 59 | static __inline__ __mmask8 __DEFAULT_FN_ATTRS | ||
| 60 | _kxnor_mask8(__mmask8 __A, __mmask8 __B) | ||
| 61 | { | ||
| 62 | return (__mmask8)__builtin_ia32_kxnorqi((__mmask8)__A, (__mmask8)__B); | ||
| 63 | } | ||
| 64 | |||
| 65 | static __inline__ __mmask8 __DEFAULT_FN_ATTRS | ||
| 66 | _kxor_mask8(__mmask8 __A, __mmask8 __B) | ||
| 67 | { | ||
| 68 | return (__mmask8)__builtin_ia32_kxorqi((__mmask8)__A, (__mmask8)__B); | ||
| 69 | } | ||
| 70 | |||
| 71 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 72 | _kortestc_mask8_u8(__mmask8 __A, __mmask8 __B) | ||
| 73 | { | ||
| 74 | return (unsigned char)__builtin_ia32_kortestcqi(__A, __B); | ||
| 75 | } | ||
| 76 | |||
| 77 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 78 | _kortestz_mask8_u8(__mmask8 __A, __mmask8 __B) | ||
| 79 | { | ||
| 80 | return (unsigned char)__builtin_ia32_kortestzqi(__A, __B); | ||
| 81 | } | ||
| 82 | |||
| 83 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 84 | _kortest_mask8_u8(__mmask8 __A, __mmask8 __B, unsigned char *__C) { | ||
| 85 | *__C = (unsigned char)__builtin_ia32_kortestcqi(__A, __B); | ||
| 86 | return (unsigned char)__builtin_ia32_kortestzqi(__A, __B); | ||
| 87 | } | ||
| 88 | |||
| 89 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 90 | _ktestc_mask8_u8(__mmask8 __A, __mmask8 __B) | ||
| 91 | { | ||
| 92 | return (unsigned char)__builtin_ia32_ktestcqi(__A, __B); | ||
| 93 | } | ||
| 94 | |||
| 95 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 96 | _ktestz_mask8_u8(__mmask8 __A, __mmask8 __B) | ||
| 97 | { | ||
| 98 | return (unsigned char)__builtin_ia32_ktestzqi(__A, __B); | ||
| 99 | } | ||
| 100 | |||
| 101 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 102 | _ktest_mask8_u8(__mmask8 __A, __mmask8 __B, unsigned char *__C) { | ||
| 103 | *__C = (unsigned char)__builtin_ia32_ktestcqi(__A, __B); | ||
| 104 | return (unsigned char)__builtin_ia32_ktestzqi(__A, __B); | ||
| 105 | } | ||
| 106 | |||
| 107 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 108 | _ktestc_mask16_u8(__mmask16 __A, __mmask16 __B) | ||
| 109 | { | ||
| 110 | return (unsigned char)__builtin_ia32_ktestchi(__A, __B); | ||
| 111 | } | ||
| 112 | |||
| 113 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 114 | _ktestz_mask16_u8(__mmask16 __A, __mmask16 __B) | ||
| 115 | { | ||
| 116 | return (unsigned char)__builtin_ia32_ktestzhi(__A, __B); | ||
| 117 | } | ||
| 118 | |||
| 119 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 120 | _ktest_mask16_u8(__mmask16 __A, __mmask16 __B, unsigned char *__C) { | ||
| 121 | *__C = (unsigned char)__builtin_ia32_ktestchi(__A, __B); | ||
| 122 | return (unsigned char)__builtin_ia32_ktestzhi(__A, __B); | ||
| 123 | } | ||
| 124 | |||
| 125 | static __inline__ __mmask8 __DEFAULT_FN_ATTRS | ||
| 126 | _kadd_mask8(__mmask8 __A, __mmask8 __B) | ||
| 127 | { | ||
| 128 | return (__mmask8)__builtin_ia32_kaddqi((__mmask8)__A, (__mmask8)__B); | ||
| 129 | } | ||
| 130 | |||
| 131 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS | ||
| 132 | _kadd_mask16(__mmask16 __A, __mmask16 __B) | ||
| 133 | { | ||
| 134 | return (__mmask16)__builtin_ia32_kaddhi((__mmask16)__A, (__mmask16)__B); | ||
| 135 | } | ||
| 136 | |||
| 137 | #define _kshiftli_mask8(A, I) \ | ||
| 138 | (__mmask8)__builtin_ia32_kshiftliqi((__mmask8)(A), (unsigned int)(I)) | ||
| 139 | |||
| 140 | #define _kshiftri_mask8(A, I) \ | ||
| 141 | (__mmask8)__builtin_ia32_kshiftriqi((__mmask8)(A), (unsigned int)(I)) | ||
| 142 | |||
| 143 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | ||
| 144 | _cvtmask8_u32(__mmask8 __A) { | ||
| 145 | return (unsigned int)__builtin_ia32_kmovb((__mmask8)__A); | ||
| 146 | } | ||
| 147 | |||
| 148 | static __inline__ __mmask8 __DEFAULT_FN_ATTRS | ||
| 149 | _cvtu32_mask8(unsigned int __A) { | ||
| 150 | return (__mmask8)__builtin_ia32_kmovb((__mmask8)__A); | ||
| 151 | } | ||
| 152 | |||
| 153 | static __inline__ __mmask8 __DEFAULT_FN_ATTRS | ||
| 154 | _load_mask8(__mmask8 *__A) { | ||
| 155 | return (__mmask8)__builtin_ia32_kmovb(*(__mmask8 *)__A); | ||
| 156 | } | ||
| 157 | |||
| 158 | static __inline__ void __DEFAULT_FN_ATTRS | ||
| 159 | _store_mask8(__mmask8 *__A, __mmask8 __B) { | ||
| 160 | *(__mmask8 *)__A = __builtin_ia32_kmovb((__mmask8)__B); | ||
| 161 | } | ||
| 162 | |||
| 163 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 | ||
| 35 | _mm512_mullo_epi64 (__m512i __A, __m512i __B) { | 164 | _mm512_mullo_epi64 (__m512i __A, __m512i __B) { |
| 36 | return (__m512i) ((__v8du) __A * (__v8du) __B); | 165 | return (__m512i) ((__v8du) __A * (__v8du) __B); |
| 37 | } | 166 | } |
| 38 | 167 | ||
| 39 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 168 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 40 | _mm512_mask_mullo_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) { | 169 | _mm512_mask_mullo_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) { |
| 41 | return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U, | 170 | return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U, |
| 42 | (__v8di)_mm512_mullo_epi64(__A, __B), | 171 | (__v8di)_mm512_mullo_epi64(__A, __B), |
| 43 | (__v8di)__W); | 172 | (__v8di)__W); |
| 44 | } | 173 | } |
| 45 | 174 | ||
| 46 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 175 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 47 | _mm512_maskz_mullo_epi64(__mmask8 __U, __m512i __A, __m512i __B) { | 176 | _mm512_maskz_mullo_epi64(__mmask8 __U, __m512i __A, __m512i __B) { |
| 48 | return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U, | 177 | return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U, |
| 49 | (__v8di)_mm512_mullo_epi64(__A, __B), | 178 | (__v8di)_mm512_mullo_epi64(__A, __B), |
| 50 | (__v8di)_mm512_setzero_si512()); | 179 | (__v8di)_mm512_setzero_si512()); |
| 51 | } | 180 | } |
| 52 | 181 | ||
| 53 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 182 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 54 | _mm512_xor_pd(__m512d __A, __m512d __B) { | 183 | _mm512_xor_pd(__m512d __A, __m512d __B) { |
| 55 | return (__m512d)((__v8du)__A ^ (__v8du)__B); | 184 | return (__m512d)((__v8du)__A ^ (__v8du)__B); |
| 56 | } | 185 | } |
| 57 | 186 | ||
| 58 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 187 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 59 | _mm512_mask_xor_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) { | 188 | _mm512_mask_xor_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) { |
| 60 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, | 189 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, |
| 61 | (__v8df)_mm512_xor_pd(__A, __B), | 190 | (__v8df)_mm512_xor_pd(__A, __B), |
| 62 | (__v8df)__W); | 191 | (__v8df)__W); |
| 63 | } | 192 | } |
| 64 | 193 | ||
| 65 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 194 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 66 | _mm512_maskz_xor_pd(__mmask8 __U, __m512d __A, __m512d __B) { | 195 | _mm512_maskz_xor_pd(__mmask8 __U, __m512d __A, __m512d __B) { |
| 67 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, | 196 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, |
| 68 | (__v8df)_mm512_xor_pd(__A, __B), | 197 | (__v8df)_mm512_xor_pd(__A, __B), |
| 69 | (__v8df)_mm512_setzero_pd()); | 198 | (__v8df)_mm512_setzero_pd()); |
| 70 | } | 199 | } |
| 71 | 200 | ||
| 72 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 201 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 73 | _mm512_xor_ps (__m512 __A, __m512 __B) { | 202 | _mm512_xor_ps (__m512 __A, __m512 __B) { |
| 74 | return (__m512)((__v16su)__A ^ (__v16su)__B); | 203 | return (__m512)((__v16su)__A ^ (__v16su)__B); |
| 75 | } | 204 | } |
| 76 | 205 | ||
| 77 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 206 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 78 | _mm512_mask_xor_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { | 207 | _mm512_mask_xor_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { |
| 79 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, | 208 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, |
| 80 | (__v16sf)_mm512_xor_ps(__A, __B), | 209 | (__v16sf)_mm512_xor_ps(__A, __B), |
| 81 | (__v16sf)__W); | 210 | (__v16sf)__W); |
| 82 | } | 211 | } |
| 83 | 212 | ||
| 84 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 213 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 85 | _mm512_maskz_xor_ps(__mmask16 __U, __m512 __A, __m512 __B) { | 214 | _mm512_maskz_xor_ps(__mmask16 __U, __m512 __A, __m512 __B) { |
| 86 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, | 215 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, |
| 87 | (__v16sf)_mm512_xor_ps(__A, __B), | 216 | (__v16sf)_mm512_xor_ps(__A, __B), |
| 88 | (__v16sf)_mm512_setzero_ps()); | 217 | (__v16sf)_mm512_setzero_ps()); |
| 89 | } | 218 | } |
| 90 | 219 | ||
| 91 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 220 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 92 | _mm512_or_pd(__m512d __A, __m512d __B) { | 221 | _mm512_or_pd(__m512d __A, __m512d __B) { |
| 93 | return (__m512d)((__v8du)__A | (__v8du)__B); | 222 | return (__m512d)((__v8du)__A | (__v8du)__B); |
| 94 | } | 223 | } |
| 95 | 224 | ||
| 96 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 225 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 97 | _mm512_mask_or_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) { | 226 | _mm512_mask_or_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) { |
| 98 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, | 227 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, |
| 99 | (__v8df)_mm512_or_pd(__A, __B), | 228 | (__v8df)_mm512_or_pd(__A, __B), |
| 100 | (__v8df)__W); | 229 | (__v8df)__W); |
| 101 | } | 230 | } |
| 102 | 231 | ||
| 103 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 232 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 104 | _mm512_maskz_or_pd(__mmask8 __U, __m512d __A, __m512d __B) { | 233 | _mm512_maskz_or_pd(__mmask8 __U, __m512d __A, __m512d __B) { |
| 105 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, | 234 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, |
| 106 | (__v8df)_mm512_or_pd(__A, __B), | 235 | (__v8df)_mm512_or_pd(__A, __B), |
| 107 | (__v8df)_mm512_setzero_pd()); | 236 | (__v8df)_mm512_setzero_pd()); |
| 108 | } | 237 | } |
| 109 | 238 | ||
| 110 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 239 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 111 | _mm512_or_ps(__m512 __A, __m512 __B) { | 240 | _mm512_or_ps(__m512 __A, __m512 __B) { |
| 112 | return (__m512)((__v16su)__A | (__v16su)__B); | 241 | return (__m512)((__v16su)__A | (__v16su)__B); |
| 113 | } | 242 | } |
| 114 | 243 | ||
| 115 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 244 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 116 | _mm512_mask_or_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { | 245 | _mm512_mask_or_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { |
| 117 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, | 246 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, |
| 118 | (__v16sf)_mm512_or_ps(__A, __B), | 247 | (__v16sf)_mm512_or_ps(__A, __B), |
| 119 | (__v16sf)__W); | 248 | (__v16sf)__W); |
| 120 | } | 249 | } |
| 121 | 250 | ||
| 122 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 251 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 123 | _mm512_maskz_or_ps(__mmask16 __U, __m512 __A, __m512 __B) { | 252 | _mm512_maskz_or_ps(__mmask16 __U, __m512 __A, __m512 __B) { |
| 124 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, | 253 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, |
| 125 | (__v16sf)_mm512_or_ps(__A, __B), | 254 | (__v16sf)_mm512_or_ps(__A, __B), |
| 126 | (__v16sf)_mm512_setzero_ps()); | 255 | (__v16sf)_mm512_setzero_ps()); |
| 127 | } | 256 | } |
| 128 | 257 | ||
| 129 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 258 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 130 | _mm512_and_pd(__m512d __A, __m512d __B) { | 259 | _mm512_and_pd(__m512d __A, __m512d __B) { |
| 131 | return (__m512d)((__v8du)__A & (__v8du)__B); | 260 | return (__m512d)((__v8du)__A & (__v8du)__B); |
| 132 | } | 261 | } |
| 133 | 262 | ||
| 134 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 263 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 135 | _mm512_mask_and_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) { | 264 | _mm512_mask_and_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) { |
| 136 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, | 265 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, |
| 137 | (__v8df)_mm512_and_pd(__A, __B), | 266 | (__v8df)_mm512_and_pd(__A, __B), |
| 138 | (__v8df)__W); | 267 | (__v8df)__W); |
| 139 | } | 268 | } |
| 140 | 269 | ||
| 141 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 270 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 142 | _mm512_maskz_and_pd(__mmask8 __U, __m512d __A, __m512d __B) { | 271 | _mm512_maskz_and_pd(__mmask8 __U, __m512d __A, __m512d __B) { |
| 143 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, | 272 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, |
| 144 | (__v8df)_mm512_and_pd(__A, __B), | 273 | (__v8df)_mm512_and_pd(__A, __B), |
| 145 | (__v8df)_mm512_setzero_pd()); | 274 | (__v8df)_mm512_setzero_pd()); |
| 146 | } | 275 | } |
| 147 | 276 | ||
| 148 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 277 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 149 | _mm512_and_ps(__m512 __A, __m512 __B) { | 278 | _mm512_and_ps(__m512 __A, __m512 __B) { |
| 150 | return (__m512)((__v16su)__A & (__v16su)__B); | 279 | return (__m512)((__v16su)__A & (__v16su)__B); |
| 151 | } | 280 | } |
| 152 | 281 | ||
| 153 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 282 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 154 | _mm512_mask_and_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { | 283 | _mm512_mask_and_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { |
| 155 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, | 284 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, |
| 156 | (__v16sf)_mm512_and_ps(__A, __B), | 285 | (__v16sf)_mm512_and_ps(__A, __B), |
| 157 | (__v16sf)__W); | 286 | (__v16sf)__W); |
| 158 | } | 287 | } |
| 159 | 288 | ||
| 160 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 289 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 161 | _mm512_maskz_and_ps(__mmask16 __U, __m512 __A, __m512 __B) { | 290 | _mm512_maskz_and_ps(__mmask16 __U, __m512 __A, __m512 __B) { |
| 162 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, | 291 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, |
| 163 | (__v16sf)_mm512_and_ps(__A, __B), | 292 | (__v16sf)_mm512_and_ps(__A, __B), |
| 164 | (__v16sf)_mm512_setzero_ps()); | 293 | (__v16sf)_mm512_setzero_ps()); |
| 165 | } | 294 | } |
| 166 | 295 | ||
| 167 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 296 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 168 | _mm512_andnot_pd(__m512d __A, __m512d __B) { | 297 | _mm512_andnot_pd(__m512d __A, __m512d __B) { |
| 169 | return (__m512d)(~(__v8du)__A & (__v8du)__B); | 298 | return (__m512d)(~(__v8du)__A & (__v8du)__B); |
| 170 | } | 299 | } |
| 171 | 300 | ||
| 172 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 301 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 173 | _mm512_mask_andnot_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) { | 302 | _mm512_mask_andnot_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) { |
| 174 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, | 303 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, |
| 175 | (__v8df)_mm512_andnot_pd(__A, __B), | 304 | (__v8df)_mm512_andnot_pd(__A, __B), |
| 176 | (__v8df)__W); | 305 | (__v8df)__W); |
| 177 | } | 306 | } |
| 178 | 307 | ||
| 179 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 308 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 180 | _mm512_maskz_andnot_pd(__mmask8 __U, __m512d __A, __m512d __B) { | 309 | _mm512_maskz_andnot_pd(__mmask8 __U, __m512d __A, __m512d __B) { |
| 181 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, | 310 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, |
| 182 | (__v8df)_mm512_andnot_pd(__A, __B), | 311 | (__v8df)_mm512_andnot_pd(__A, __B), |
| 183 | (__v8df)_mm512_setzero_pd()); | 312 | (__v8df)_mm512_setzero_pd()); |
| 184 | } | 313 | } |
| 185 | 314 | ||
| 186 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 315 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 187 | _mm512_andnot_ps(__m512 __A, __m512 __B) { | 316 | _mm512_andnot_ps(__m512 __A, __m512 __B) { |
| 188 | return (__m512)(~(__v16su)__A & (__v16su)__B); | 317 | return (__m512)(~(__v16su)__A & (__v16su)__B); |
| 189 | } | 318 | } |
| 190 | 319 | ||
| 191 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 320 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 192 | _mm512_mask_andnot_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { | 321 | _mm512_mask_andnot_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) { |
| 193 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, | 322 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, |
| 194 | (__v16sf)_mm512_andnot_ps(__A, __B), | 323 | (__v16sf)_mm512_andnot_ps(__A, __B), |
| 195 | (__v16sf)__W); | 324 | (__v16sf)__W); |
| 196 | } | 325 | } |
| 197 | 326 | ||
| 198 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 327 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 199 | _mm512_maskz_andnot_ps(__mmask16 __U, __m512 __A, __m512 __B) { | 328 | _mm512_maskz_andnot_ps(__mmask16 __U, __m512 __A, __m512 __B) { |
| 200 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, | 329 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, |
| 201 | (__v16sf)_mm512_andnot_ps(__A, __B), | 330 | (__v16sf)_mm512_andnot_ps(__A, __B), |
| 202 | (__v16sf)_mm512_setzero_ps()); | 331 | (__v16sf)_mm512_setzero_ps()); |
| 203 | } | 332 | } |
| 204 | 333 | ||
| 205 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 334 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 206 | _mm512_cvtpd_epi64 (__m512d __A) { | 335 | _mm512_cvtpd_epi64 (__m512d __A) { |
| 207 | return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, | 336 | return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, |
| 208 | (__v8di) _mm512_setzero_si512(), | 337 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -210,7 +339,7 @@ _mm512_cvtpd_epi64 (__m512d __A) { | ... | @@ -210,7 +339,7 @@ _mm512_cvtpd_epi64 (__m512d __A) { |
| 210 | _MM_FROUND_CUR_DIRECTION); | 339 | _MM_FROUND_CUR_DIRECTION); |
| 211 | } | 340 | } |
| 212 | 341 | ||
| 213 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 342 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 214 | _mm512_mask_cvtpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A) { | 343 | _mm512_mask_cvtpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A) { |
| 215 | return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, | 344 | return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, |
| 216 | (__v8di) __W, | 345 | (__v8di) __W, |
| ... | @@ -218,7 +347,7 @@ _mm512_mask_cvtpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A) { | ... | @@ -218,7 +347,7 @@ _mm512_mask_cvtpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A) { |
| 218 | _MM_FROUND_CUR_DIRECTION); | 347 | _MM_FROUND_CUR_DIRECTION); |
| 219 | } | 348 | } |
| 220 | 349 | ||
| 221 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 350 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 222 | _mm512_maskz_cvtpd_epi64 (__mmask8 __U, __m512d __A) { | 351 | _mm512_maskz_cvtpd_epi64 (__mmask8 __U, __m512d __A) { |
| 223 | return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, | 352 | return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, |
| 224 | (__v8di) _mm512_setzero_si512(), | 353 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -241,7 +370,7 @@ _mm512_maskz_cvtpd_epi64 (__mmask8 __U, __m512d __A) { | ... | @@ -241,7 +370,7 @@ _mm512_maskz_cvtpd_epi64 (__mmask8 __U, __m512d __A) { |
| 241 | (__v8di)_mm512_setzero_si512(), \ | 370 | (__v8di)_mm512_setzero_si512(), \ |
| 242 | (__mmask8)(U), (int)(R)) | 371 | (__mmask8)(U), (int)(R)) |
| 243 | 372 | ||
| 244 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 373 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 245 | _mm512_cvtpd_epu64 (__m512d __A) { | 374 | _mm512_cvtpd_epu64 (__m512d __A) { |
| 246 | return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, | 375 | return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, |
| 247 | (__v8di) _mm512_setzero_si512(), | 376 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -249,7 +378,7 @@ _mm512_cvtpd_epu64 (__m512d __A) { | ... | @@ -249,7 +378,7 @@ _mm512_cvtpd_epu64 (__m512d __A) { |
| 249 | _MM_FROUND_CUR_DIRECTION); | 378 | _MM_FROUND_CUR_DIRECTION); |
| 250 | } | 379 | } |
| 251 | 380 | ||
| 252 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 381 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 253 | _mm512_mask_cvtpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A) { | 382 | _mm512_mask_cvtpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A) { |
| 254 | return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, | 383 | return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, |
| 255 | (__v8di) __W, | 384 | (__v8di) __W, |
| ... | @@ -257,7 +386,7 @@ _mm512_mask_cvtpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A) { | ... | @@ -257,7 +386,7 @@ _mm512_mask_cvtpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A) { |
| 257 | _MM_FROUND_CUR_DIRECTION); | 386 | _MM_FROUND_CUR_DIRECTION); |
| 258 | } | 387 | } |
| 259 | 388 | ||
| 260 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 389 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 261 | _mm512_maskz_cvtpd_epu64 (__mmask8 __U, __m512d __A) { | 390 | _mm512_maskz_cvtpd_epu64 (__mmask8 __U, __m512d __A) { |
| 262 | return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, | 391 | return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, |
| 263 | (__v8di) _mm512_setzero_si512(), | 392 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -280,7 +409,7 @@ _mm512_maskz_cvtpd_epu64 (__mmask8 __U, __m512d __A) { | ... | @@ -280,7 +409,7 @@ _mm512_maskz_cvtpd_epu64 (__mmask8 __U, __m512d __A) { |
| 280 | (__v8di)_mm512_setzero_si512(), \ | 409 | (__v8di)_mm512_setzero_si512(), \ |
| 281 | (__mmask8)(U), (int)(R)) | 410 | (__mmask8)(U), (int)(R)) |
| 282 | 411 | ||
| 283 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 412 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 284 | _mm512_cvtps_epi64 (__m256 __A) { | 413 | _mm512_cvtps_epi64 (__m256 __A) { |
| 285 | return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, | 414 | return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, |
| 286 | (__v8di) _mm512_setzero_si512(), | 415 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -288,7 +417,7 @@ _mm512_cvtps_epi64 (__m256 __A) { | ... | @@ -288,7 +417,7 @@ _mm512_cvtps_epi64 (__m256 __A) { |
| 288 | _MM_FROUND_CUR_DIRECTION); | 417 | _MM_FROUND_CUR_DIRECTION); |
| 289 | } | 418 | } |
| 290 | 419 | ||
| 291 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 420 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 292 | _mm512_mask_cvtps_epi64 (__m512i __W, __mmask8 __U, __m256 __A) { | 421 | _mm512_mask_cvtps_epi64 (__m512i __W, __mmask8 __U, __m256 __A) { |
| 293 | return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, | 422 | return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, |
| 294 | (__v8di) __W, | 423 | (__v8di) __W, |
| ... | @@ -296,7 +425,7 @@ _mm512_mask_cvtps_epi64 (__m512i __W, __mmask8 __U, __m256 __A) { | ... | @@ -296,7 +425,7 @@ _mm512_mask_cvtps_epi64 (__m512i __W, __mmask8 __U, __m256 __A) { |
| 296 | _MM_FROUND_CUR_DIRECTION); | 425 | _MM_FROUND_CUR_DIRECTION); |
| 297 | } | 426 | } |
| 298 | 427 | ||
| 299 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 428 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 300 | _mm512_maskz_cvtps_epi64 (__mmask8 __U, __m256 __A) { | 429 | _mm512_maskz_cvtps_epi64 (__mmask8 __U, __m256 __A) { |
| 301 | return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, | 430 | return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, |
| 302 | (__v8di) _mm512_setzero_si512(), | 431 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -319,7 +448,7 @@ _mm512_maskz_cvtps_epi64 (__mmask8 __U, __m256 __A) { | ... | @@ -319,7 +448,7 @@ _mm512_maskz_cvtps_epi64 (__mmask8 __U, __m256 __A) { |
| 319 | (__v8di)_mm512_setzero_si512(), \ | 448 | (__v8di)_mm512_setzero_si512(), \ |
| 320 | (__mmask8)(U), (int)(R)) | 449 | (__mmask8)(U), (int)(R)) |
| 321 | 450 | ||
| 322 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 451 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 323 | _mm512_cvtps_epu64 (__m256 __A) { | 452 | _mm512_cvtps_epu64 (__m256 __A) { |
| 324 | return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, | 453 | return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, |
| 325 | (__v8di) _mm512_setzero_si512(), | 454 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -327,7 +456,7 @@ _mm512_cvtps_epu64 (__m256 __A) { | ... | @@ -327,7 +456,7 @@ _mm512_cvtps_epu64 (__m256 __A) { |
| 327 | _MM_FROUND_CUR_DIRECTION); | 456 | _MM_FROUND_CUR_DIRECTION); |
| 328 | } | 457 | } |
| 329 | 458 | ||
| 330 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 459 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 331 | _mm512_mask_cvtps_epu64 (__m512i __W, __mmask8 __U, __m256 __A) { | 460 | _mm512_mask_cvtps_epu64 (__m512i __W, __mmask8 __U, __m256 __A) { |
| 332 | return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, | 461 | return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, |
| 333 | (__v8di) __W, | 462 | (__v8di) __W, |
| ... | @@ -335,7 +464,7 @@ _mm512_mask_cvtps_epu64 (__m512i __W, __mmask8 __U, __m256 __A) { | ... | @@ -335,7 +464,7 @@ _mm512_mask_cvtps_epu64 (__m512i __W, __mmask8 __U, __m256 __A) { |
| 335 | _MM_FROUND_CUR_DIRECTION); | 464 | _MM_FROUND_CUR_DIRECTION); |
| 336 | } | 465 | } |
| 337 | 466 | ||
| 338 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 467 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 339 | _mm512_maskz_cvtps_epu64 (__mmask8 __U, __m256 __A) { | 468 | _mm512_maskz_cvtps_epu64 (__mmask8 __U, __m256 __A) { |
| 340 | return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, | 469 | return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, |
| 341 | (__v8di) _mm512_setzero_si512(), | 470 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -359,19 +488,19 @@ _mm512_maskz_cvtps_epu64 (__mmask8 __U, __m256 __A) { | ... | @@ -359,19 +488,19 @@ _mm512_maskz_cvtps_epu64 (__mmask8 __U, __m256 __A) { |
| 359 | (__mmask8)(U), (int)(R)) | 488 | (__mmask8)(U), (int)(R)) |
| 360 | 489 | ||
| 361 | 490 | ||
| 362 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 491 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 363 | _mm512_cvtepi64_pd (__m512i __A) { | 492 | _mm512_cvtepi64_pd (__m512i __A) { |
| 364 | return (__m512d)__builtin_convertvector((__v8di)__A, __v8df); | 493 | return (__m512d)__builtin_convertvector((__v8di)__A, __v8df); |
| 365 | } | 494 | } |
| 366 | 495 | ||
| 367 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 496 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 368 | _mm512_mask_cvtepi64_pd (__m512d __W, __mmask8 __U, __m512i __A) { | 497 | _mm512_mask_cvtepi64_pd (__m512d __W, __mmask8 __U, __m512i __A) { |
| 369 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, | 498 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, |
| 370 | (__v8df)_mm512_cvtepi64_pd(__A), | 499 | (__v8df)_mm512_cvtepi64_pd(__A), |
| 371 | (__v8df)__W); | 500 | (__v8df)__W); |
| 372 | } | 501 | } |
| 373 | 502 | ||
| 374 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 503 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 375 | _mm512_maskz_cvtepi64_pd (__mmask8 __U, __m512i __A) { | 504 | _mm512_maskz_cvtepi64_pd (__mmask8 __U, __m512i __A) { |
| 376 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, | 505 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, |
| 377 | (__v8df)_mm512_cvtepi64_pd(__A), | 506 | (__v8df)_mm512_cvtepi64_pd(__A), |
| ... | @@ -393,7 +522,7 @@ _mm512_maskz_cvtepi64_pd (__mmask8 __U, __m512i __A) { | ... | @@ -393,7 +522,7 @@ _mm512_maskz_cvtepi64_pd (__mmask8 __U, __m512i __A) { |
| 393 | (__v8df)_mm512_setzero_pd(), \ | 522 | (__v8df)_mm512_setzero_pd(), \ |
| 394 | (__mmask8)(U), (int)(R)) | 523 | (__mmask8)(U), (int)(R)) |
| 395 | 524 | ||
| 396 | static __inline__ __m256 __DEFAULT_FN_ATTRS | 525 | static __inline__ __m256 __DEFAULT_FN_ATTRS512 |
| 397 | _mm512_cvtepi64_ps (__m512i __A) { | 526 | _mm512_cvtepi64_ps (__m512i __A) { |
| 398 | return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, | 527 | return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, |
| 399 | (__v8sf) _mm256_setzero_ps(), | 528 | (__v8sf) _mm256_setzero_ps(), |
| ... | @@ -401,7 +530,7 @@ _mm512_cvtepi64_ps (__m512i __A) { | ... | @@ -401,7 +530,7 @@ _mm512_cvtepi64_ps (__m512i __A) { |
| 401 | _MM_FROUND_CUR_DIRECTION); | 530 | _MM_FROUND_CUR_DIRECTION); |
| 402 | } | 531 | } |
| 403 | 532 | ||
| 404 | static __inline__ __m256 __DEFAULT_FN_ATTRS | 533 | static __inline__ __m256 __DEFAULT_FN_ATTRS512 |
| 405 | _mm512_mask_cvtepi64_ps (__m256 __W, __mmask8 __U, __m512i __A) { | 534 | _mm512_mask_cvtepi64_ps (__m256 __W, __mmask8 __U, __m512i __A) { |
| 406 | return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, | 535 | return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, |
| 407 | (__v8sf) __W, | 536 | (__v8sf) __W, |
| ... | @@ -409,7 +538,7 @@ _mm512_mask_cvtepi64_ps (__m256 __W, __mmask8 __U, __m512i __A) { | ... | @@ -409,7 +538,7 @@ _mm512_mask_cvtepi64_ps (__m256 __W, __mmask8 __U, __m512i __A) { |
| 409 | _MM_FROUND_CUR_DIRECTION); | 538 | _MM_FROUND_CUR_DIRECTION); |
| 410 | } | 539 | } |
| 411 | 540 | ||
| 412 | static __inline__ __m256 __DEFAULT_FN_ATTRS | 541 | static __inline__ __m256 __DEFAULT_FN_ATTRS512 |
| 413 | _mm512_maskz_cvtepi64_ps (__mmask8 __U, __m512i __A) { | 542 | _mm512_maskz_cvtepi64_ps (__mmask8 __U, __m512i __A) { |
| 414 | return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, | 543 | return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, |
| 415 | (__v8sf) _mm256_setzero_ps(), | 544 | (__v8sf) _mm256_setzero_ps(), |
| ... | @@ -433,7 +562,7 @@ _mm512_maskz_cvtepi64_ps (__mmask8 __U, __m512i __A) { | ... | @@ -433,7 +562,7 @@ _mm512_maskz_cvtepi64_ps (__mmask8 __U, __m512i __A) { |
| 433 | (__mmask8)(U), (int)(R)) | 562 | (__mmask8)(U), (int)(R)) |
| 434 | 563 | ||
| 435 | 564 | ||
| 436 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 565 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 437 | _mm512_cvttpd_epi64 (__m512d __A) { | 566 | _mm512_cvttpd_epi64 (__m512d __A) { |
| 438 | return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, | 567 | return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, |
| 439 | (__v8di) _mm512_setzero_si512(), | 568 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -441,7 +570,7 @@ _mm512_cvttpd_epi64 (__m512d __A) { | ... | @@ -441,7 +570,7 @@ _mm512_cvttpd_epi64 (__m512d __A) { |
| 441 | _MM_FROUND_CUR_DIRECTION); | 570 | _MM_FROUND_CUR_DIRECTION); |
| 442 | } | 571 | } |
| 443 | 572 | ||
| 444 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 573 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 445 | _mm512_mask_cvttpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A) { | 574 | _mm512_mask_cvttpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A) { |
| 446 | return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, | 575 | return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, |
| 447 | (__v8di) __W, | 576 | (__v8di) __W, |
| ... | @@ -449,7 +578,7 @@ _mm512_mask_cvttpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A) { | ... | @@ -449,7 +578,7 @@ _mm512_mask_cvttpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A) { |
| 449 | _MM_FROUND_CUR_DIRECTION); | 578 | _MM_FROUND_CUR_DIRECTION); |
| 450 | } | 579 | } |
| 451 | 580 | ||
| 452 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 581 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 453 | _mm512_maskz_cvttpd_epi64 (__mmask8 __U, __m512d __A) { | 582 | _mm512_maskz_cvttpd_epi64 (__mmask8 __U, __m512d __A) { |
| 454 | return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, | 583 | return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, |
| 455 | (__v8di) _mm512_setzero_si512(), | 584 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -472,7 +601,7 @@ _mm512_maskz_cvttpd_epi64 (__mmask8 __U, __m512d __A) { | ... | @@ -472,7 +601,7 @@ _mm512_maskz_cvttpd_epi64 (__mmask8 __U, __m512d __A) { |
| 472 | (__v8di)_mm512_setzero_si512(), \ | 601 | (__v8di)_mm512_setzero_si512(), \ |
| 473 | (__mmask8)(U), (int)(R)) | 602 | (__mmask8)(U), (int)(R)) |
| 474 | 603 | ||
| 475 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 604 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 476 | _mm512_cvttpd_epu64 (__m512d __A) { | 605 | _mm512_cvttpd_epu64 (__m512d __A) { |
| 477 | return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, | 606 | return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, |
| 478 | (__v8di) _mm512_setzero_si512(), | 607 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -480,7 +609,7 @@ _mm512_cvttpd_epu64 (__m512d __A) { | ... | @@ -480,7 +609,7 @@ _mm512_cvttpd_epu64 (__m512d __A) { |
| 480 | _MM_FROUND_CUR_DIRECTION); | 609 | _MM_FROUND_CUR_DIRECTION); |
| 481 | } | 610 | } |
| 482 | 611 | ||
| 483 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 612 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 484 | _mm512_mask_cvttpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A) { | 613 | _mm512_mask_cvttpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A) { |
| 485 | return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, | 614 | return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, |
| 486 | (__v8di) __W, | 615 | (__v8di) __W, |
| ... | @@ -488,7 +617,7 @@ _mm512_mask_cvttpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A) { | ... | @@ -488,7 +617,7 @@ _mm512_mask_cvttpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A) { |
| 488 | _MM_FROUND_CUR_DIRECTION); | 617 | _MM_FROUND_CUR_DIRECTION); |
| 489 | } | 618 | } |
| 490 | 619 | ||
| 491 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 620 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 492 | _mm512_maskz_cvttpd_epu64 (__mmask8 __U, __m512d __A) { | 621 | _mm512_maskz_cvttpd_epu64 (__mmask8 __U, __m512d __A) { |
| 493 | return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, | 622 | return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, |
| 494 | (__v8di) _mm512_setzero_si512(), | 623 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -511,7 +640,7 @@ _mm512_maskz_cvttpd_epu64 (__mmask8 __U, __m512d __A) { | ... | @@ -511,7 +640,7 @@ _mm512_maskz_cvttpd_epu64 (__mmask8 __U, __m512d __A) { |
| 511 | (__v8di)_mm512_setzero_si512(), \ | 640 | (__v8di)_mm512_setzero_si512(), \ |
| 512 | (__mmask8)(U), (int)(R)) | 641 | (__mmask8)(U), (int)(R)) |
| 513 | 642 | ||
| 514 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 643 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 515 | _mm512_cvttps_epi64 (__m256 __A) { | 644 | _mm512_cvttps_epi64 (__m256 __A) { |
| 516 | return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, | 645 | return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, |
| 517 | (__v8di) _mm512_setzero_si512(), | 646 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -519,7 +648,7 @@ _mm512_cvttps_epi64 (__m256 __A) { | ... | @@ -519,7 +648,7 @@ _mm512_cvttps_epi64 (__m256 __A) { |
| 519 | _MM_FROUND_CUR_DIRECTION); | 648 | _MM_FROUND_CUR_DIRECTION); |
| 520 | } | 649 | } |
| 521 | 650 | ||
| 522 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 651 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 523 | _mm512_mask_cvttps_epi64 (__m512i __W, __mmask8 __U, __m256 __A) { | 652 | _mm512_mask_cvttps_epi64 (__m512i __W, __mmask8 __U, __m256 __A) { |
| 524 | return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, | 653 | return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, |
| 525 | (__v8di) __W, | 654 | (__v8di) __W, |
| ... | @@ -527,7 +656,7 @@ _mm512_mask_cvttps_epi64 (__m512i __W, __mmask8 __U, __m256 __A) { | ... | @@ -527,7 +656,7 @@ _mm512_mask_cvttps_epi64 (__m512i __W, __mmask8 __U, __m256 __A) { |
| 527 | _MM_FROUND_CUR_DIRECTION); | 656 | _MM_FROUND_CUR_DIRECTION); |
| 528 | } | 657 | } |
| 529 | 658 | ||
| 530 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 659 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 531 | _mm512_maskz_cvttps_epi64 (__mmask8 __U, __m256 __A) { | 660 | _mm512_maskz_cvttps_epi64 (__mmask8 __U, __m256 __A) { |
| 532 | return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, | 661 | return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, |
| 533 | (__v8di) _mm512_setzero_si512(), | 662 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -550,7 +679,7 @@ _mm512_maskz_cvttps_epi64 (__mmask8 __U, __m256 __A) { | ... | @@ -550,7 +679,7 @@ _mm512_maskz_cvttps_epi64 (__mmask8 __U, __m256 __A) { |
| 550 | (__v8di)_mm512_setzero_si512(), \ | 679 | (__v8di)_mm512_setzero_si512(), \ |
| 551 | (__mmask8)(U), (int)(R)) | 680 | (__mmask8)(U), (int)(R)) |
| 552 | 681 | ||
| 553 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 682 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 554 | _mm512_cvttps_epu64 (__m256 __A) { | 683 | _mm512_cvttps_epu64 (__m256 __A) { |
| 555 | return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, | 684 | return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, |
| 556 | (__v8di) _mm512_setzero_si512(), | 685 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -558,7 +687,7 @@ _mm512_cvttps_epu64 (__m256 __A) { | ... | @@ -558,7 +687,7 @@ _mm512_cvttps_epu64 (__m256 __A) { |
| 558 | _MM_FROUND_CUR_DIRECTION); | 687 | _MM_FROUND_CUR_DIRECTION); |
| 559 | } | 688 | } |
| 560 | 689 | ||
| 561 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 690 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 562 | _mm512_mask_cvttps_epu64 (__m512i __W, __mmask8 __U, __m256 __A) { | 691 | _mm512_mask_cvttps_epu64 (__m512i __W, __mmask8 __U, __m256 __A) { |
| 563 | return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, | 692 | return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, |
| 564 | (__v8di) __W, | 693 | (__v8di) __W, |
| ... | @@ -566,7 +695,7 @@ _mm512_mask_cvttps_epu64 (__m512i __W, __mmask8 __U, __m256 __A) { | ... | @@ -566,7 +695,7 @@ _mm512_mask_cvttps_epu64 (__m512i __W, __mmask8 __U, __m256 __A) { |
| 566 | _MM_FROUND_CUR_DIRECTION); | 695 | _MM_FROUND_CUR_DIRECTION); |
| 567 | } | 696 | } |
| 568 | 697 | ||
| 569 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 698 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 570 | _mm512_maskz_cvttps_epu64 (__mmask8 __U, __m256 __A) { | 699 | _mm512_maskz_cvttps_epu64 (__mmask8 __U, __m256 __A) { |
| 571 | return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, | 700 | return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, |
| 572 | (__v8di) _mm512_setzero_si512(), | 701 | (__v8di) _mm512_setzero_si512(), |
| ... | @@ -589,19 +718,19 @@ _mm512_maskz_cvttps_epu64 (__mmask8 __U, __m256 __A) { | ... | @@ -589,19 +718,19 @@ _mm512_maskz_cvttps_epu64 (__mmask8 __U, __m256 __A) { |
| 589 | (__v8di)_mm512_setzero_si512(), \ | 718 | (__v8di)_mm512_setzero_si512(), \ |
| 590 | (__mmask8)(U), (int)(R)) | 719 | (__mmask8)(U), (int)(R)) |
| 591 | 720 | ||
| 592 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 721 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 593 | _mm512_cvtepu64_pd (__m512i __A) { | 722 | _mm512_cvtepu64_pd (__m512i __A) { |
| 594 | return (__m512d)__builtin_convertvector((__v8du)__A, __v8df); | 723 | return (__m512d)__builtin_convertvector((__v8du)__A, __v8df); |
| 595 | } | 724 | } |
| 596 | 725 | ||
| 597 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 726 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 598 | _mm512_mask_cvtepu64_pd (__m512d __W, __mmask8 __U, __m512i __A) { | 727 | _mm512_mask_cvtepu64_pd (__m512d __W, __mmask8 __U, __m512i __A) { |
| 599 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, | 728 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, |
| 600 | (__v8df)_mm512_cvtepu64_pd(__A), | 729 | (__v8df)_mm512_cvtepu64_pd(__A), |
| 601 | (__v8df)__W); | 730 | (__v8df)__W); |
| 602 | } | 731 | } |
| 603 | 732 | ||
| 604 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 733 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 605 | _mm512_maskz_cvtepu64_pd (__mmask8 __U, __m512i __A) { | 734 | _mm512_maskz_cvtepu64_pd (__mmask8 __U, __m512i __A) { |
| 606 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, | 735 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, |
| 607 | (__v8df)_mm512_cvtepu64_pd(__A), | 736 | (__v8df)_mm512_cvtepu64_pd(__A), |
| ... | @@ -625,7 +754,7 @@ _mm512_maskz_cvtepu64_pd (__mmask8 __U, __m512i __A) { | ... | @@ -625,7 +754,7 @@ _mm512_maskz_cvtepu64_pd (__mmask8 __U, __m512i __A) { |
| 625 | (__mmask8)(U), (int)(R)) | 754 | (__mmask8)(U), (int)(R)) |
| 626 | 755 | ||
| 627 | 756 | ||
| 628 | static __inline__ __m256 __DEFAULT_FN_ATTRS | 757 | static __inline__ __m256 __DEFAULT_FN_ATTRS512 |
| 629 | _mm512_cvtepu64_ps (__m512i __A) { | 758 | _mm512_cvtepu64_ps (__m512i __A) { |
| 630 | return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, | 759 | return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, |
| 631 | (__v8sf) _mm256_setzero_ps(), | 760 | (__v8sf) _mm256_setzero_ps(), |
| ... | @@ -633,7 +762,7 @@ _mm512_cvtepu64_ps (__m512i __A) { | ... | @@ -633,7 +762,7 @@ _mm512_cvtepu64_ps (__m512i __A) { |
| 633 | _MM_FROUND_CUR_DIRECTION); | 762 | _MM_FROUND_CUR_DIRECTION); |
| 634 | } | 763 | } |
| 635 | 764 | ||
| 636 | static __inline__ __m256 __DEFAULT_FN_ATTRS | 765 | static __inline__ __m256 __DEFAULT_FN_ATTRS512 |
| 637 | _mm512_mask_cvtepu64_ps (__m256 __W, __mmask8 __U, __m512i __A) { | 766 | _mm512_mask_cvtepu64_ps (__m256 __W, __mmask8 __U, __m512i __A) { |
| 638 | return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, | 767 | return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, |
| 639 | (__v8sf) __W, | 768 | (__v8sf) __W, |
| ... | @@ -641,7 +770,7 @@ _mm512_mask_cvtepu64_ps (__m256 __W, __mmask8 __U, __m512i __A) { | ... | @@ -641,7 +770,7 @@ _mm512_mask_cvtepu64_ps (__m256 __W, __mmask8 __U, __m512i __A) { |
| 641 | _MM_FROUND_CUR_DIRECTION); | 770 | _MM_FROUND_CUR_DIRECTION); |
| 642 | } | 771 | } |
| 643 | 772 | ||
| 644 | static __inline__ __m256 __DEFAULT_FN_ATTRS | 773 | static __inline__ __m256 __DEFAULT_FN_ATTRS512 |
| 645 | _mm512_maskz_cvtepu64_ps (__mmask8 __U, __m512i __A) { | 774 | _mm512_maskz_cvtepu64_ps (__mmask8 __U, __m512i __A) { |
| 646 | return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, | 775 | return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, |
| 647 | (__v8sf) _mm256_setzero_ps(), | 776 | (__v8sf) _mm256_setzero_ps(), |
| ... | @@ -935,32 +1064,32 @@ _mm512_maskz_cvtepu64_ps (__mmask8 __U, __m512i __A) { | ... | @@ -935,32 +1064,32 @@ _mm512_maskz_cvtepu64_ps (__mmask8 __U, __m512i __A) { |
| 935 | (__v2df)_mm_setzero_pd(), \ | 1064 | (__v2df)_mm_setzero_pd(), \ |
| 936 | (__mmask8)(U), (int)(C), (int)(R)) | 1065 | (__mmask8)(U), (int)(C), (int)(R)) |
| 937 | 1066 | ||
| 938 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS | 1067 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS512 |
| 939 | _mm512_movepi32_mask (__m512i __A) | 1068 | _mm512_movepi32_mask (__m512i __A) |
| 940 | { | 1069 | { |
| 941 | return (__mmask16) __builtin_ia32_cvtd2mask512 ((__v16si) __A); | 1070 | return (__mmask16) __builtin_ia32_cvtd2mask512 ((__v16si) __A); |
| 942 | } | 1071 | } |
| 943 | 1072 | ||
| 944 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1073 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 945 | _mm512_movm_epi32 (__mmask16 __A) | 1074 | _mm512_movm_epi32 (__mmask16 __A) |
| 946 | { | 1075 | { |
| 947 | return (__m512i) __builtin_ia32_cvtmask2d512 (__A); | 1076 | return (__m512i) __builtin_ia32_cvtmask2d512 (__A); |
| 948 | } | 1077 | } |
| 949 | 1078 | ||
| 950 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1079 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 951 | _mm512_movm_epi64 (__mmask8 __A) | 1080 | _mm512_movm_epi64 (__mmask8 __A) |
| 952 | { | 1081 | { |
| 953 | return (__m512i) __builtin_ia32_cvtmask2q512 (__A); | 1082 | return (__m512i) __builtin_ia32_cvtmask2q512 (__A); |
| 954 | } | 1083 | } |
| 955 | 1084 | ||
| 956 | static __inline__ __mmask8 __DEFAULT_FN_ATTRS | 1085 | static __inline__ __mmask8 __DEFAULT_FN_ATTRS512 |
| 957 | _mm512_movepi64_mask (__m512i __A) | 1086 | _mm512_movepi64_mask (__m512i __A) |
| 958 | { | 1087 | { |
| 959 | return (__mmask8) __builtin_ia32_cvtq2mask512 ((__v8di) __A); | 1088 | return (__mmask8) __builtin_ia32_cvtq2mask512 ((__v8di) __A); |
| 960 | } | 1089 | } |
| 961 | 1090 | ||
| 962 | 1091 | ||
| 963 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 1092 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 964 | _mm512_broadcast_f32x2 (__m128 __A) | 1093 | _mm512_broadcast_f32x2 (__m128 __A) |
| 965 | { | 1094 | { |
| 966 | return (__m512)__builtin_shufflevector((__v4sf)__A, (__v4sf)__A, | 1095 | return (__m512)__builtin_shufflevector((__v4sf)__A, (__v4sf)__A, |
| ... | @@ -968,7 +1097,7 @@ _mm512_broadcast_f32x2 (__m128 __A) | ... | @@ -968,7 +1097,7 @@ _mm512_broadcast_f32x2 (__m128 __A) |
| 968 | 0, 1, 0, 1, 0, 1, 0, 1); | 1097 | 0, 1, 0, 1, 0, 1, 0, 1); |
| 969 | } | 1098 | } |
| 970 | 1099 | ||
| 971 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 1100 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 972 | _mm512_mask_broadcast_f32x2 (__m512 __O, __mmask16 __M, __m128 __A) | 1101 | _mm512_mask_broadcast_f32x2 (__m512 __O, __mmask16 __M, __m128 __A) |
| 973 | { | 1102 | { |
| 974 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__M, | 1103 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__M, |
| ... | @@ -976,7 +1105,7 @@ _mm512_mask_broadcast_f32x2 (__m512 __O, __mmask16 __M, __m128 __A) | ... | @@ -976,7 +1105,7 @@ _mm512_mask_broadcast_f32x2 (__m512 __O, __mmask16 __M, __m128 __A) |
| 976 | (__v16sf)__O); | 1105 | (__v16sf)__O); |
| 977 | } | 1106 | } |
| 978 | 1107 | ||
| 979 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 1108 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 980 | _mm512_maskz_broadcast_f32x2 (__mmask16 __M, __m128 __A) | 1109 | _mm512_maskz_broadcast_f32x2 (__mmask16 __M, __m128 __A) |
| 981 | { | 1110 | { |
| 982 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__M, | 1111 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__M, |
| ... | @@ -984,7 +1113,7 @@ _mm512_maskz_broadcast_f32x2 (__mmask16 __M, __m128 __A) | ... | @@ -984,7 +1113,7 @@ _mm512_maskz_broadcast_f32x2 (__mmask16 __M, __m128 __A) |
| 984 | (__v16sf)_mm512_setzero_ps()); | 1113 | (__v16sf)_mm512_setzero_ps()); |
| 985 | } | 1114 | } |
| 986 | 1115 | ||
| 987 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 1116 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 988 | _mm512_broadcast_f32x8(__m256 __A) | 1117 | _mm512_broadcast_f32x8(__m256 __A) |
| 989 | { | 1118 | { |
| 990 | return (__m512)__builtin_shufflevector((__v8sf)__A, (__v8sf)__A, | 1119 | return (__m512)__builtin_shufflevector((__v8sf)__A, (__v8sf)__A, |
| ... | @@ -992,7 +1121,7 @@ _mm512_broadcast_f32x8(__m256 __A) | ... | @@ -992,7 +1121,7 @@ _mm512_broadcast_f32x8(__m256 __A) |
| 992 | 0, 1, 2, 3, 4, 5, 6, 7); | 1121 | 0, 1, 2, 3, 4, 5, 6, 7); |
| 993 | } | 1122 | } |
| 994 | 1123 | ||
| 995 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 1124 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 996 | _mm512_mask_broadcast_f32x8(__m512 __O, __mmask16 __M, __m256 __A) | 1125 | _mm512_mask_broadcast_f32x8(__m512 __O, __mmask16 __M, __m256 __A) |
| 997 | { | 1126 | { |
| 998 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__M, | 1127 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__M, |
| ... | @@ -1000,7 +1129,7 @@ _mm512_mask_broadcast_f32x8(__m512 __O, __mmask16 __M, __m256 __A) | ... | @@ -1000,7 +1129,7 @@ _mm512_mask_broadcast_f32x8(__m512 __O, __mmask16 __M, __m256 __A) |
| 1000 | (__v16sf)__O); | 1129 | (__v16sf)__O); |
| 1001 | } | 1130 | } |
| 1002 | 1131 | ||
| 1003 | static __inline__ __m512 __DEFAULT_FN_ATTRS | 1132 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 1004 | _mm512_maskz_broadcast_f32x8(__mmask16 __M, __m256 __A) | 1133 | _mm512_maskz_broadcast_f32x8(__mmask16 __M, __m256 __A) |
| 1005 | { | 1134 | { |
| 1006 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__M, | 1135 | return (__m512)__builtin_ia32_selectps_512((__mmask16)__M, |
| ... | @@ -1008,14 +1137,14 @@ _mm512_maskz_broadcast_f32x8(__mmask16 __M, __m256 __A) | ... | @@ -1008,14 +1137,14 @@ _mm512_maskz_broadcast_f32x8(__mmask16 __M, __m256 __A) |
| 1008 | (__v16sf)_mm512_setzero_ps()); | 1137 | (__v16sf)_mm512_setzero_ps()); |
| 1009 | } | 1138 | } |
| 1010 | 1139 | ||
| 1011 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 1140 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 1012 | _mm512_broadcast_f64x2(__m128d __A) | 1141 | _mm512_broadcast_f64x2(__m128d __A) |
| 1013 | { | 1142 | { |
| 1014 | return (__m512d)__builtin_shufflevector((__v2df)__A, (__v2df)__A, | 1143 | return (__m512d)__builtin_shufflevector((__v2df)__A, (__v2df)__A, |
| 1015 | 0, 1, 0, 1, 0, 1, 0, 1); | 1144 | 0, 1, 0, 1, 0, 1, 0, 1); |
| 1016 | } | 1145 | } |
| 1017 | 1146 | ||
| 1018 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 1147 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 1019 | _mm512_mask_broadcast_f64x2(__m512d __O, __mmask8 __M, __m128d __A) | 1148 | _mm512_mask_broadcast_f64x2(__m512d __O, __mmask8 __M, __m128d __A) |
| 1020 | { | 1149 | { |
| 1021 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__M, | 1150 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__M, |
| ... | @@ -1023,7 +1152,7 @@ _mm512_mask_broadcast_f64x2(__m512d __O, __mmask8 __M, __m128d __A) | ... | @@ -1023,7 +1152,7 @@ _mm512_mask_broadcast_f64x2(__m512d __O, __mmask8 __M, __m128d __A) |
| 1023 | (__v8df)__O); | 1152 | (__v8df)__O); |
| 1024 | } | 1153 | } |
| 1025 | 1154 | ||
| 1026 | static __inline__ __m512d __DEFAULT_FN_ATTRS | 1155 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 1027 | _mm512_maskz_broadcast_f64x2(__mmask8 __M, __m128d __A) | 1156 | _mm512_maskz_broadcast_f64x2(__mmask8 __M, __m128d __A) |
| 1028 | { | 1157 | { |
| 1029 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__M, | 1158 | return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__M, |
| ... | @@ -1031,7 +1160,7 @@ _mm512_maskz_broadcast_f64x2(__mmask8 __M, __m128d __A) | ... | @@ -1031,7 +1160,7 @@ _mm512_maskz_broadcast_f64x2(__mmask8 __M, __m128d __A) |
| 1031 | (__v8df)_mm512_setzero_pd()); | 1160 | (__v8df)_mm512_setzero_pd()); |
| 1032 | } | 1161 | } |
| 1033 | 1162 | ||
| 1034 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1163 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1035 | _mm512_broadcast_i32x2 (__m128i __A) | 1164 | _mm512_broadcast_i32x2 (__m128i __A) |
| 1036 | { | 1165 | { |
| 1037 | return (__m512i)__builtin_shufflevector((__v4si)__A, (__v4si)__A, | 1166 | return (__m512i)__builtin_shufflevector((__v4si)__A, (__v4si)__A, |
| ... | @@ -1039,7 +1168,7 @@ _mm512_broadcast_i32x2 (__m128i __A) | ... | @@ -1039,7 +1168,7 @@ _mm512_broadcast_i32x2 (__m128i __A) |
| 1039 | 0, 1, 0, 1, 0, 1, 0, 1); | 1168 | 0, 1, 0, 1, 0, 1, 0, 1); |
| 1040 | } | 1169 | } |
| 1041 | 1170 | ||
| 1042 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1171 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1043 | _mm512_mask_broadcast_i32x2 (__m512i __O, __mmask16 __M, __m128i __A) | 1172 | _mm512_mask_broadcast_i32x2 (__m512i __O, __mmask16 __M, __m128i __A) |
| 1044 | { | 1173 | { |
| 1045 | return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M, | 1174 | return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M, |
| ... | @@ -1047,7 +1176,7 @@ _mm512_mask_broadcast_i32x2 (__m512i __O, __mmask16 __M, __m128i __A) | ... | @@ -1047,7 +1176,7 @@ _mm512_mask_broadcast_i32x2 (__m512i __O, __mmask16 __M, __m128i __A) |
| 1047 | (__v16si)__O); | 1176 | (__v16si)__O); |
| 1048 | } | 1177 | } |
| 1049 | 1178 | ||
| 1050 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1179 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1051 | _mm512_maskz_broadcast_i32x2 (__mmask16 __M, __m128i __A) | 1180 | _mm512_maskz_broadcast_i32x2 (__mmask16 __M, __m128i __A) |
| 1052 | { | 1181 | { |
| 1053 | return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M, | 1182 | return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M, |
| ... | @@ -1055,7 +1184,7 @@ _mm512_maskz_broadcast_i32x2 (__mmask16 __M, __m128i __A) | ... | @@ -1055,7 +1184,7 @@ _mm512_maskz_broadcast_i32x2 (__mmask16 __M, __m128i __A) |
| 1055 | (__v16si)_mm512_setzero_si512()); | 1184 | (__v16si)_mm512_setzero_si512()); |
| 1056 | } | 1185 | } |
| 1057 | 1186 | ||
| 1058 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1187 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1059 | _mm512_broadcast_i32x8(__m256i __A) | 1188 | _mm512_broadcast_i32x8(__m256i __A) |
| 1060 | { | 1189 | { |
| 1061 | return (__m512i)__builtin_shufflevector((__v8si)__A, (__v8si)__A, | 1190 | return (__m512i)__builtin_shufflevector((__v8si)__A, (__v8si)__A, |
| ... | @@ -1063,7 +1192,7 @@ _mm512_broadcast_i32x8(__m256i __A) | ... | @@ -1063,7 +1192,7 @@ _mm512_broadcast_i32x8(__m256i __A) |
| 1063 | 0, 1, 2, 3, 4, 5, 6, 7); | 1192 | 0, 1, 2, 3, 4, 5, 6, 7); |
| 1064 | } | 1193 | } |
| 1065 | 1194 | ||
| 1066 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1195 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1067 | _mm512_mask_broadcast_i32x8(__m512i __O, __mmask16 __M, __m256i __A) | 1196 | _mm512_mask_broadcast_i32x8(__m512i __O, __mmask16 __M, __m256i __A) |
| 1068 | { | 1197 | { |
| 1069 | return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M, | 1198 | return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M, |
| ... | @@ -1071,7 +1200,7 @@ _mm512_mask_broadcast_i32x8(__m512i __O, __mmask16 __M, __m256i __A) | ... | @@ -1071,7 +1200,7 @@ _mm512_mask_broadcast_i32x8(__m512i __O, __mmask16 __M, __m256i __A) |
| 1071 | (__v16si)__O); | 1200 | (__v16si)__O); |
| 1072 | } | 1201 | } |
| 1073 | 1202 | ||
| 1074 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1203 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1075 | _mm512_maskz_broadcast_i32x8(__mmask16 __M, __m256i __A) | 1204 | _mm512_maskz_broadcast_i32x8(__mmask16 __M, __m256i __A) |
| 1076 | { | 1205 | { |
| 1077 | return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M, | 1206 | return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M, |
| ... | @@ -1079,14 +1208,14 @@ _mm512_maskz_broadcast_i32x8(__mmask16 __M, __m256i __A) | ... | @@ -1079,14 +1208,14 @@ _mm512_maskz_broadcast_i32x8(__mmask16 __M, __m256i __A) |
| 1079 | (__v16si)_mm512_setzero_si512()); | 1208 | (__v16si)_mm512_setzero_si512()); |
| 1080 | } | 1209 | } |
| 1081 | 1210 | ||
| 1082 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1211 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1083 | _mm512_broadcast_i64x2(__m128i __A) | 1212 | _mm512_broadcast_i64x2(__m128i __A) |
| 1084 | { | 1213 | { |
| 1085 | return (__m512i)__builtin_shufflevector((__v2di)__A, (__v2di)__A, | 1214 | return (__m512i)__builtin_shufflevector((__v2di)__A, (__v2di)__A, |
| 1086 | 0, 1, 0, 1, 0, 1, 0, 1); | 1215 | 0, 1, 0, 1, 0, 1, 0, 1); |
| 1087 | } | 1216 | } |
| 1088 | 1217 | ||
| 1089 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1218 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1090 | _mm512_mask_broadcast_i64x2(__m512i __O, __mmask8 __M, __m128i __A) | 1219 | _mm512_mask_broadcast_i64x2(__m512i __O, __mmask8 __M, __m128i __A) |
| 1091 | { | 1220 | { |
| 1092 | return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M, | 1221 | return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M, |
| ... | @@ -1094,7 +1223,7 @@ _mm512_mask_broadcast_i64x2(__m512i __O, __mmask8 __M, __m128i __A) | ... | @@ -1094,7 +1223,7 @@ _mm512_mask_broadcast_i64x2(__m512i __O, __mmask8 __M, __m128i __A) |
| 1094 | (__v8di)__O); | 1223 | (__v8di)__O); |
| 1095 | } | 1224 | } |
| 1096 | 1225 | ||
| 1097 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 1226 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 1098 | _mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i __A) | 1227 | _mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i __A) |
| 1099 | { | 1228 | { |
| 1100 | return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M, | 1229 | return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M, |
| ... | @@ -1256,6 +1385,7 @@ _mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i __A) | ... | @@ -1256,6 +1385,7 @@ _mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i __A) |
| 1256 | (__mmask8)__builtin_ia32_fpclassss_mask((__v4sf)(__m128)(A), (int)(imm), \ | 1385 | (__mmask8)__builtin_ia32_fpclassss_mask((__v4sf)(__m128)(A), (int)(imm), \ |
| 1257 | (__mmask8)(U)) | 1386 | (__mmask8)(U)) |
| 1258 | 1387 | ||
| 1388 | #undef __DEFAULT_FN_ATTRS512 | ||
| 1259 | #undef __DEFAULT_FN_ATTRS | 1389 | #undef __DEFAULT_FN_ATTRS |
| 1260 | 1390 | ||
| 1261 | #endif | 1391 | #endif |
c_headers/avx512fintrin.h+133-44
| ... | @@ -175,6 +175,7 @@ typedef enum | ... | @@ -175,6 +175,7 @@ typedef enum |
| 175 | /* Define the default attributes for the functions in this file. */ | 175 | /* Define the default attributes for the functions in this file. */ |
| 176 | #define __DEFAULT_FN_ATTRS512 __attribute__((__always_inline__, __nodebug__, __target__("avx512f"), __min_vector_width__(512))) | 176 | #define __DEFAULT_FN_ATTRS512 __attribute__((__always_inline__, __nodebug__, __target__("avx512f"), __min_vector_width__(512))) |
| 177 | #define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avx512f"), __min_vector_width__(128))) | 177 | #define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avx512f"), __min_vector_width__(128))) |
| 178 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("avx512f"))) | ||
| 178 | 179 | ||
| 179 | /* Create vectors with repeated elements */ | 180 | /* Create vectors with repeated elements */ |
| 180 | 181 | ||
| ... | @@ -508,13 +509,13 @@ _mm512_castsi512_si256 (__m512i __A) | ... | @@ -508,13 +509,13 @@ _mm512_castsi512_si256 (__m512i __A) |
| 508 | return (__m256i)__builtin_shufflevector(__A, __A , 0, 1, 2, 3); | 509 | return (__m256i)__builtin_shufflevector(__A, __A , 0, 1, 2, 3); |
| 509 | } | 510 | } |
| 510 | 511 | ||
| 511 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS512 | 512 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS |
| 512 | _mm512_int2mask(int __a) | 513 | _mm512_int2mask(int __a) |
| 513 | { | 514 | { |
| 514 | return (__mmask16)__a; | 515 | return (__mmask16)__a; |
| 515 | } | 516 | } |
| 516 | 517 | ||
| 517 | static __inline__ int __DEFAULT_FN_ATTRS512 | 518 | static __inline__ int __DEFAULT_FN_ATTRS |
| 518 | _mm512_mask2int(__mmask16 __a) | 519 | _mm512_mask2int(__mmask16 __a) |
| 519 | { | 520 | { |
| 520 | return (int)__a; | 521 | return (int)__a; |
| ... | @@ -4328,6 +4329,15 @@ _mm512_loadu_si512 (void const *__P) | ... | @@ -4328,6 +4329,15 @@ _mm512_loadu_si512 (void const *__P) |
| 4328 | return ((struct __loadu_si512*)__P)->__v; | 4329 | return ((struct __loadu_si512*)__P)->__v; |
| 4329 | } | 4330 | } |
| 4330 | 4331 | ||
| 4332 | static __inline __m512i __DEFAULT_FN_ATTRS512 | ||
| 4333 | _mm512_loadu_epi32 (void const *__P) | ||
| 4334 | { | ||
| 4335 | struct __loadu_epi32 { | ||
| 4336 | __m512i __v; | ||
| 4337 | } __attribute__((__packed__, __may_alias__)); | ||
| 4338 | return ((struct __loadu_epi32*)__P)->__v; | ||
| 4339 | } | ||
| 4340 | |||
| 4331 | static __inline __m512i __DEFAULT_FN_ATTRS512 | 4341 | static __inline __m512i __DEFAULT_FN_ATTRS512 |
| 4332 | _mm512_mask_loadu_epi32 (__m512i __W, __mmask16 __U, void const *__P) | 4342 | _mm512_mask_loadu_epi32 (__m512i __W, __mmask16 __U, void const *__P) |
| 4333 | { | 4343 | { |
| ... | @@ -4346,6 +4356,15 @@ _mm512_maskz_loadu_epi32(__mmask16 __U, void const *__P) | ... | @@ -4346,6 +4356,15 @@ _mm512_maskz_loadu_epi32(__mmask16 __U, void const *__P) |
| 4346 | (__mmask16) __U); | 4356 | (__mmask16) __U); |
| 4347 | } | 4357 | } |
| 4348 | 4358 | ||
| 4359 | static __inline __m512i __DEFAULT_FN_ATTRS512 | ||
| 4360 | _mm512_loadu_epi64 (void const *__P) | ||
| 4361 | { | ||
| 4362 | struct __loadu_epi64 { | ||
| 4363 | __m512i __v; | ||
| 4364 | } __attribute__((__packed__, __may_alias__)); | ||
| 4365 | return ((struct __loadu_epi64*)__P)->__v; | ||
| 4366 | } | ||
| 4367 | |||
| 4349 | static __inline __m512i __DEFAULT_FN_ATTRS512 | 4368 | static __inline __m512i __DEFAULT_FN_ATTRS512 |
| 4350 | _mm512_mask_loadu_epi64 (__m512i __W, __mmask8 __U, void const *__P) | 4369 | _mm512_mask_loadu_epi64 (__m512i __W, __mmask8 __U, void const *__P) |
| 4351 | { | 4370 | { |
| ... | @@ -4481,6 +4500,15 @@ _mm512_load_epi64 (void const *__P) | ... | @@ -4481,6 +4500,15 @@ _mm512_load_epi64 (void const *__P) |
| 4481 | 4500 | ||
| 4482 | /* SIMD store ops */ | 4501 | /* SIMD store ops */ |
| 4483 | 4502 | ||
| 4503 | static __inline void __DEFAULT_FN_ATTRS512 | ||
| 4504 | _mm512_storeu_epi64 (void *__P, __m512i __A) | ||
| 4505 | { | ||
| 4506 | struct __storeu_epi64 { | ||
| 4507 | __m512i __v; | ||
| 4508 | } __attribute__((__packed__, __may_alias__)); | ||
| 4509 | ((struct __storeu_epi64*)__P)->__v = __A; | ||
| 4510 | } | ||
| 4511 | |||
| 4484 | static __inline void __DEFAULT_FN_ATTRS512 | 4512 | static __inline void __DEFAULT_FN_ATTRS512 |
| 4485 | _mm512_mask_storeu_epi64(void *__P, __mmask8 __U, __m512i __A) | 4513 | _mm512_mask_storeu_epi64(void *__P, __mmask8 __U, __m512i __A) |
| 4486 | { | 4514 | { |
| ... | @@ -4497,6 +4525,15 @@ _mm512_storeu_si512 (void *__P, __m512i __A) | ... | @@ -4497,6 +4525,15 @@ _mm512_storeu_si512 (void *__P, __m512i __A) |
| 4497 | ((struct __storeu_si512*)__P)->__v = __A; | 4525 | ((struct __storeu_si512*)__P)->__v = __A; |
| 4498 | } | 4526 | } |
| 4499 | 4527 | ||
| 4528 | static __inline void __DEFAULT_FN_ATTRS512 | ||
| 4529 | _mm512_storeu_epi32 (void *__P, __m512i __A) | ||
| 4530 | { | ||
| 4531 | struct __storeu_epi32 { | ||
| 4532 | __m512i __v; | ||
| 4533 | } __attribute__((__packed__, __may_alias__)); | ||
| 4534 | ((struct __storeu_epi32*)__P)->__v = __A; | ||
| 4535 | } | ||
| 4536 | |||
| 4500 | static __inline void __DEFAULT_FN_ATTRS512 | 4537 | static __inline void __DEFAULT_FN_ATTRS512 |
| 4501 | _mm512_mask_storeu_epi32(void *__P, __mmask16 __U, __m512i __A) | 4538 | _mm512_mask_storeu_epi32(void *__P, __mmask16 __U, __m512i __A) |
| 4502 | { | 4539 | { |
| ... | @@ -4580,7 +4617,7 @@ _mm512_store_epi64 (void *__P, __m512i __A) | ... | @@ -4580,7 +4617,7 @@ _mm512_store_epi64 (void *__P, __m512i __A) |
| 4580 | 4617 | ||
| 4581 | /* Mask ops */ | 4618 | /* Mask ops */ |
| 4582 | 4619 | ||
| 4583 | static __inline __mmask16 __DEFAULT_FN_ATTRS512 | 4620 | static __inline __mmask16 __DEFAULT_FN_ATTRS |
| 4584 | _mm512_knot(__mmask16 __M) | 4621 | _mm512_knot(__mmask16 __M) |
| 4585 | { | 4622 | { |
| 4586 | return __builtin_ia32_knothi(__M); | 4623 | return __builtin_ia32_knothi(__M); |
| ... | @@ -5622,7 +5659,7 @@ _mm_maskz_getexp_ss (__mmask8 __U, __m128 __A, __m128 __B) | ... | @@ -5622,7 +5659,7 @@ _mm_maskz_getexp_ss (__mmask8 __U, __m128 __A, __m128 __B) |
| 5622 | (__v4sf)_mm_setzero_ps(), \ | 5659 | (__v4sf)_mm_setzero_ps(), \ |
| 5623 | (__mmask8)(U), (int)(R)) | 5660 | (__mmask8)(U), (int)(R)) |
| 5624 | 5661 | ||
| 5625 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS512 | 5662 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS |
| 5626 | _mm512_kmov (__mmask16 __A) | 5663 | _mm512_kmov (__mmask16 __A) |
| 5627 | { | 5664 | { |
| 5628 | return __A; | 5665 | return __A; |
| ... | @@ -7593,177 +7630,177 @@ _mm512_maskz_getexp_ps (__mmask16 __U, __m512 __A) | ... | @@ -7593,177 +7630,177 @@ _mm512_maskz_getexp_ps (__mmask16 __U, __m512 __A) |
| 7593 | 7630 | ||
| 7594 | #define _mm512_i64gather_ps(index, addr, scale) \ | 7631 | #define _mm512_i64gather_ps(index, addr, scale) \ |
| 7595 | (__m256)__builtin_ia32_gatherdiv16sf((__v8sf)_mm256_undefined_ps(), \ | 7632 | (__m256)__builtin_ia32_gatherdiv16sf((__v8sf)_mm256_undefined_ps(), \ |
| 7596 | (float const *)(addr), \ | 7633 | (void const *)(addr), \ |
| 7597 | (__v8di)(__m512i)(index), (__mmask8)-1, \ | 7634 | (__v8di)(__m512i)(index), (__mmask8)-1, \ |
| 7598 | (int)(scale)) | 7635 | (int)(scale)) |
| 7599 | 7636 | ||
| 7600 | #define _mm512_mask_i64gather_ps(v1_old, mask, index, addr, scale) \ | 7637 | #define _mm512_mask_i64gather_ps(v1_old, mask, index, addr, scale) \ |
| 7601 | (__m256)__builtin_ia32_gatherdiv16sf((__v8sf)(__m256)(v1_old),\ | 7638 | (__m256)__builtin_ia32_gatherdiv16sf((__v8sf)(__m256)(v1_old),\ |
| 7602 | (float const *)(addr), \ | 7639 | (void const *)(addr), \ |
| 7603 | (__v8di)(__m512i)(index), \ | 7640 | (__v8di)(__m512i)(index), \ |
| 7604 | (__mmask8)(mask), (int)(scale)) | 7641 | (__mmask8)(mask), (int)(scale)) |
| 7605 | 7642 | ||
| 7606 | #define _mm512_i64gather_epi32(index, addr, scale) \ | 7643 | #define _mm512_i64gather_epi32(index, addr, scale) \ |
| 7607 | (__m256i)__builtin_ia32_gatherdiv16si((__v8si)_mm256_undefined_si256(), \ | 7644 | (__m256i)__builtin_ia32_gatherdiv16si((__v8si)_mm256_undefined_si256(), \ |
| 7608 | (int const *)(addr), \ | 7645 | (void const *)(addr), \ |
| 7609 | (__v8di)(__m512i)(index), \ | 7646 | (__v8di)(__m512i)(index), \ |
| 7610 | (__mmask8)-1, (int)(scale)) | 7647 | (__mmask8)-1, (int)(scale)) |
| 7611 | 7648 | ||
| 7612 | #define _mm512_mask_i64gather_epi32(v1_old, mask, index, addr, scale) \ | 7649 | #define _mm512_mask_i64gather_epi32(v1_old, mask, index, addr, scale) \ |
| 7613 | (__m256i)__builtin_ia32_gatherdiv16si((__v8si)(__m256i)(v1_old), \ | 7650 | (__m256i)__builtin_ia32_gatherdiv16si((__v8si)(__m256i)(v1_old), \ |
| 7614 | (int const *)(addr), \ | 7651 | (void const *)(addr), \ |
| 7615 | (__v8di)(__m512i)(index), \ | 7652 | (__v8di)(__m512i)(index), \ |
| 7616 | (__mmask8)(mask), (int)(scale)) | 7653 | (__mmask8)(mask), (int)(scale)) |
| 7617 | 7654 | ||
| 7618 | #define _mm512_i64gather_pd(index, addr, scale) \ | 7655 | #define _mm512_i64gather_pd(index, addr, scale) \ |
| 7619 | (__m512d)__builtin_ia32_gatherdiv8df((__v8df)_mm512_undefined_pd(), \ | 7656 | (__m512d)__builtin_ia32_gatherdiv8df((__v8df)_mm512_undefined_pd(), \ |
| 7620 | (double const *)(addr), \ | 7657 | (void const *)(addr), \ |
| 7621 | (__v8di)(__m512i)(index), (__mmask8)-1, \ | 7658 | (__v8di)(__m512i)(index), (__mmask8)-1, \ |
| 7622 | (int)(scale)) | 7659 | (int)(scale)) |
| 7623 | 7660 | ||
| 7624 | #define _mm512_mask_i64gather_pd(v1_old, mask, index, addr, scale) \ | 7661 | #define _mm512_mask_i64gather_pd(v1_old, mask, index, addr, scale) \ |
| 7625 | (__m512d)__builtin_ia32_gatherdiv8df((__v8df)(__m512d)(v1_old), \ | 7662 | (__m512d)__builtin_ia32_gatherdiv8df((__v8df)(__m512d)(v1_old), \ |
| 7626 | (double const *)(addr), \ | 7663 | (void const *)(addr), \ |
| 7627 | (__v8di)(__m512i)(index), \ | 7664 | (__v8di)(__m512i)(index), \ |
| 7628 | (__mmask8)(mask), (int)(scale)) | 7665 | (__mmask8)(mask), (int)(scale)) |
| 7629 | 7666 | ||
| 7630 | #define _mm512_i64gather_epi64(index, addr, scale) \ | 7667 | #define _mm512_i64gather_epi64(index, addr, scale) \ |
| 7631 | (__m512i)__builtin_ia32_gatherdiv8di((__v8di)_mm512_undefined_epi32(), \ | 7668 | (__m512i)__builtin_ia32_gatherdiv8di((__v8di)_mm512_undefined_epi32(), \ |
| 7632 | (long long const *)(addr), \ | 7669 | (void const *)(addr), \ |
| 7633 | (__v8di)(__m512i)(index), (__mmask8)-1, \ | 7670 | (__v8di)(__m512i)(index), (__mmask8)-1, \ |
| 7634 | (int)(scale)) | 7671 | (int)(scale)) |
| 7635 | 7672 | ||
| 7636 | #define _mm512_mask_i64gather_epi64(v1_old, mask, index, addr, scale) \ | 7673 | #define _mm512_mask_i64gather_epi64(v1_old, mask, index, addr, scale) \ |
| 7637 | (__m512i)__builtin_ia32_gatherdiv8di((__v8di)(__m512i)(v1_old), \ | 7674 | (__m512i)__builtin_ia32_gatherdiv8di((__v8di)(__m512i)(v1_old), \ |
| 7638 | (long long const *)(addr), \ | 7675 | (void const *)(addr), \ |
| 7639 | (__v8di)(__m512i)(index), \ | 7676 | (__v8di)(__m512i)(index), \ |
| 7640 | (__mmask8)(mask), (int)(scale)) | 7677 | (__mmask8)(mask), (int)(scale)) |
| 7641 | 7678 | ||
| 7642 | #define _mm512_i32gather_ps(index, addr, scale) \ | 7679 | #define _mm512_i32gather_ps(index, addr, scale) \ |
| 7643 | (__m512)__builtin_ia32_gathersiv16sf((__v16sf)_mm512_undefined_ps(), \ | 7680 | (__m512)__builtin_ia32_gathersiv16sf((__v16sf)_mm512_undefined_ps(), \ |
| 7644 | (float const *)(addr), \ | 7681 | (void const *)(addr), \ |
| 7645 | (__v16sf)(__m512)(index), \ | 7682 | (__v16sf)(__m512)(index), \ |
| 7646 | (__mmask16)-1, (int)(scale)) | 7683 | (__mmask16)-1, (int)(scale)) |
| 7647 | 7684 | ||
| 7648 | #define _mm512_mask_i32gather_ps(v1_old, mask, index, addr, scale) \ | 7685 | #define _mm512_mask_i32gather_ps(v1_old, mask, index, addr, scale) \ |
| 7649 | (__m512)__builtin_ia32_gathersiv16sf((__v16sf)(__m512)(v1_old), \ | 7686 | (__m512)__builtin_ia32_gathersiv16sf((__v16sf)(__m512)(v1_old), \ |
| 7650 | (float const *)(addr), \ | 7687 | (void const *)(addr), \ |
| 7651 | (__v16sf)(__m512)(index), \ | 7688 | (__v16sf)(__m512)(index), \ |
| 7652 | (__mmask16)(mask), (int)(scale)) | 7689 | (__mmask16)(mask), (int)(scale)) |
| 7653 | 7690 | ||
| 7654 | #define _mm512_i32gather_epi32(index, addr, scale) \ | 7691 | #define _mm512_i32gather_epi32(index, addr, scale) \ |
| 7655 | (__m512i)__builtin_ia32_gathersiv16si((__v16si)_mm512_undefined_epi32(), \ | 7692 | (__m512i)__builtin_ia32_gathersiv16si((__v16si)_mm512_undefined_epi32(), \ |
| 7656 | (int const *)(addr), \ | 7693 | (void const *)(addr), \ |
| 7657 | (__v16si)(__m512i)(index), \ | 7694 | (__v16si)(__m512i)(index), \ |
| 7658 | (__mmask16)-1, (int)(scale)) | 7695 | (__mmask16)-1, (int)(scale)) |
| 7659 | 7696 | ||
| 7660 | #define _mm512_mask_i32gather_epi32(v1_old, mask, index, addr, scale) \ | 7697 | #define _mm512_mask_i32gather_epi32(v1_old, mask, index, addr, scale) \ |
| 7661 | (__m512i)__builtin_ia32_gathersiv16si((__v16si)(__m512i)(v1_old), \ | 7698 | (__m512i)__builtin_ia32_gathersiv16si((__v16si)(__m512i)(v1_old), \ |
| 7662 | (int const *)(addr), \ | 7699 | (void const *)(addr), \ |
| 7663 | (__v16si)(__m512i)(index), \ | 7700 | (__v16si)(__m512i)(index), \ |
| 7664 | (__mmask16)(mask), (int)(scale)) | 7701 | (__mmask16)(mask), (int)(scale)) |
| 7665 | 7702 | ||
| 7666 | #define _mm512_i32gather_pd(index, addr, scale) \ | 7703 | #define _mm512_i32gather_pd(index, addr, scale) \ |
| 7667 | (__m512d)__builtin_ia32_gathersiv8df((__v8df)_mm512_undefined_pd(), \ | 7704 | (__m512d)__builtin_ia32_gathersiv8df((__v8df)_mm512_undefined_pd(), \ |
| 7668 | (double const *)(addr), \ | 7705 | (void const *)(addr), \ |
| 7669 | (__v8si)(__m256i)(index), (__mmask8)-1, \ | 7706 | (__v8si)(__m256i)(index), (__mmask8)-1, \ |
| 7670 | (int)(scale)) | 7707 | (int)(scale)) |
| 7671 | 7708 | ||
| 7672 | #define _mm512_mask_i32gather_pd(v1_old, mask, index, addr, scale) \ | 7709 | #define _mm512_mask_i32gather_pd(v1_old, mask, index, addr, scale) \ |
| 7673 | (__m512d)__builtin_ia32_gathersiv8df((__v8df)(__m512d)(v1_old), \ | 7710 | (__m512d)__builtin_ia32_gathersiv8df((__v8df)(__m512d)(v1_old), \ |
| 7674 | (double const *)(addr), \ | 7711 | (void const *)(addr), \ |
| 7675 | (__v8si)(__m256i)(index), \ | 7712 | (__v8si)(__m256i)(index), \ |
| 7676 | (__mmask8)(mask), (int)(scale)) | 7713 | (__mmask8)(mask), (int)(scale)) |
| 7677 | 7714 | ||
| 7678 | #define _mm512_i32gather_epi64(index, addr, scale) \ | 7715 | #define _mm512_i32gather_epi64(index, addr, scale) \ |
| 7679 | (__m512i)__builtin_ia32_gathersiv8di((__v8di)_mm512_undefined_epi32(), \ | 7716 | (__m512i)__builtin_ia32_gathersiv8di((__v8di)_mm512_undefined_epi32(), \ |
| 7680 | (long long const *)(addr), \ | 7717 | (void const *)(addr), \ |
| 7681 | (__v8si)(__m256i)(index), (__mmask8)-1, \ | 7718 | (__v8si)(__m256i)(index), (__mmask8)-1, \ |
| 7682 | (int)(scale)) | 7719 | (int)(scale)) |
| 7683 | 7720 | ||
| 7684 | #define _mm512_mask_i32gather_epi64(v1_old, mask, index, addr, scale) \ | 7721 | #define _mm512_mask_i32gather_epi64(v1_old, mask, index, addr, scale) \ |
| 7685 | (__m512i)__builtin_ia32_gathersiv8di((__v8di)(__m512i)(v1_old), \ | 7722 | (__m512i)__builtin_ia32_gathersiv8di((__v8di)(__m512i)(v1_old), \ |
| 7686 | (long long const *)(addr), \ | 7723 | (void const *)(addr), \ |
| 7687 | (__v8si)(__m256i)(index), \ | 7724 | (__v8si)(__m256i)(index), \ |
| 7688 | (__mmask8)(mask), (int)(scale)) | 7725 | (__mmask8)(mask), (int)(scale)) |
| 7689 | 7726 | ||
| 7690 | #define _mm512_i64scatter_ps(addr, index, v1, scale) \ | 7727 | #define _mm512_i64scatter_ps(addr, index, v1, scale) \ |
| 7691 | __builtin_ia32_scatterdiv16sf((float *)(addr), (__mmask8)-1, \ | 7728 | __builtin_ia32_scatterdiv16sf((void *)(addr), (__mmask8)-1, \ |
| 7692 | (__v8di)(__m512i)(index), \ | 7729 | (__v8di)(__m512i)(index), \ |
| 7693 | (__v8sf)(__m256)(v1), (int)(scale)) | 7730 | (__v8sf)(__m256)(v1), (int)(scale)) |
| 7694 | 7731 | ||
| 7695 | #define _mm512_mask_i64scatter_ps(addr, mask, index, v1, scale) \ | 7732 | #define _mm512_mask_i64scatter_ps(addr, mask, index, v1, scale) \ |
| 7696 | __builtin_ia32_scatterdiv16sf((float *)(addr), (__mmask8)(mask), \ | 7733 | __builtin_ia32_scatterdiv16sf((void *)(addr), (__mmask8)(mask), \ |
| 7697 | (__v8di)(__m512i)(index), \ | 7734 | (__v8di)(__m512i)(index), \ |
| 7698 | (__v8sf)(__m256)(v1), (int)(scale)) | 7735 | (__v8sf)(__m256)(v1), (int)(scale)) |
| 7699 | 7736 | ||
| 7700 | #define _mm512_i64scatter_epi32(addr, index, v1, scale) \ | 7737 | #define _mm512_i64scatter_epi32(addr, index, v1, scale) \ |
| 7701 | __builtin_ia32_scatterdiv16si((int *)(addr), (__mmask8)-1, \ | 7738 | __builtin_ia32_scatterdiv16si((void *)(addr), (__mmask8)-1, \ |
| 7702 | (__v8di)(__m512i)(index), \ | 7739 | (__v8di)(__m512i)(index), \ |
| 7703 | (__v8si)(__m256i)(v1), (int)(scale)) | 7740 | (__v8si)(__m256i)(v1), (int)(scale)) |
| 7704 | 7741 | ||
| 7705 | #define _mm512_mask_i64scatter_epi32(addr, mask, index, v1, scale) \ | 7742 | #define _mm512_mask_i64scatter_epi32(addr, mask, index, v1, scale) \ |
| 7706 | __builtin_ia32_scatterdiv16si((int *)(addr), (__mmask8)(mask), \ | 7743 | __builtin_ia32_scatterdiv16si((void *)(addr), (__mmask8)(mask), \ |
| 7707 | (__v8di)(__m512i)(index), \ | 7744 | (__v8di)(__m512i)(index), \ |
| 7708 | (__v8si)(__m256i)(v1), (int)(scale)) | 7745 | (__v8si)(__m256i)(v1), (int)(scale)) |
| 7709 | 7746 | ||
| 7710 | #define _mm512_i64scatter_pd(addr, index, v1, scale) \ | 7747 | #define _mm512_i64scatter_pd(addr, index, v1, scale) \ |
| 7711 | __builtin_ia32_scatterdiv8df((double *)(addr), (__mmask8)-1, \ | 7748 | __builtin_ia32_scatterdiv8df((void *)(addr), (__mmask8)-1, \ |
| 7712 | (__v8di)(__m512i)(index), \ | 7749 | (__v8di)(__m512i)(index), \ |
| 7713 | (__v8df)(__m512d)(v1), (int)(scale)) | 7750 | (__v8df)(__m512d)(v1), (int)(scale)) |
| 7714 | 7751 | ||
| 7715 | #define _mm512_mask_i64scatter_pd(addr, mask, index, v1, scale) \ | 7752 | #define _mm512_mask_i64scatter_pd(addr, mask, index, v1, scale) \ |
| 7716 | __builtin_ia32_scatterdiv8df((double *)(addr), (__mmask8)(mask), \ | 7753 | __builtin_ia32_scatterdiv8df((void *)(addr), (__mmask8)(mask), \ |
| 7717 | (__v8di)(__m512i)(index), \ | 7754 | (__v8di)(__m512i)(index), \ |
| 7718 | (__v8df)(__m512d)(v1), (int)(scale)) | 7755 | (__v8df)(__m512d)(v1), (int)(scale)) |
| 7719 | 7756 | ||
| 7720 | #define _mm512_i64scatter_epi64(addr, index, v1, scale) \ | 7757 | #define _mm512_i64scatter_epi64(addr, index, v1, scale) \ |
| 7721 | __builtin_ia32_scatterdiv8di((long long *)(addr), (__mmask8)-1, \ | 7758 | __builtin_ia32_scatterdiv8di((void *)(addr), (__mmask8)-1, \ |
| 7722 | (__v8di)(__m512i)(index), \ | 7759 | (__v8di)(__m512i)(index), \ |
| 7723 | (__v8di)(__m512i)(v1), (int)(scale)) | 7760 | (__v8di)(__m512i)(v1), (int)(scale)) |
| 7724 | 7761 | ||
| 7725 | #define _mm512_mask_i64scatter_epi64(addr, mask, index, v1, scale) \ | 7762 | #define _mm512_mask_i64scatter_epi64(addr, mask, index, v1, scale) \ |
| 7726 | __builtin_ia32_scatterdiv8di((long long *)(addr), (__mmask8)(mask), \ | 7763 | __builtin_ia32_scatterdiv8di((void *)(addr), (__mmask8)(mask), \ |
| 7727 | (__v8di)(__m512i)(index), \ | 7764 | (__v8di)(__m512i)(index), \ |
| 7728 | (__v8di)(__m512i)(v1), (int)(scale)) | 7765 | (__v8di)(__m512i)(v1), (int)(scale)) |
| 7729 | 7766 | ||
| 7730 | #define _mm512_i32scatter_ps(addr, index, v1, scale) \ | 7767 | #define _mm512_i32scatter_ps(addr, index, v1, scale) \ |
| 7731 | __builtin_ia32_scattersiv16sf((float *)(addr), (__mmask16)-1, \ | 7768 | __builtin_ia32_scattersiv16sf((void *)(addr), (__mmask16)-1, \ |
| 7732 | (__v16si)(__m512i)(index), \ | 7769 | (__v16si)(__m512i)(index), \ |
| 7733 | (__v16sf)(__m512)(v1), (int)(scale)) | 7770 | (__v16sf)(__m512)(v1), (int)(scale)) |
| 7734 | 7771 | ||
| 7735 | #define _mm512_mask_i32scatter_ps(addr, mask, index, v1, scale) \ | 7772 | #define _mm512_mask_i32scatter_ps(addr, mask, index, v1, scale) \ |
| 7736 | __builtin_ia32_scattersiv16sf((float *)(addr), (__mmask16)(mask), \ | 7773 | __builtin_ia32_scattersiv16sf((void *)(addr), (__mmask16)(mask), \ |
| 7737 | (__v16si)(__m512i)(index), \ | 7774 | (__v16si)(__m512i)(index), \ |
| 7738 | (__v16sf)(__m512)(v1), (int)(scale)) | 7775 | (__v16sf)(__m512)(v1), (int)(scale)) |
| 7739 | 7776 | ||
| 7740 | #define _mm512_i32scatter_epi32(addr, index, v1, scale) \ | 7777 | #define _mm512_i32scatter_epi32(addr, index, v1, scale) \ |
| 7741 | __builtin_ia32_scattersiv16si((int *)(addr), (__mmask16)-1, \ | 7778 | __builtin_ia32_scattersiv16si((void *)(addr), (__mmask16)-1, \ |
| 7742 | (__v16si)(__m512i)(index), \ | 7779 | (__v16si)(__m512i)(index), \ |
| 7743 | (__v16si)(__m512i)(v1), (int)(scale)) | 7780 | (__v16si)(__m512i)(v1), (int)(scale)) |
| 7744 | 7781 | ||
| 7745 | #define _mm512_mask_i32scatter_epi32(addr, mask, index, v1, scale) \ | 7782 | #define _mm512_mask_i32scatter_epi32(addr, mask, index, v1, scale) \ |
| 7746 | __builtin_ia32_scattersiv16si((int *)(addr), (__mmask16)(mask), \ | 7783 | __builtin_ia32_scattersiv16si((void *)(addr), (__mmask16)(mask), \ |
| 7747 | (__v16si)(__m512i)(index), \ | 7784 | (__v16si)(__m512i)(index), \ |
| 7748 | (__v16si)(__m512i)(v1), (int)(scale)) | 7785 | (__v16si)(__m512i)(v1), (int)(scale)) |
| 7749 | 7786 | ||
| 7750 | #define _mm512_i32scatter_pd(addr, index, v1, scale) \ | 7787 | #define _mm512_i32scatter_pd(addr, index, v1, scale) \ |
| 7751 | __builtin_ia32_scattersiv8df((double *)(addr), (__mmask8)-1, \ | 7788 | __builtin_ia32_scattersiv8df((void *)(addr), (__mmask8)-1, \ |
| 7752 | (__v8si)(__m256i)(index), \ | 7789 | (__v8si)(__m256i)(index), \ |
| 7753 | (__v8df)(__m512d)(v1), (int)(scale)) | 7790 | (__v8df)(__m512d)(v1), (int)(scale)) |
| 7754 | 7791 | ||
| 7755 | #define _mm512_mask_i32scatter_pd(addr, mask, index, v1, scale) \ | 7792 | #define _mm512_mask_i32scatter_pd(addr, mask, index, v1, scale) \ |
| 7756 | __builtin_ia32_scattersiv8df((double *)(addr), (__mmask8)(mask), \ | 7793 | __builtin_ia32_scattersiv8df((void *)(addr), (__mmask8)(mask), \ |
| 7757 | (__v8si)(__m256i)(index), \ | 7794 | (__v8si)(__m256i)(index), \ |
| 7758 | (__v8df)(__m512d)(v1), (int)(scale)) | 7795 | (__v8df)(__m512d)(v1), (int)(scale)) |
| 7759 | 7796 | ||
| 7760 | #define _mm512_i32scatter_epi64(addr, index, v1, scale) \ | 7797 | #define _mm512_i32scatter_epi64(addr, index, v1, scale) \ |
| 7761 | __builtin_ia32_scattersiv8di((long long *)(addr), (__mmask8)-1, \ | 7798 | __builtin_ia32_scattersiv8di((void *)(addr), (__mmask8)-1, \ |
| 7762 | (__v8si)(__m256i)(index), \ | 7799 | (__v8si)(__m256i)(index), \ |
| 7763 | (__v8di)(__m512i)(v1), (int)(scale)) | 7800 | (__v8di)(__m512i)(v1), (int)(scale)) |
| 7764 | 7801 | ||
| 7765 | #define _mm512_mask_i32scatter_epi64(addr, mask, index, v1, scale) \ | 7802 | #define _mm512_mask_i32scatter_epi64(addr, mask, index, v1, scale) \ |
| 7766 | __builtin_ia32_scattersiv8di((long long *)(addr), (__mmask8)(mask), \ | 7803 | __builtin_ia32_scattersiv8di((void *)(addr), (__mmask8)(mask), \ |
| 7767 | (__v8si)(__m256i)(index), \ | 7804 | (__v8si)(__m256i)(index), \ |
| 7768 | (__v8di)(__m512i)(v1), (int)(scale)) | 7805 | (__v8di)(__m512i)(v1), (int)(scale)) |
| 7769 | 7806 | ||
| ... | @@ -8320,54 +8357,105 @@ _mm512_mask_permutexvar_epi32 (__m512i __W, __mmask16 __M, __m512i __X, | ... | @@ -8320,54 +8357,105 @@ _mm512_mask_permutexvar_epi32 (__m512i __W, __mmask16 __M, __m512i __X, |
| 8320 | 8357 | ||
| 8321 | #define _mm512_mask_permutevar_epi32 _mm512_mask_permutexvar_epi32 | 8358 | #define _mm512_mask_permutevar_epi32 _mm512_mask_permutexvar_epi32 |
| 8322 | 8359 | ||
| 8323 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS512 | 8360 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS |
| 8324 | _mm512_kand (__mmask16 __A, __mmask16 __B) | 8361 | _mm512_kand (__mmask16 __A, __mmask16 __B) |
| 8325 | { | 8362 | { |
| 8326 | return (__mmask16) __builtin_ia32_kandhi ((__mmask16) __A, (__mmask16) __B); | 8363 | return (__mmask16) __builtin_ia32_kandhi ((__mmask16) __A, (__mmask16) __B); |
| 8327 | } | 8364 | } |
| 8328 | 8365 | ||
| 8329 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS512 | 8366 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS |
| 8330 | _mm512_kandn (__mmask16 __A, __mmask16 __B) | 8367 | _mm512_kandn (__mmask16 __A, __mmask16 __B) |
| 8331 | { | 8368 | { |
| 8332 | return (__mmask16) __builtin_ia32_kandnhi ((__mmask16) __A, (__mmask16) __B); | 8369 | return (__mmask16) __builtin_ia32_kandnhi ((__mmask16) __A, (__mmask16) __B); |
| 8333 | } | 8370 | } |
| 8334 | 8371 | ||
| 8335 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS512 | 8372 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS |
| 8336 | _mm512_kor (__mmask16 __A, __mmask16 __B) | 8373 | _mm512_kor (__mmask16 __A, __mmask16 __B) |
| 8337 | { | 8374 | { |
| 8338 | return (__mmask16) __builtin_ia32_korhi ((__mmask16) __A, (__mmask16) __B); | 8375 | return (__mmask16) __builtin_ia32_korhi ((__mmask16) __A, (__mmask16) __B); |
| 8339 | } | 8376 | } |
| 8340 | 8377 | ||
| 8341 | static __inline__ int __DEFAULT_FN_ATTRS512 | 8378 | static __inline__ int __DEFAULT_FN_ATTRS |
| 8342 | _mm512_kortestc (__mmask16 __A, __mmask16 __B) | 8379 | _mm512_kortestc (__mmask16 __A, __mmask16 __B) |
| 8343 | { | 8380 | { |
| 8344 | return __builtin_ia32_kortestchi ((__mmask16) __A, (__mmask16) __B); | 8381 | return __builtin_ia32_kortestchi ((__mmask16) __A, (__mmask16) __B); |
| 8345 | } | 8382 | } |
| 8346 | 8383 | ||
| 8347 | static __inline__ int __DEFAULT_FN_ATTRS512 | 8384 | static __inline__ int __DEFAULT_FN_ATTRS |
| 8348 | _mm512_kortestz (__mmask16 __A, __mmask16 __B) | 8385 | _mm512_kortestz (__mmask16 __A, __mmask16 __B) |
| 8349 | { | 8386 | { |
| 8350 | return __builtin_ia32_kortestzhi ((__mmask16) __A, (__mmask16) __B); | 8387 | return __builtin_ia32_kortestzhi ((__mmask16) __A, (__mmask16) __B); |
| 8351 | } | 8388 | } |
| 8352 | 8389 | ||
| 8353 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS512 | 8390 | static __inline__ unsigned char __DEFAULT_FN_ATTRS |
| 8391 | _kortestc_mask16_u8(__mmask16 __A, __mmask16 __B) | ||
| 8392 | { | ||
| 8393 | return (unsigned char)__builtin_ia32_kortestchi(__A, __B); | ||
| 8394 | } | ||
| 8395 | |||
| 8396 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 8397 | _kortestz_mask16_u8(__mmask16 __A, __mmask16 __B) | ||
| 8398 | { | ||
| 8399 | return (unsigned char)__builtin_ia32_kortestzhi(__A, __B); | ||
| 8400 | } | ||
| 8401 | |||
| 8402 | static __inline__ unsigned char __DEFAULT_FN_ATTRS | ||
| 8403 | _kortest_mask16_u8(__mmask16 __A, __mmask16 __B, unsigned char *__C) { | ||
| 8404 | *__C = (unsigned char)__builtin_ia32_kortestchi(__A, __B); | ||
| 8405 | return (unsigned char)__builtin_ia32_kortestzhi(__A, __B); | ||
| 8406 | } | ||
| 8407 | |||
| 8408 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS | ||
| 8354 | _mm512_kunpackb (__mmask16 __A, __mmask16 __B) | 8409 | _mm512_kunpackb (__mmask16 __A, __mmask16 __B) |
| 8355 | { | 8410 | { |
| 8356 | return (__mmask16) __builtin_ia32_kunpckhi ((__mmask16) __A, (__mmask16) __B); | 8411 | return (__mmask16) __builtin_ia32_kunpckhi ((__mmask16) __A, (__mmask16) __B); |
| 8357 | } | 8412 | } |
| 8358 | 8413 | ||
| 8359 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS512 | 8414 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS |
| 8360 | _mm512_kxnor (__mmask16 __A, __mmask16 __B) | 8415 | _mm512_kxnor (__mmask16 __A, __mmask16 __B) |
| 8361 | { | 8416 | { |
| 8362 | return (__mmask16) __builtin_ia32_kxnorhi ((__mmask16) __A, (__mmask16) __B); | 8417 | return (__mmask16) __builtin_ia32_kxnorhi ((__mmask16) __A, (__mmask16) __B); |
| 8363 | } | 8418 | } |
| 8364 | 8419 | ||
| 8365 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS512 | 8420 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS |
| 8366 | _mm512_kxor (__mmask16 __A, __mmask16 __B) | 8421 | _mm512_kxor (__mmask16 __A, __mmask16 __B) |
| 8367 | { | 8422 | { |
| 8368 | return (__mmask16) __builtin_ia32_kxorhi ((__mmask16) __A, (__mmask16) __B); | 8423 | return (__mmask16) __builtin_ia32_kxorhi ((__mmask16) __A, (__mmask16) __B); |
| 8369 | } | 8424 | } |
| 8370 | 8425 | ||
| 8426 | #define _kand_mask16 _mm512_kand | ||
| 8427 | #define _kandn_mask16 _mm512_kandn | ||
| 8428 | #define _knot_mask16 _mm512_knot | ||
| 8429 | #define _kor_mask16 _mm512_kor | ||
| 8430 | #define _kxnor_mask16 _mm512_kxnor | ||
| 8431 | #define _kxor_mask16 _mm512_kxor | ||
| 8432 | |||
| 8433 | #define _kshiftli_mask16(A, I) \ | ||
| 8434 | (__mmask16)__builtin_ia32_kshiftlihi((__mmask16)(A), (unsigned int)(I)) | ||
| 8435 | |||
| 8436 | #define _kshiftri_mask16(A, I) \ | ||
| 8437 | (__mmask16)__builtin_ia32_kshiftrihi((__mmask16)(A), (unsigned int)(I)) | ||
| 8438 | |||
| 8439 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | ||
| 8440 | _cvtmask16_u32(__mmask16 __A) { | ||
| 8441 | return (unsigned int)__builtin_ia32_kmovw((__mmask16)__A); | ||
| 8442 | } | ||
| 8443 | |||
| 8444 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS | ||
| 8445 | _cvtu32_mask16(unsigned int __A) { | ||
| 8446 | return (__mmask16)__builtin_ia32_kmovw((__mmask16)__A); | ||
| 8447 | } | ||
| 8448 | |||
| 8449 | static __inline__ __mmask16 __DEFAULT_FN_ATTRS | ||
| 8450 | _load_mask16(__mmask16 *__A) { | ||
| 8451 | return (__mmask16)__builtin_ia32_kmovw(*(__mmask16 *)__A); | ||
| 8452 | } | ||
| 8453 | |||
| 8454 | static __inline__ void __DEFAULT_FN_ATTRS | ||
| 8455 | _store_mask16(__mmask16 *__A, __mmask16 __B) { | ||
| 8456 | *(__mmask16 *)__A = __builtin_ia32_kmovw((__mmask16)__B); | ||
| 8457 | } | ||
| 8458 | |||
| 8371 | static __inline__ void __DEFAULT_FN_ATTRS512 | 8459 | static __inline__ void __DEFAULT_FN_ATTRS512 |
| 8372 | _mm512_stream_si512 (__m512i * __P, __m512i __A) | 8460 | _mm512_stream_si512 (__m512i * __P, __m512i __A) |
| 8373 | { | 8461 | { |
| ... | @@ -9594,5 +9682,6 @@ _mm512_mask_reduce_min_ps(__mmask16 __M, __m512 __V) { | ... | @@ -9594,5 +9682,6 @@ _mm512_mask_reduce_min_ps(__mmask16 __M, __m512 __V) { |
| 9594 | 9682 | ||
| 9595 | #undef __DEFAULT_FN_ATTRS512 | 9683 | #undef __DEFAULT_FN_ATTRS512 |
| 9596 | #undef __DEFAULT_FN_ATTRS128 | 9684 | #undef __DEFAULT_FN_ATTRS128 |
| 9685 | #undef __DEFAULT_FN_ATTRS | ||
| 9597 | 9686 | ||
| 9598 | #endif /* __AVX512FINTRIN_H */ | 9687 | #endif /* __AVX512FINTRIN_H */ |
c_headers/avx512pfintrin.h+16-16
| ... | @@ -33,78 +33,78 @@ | ... | @@ -33,78 +33,78 @@ |
| 33 | 33 | ||
| 34 | #define _mm512_mask_prefetch_i32gather_pd(index, mask, addr, scale, hint) \ | 34 | #define _mm512_mask_prefetch_i32gather_pd(index, mask, addr, scale, hint) \ |
| 35 | __builtin_ia32_gatherpfdpd((__mmask8)(mask), (__v8si)(__m256i)(index), \ | 35 | __builtin_ia32_gatherpfdpd((__mmask8)(mask), (__v8si)(__m256i)(index), \ |
| 36 | (long long const *)(addr), (int)(scale), \ | 36 | (void const *)(addr), (int)(scale), \ |
| 37 | (int)(hint)) | 37 | (int)(hint)) |
| 38 | 38 | ||
| 39 | #define _mm512_prefetch_i32gather_pd(index, addr, scale, hint) \ | 39 | #define _mm512_prefetch_i32gather_pd(index, addr, scale, hint) \ |
| 40 | __builtin_ia32_gatherpfdpd((__mmask8) -1, (__v8si)(__m256i)(index), \ | 40 | __builtin_ia32_gatherpfdpd((__mmask8) -1, (__v8si)(__m256i)(index), \ |
| 41 | (long long const *)(addr), (int)(scale), \ | 41 | (void const *)(addr), (int)(scale), \ |
| 42 | (int)(hint)) | 42 | (int)(hint)) |
| 43 | 43 | ||
| 44 | #define _mm512_mask_prefetch_i32gather_ps(index, mask, addr, scale, hint) \ | 44 | #define _mm512_mask_prefetch_i32gather_ps(index, mask, addr, scale, hint) \ |
| 45 | __builtin_ia32_gatherpfdps((__mmask16)(mask), \ | 45 | __builtin_ia32_gatherpfdps((__mmask16)(mask), \ |
| 46 | (__v16si)(__m512i)(index), (int const *)(addr), \ | 46 | (__v16si)(__m512i)(index), (void const *)(addr), \ |
| 47 | (int)(scale), (int)(hint)) | 47 | (int)(scale), (int)(hint)) |
| 48 | 48 | ||
| 49 | #define _mm512_prefetch_i32gather_ps(index, addr, scale, hint) \ | 49 | #define _mm512_prefetch_i32gather_ps(index, addr, scale, hint) \ |
| 50 | __builtin_ia32_gatherpfdps((__mmask16) -1, \ | 50 | __builtin_ia32_gatherpfdps((__mmask16) -1, \ |
| 51 | (__v16si)(__m512i)(index), (int const *)(addr), \ | 51 | (__v16si)(__m512i)(index), (void const *)(addr), \ |
| 52 | (int)(scale), (int)(hint)) | 52 | (int)(scale), (int)(hint)) |
| 53 | 53 | ||
| 54 | #define _mm512_mask_prefetch_i64gather_pd(index, mask, addr, scale, hint) \ | 54 | #define _mm512_mask_prefetch_i64gather_pd(index, mask, addr, scale, hint) \ |
| 55 | __builtin_ia32_gatherpfqpd((__mmask8)(mask), (__v8di)(__m512i)(index), \ | 55 | __builtin_ia32_gatherpfqpd((__mmask8)(mask), (__v8di)(__m512i)(index), \ |
| 56 | (long long const *)(addr), (int)(scale), \ | 56 | (void const *)(addr), (int)(scale), \ |
| 57 | (int)(hint)) | 57 | (int)(hint)) |
| 58 | 58 | ||
| 59 | #define _mm512_prefetch_i64gather_pd(index, addr, scale, hint) \ | 59 | #define _mm512_prefetch_i64gather_pd(index, addr, scale, hint) \ |
| 60 | __builtin_ia32_gatherpfqpd((__mmask8) -1, (__v8di)(__m512i)(index), \ | 60 | __builtin_ia32_gatherpfqpd((__mmask8) -1, (__v8di)(__m512i)(index), \ |
| 61 | (long long const *)(addr), (int)(scale), \ | 61 | (void const *)(addr), (int)(scale), \ |
| 62 | (int)(hint)) | 62 | (int)(hint)) |
| 63 | 63 | ||
| 64 | #define _mm512_mask_prefetch_i64gather_ps(index, mask, addr, scale, hint) \ | 64 | #define _mm512_mask_prefetch_i64gather_ps(index, mask, addr, scale, hint) \ |
| 65 | __builtin_ia32_gatherpfqps((__mmask8)(mask), (__v8di)(__m512i)(index), \ | 65 | __builtin_ia32_gatherpfqps((__mmask8)(mask), (__v8di)(__m512i)(index), \ |
| 66 | (int const *)(addr), (int)(scale), (int)(hint)) | 66 | (void const *)(addr), (int)(scale), (int)(hint)) |
| 67 | 67 | ||
| 68 | #define _mm512_prefetch_i64gather_ps(index, addr, scale, hint) \ | 68 | #define _mm512_prefetch_i64gather_ps(index, addr, scale, hint) \ |
| 69 | __builtin_ia32_gatherpfqps((__mmask8) -1, (__v8di)(__m512i)(index), \ | 69 | __builtin_ia32_gatherpfqps((__mmask8) -1, (__v8di)(__m512i)(index), \ |
| 70 | (int const *)(addr), (int)(scale), (int)(hint)) | 70 | (void const *)(addr), (int)(scale), (int)(hint)) |
| 71 | 71 | ||
| 72 | #define _mm512_prefetch_i32scatter_pd(addr, index, scale, hint) \ | 72 | #define _mm512_prefetch_i32scatter_pd(addr, index, scale, hint) \ |
| 73 | __builtin_ia32_scatterpfdpd((__mmask8)-1, (__v8si)(__m256i)(index), \ | 73 | __builtin_ia32_scatterpfdpd((__mmask8)-1, (__v8si)(__m256i)(index), \ |
| 74 | (long long *)(addr), (int)(scale), \ | 74 | (void *)(addr), (int)(scale), \ |
| 75 | (int)(hint)) | 75 | (int)(hint)) |
| 76 | 76 | ||
| 77 | #define _mm512_mask_prefetch_i32scatter_pd(addr, mask, index, scale, hint) \ | 77 | #define _mm512_mask_prefetch_i32scatter_pd(addr, mask, index, scale, hint) \ |
| 78 | __builtin_ia32_scatterpfdpd((__mmask8)(mask), (__v8si)(__m256i)(index), \ | 78 | __builtin_ia32_scatterpfdpd((__mmask8)(mask), (__v8si)(__m256i)(index), \ |
| 79 | (long long *)(addr), (int)(scale), \ | 79 | (void *)(addr), (int)(scale), \ |
| 80 | (int)(hint)) | 80 | (int)(hint)) |
| 81 | 81 | ||
| 82 | #define _mm512_prefetch_i32scatter_ps(addr, index, scale, hint) \ | 82 | #define _mm512_prefetch_i32scatter_ps(addr, index, scale, hint) \ |
| 83 | __builtin_ia32_scatterpfdps((__mmask16)-1, (__v16si)(__m512i)(index), \ | 83 | __builtin_ia32_scatterpfdps((__mmask16)-1, (__v16si)(__m512i)(index), \ |
| 84 | (int *)(addr), (int)(scale), (int)(hint)) | 84 | (void *)(addr), (int)(scale), (int)(hint)) |
| 85 | 85 | ||
| 86 | #define _mm512_mask_prefetch_i32scatter_ps(addr, mask, index, scale, hint) \ | 86 | #define _mm512_mask_prefetch_i32scatter_ps(addr, mask, index, scale, hint) \ |
| 87 | __builtin_ia32_scatterpfdps((__mmask16)(mask), \ | 87 | __builtin_ia32_scatterpfdps((__mmask16)(mask), \ |
| 88 | (__v16si)(__m512i)(index), (int *)(addr), \ | 88 | (__v16si)(__m512i)(index), (void *)(addr), \ |
| 89 | (int)(scale), (int)(hint)) | 89 | (int)(scale), (int)(hint)) |
| 90 | 90 | ||
| 91 | #define _mm512_prefetch_i64scatter_pd(addr, index, scale, hint) \ | 91 | #define _mm512_prefetch_i64scatter_pd(addr, index, scale, hint) \ |
| 92 | __builtin_ia32_scatterpfqpd((__mmask8)-1, (__v8di)(__m512i)(index), \ | 92 | __builtin_ia32_scatterpfqpd((__mmask8)-1, (__v8di)(__m512i)(index), \ |
| 93 | (long long *)(addr), (int)(scale), \ | 93 | (void *)(addr), (int)(scale), \ |
| 94 | (int)(hint)) | 94 | (int)(hint)) |
| 95 | 95 | ||
| 96 | #define _mm512_mask_prefetch_i64scatter_pd(addr, mask, index, scale, hint) \ | 96 | #define _mm512_mask_prefetch_i64scatter_pd(addr, mask, index, scale, hint) \ |
| 97 | __builtin_ia32_scatterpfqpd((__mmask8)(mask), (__v8di)(__m512i)(index), \ | 97 | __builtin_ia32_scatterpfqpd((__mmask8)(mask), (__v8di)(__m512i)(index), \ |
| 98 | (long long *)(addr), (int)(scale), \ | 98 | (void *)(addr), (int)(scale), \ |
| 99 | (int)(hint)) | 99 | (int)(hint)) |
| 100 | 100 | ||
| 101 | #define _mm512_prefetch_i64scatter_ps(addr, index, scale, hint) \ | 101 | #define _mm512_prefetch_i64scatter_ps(addr, index, scale, hint) \ |
| 102 | __builtin_ia32_scatterpfqps((__mmask8)-1, (__v8di)(__m512i)(index), \ | 102 | __builtin_ia32_scatterpfqps((__mmask8)-1, (__v8di)(__m512i)(index), \ |
| 103 | (int *)(addr), (int)(scale), (int)(hint)) | 103 | (void *)(addr), (int)(scale), (int)(hint)) |
| 104 | 104 | ||
| 105 | #define _mm512_mask_prefetch_i64scatter_ps(addr, mask, index, scale, hint) \ | 105 | #define _mm512_mask_prefetch_i64scatter_ps(addr, mask, index, scale, hint) \ |
| 106 | __builtin_ia32_scatterpfqps((__mmask8)(mask), (__v8di)(__m512i)(index), \ | 106 | __builtin_ia32_scatterpfqps((__mmask8)(mask), (__v8di)(__m512i)(index), \ |
| 107 | (int *)(addr), (int)(scale), (int)(hint)) | 107 | (void *)(addr), (int)(scale), (int)(hint)) |
| 108 | 108 | ||
| 109 | #undef __DEFAULT_FN_ATTRS | 109 | #undef __DEFAULT_FN_ATTRS |
| 110 | 110 |
c_headers/avx512vbmi2intrin.h+66-92
| ... | @@ -227,167 +227,141 @@ _mm512_maskz_expandloadu_epi8(__mmask64 __U, void const *__P) | ... | @@ -227,167 +227,141 @@ _mm512_maskz_expandloadu_epi8(__mmask64 __U, void const *__P) |
| 227 | (__v32hi)_mm512_setzero_si512()) | 227 | (__v32hi)_mm512_setzero_si512()) |
| 228 | 228 | ||
| 229 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 229 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 230 | _mm512_mask_shldv_epi64(__m512i __S, __mmask8 __U, __m512i __A, __m512i __B) | 230 | _mm512_shldv_epi64(__m512i __A, __m512i __B, __m512i __C) |
| 231 | { | 231 | { |
| 232 | return (__m512i) __builtin_ia32_vpshldvq512_mask ((__v8di) __S, | 232 | return (__m512i)__builtin_ia32_vpshldvq512((__v8di)__A, (__v8di)__B, |
| 233 | (__v8di) __A, | 233 | (__v8di)__C); |
| 234 | (__v8di) __B, | ||
| 235 | __U); | ||
| 236 | } | 234 | } |
| 237 | 235 | ||
| 238 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 236 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 239 | _mm512_maskz_shldv_epi64(__mmask8 __U, __m512i __S, __m512i __A, __m512i __B) | 237 | _mm512_mask_shldv_epi64(__m512i __A, __mmask8 __U, __m512i __B, __m512i __C) |
| 240 | { | 238 | { |
| 241 | return (__m512i) __builtin_ia32_vpshldvq512_maskz ((__v8di) __S, | 239 | return (__m512i)__builtin_ia32_selectq_512(__U, |
| 242 | (__v8di) __A, | 240 | (__v8di)_mm512_shldv_epi64(__A, __B, __C), |
| 243 | (__v8di) __B, | 241 | (__v8di)__A); |
| 244 | __U); | ||
| 245 | } | 242 | } |
| 246 | 243 | ||
| 247 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 244 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 248 | _mm512_shldv_epi64(__m512i __S, __m512i __A, __m512i __B) | 245 | _mm512_maskz_shldv_epi64(__mmask8 __U, __m512i __A, __m512i __B, __m512i __C) |
| 249 | { | 246 | { |
| 250 | return (__m512i) __builtin_ia32_vpshldvq512_mask ((__v8di) __S, | 247 | return (__m512i)__builtin_ia32_selectq_512(__U, |
| 251 | (__v8di) __A, | 248 | (__v8di)_mm512_shldv_epi64(__A, __B, __C), |
| 252 | (__v8di) __B, | 249 | (__v8di)_mm512_setzero_si512()); |
| 253 | (__mmask8) -1); | ||
| 254 | } | 250 | } |
| 255 | 251 | ||
| 256 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 252 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 257 | _mm512_mask_shldv_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) | 253 | _mm512_shldv_epi32(__m512i __A, __m512i __B, __m512i __C) |
| 258 | { | 254 | { |
| 259 | return (__m512i) __builtin_ia32_vpshldvd512_mask ((__v16si) __S, | 255 | return (__m512i)__builtin_ia32_vpshldvd512((__v16si)__A, (__v16si)__B, |
| 260 | (__v16si) __A, | 256 | (__v16si)__C); |
| 261 | (__v16si) __B, | ||
| 262 | __U); | ||
| 263 | } | 257 | } |
| 264 | 258 | ||
| 265 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 259 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 266 | _mm512_maskz_shldv_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) | 260 | _mm512_mask_shldv_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) |
| 267 | { | 261 | { |
| 268 | return (__m512i) __builtin_ia32_vpshldvd512_maskz ((__v16si) __S, | 262 | return (__m512i)__builtin_ia32_selectd_512(__U, |
| 269 | (__v16si) __A, | 263 | (__v16si)_mm512_shldv_epi32(__A, __B, __C), |
| 270 | (__v16si) __B, | 264 | (__v16si)__A); |
| 271 | __U); | ||
| 272 | } | 265 | } |
| 273 | 266 | ||
| 274 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 267 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 275 | _mm512_shldv_epi32(__m512i __S, __m512i __A, __m512i __B) | 268 | _mm512_maskz_shldv_epi32(__mmask16 __U, __m512i __A, __m512i __B, __m512i __C) |
| 276 | { | 269 | { |
| 277 | return (__m512i) __builtin_ia32_vpshldvd512_mask ((__v16si) __S, | 270 | return (__m512i)__builtin_ia32_selectd_512(__U, |
| 278 | (__v16si) __A, | 271 | (__v16si)_mm512_shldv_epi32(__A, __B, __C), |
| 279 | (__v16si) __B, | 272 | (__v16si)_mm512_setzero_si512()); |
| 280 | (__mmask16) -1); | ||
| 281 | } | 273 | } |
| 282 | 274 | ||
| 283 | |||
| 284 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 275 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 285 | _mm512_mask_shldv_epi16(__m512i __S, __mmask32 __U, __m512i __A, __m512i __B) | 276 | _mm512_shldv_epi16(__m512i __A, __m512i __B, __m512i __C) |
| 286 | { | 277 | { |
| 287 | return (__m512i) __builtin_ia32_vpshldvw512_mask ((__v32hi) __S, | 278 | return (__m512i)__builtin_ia32_vpshldvw512((__v32hi)__A, (__v32hi)__B, |
| 288 | (__v32hi) __A, | 279 | (__v32hi)__C); |
| 289 | (__v32hi) __B, | ||
| 290 | __U); | ||
| 291 | } | 280 | } |
| 292 | 281 | ||
| 293 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 282 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 294 | _mm512_maskz_shldv_epi16(__mmask32 __U, __m512i __S, __m512i __A, __m512i __B) | 283 | _mm512_mask_shldv_epi16(__m512i __A, __mmask32 __U, __m512i __B, __m512i __C) |
| 295 | { | 284 | { |
| 296 | return (__m512i) __builtin_ia32_vpshldvw512_maskz ((__v32hi) __S, | 285 | return (__m512i)__builtin_ia32_selectw_512(__U, |
| 297 | (__v32hi) __A, | 286 | (__v32hi)_mm512_shldv_epi16(__A, __B, __C), |
| 298 | (__v32hi) __B, | 287 | (__v32hi)__A); |
| 299 | __U); | ||
| 300 | } | 288 | } |
| 301 | 289 | ||
| 302 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 290 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 303 | _mm512_shldv_epi16(__m512i __S, __m512i __A, __m512i __B) | 291 | _mm512_maskz_shldv_epi16(__mmask32 __U, __m512i __A, __m512i __B, __m512i __C) |
| 304 | { | 292 | { |
| 305 | return (__m512i) __builtin_ia32_vpshldvw512_mask ((__v32hi) __S, | 293 | return (__m512i)__builtin_ia32_selectw_512(__U, |
| 306 | (__v32hi) __A, | 294 | (__v32hi)_mm512_shldv_epi16(__A, __B, __C), |
| 307 | (__v32hi) __B, | 295 | (__v32hi)_mm512_setzero_si512()); |
| 308 | (__mmask32) -1); | ||
| 309 | } | 296 | } |
| 310 | 297 | ||
| 311 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 298 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 312 | _mm512_mask_shrdv_epi64(__m512i __S, __mmask8 __U, __m512i __A, __m512i __B) | 299 | _mm512_shrdv_epi64(__m512i __A, __m512i __B, __m512i __C) |
| 313 | { | 300 | { |
| 314 | return (__m512i) __builtin_ia32_vpshrdvq512_mask ((__v8di) __S, | 301 | return (__m512i)__builtin_ia32_vpshrdvq512((__v8di)__A, (__v8di)__B, |
| 315 | (__v8di) __A, | 302 | (__v8di)__C); |
| 316 | (__v8di) __B, | ||
| 317 | __U); | ||
| 318 | } | 303 | } |
| 319 | 304 | ||
| 320 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 305 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 321 | _mm512_maskz_shrdv_epi64(__mmask8 __U, __m512i __S, __m512i __A, __m512i __B) | 306 | _mm512_mask_shrdv_epi64(__m512i __A, __mmask8 __U, __m512i __B, __m512i __C) |
| 322 | { | 307 | { |
| 323 | return (__m512i) __builtin_ia32_vpshrdvq512_maskz ((__v8di) __S, | 308 | return (__m512i)__builtin_ia32_selectq_512(__U, |
| 324 | (__v8di) __A, | 309 | (__v8di)_mm512_shrdv_epi64(__A, __B, __C), |
| 325 | (__v8di) __B, | 310 | (__v8di)__A); |
| 326 | __U); | ||
| 327 | } | 311 | } |
| 328 | 312 | ||
| 329 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 313 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 330 | _mm512_shrdv_epi64(__m512i __S, __m512i __A, __m512i __B) | 314 | _mm512_maskz_shrdv_epi64(__mmask8 __U, __m512i __A, __m512i __B, __m512i __C) |
| 331 | { | 315 | { |
| 332 | return (__m512i) __builtin_ia32_vpshrdvq512_mask ((__v8di) __S, | 316 | return (__m512i)__builtin_ia32_selectq_512(__U, |
| 333 | (__v8di) __A, | 317 | (__v8di)_mm512_shrdv_epi64(__A, __B, __C), |
| 334 | (__v8di) __B, | 318 | (__v8di)_mm512_setzero_si512()); |
| 335 | (__mmask8) -1); | ||
| 336 | } | 319 | } |
| 337 | 320 | ||
| 338 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 321 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 339 | _mm512_mask_shrdv_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) | 322 | _mm512_shrdv_epi32(__m512i __A, __m512i __B, __m512i __C) |
| 340 | { | 323 | { |
| 341 | return (__m512i) __builtin_ia32_vpshrdvd512_mask ((__v16si) __S, | 324 | return (__m512i)__builtin_ia32_vpshrdvd512((__v16si)__A, (__v16si)__B, |
| 342 | (__v16si) __A, | 325 | (__v16si)__C); |
| 343 | (__v16si) __B, | ||
| 344 | __U); | ||
| 345 | } | 326 | } |
| 346 | 327 | ||
| 347 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 328 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 348 | _mm512_maskz_shrdv_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) | 329 | _mm512_mask_shrdv_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) |
| 349 | { | 330 | { |
| 350 | return (__m512i) __builtin_ia32_vpshrdvd512_maskz ((__v16si) __S, | 331 | return (__m512i) __builtin_ia32_selectd_512(__U, |
| 351 | (__v16si) __A, | 332 | (__v16si)_mm512_shrdv_epi32(__A, __B, __C), |
| 352 | (__v16si) __B, | 333 | (__v16si)__A); |
| 353 | __U); | ||
| 354 | } | 334 | } |
| 355 | 335 | ||
| 356 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 336 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 357 | _mm512_shrdv_epi32(__m512i __S, __m512i __A, __m512i __B) | 337 | _mm512_maskz_shrdv_epi32(__mmask16 __U, __m512i __A, __m512i __B, __m512i __C) |
| 358 | { | 338 | { |
| 359 | return (__m512i) __builtin_ia32_vpshrdvd512_mask ((__v16si) __S, | 339 | return (__m512i) __builtin_ia32_selectd_512(__U, |
| 360 | (__v16si) __A, | 340 | (__v16si)_mm512_shrdv_epi32(__A, __B, __C), |
| 361 | (__v16si) __B, | 341 | (__v16si)_mm512_setzero_si512()); |
| 362 | (__mmask16) -1); | ||
| 363 | } | 342 | } |
| 364 | 343 | ||
| 365 | |||
| 366 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 344 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 367 | _mm512_mask_shrdv_epi16(__m512i __S, __mmask32 __U, __m512i __A, __m512i __B) | 345 | _mm512_shrdv_epi16(__m512i __A, __m512i __B, __m512i __C) |
| 368 | { | 346 | { |
| 369 | return (__m512i) __builtin_ia32_vpshrdvw512_mask ((__v32hi) __S, | 347 | return (__m512i)__builtin_ia32_vpshrdvw512((__v32hi)__A, (__v32hi)__B, |
| 370 | (__v32hi) __A, | 348 | (__v32hi)__C); |
| 371 | (__v32hi) __B, | ||
| 372 | __U); | ||
| 373 | } | 349 | } |
| 374 | 350 | ||
| 375 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 351 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 376 | _mm512_maskz_shrdv_epi16(__mmask32 __U, __m512i __S, __m512i __A, __m512i __B) | 352 | _mm512_mask_shrdv_epi16(__m512i __A, __mmask32 __U, __m512i __B, __m512i __C) |
| 377 | { | 353 | { |
| 378 | return (__m512i) __builtin_ia32_vpshrdvw512_maskz ((__v32hi) __S, | 354 | return (__m512i)__builtin_ia32_selectw_512(__U, |
| 379 | (__v32hi) __A, | 355 | (__v32hi)_mm512_shrdv_epi16(__A, __B, __C), |
| 380 | (__v32hi) __B, | 356 | (__v32hi)__A); |
| 381 | __U); | ||
| 382 | } | 357 | } |
| 383 | 358 | ||
| 384 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 359 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 385 | _mm512_shrdv_epi16(__m512i __S, __m512i __A, __m512i __B) | 360 | _mm512_maskz_shrdv_epi16(__mmask32 __U, __m512i __A, __m512i __B, __m512i __C) |
| 386 | { | 361 | { |
| 387 | return (__m512i) __builtin_ia32_vpshrdvw512_mask ((__v32hi) __S, | 362 | return (__m512i)__builtin_ia32_selectw_512(__U, |
| 388 | (__v32hi) __A, | 363 | (__v32hi)_mm512_shrdv_epi16(__A, __B, __C), |
| 389 | (__v32hi) __B, | 364 | (__v32hi)_mm512_setzero_si512()); |
| 390 | (__mmask32) -1); | ||
| 391 | } | 365 | } |
| 392 | 366 | ||
| 393 | 367 |
c_headers/avx512vbmiintrin.h+11-15
| ... | @@ -91,30 +91,26 @@ _mm512_mask_permutexvar_epi8 (__m512i __W, __mmask64 __M, __m512i __A, | ... | @@ -91,30 +91,26 @@ _mm512_mask_permutexvar_epi8 (__m512i __W, __mmask64 __M, __m512i __A, |
| 91 | } | 91 | } |
| 92 | 92 | ||
| 93 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 93 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 94 | _mm512_mask_multishift_epi64_epi8 (__m512i __W, __mmask64 __M, __m512i __X, __m512i __Y) | 94 | _mm512_multishift_epi64_epi8(__m512i __X, __m512i __Y) |
| 95 | { | 95 | { |
| 96 | return (__m512i) __builtin_ia32_vpmultishiftqb512_mask ((__v64qi) __X, | 96 | return (__m512i)__builtin_ia32_vpmultishiftqb512((__v64qi)__X, (__v64qi) __Y); |
| 97 | (__v64qi) __Y, | ||
| 98 | (__v64qi) __W, | ||
| 99 | (__mmask64) __M); | ||
| 100 | } | 97 | } |
| 101 | 98 | ||
| 102 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 99 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 103 | _mm512_maskz_multishift_epi64_epi8 (__mmask64 __M, __m512i __X, __m512i __Y) | 100 | _mm512_mask_multishift_epi64_epi8(__m512i __W, __mmask64 __M, __m512i __X, |
| 101 | __m512i __Y) | ||
| 104 | { | 102 | { |
| 105 | return (__m512i) __builtin_ia32_vpmultishiftqb512_mask ((__v64qi) __X, | 103 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, |
| 106 | (__v64qi) __Y, | 104 | (__v64qi)_mm512_multishift_epi64_epi8(__X, __Y), |
| 107 | (__v64qi) _mm512_setzero_si512 (), | 105 | (__v64qi)__W); |
| 108 | (__mmask64) __M); | ||
| 109 | } | 106 | } |
| 110 | 107 | ||
| 111 | static __inline__ __m512i __DEFAULT_FN_ATTRS | 108 | static __inline__ __m512i __DEFAULT_FN_ATTRS |
| 112 | _mm512_multishift_epi64_epi8 (__m512i __X, __m512i __Y) | 109 | _mm512_maskz_multishift_epi64_epi8(__mmask64 __M, __m512i __X, __m512i __Y) |
| 113 | { | 110 | { |
| 114 | return (__m512i) __builtin_ia32_vpmultishiftqb512_mask ((__v64qi) __X, | 111 | return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M, |
| 115 | (__v64qi) __Y, | 112 | (__v64qi)_mm512_multishift_epi64_epi8(__X, __Y), |
| 116 | (__v64qi) _mm512_undefined_epi32 (), | 113 | (__v64qi)_mm512_setzero_si512()); |
| 117 | (__mmask64) -1); | ||
| 118 | } | 114 | } |
| 119 | 115 | ||
| 120 | 116 |
c_headers/avx512vbmivlintrin.h+22-34
| ... | @@ -150,61 +150,49 @@ _mm256_mask_permutexvar_epi8 (__m256i __W, __mmask32 __M, __m256i __A, | ... | @@ -150,61 +150,49 @@ _mm256_mask_permutexvar_epi8 (__m256i __W, __mmask32 __M, __m256i __A, |
| 150 | } | 150 | } |
| 151 | 151 | ||
| 152 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 152 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 153 | _mm_mask_multishift_epi64_epi8 (__m128i __W, __mmask16 __M, __m128i __X, __m128i __Y) | 153 | _mm_multishift_epi64_epi8(__m128i __X, __m128i __Y) |
| 154 | { | 154 | { |
| 155 | return (__m128i) __builtin_ia32_vpmultishiftqb128_mask ((__v16qi) __X, | 155 | return (__m128i)__builtin_ia32_vpmultishiftqb128((__v16qi)__X, (__v16qi)__Y); |
| 156 | (__v16qi) __Y, | ||
| 157 | (__v16qi) __W, | ||
| 158 | (__mmask16) __M); | ||
| 159 | } | 156 | } |
| 160 | 157 | ||
| 161 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 158 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 162 | _mm_maskz_multishift_epi64_epi8 (__mmask16 __M, __m128i __X, __m128i __Y) | 159 | _mm_mask_multishift_epi64_epi8(__m128i __W, __mmask16 __M, __m128i __X, |
| 160 | __m128i __Y) | ||
| 163 | { | 161 | { |
| 164 | return (__m128i) __builtin_ia32_vpmultishiftqb128_mask ((__v16qi) __X, | 162 | return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M, |
| 165 | (__v16qi) __Y, | 163 | (__v16qi)_mm_multishift_epi64_epi8(__X, __Y), |
| 166 | (__v16qi) | 164 | (__v16qi)__W); |
| 167 | _mm_setzero_si128 (), | ||
| 168 | (__mmask16) __M); | ||
| 169 | } | 165 | } |
| 170 | 166 | ||
| 171 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 167 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 172 | _mm_multishift_epi64_epi8 (__m128i __X, __m128i __Y) | 168 | _mm_maskz_multishift_epi64_epi8(__mmask16 __M, __m128i __X, __m128i __Y) |
| 173 | { | 169 | { |
| 174 | return (__m128i) __builtin_ia32_vpmultishiftqb128_mask ((__v16qi) __X, | 170 | return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M, |
| 175 | (__v16qi) __Y, | 171 | (__v16qi)_mm_multishift_epi64_epi8(__X, __Y), |
| 176 | (__v16qi) | 172 | (__v16qi)_mm_setzero_si128()); |
| 177 | _mm_undefined_si128 (), | ||
| 178 | (__mmask16) -1); | ||
| 179 | } | 173 | } |
| 180 | 174 | ||
| 181 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 175 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 182 | _mm256_mask_multishift_epi64_epi8 (__m256i __W, __mmask32 __M, __m256i __X, __m256i __Y) | 176 | _mm256_multishift_epi64_epi8(__m256i __X, __m256i __Y) |
| 183 | { | 177 | { |
| 184 | return (__m256i) __builtin_ia32_vpmultishiftqb256_mask ((__v32qi) __X, | 178 | return (__m256i)__builtin_ia32_vpmultishiftqb256((__v32qi)__X, (__v32qi)__Y); |
| 185 | (__v32qi) __Y, | ||
| 186 | (__v32qi) __W, | ||
| 187 | (__mmask32) __M); | ||
| 188 | } | 179 | } |
| 189 | 180 | ||
| 190 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 181 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 191 | _mm256_maskz_multishift_epi64_epi8 (__mmask32 __M, __m256i __X, __m256i __Y) | 182 | _mm256_mask_multishift_epi64_epi8(__m256i __W, __mmask32 __M, __m256i __X, |
| 183 | __m256i __Y) | ||
| 192 | { | 184 | { |
| 193 | return (__m256i) __builtin_ia32_vpmultishiftqb256_mask ((__v32qi) __X, | 185 | return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M, |
| 194 | (__v32qi) __Y, | 186 | (__v32qi)_mm256_multishift_epi64_epi8(__X, __Y), |
| 195 | (__v32qi) | 187 | (__v32qi)__W); |
| 196 | _mm256_setzero_si256 (), | ||
| 197 | (__mmask32) __M); | ||
| 198 | } | 188 | } |
| 199 | 189 | ||
| 200 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 190 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 201 | _mm256_multishift_epi64_epi8 (__m256i __X, __m256i __Y) | 191 | _mm256_maskz_multishift_epi64_epi8(__mmask32 __M, __m256i __X, __m256i __Y) |
| 202 | { | 192 | { |
| 203 | return (__m256i) __builtin_ia32_vpmultishiftqb256_mask ((__v32qi) __X, | 193 | return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M, |
| 204 | (__v32qi) __Y, | 194 | (__v32qi)_mm256_multishift_epi64_epi8(__X, __Y), |
| 205 | (__v32qi) | 195 | (__v32qi)_mm256_setzero_si256()); |
| 206 | _mm256_undefined_si256 (), | ||
| 207 | (__mmask32) -1); | ||
| 208 | } | 196 | } |
| 209 | 197 | ||
| 210 | 198 |
c_headers/avx512vlbwintrin.h+74-1
| ... | @@ -2297,6 +2297,15 @@ _mm256_maskz_set1_epi8 (__mmask32 __M, char __A) | ... | @@ -2297,6 +2297,15 @@ _mm256_maskz_set1_epi8 (__mmask32 __M, char __A) |
| 2297 | (__v32qi) _mm256_setzero_si256()); | 2297 | (__v32qi) _mm256_setzero_si256()); |
| 2298 | } | 2298 | } |
| 2299 | 2299 | ||
| 2300 | static __inline __m128i __DEFAULT_FN_ATTRS128 | ||
| 2301 | _mm_loadu_epi16 (void const *__P) | ||
| 2302 | { | ||
| 2303 | struct __loadu_epi16 { | ||
| 2304 | __m128i __v; | ||
| 2305 | } __attribute__((__packed__, __may_alias__)); | ||
| 2306 | return ((struct __loadu_epi16*)__P)->__v; | ||
| 2307 | } | ||
| 2308 | |||
| 2300 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 2309 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 2301 | _mm_mask_loadu_epi16 (__m128i __W, __mmask8 __U, void const *__P) | 2310 | _mm_mask_loadu_epi16 (__m128i __W, __mmask8 __U, void const *__P) |
| 2302 | { | 2311 | { |
| ... | @@ -2314,6 +2323,15 @@ _mm_maskz_loadu_epi16 (__mmask8 __U, void const *__P) | ... | @@ -2314,6 +2323,15 @@ _mm_maskz_loadu_epi16 (__mmask8 __U, void const *__P) |
| 2314 | (__mmask8) __U); | 2323 | (__mmask8) __U); |
| 2315 | } | 2324 | } |
| 2316 | 2325 | ||
| 2326 | static __inline __m256i __DEFAULT_FN_ATTRS256 | ||
| 2327 | _mm256_loadu_epi16 (void const *__P) | ||
| 2328 | { | ||
| 2329 | struct __loadu_epi16 { | ||
| 2330 | __m256i __v; | ||
| 2331 | } __attribute__((__packed__, __may_alias__)); | ||
| 2332 | return ((struct __loadu_epi16*)__P)->__v; | ||
| 2333 | } | ||
| 2334 | |||
| 2317 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2335 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 2318 | _mm256_mask_loadu_epi16 (__m256i __W, __mmask16 __U, void const *__P) | 2336 | _mm256_mask_loadu_epi16 (__m256i __W, __mmask16 __U, void const *__P) |
| 2319 | { | 2337 | { |
| ... | @@ -2331,6 +2349,15 @@ _mm256_maskz_loadu_epi16 (__mmask16 __U, void const *__P) | ... | @@ -2331,6 +2349,15 @@ _mm256_maskz_loadu_epi16 (__mmask16 __U, void const *__P) |
| 2331 | (__mmask16) __U); | 2349 | (__mmask16) __U); |
| 2332 | } | 2350 | } |
| 2333 | 2351 | ||
| 2352 | static __inline __m128i __DEFAULT_FN_ATTRS128 | ||
| 2353 | _mm_loadu_epi8 (void const *__P) | ||
| 2354 | { | ||
| 2355 | struct __loadu_epi8 { | ||
| 2356 | __m128i __v; | ||
| 2357 | } __attribute__((__packed__, __may_alias__)); | ||
| 2358 | return ((struct __loadu_epi8*)__P)->__v; | ||
| 2359 | } | ||
| 2360 | |||
| 2334 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 2361 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 2335 | _mm_mask_loadu_epi8 (__m128i __W, __mmask16 __U, void const *__P) | 2362 | _mm_mask_loadu_epi8 (__m128i __W, __mmask16 __U, void const *__P) |
| 2336 | { | 2363 | { |
| ... | @@ -2348,6 +2375,15 @@ _mm_maskz_loadu_epi8 (__mmask16 __U, void const *__P) | ... | @@ -2348,6 +2375,15 @@ _mm_maskz_loadu_epi8 (__mmask16 __U, void const *__P) |
| 2348 | (__mmask16) __U); | 2375 | (__mmask16) __U); |
| 2349 | } | 2376 | } |
| 2350 | 2377 | ||
| 2378 | static __inline __m256i __DEFAULT_FN_ATTRS256 | ||
| 2379 | _mm256_loadu_epi8 (void const *__P) | ||
| 2380 | { | ||
| 2381 | struct __loadu_epi8 { | ||
| 2382 | __m256i __v; | ||
| 2383 | } __attribute__((__packed__, __may_alias__)); | ||
| 2384 | return ((struct __loadu_epi8*)__P)->__v; | ||
| 2385 | } | ||
| 2386 | |||
| 2351 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2387 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 2352 | _mm256_mask_loadu_epi8 (__m256i __W, __mmask32 __U, void const *__P) | 2388 | _mm256_mask_loadu_epi8 (__m256i __W, __mmask32 __U, void const *__P) |
| 2353 | { | 2389 | { |
| ... | @@ -2364,7 +2400,17 @@ _mm256_maskz_loadu_epi8 (__mmask32 __U, void const *__P) | ... | @@ -2364,7 +2400,17 @@ _mm256_maskz_loadu_epi8 (__mmask32 __U, void const *__P) |
| 2364 | _mm256_setzero_si256 (), | 2400 | _mm256_setzero_si256 (), |
| 2365 | (__mmask32) __U); | 2401 | (__mmask32) __U); |
| 2366 | } | 2402 | } |
| 2367 | static __inline__ void __DEFAULT_FN_ATTRS256 | 2403 | |
| 2404 | static __inline void __DEFAULT_FN_ATTRS128 | ||
| 2405 | _mm_storeu_epi16 (void *__P, __m128i __A) | ||
| 2406 | { | ||
| 2407 | struct __storeu_epi16 { | ||
| 2408 | __m128i __v; | ||
| 2409 | } __attribute__((__packed__, __may_alias__)); | ||
| 2410 | ((struct __storeu_epi16*)__P)->__v = __A; | ||
| 2411 | } | ||
| 2412 | |||
| 2413 | static __inline__ void __DEFAULT_FN_ATTRS128 | ||
| 2368 | _mm_mask_storeu_epi16 (void *__P, __mmask8 __U, __m128i __A) | 2414 | _mm_mask_storeu_epi16 (void *__P, __mmask8 __U, __m128i __A) |
| 2369 | { | 2415 | { |
| 2370 | __builtin_ia32_storedquhi128_mask ((__v8hi *) __P, | 2416 | __builtin_ia32_storedquhi128_mask ((__v8hi *) __P, |
| ... | @@ -2372,6 +2418,15 @@ _mm_mask_storeu_epi16 (void *__P, __mmask8 __U, __m128i __A) | ... | @@ -2372,6 +2418,15 @@ _mm_mask_storeu_epi16 (void *__P, __mmask8 __U, __m128i __A) |
| 2372 | (__mmask8) __U); | 2418 | (__mmask8) __U); |
| 2373 | } | 2419 | } |
| 2374 | 2420 | ||
| 2421 | static __inline void __DEFAULT_FN_ATTRS256 | ||
| 2422 | _mm256_storeu_epi16 (void *__P, __m256i __A) | ||
| 2423 | { | ||
| 2424 | struct __storeu_epi16 { | ||
| 2425 | __m256i __v; | ||
| 2426 | } __attribute__((__packed__, __may_alias__)); | ||
| 2427 | ((struct __storeu_epi16*)__P)->__v = __A; | ||
| 2428 | } | ||
| 2429 | |||
| 2375 | static __inline__ void __DEFAULT_FN_ATTRS256 | 2430 | static __inline__ void __DEFAULT_FN_ATTRS256 |
| 2376 | _mm256_mask_storeu_epi16 (void *__P, __mmask16 __U, __m256i __A) | 2431 | _mm256_mask_storeu_epi16 (void *__P, __mmask16 __U, __m256i __A) |
| 2377 | { | 2432 | { |
| ... | @@ -2380,6 +2435,15 @@ _mm256_mask_storeu_epi16 (void *__P, __mmask16 __U, __m256i __A) | ... | @@ -2380,6 +2435,15 @@ _mm256_mask_storeu_epi16 (void *__P, __mmask16 __U, __m256i __A) |
| 2380 | (__mmask16) __U); | 2435 | (__mmask16) __U); |
| 2381 | } | 2436 | } |
| 2382 | 2437 | ||
| 2438 | static __inline void __DEFAULT_FN_ATTRS128 | ||
| 2439 | _mm_storeu_epi8 (void *__P, __m128i __A) | ||
| 2440 | { | ||
| 2441 | struct __storeu_epi8 { | ||
| 2442 | __m128i __v; | ||
| 2443 | } __attribute__((__packed__, __may_alias__)); | ||
| 2444 | ((struct __storeu_epi8*)__P)->__v = __A; | ||
| 2445 | } | ||
| 2446 | |||
| 2383 | static __inline__ void __DEFAULT_FN_ATTRS128 | 2447 | static __inline__ void __DEFAULT_FN_ATTRS128 |
| 2384 | _mm_mask_storeu_epi8 (void *__P, __mmask16 __U, __m128i __A) | 2448 | _mm_mask_storeu_epi8 (void *__P, __mmask16 __U, __m128i __A) |
| 2385 | { | 2449 | { |
| ... | @@ -2388,6 +2452,15 @@ _mm_mask_storeu_epi8 (void *__P, __mmask16 __U, __m128i __A) | ... | @@ -2388,6 +2452,15 @@ _mm_mask_storeu_epi8 (void *__P, __mmask16 __U, __m128i __A) |
| 2388 | (__mmask16) __U); | 2452 | (__mmask16) __U); |
| 2389 | } | 2453 | } |
| 2390 | 2454 | ||
| 2455 | static __inline void __DEFAULT_FN_ATTRS256 | ||
| 2456 | _mm256_storeu_epi8 (void *__P, __m256i __A) | ||
| 2457 | { | ||
| 2458 | struct __storeu_epi8 { | ||
| 2459 | __m256i __v; | ||
| 2460 | } __attribute__((__packed__, __may_alias__)); | ||
| 2461 | ((struct __storeu_epi8*)__P)->__v = __A; | ||
| 2462 | } | ||
| 2463 | |||
| 2391 | static __inline__ void __DEFAULT_FN_ATTRS256 | 2464 | static __inline__ void __DEFAULT_FN_ATTRS256 |
| 2392 | _mm256_mask_storeu_epi8 (void *__P, __mmask32 __U, __m256i __A) | 2465 | _mm256_mask_storeu_epi8 (void *__P, __mmask32 __U, __m256i __A) |
| 2393 | { | 2466 | { |
c_headers/avx512vlintrin.h+282-67
| ... | @@ -461,11 +461,17 @@ _mm_mask_mullo_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) | ... | @@ -461,11 +461,17 @@ _mm_mask_mullo_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) |
| 461 | (__v4si)__W); | 461 | (__v4si)__W); |
| 462 | } | 462 | } |
| 463 | 463 | ||
| 464 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 465 | _mm256_and_epi32(__m256i __a, __m256i __b) | ||
| 466 | { | ||
| 467 | return (__m256i)((__v8su)__a & (__v8su)__b); | ||
| 468 | } | ||
| 469 | |||
| 464 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 470 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 465 | _mm256_mask_and_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) | 471 | _mm256_mask_and_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) |
| 466 | { | 472 | { |
| 467 | return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U, | 473 | return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U, |
| 468 | (__v8si)_mm256_and_si256(__A, __B), | 474 | (__v8si)_mm256_and_epi32(__A, __B), |
| 469 | (__v8si)__W); | 475 | (__v8si)__W); |
| 470 | } | 476 | } |
| 471 | 477 | ||
| ... | @@ -475,11 +481,17 @@ _mm256_maskz_and_epi32(__mmask8 __U, __m256i __A, __m256i __B) | ... | @@ -475,11 +481,17 @@ _mm256_maskz_and_epi32(__mmask8 __U, __m256i __A, __m256i __B) |
| 475 | return (__m256i)_mm256_mask_and_epi32(_mm256_setzero_si256(), __U, __A, __B); | 481 | return (__m256i)_mm256_mask_and_epi32(_mm256_setzero_si256(), __U, __A, __B); |
| 476 | } | 482 | } |
| 477 | 483 | ||
| 484 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | ||
| 485 | _mm_and_epi32(__m128i __a, __m128i __b) | ||
| 486 | { | ||
| 487 | return (__m128i)((__v4su)__a & (__v4su)__b); | ||
| 488 | } | ||
| 489 | |||
| 478 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 490 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 479 | _mm_mask_and_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) | 491 | _mm_mask_and_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) |
| 480 | { | 492 | { |
| 481 | return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U, | 493 | return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U, |
| 482 | (__v4si)_mm_and_si128(__A, __B), | 494 | (__v4si)_mm_and_epi32(__A, __B), |
| 483 | (__v4si)__W); | 495 | (__v4si)__W); |
| 484 | } | 496 | } |
| 485 | 497 | ||
| ... | @@ -489,11 +501,17 @@ _mm_maskz_and_epi32(__mmask8 __U, __m128i __A, __m128i __B) | ... | @@ -489,11 +501,17 @@ _mm_maskz_and_epi32(__mmask8 __U, __m128i __A, __m128i __B) |
| 489 | return (__m128i)_mm_mask_and_epi32(_mm_setzero_si128(), __U, __A, __B); | 501 | return (__m128i)_mm_mask_and_epi32(_mm_setzero_si128(), __U, __A, __B); |
| 490 | } | 502 | } |
| 491 | 503 | ||
| 504 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 505 | _mm256_andnot_epi32(__m256i __A, __m256i __B) | ||
| 506 | { | ||
| 507 | return (__m256i)(~(__v8su)__A & (__v8su)__B); | ||
| 508 | } | ||
| 509 | |||
| 492 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 510 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 493 | _mm256_mask_andnot_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) | 511 | _mm256_mask_andnot_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) |
| 494 | { | 512 | { |
| 495 | return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U, | 513 | return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U, |
| 496 | (__v8si)_mm256_andnot_si256(__A, __B), | 514 | (__v8si)_mm256_andnot_epi32(__A, __B), |
| 497 | (__v8si)__W); | 515 | (__v8si)__W); |
| 498 | } | 516 | } |
| 499 | 517 | ||
| ... | @@ -504,25 +522,37 @@ _mm256_maskz_andnot_epi32(__mmask8 __U, __m256i __A, __m256i __B) | ... | @@ -504,25 +522,37 @@ _mm256_maskz_andnot_epi32(__mmask8 __U, __m256i __A, __m256i __B) |
| 504 | __U, __A, __B); | 522 | __U, __A, __B); |
| 505 | } | 523 | } |
| 506 | 524 | ||
| 525 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | ||
| 526 | _mm_andnot_epi32(__m128i __A, __m128i __B) | ||
| 527 | { | ||
| 528 | return (__m128i)(~(__v4su)__A & (__v4su)__B); | ||
| 529 | } | ||
| 530 | |||
| 507 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 531 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 508 | _mm_mask_andnot_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) | 532 | _mm_mask_andnot_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) |
| 509 | { | 533 | { |
| 510 | return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U, | 534 | return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U, |
| 511 | (__v4si)_mm_andnot_si128(__A, __B), | 535 | (__v4si)_mm_andnot_epi32(__A, __B), |
| 512 | (__v4si)__W); | 536 | (__v4si)__W); |
| 513 | } | 537 | } |
| 514 | 538 | ||
| 515 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 539 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 516 | _mm_maskz_andnot_epi32 (__mmask8 __U, __m128i __A, __m128i __B) | 540 | _mm_maskz_andnot_epi32(__mmask8 __U, __m128i __A, __m128i __B) |
| 517 | { | 541 | { |
| 518 | return (__m128i)_mm_mask_andnot_epi32(_mm_setzero_si128(), __U, __A, __B); | 542 | return (__m128i)_mm_mask_andnot_epi32(_mm_setzero_si128(), __U, __A, __B); |
| 519 | } | 543 | } |
| 520 | 544 | ||
| 545 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 546 | _mm256_or_epi32(__m256i __a, __m256i __b) | ||
| 547 | { | ||
| 548 | return (__m256i)((__v8su)__a | (__v8su)__b); | ||
| 549 | } | ||
| 550 | |||
| 521 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 551 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 522 | _mm256_mask_or_epi32 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) | 552 | _mm256_mask_or_epi32 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) |
| 523 | { | 553 | { |
| 524 | return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U, | 554 | return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U, |
| 525 | (__v8si)_mm256_or_si256(__A, __B), | 555 | (__v8si)_mm256_or_epi32(__A, __B), |
| 526 | (__v8si)__W); | 556 | (__v8si)__W); |
| 527 | } | 557 | } |
| 528 | 558 | ||
| ... | @@ -532,11 +562,17 @@ _mm256_maskz_or_epi32(__mmask8 __U, __m256i __A, __m256i __B) | ... | @@ -532,11 +562,17 @@ _mm256_maskz_or_epi32(__mmask8 __U, __m256i __A, __m256i __B) |
| 532 | return (__m256i)_mm256_mask_or_epi32(_mm256_setzero_si256(), __U, __A, __B); | 562 | return (__m256i)_mm256_mask_or_epi32(_mm256_setzero_si256(), __U, __A, __B); |
| 533 | } | 563 | } |
| 534 | 564 | ||
| 565 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | ||
| 566 | _mm_or_epi32(__m128i __a, __m128i __b) | ||
| 567 | { | ||
| 568 | return (__m128i)((__v4su)__a | (__v4su)__b); | ||
| 569 | } | ||
| 570 | |||
| 535 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 571 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 536 | _mm_mask_or_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) | 572 | _mm_mask_or_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) |
| 537 | { | 573 | { |
| 538 | return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U, | 574 | return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U, |
| 539 | (__v4si)_mm_or_si128(__A, __B), | 575 | (__v4si)_mm_or_epi32(__A, __B), |
| 540 | (__v4si)__W); | 576 | (__v4si)__W); |
| 541 | } | 577 | } |
| 542 | 578 | ||
| ... | @@ -546,11 +582,17 @@ _mm_maskz_or_epi32(__mmask8 __U, __m128i __A, __m128i __B) | ... | @@ -546,11 +582,17 @@ _mm_maskz_or_epi32(__mmask8 __U, __m128i __A, __m128i __B) |
| 546 | return (__m128i)_mm_mask_or_epi32(_mm_setzero_si128(), __U, __A, __B); | 582 | return (__m128i)_mm_mask_or_epi32(_mm_setzero_si128(), __U, __A, __B); |
| 547 | } | 583 | } |
| 548 | 584 | ||
| 585 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 586 | _mm256_xor_epi32(__m256i __a, __m256i __b) | ||
| 587 | { | ||
| 588 | return (__m256i)((__v8su)__a ^ (__v8su)__b); | ||
| 589 | } | ||
| 590 | |||
| 549 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 591 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 550 | _mm256_mask_xor_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) | 592 | _mm256_mask_xor_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) |
| 551 | { | 593 | { |
| 552 | return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U, | 594 | return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U, |
| 553 | (__v8si)_mm256_xor_si256(__A, __B), | 595 | (__v8si)_mm256_xor_epi32(__A, __B), |
| 554 | (__v8si)__W); | 596 | (__v8si)__W); |
| 555 | } | 597 | } |
| 556 | 598 | ||
| ... | @@ -561,11 +603,16 @@ _mm256_maskz_xor_epi32(__mmask8 __U, __m256i __A, __m256i __B) | ... | @@ -561,11 +603,16 @@ _mm256_maskz_xor_epi32(__mmask8 __U, __m256i __A, __m256i __B) |
| 561 | } | 603 | } |
| 562 | 604 | ||
| 563 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 605 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 564 | _mm_mask_xor_epi32(__m128i __W, __mmask8 __U, __m128i __A, | 606 | _mm_xor_epi32(__m128i __a, __m128i __b) |
| 565 | __m128i __B) | 607 | { |
| 608 | return (__m128i)((__v4su)__a ^ (__v4su)__b); | ||
| 609 | } | ||
| 610 | |||
| 611 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | ||
| 612 | _mm_mask_xor_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) | ||
| 566 | { | 613 | { |
| 567 | return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U, | 614 | return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U, |
| 568 | (__v4si)_mm_xor_si128(__A, __B), | 615 | (__v4si)_mm_xor_epi32(__A, __B), |
| 569 | (__v4si)__W); | 616 | (__v4si)__W); |
| 570 | } | 617 | } |
| 571 | 618 | ||
| ... | @@ -575,11 +622,17 @@ _mm_maskz_xor_epi32(__mmask8 __U, __m128i __A, __m128i __B) | ... | @@ -575,11 +622,17 @@ _mm_maskz_xor_epi32(__mmask8 __U, __m128i __A, __m128i __B) |
| 575 | return (__m128i)_mm_mask_xor_epi32(_mm_setzero_si128(), __U, __A, __B); | 622 | return (__m128i)_mm_mask_xor_epi32(_mm_setzero_si128(), __U, __A, __B); |
| 576 | } | 623 | } |
| 577 | 624 | ||
| 625 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 626 | _mm256_and_epi64(__m256i __a, __m256i __b) | ||
| 627 | { | ||
| 628 | return (__m256i)((__v4du)__a & (__v4du)__b); | ||
| 629 | } | ||
| 630 | |||
| 578 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 631 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 579 | _mm256_mask_and_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) | 632 | _mm256_mask_and_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) |
| 580 | { | 633 | { |
| 581 | return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, | 634 | return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, |
| 582 | (__v4di)_mm256_and_si256(__A, __B), | 635 | (__v4di)_mm256_and_epi64(__A, __B), |
| 583 | (__v4di)__W); | 636 | (__v4di)__W); |
| 584 | } | 637 | } |
| 585 | 638 | ||
| ... | @@ -589,11 +642,17 @@ _mm256_maskz_and_epi64(__mmask8 __U, __m256i __A, __m256i __B) | ... | @@ -589,11 +642,17 @@ _mm256_maskz_and_epi64(__mmask8 __U, __m256i __A, __m256i __B) |
| 589 | return (__m256i)_mm256_mask_and_epi64(_mm256_setzero_si256(), __U, __A, __B); | 642 | return (__m256i)_mm256_mask_and_epi64(_mm256_setzero_si256(), __U, __A, __B); |
| 590 | } | 643 | } |
| 591 | 644 | ||
| 645 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | ||
| 646 | _mm_and_epi64(__m128i __a, __m128i __b) | ||
| 647 | { | ||
| 648 | return (__m128i)((__v2du)__a & (__v2du)__b); | ||
| 649 | } | ||
| 650 | |||
| 592 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 651 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 593 | _mm_mask_and_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) | 652 | _mm_mask_and_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) |
| 594 | { | 653 | { |
| 595 | return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, | 654 | return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, |
| 596 | (__v2di)_mm_and_si128(__A, __B), | 655 | (__v2di)_mm_and_epi64(__A, __B), |
| 597 | (__v2di)__W); | 656 | (__v2di)__W); |
| 598 | } | 657 | } |
| 599 | 658 | ||
| ... | @@ -603,11 +662,17 @@ _mm_maskz_and_epi64(__mmask8 __U, __m128i __A, __m128i __B) | ... | @@ -603,11 +662,17 @@ _mm_maskz_and_epi64(__mmask8 __U, __m128i __A, __m128i __B) |
| 603 | return (__m128i)_mm_mask_and_epi64(_mm_setzero_si128(), __U, __A, __B); | 662 | return (__m128i)_mm_mask_and_epi64(_mm_setzero_si128(), __U, __A, __B); |
| 604 | } | 663 | } |
| 605 | 664 | ||
| 665 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 666 | _mm256_andnot_epi64(__m256i __A, __m256i __B) | ||
| 667 | { | ||
| 668 | return (__m256i)(~(__v4du)__A & (__v4du)__B); | ||
| 669 | } | ||
| 670 | |||
| 606 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 671 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 607 | _mm256_mask_andnot_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) | 672 | _mm256_mask_andnot_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) |
| 608 | { | 673 | { |
| 609 | return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, | 674 | return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, |
| 610 | (__v4di)_mm256_andnot_si256(__A, __B), | 675 | (__v4di)_mm256_andnot_epi64(__A, __B), |
| 611 | (__v4di)__W); | 676 | (__v4di)__W); |
| 612 | } | 677 | } |
| 613 | 678 | ||
| ... | @@ -618,11 +683,17 @@ _mm256_maskz_andnot_epi64(__mmask8 __U, __m256i __A, __m256i __B) | ... | @@ -618,11 +683,17 @@ _mm256_maskz_andnot_epi64(__mmask8 __U, __m256i __A, __m256i __B) |
| 618 | __U, __A, __B); | 683 | __U, __A, __B); |
| 619 | } | 684 | } |
| 620 | 685 | ||
| 686 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | ||
| 687 | _mm_andnot_epi64(__m128i __A, __m128i __B) | ||
| 688 | { | ||
| 689 | return (__m128i)(~(__v2du)__A & (__v2du)__B); | ||
| 690 | } | ||
| 691 | |||
| 621 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 692 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 622 | _mm_mask_andnot_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) | 693 | _mm_mask_andnot_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) |
| 623 | { | 694 | { |
| 624 | return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, | 695 | return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, |
| 625 | (__v2di)_mm_andnot_si128(__A, __B), | 696 | (__v2di)_mm_andnot_epi64(__A, __B), |
| 626 | (__v2di)__W); | 697 | (__v2di)__W); |
| 627 | } | 698 | } |
| 628 | 699 | ||
| ... | @@ -632,11 +703,17 @@ _mm_maskz_andnot_epi64(__mmask8 __U, __m128i __A, __m128i __B) | ... | @@ -632,11 +703,17 @@ _mm_maskz_andnot_epi64(__mmask8 __U, __m128i __A, __m128i __B) |
| 632 | return (__m128i)_mm_mask_andnot_epi64(_mm_setzero_si128(), __U, __A, __B); | 703 | return (__m128i)_mm_mask_andnot_epi64(_mm_setzero_si128(), __U, __A, __B); |
| 633 | } | 704 | } |
| 634 | 705 | ||
| 706 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 707 | _mm256_or_epi64(__m256i __a, __m256i __b) | ||
| 708 | { | ||
| 709 | return (__m256i)((__v4du)__a | (__v4du)__b); | ||
| 710 | } | ||
| 711 | |||
| 635 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 712 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 636 | _mm256_mask_or_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) | 713 | _mm256_mask_or_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) |
| 637 | { | 714 | { |
| 638 | return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, | 715 | return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, |
| 639 | (__v4di)_mm256_or_si256(__A, __B), | 716 | (__v4di)_mm256_or_epi64(__A, __B), |
| 640 | (__v4di)__W); | 717 | (__v4di)__W); |
| 641 | } | 718 | } |
| 642 | 719 | ||
| ... | @@ -646,11 +723,17 @@ _mm256_maskz_or_epi64(__mmask8 __U, __m256i __A, __m256i __B) | ... | @@ -646,11 +723,17 @@ _mm256_maskz_or_epi64(__mmask8 __U, __m256i __A, __m256i __B) |
| 646 | return (__m256i)_mm256_mask_or_epi64(_mm256_setzero_si256(), __U, __A, __B); | 723 | return (__m256i)_mm256_mask_or_epi64(_mm256_setzero_si256(), __U, __A, __B); |
| 647 | } | 724 | } |
| 648 | 725 | ||
| 726 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | ||
| 727 | _mm_or_epi64(__m128i __a, __m128i __b) | ||
| 728 | { | ||
| 729 | return (__m128i)((__v2du)__a | (__v2du)__b); | ||
| 730 | } | ||
| 731 | |||
| 649 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 732 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 650 | _mm_mask_or_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) | 733 | _mm_mask_or_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) |
| 651 | { | 734 | { |
| 652 | return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, | 735 | return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, |
| 653 | (__v2di)_mm_or_si128(__A, __B), | 736 | (__v2di)_mm_or_epi64(__A, __B), |
| 654 | (__v2di)__W); | 737 | (__v2di)__W); |
| 655 | } | 738 | } |
| 656 | 739 | ||
| ... | @@ -660,11 +743,17 @@ _mm_maskz_or_epi64(__mmask8 __U, __m128i __A, __m128i __B) | ... | @@ -660,11 +743,17 @@ _mm_maskz_or_epi64(__mmask8 __U, __m128i __A, __m128i __B) |
| 660 | return (__m128i)_mm_mask_or_epi64(_mm_setzero_si128(), __U, __A, __B); | 743 | return (__m128i)_mm_mask_or_epi64(_mm_setzero_si128(), __U, __A, __B); |
| 661 | } | 744 | } |
| 662 | 745 | ||
| 746 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 747 | _mm256_xor_epi64(__m256i __a, __m256i __b) | ||
| 748 | { | ||
| 749 | return (__m256i)((__v4du)__a ^ (__v4du)__b); | ||
| 750 | } | ||
| 751 | |||
| 663 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 752 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 664 | _mm256_mask_xor_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) | 753 | _mm256_mask_xor_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) |
| 665 | { | 754 | { |
| 666 | return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, | 755 | return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, |
| 667 | (__v4di)_mm256_xor_si256(__A, __B), | 756 | (__v4di)_mm256_xor_epi64(__A, __B), |
| 668 | (__v4di)__W); | 757 | (__v4di)__W); |
| 669 | } | 758 | } |
| 670 | 759 | ||
| ... | @@ -674,12 +763,18 @@ _mm256_maskz_xor_epi64(__mmask8 __U, __m256i __A, __m256i __B) | ... | @@ -674,12 +763,18 @@ _mm256_maskz_xor_epi64(__mmask8 __U, __m256i __A, __m256i __B) |
| 674 | return (__m256i)_mm256_mask_xor_epi64(_mm256_setzero_si256(), __U, __A, __B); | 763 | return (__m256i)_mm256_mask_xor_epi64(_mm256_setzero_si256(), __U, __A, __B); |
| 675 | } | 764 | } |
| 676 | 765 | ||
| 766 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | ||
| 767 | _mm_xor_epi64(__m128i __a, __m128i __b) | ||
| 768 | { | ||
| 769 | return (__m128i)((__v2du)__a ^ (__v2du)__b); | ||
| 770 | } | ||
| 771 | |||
| 677 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 772 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 678 | _mm_mask_xor_epi64(__m128i __W, __mmask8 __U, __m128i __A, | 773 | _mm_mask_xor_epi64(__m128i __W, __mmask8 __U, __m128i __A, |
| 679 | __m128i __B) | 774 | __m128i __B) |
| 680 | { | 775 | { |
| 681 | return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, | 776 | return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, |
| 682 | (__v2di)_mm_xor_si128(__A, __B), | 777 | (__v2di)_mm_xor_epi64(__A, __B), |
| 683 | (__v2di)__W); | 778 | (__v2di)__W); |
| 684 | } | 779 | } |
| 685 | 780 | ||
| ... | @@ -3389,162 +3484,162 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) { | ... | @@ -3389,162 +3484,162 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) { |
| 3389 | } | 3484 | } |
| 3390 | 3485 | ||
| 3391 | #define _mm_i64scatter_pd(addr, index, v1, scale) \ | 3486 | #define _mm_i64scatter_pd(addr, index, v1, scale) \ |
| 3392 | __builtin_ia32_scatterdiv2df((double *)(addr), (__mmask8)-1, \ | 3487 | __builtin_ia32_scatterdiv2df((void *)(addr), (__mmask8)-1, \ |
| 3393 | (__v2di)(__m128i)(index), \ | 3488 | (__v2di)(__m128i)(index), \ |
| 3394 | (__v2df)(__m128d)(v1), (int)(scale)) | 3489 | (__v2df)(__m128d)(v1), (int)(scale)) |
| 3395 | 3490 | ||
| 3396 | #define _mm_mask_i64scatter_pd(addr, mask, index, v1, scale) \ | 3491 | #define _mm_mask_i64scatter_pd(addr, mask, index, v1, scale) \ |
| 3397 | __builtin_ia32_scatterdiv2df((double *)(addr), (__mmask8)(mask), \ | 3492 | __builtin_ia32_scatterdiv2df((void *)(addr), (__mmask8)(mask), \ |
| 3398 | (__v2di)(__m128i)(index), \ | 3493 | (__v2di)(__m128i)(index), \ |
| 3399 | (__v2df)(__m128d)(v1), (int)(scale)) | 3494 | (__v2df)(__m128d)(v1), (int)(scale)) |
| 3400 | 3495 | ||
| 3401 | #define _mm_i64scatter_epi64(addr, index, v1, scale) \ | 3496 | #define _mm_i64scatter_epi64(addr, index, v1, scale) \ |
| 3402 | __builtin_ia32_scatterdiv2di((long long *)(addr), (__mmask8)-1, \ | 3497 | __builtin_ia32_scatterdiv2di((void *)(addr), (__mmask8)-1, \ |
| 3403 | (__v2di)(__m128i)(index), \ | 3498 | (__v2di)(__m128i)(index), \ |
| 3404 | (__v2di)(__m128i)(v1), (int)(scale)) | 3499 | (__v2di)(__m128i)(v1), (int)(scale)) |
| 3405 | 3500 | ||
| 3406 | #define _mm_mask_i64scatter_epi64(addr, mask, index, v1, scale) \ | 3501 | #define _mm_mask_i64scatter_epi64(addr, mask, index, v1, scale) \ |
| 3407 | __builtin_ia32_scatterdiv2di((long long *)(addr), (__mmask8)(mask), \ | 3502 | __builtin_ia32_scatterdiv2di((void *)(addr), (__mmask8)(mask), \ |
| 3408 | (__v2di)(__m128i)(index), \ | 3503 | (__v2di)(__m128i)(index), \ |
| 3409 | (__v2di)(__m128i)(v1), (int)(scale)) | 3504 | (__v2di)(__m128i)(v1), (int)(scale)) |
| 3410 | 3505 | ||
| 3411 | #define _mm256_i64scatter_pd(addr, index, v1, scale) \ | 3506 | #define _mm256_i64scatter_pd(addr, index, v1, scale) \ |
| 3412 | __builtin_ia32_scatterdiv4df((double *)(addr), (__mmask8)-1, \ | 3507 | __builtin_ia32_scatterdiv4df((void *)(addr), (__mmask8)-1, \ |
| 3413 | (__v4di)(__m256i)(index), \ | 3508 | (__v4di)(__m256i)(index), \ |
| 3414 | (__v4df)(__m256d)(v1), (int)(scale)) | 3509 | (__v4df)(__m256d)(v1), (int)(scale)) |
| 3415 | 3510 | ||
| 3416 | #define _mm256_mask_i64scatter_pd(addr, mask, index, v1, scale) \ | 3511 | #define _mm256_mask_i64scatter_pd(addr, mask, index, v1, scale) \ |
| 3417 | __builtin_ia32_scatterdiv4df((double *)(addr), (__mmask8)(mask), \ | 3512 | __builtin_ia32_scatterdiv4df((void *)(addr), (__mmask8)(mask), \ |
| 3418 | (__v4di)(__m256i)(index), \ | 3513 | (__v4di)(__m256i)(index), \ |
| 3419 | (__v4df)(__m256d)(v1), (int)(scale)) | 3514 | (__v4df)(__m256d)(v1), (int)(scale)) |
| 3420 | 3515 | ||
| 3421 | #define _mm256_i64scatter_epi64(addr, index, v1, scale) \ | 3516 | #define _mm256_i64scatter_epi64(addr, index, v1, scale) \ |
| 3422 | __builtin_ia32_scatterdiv4di((long long *)(addr), (__mmask8)-1, \ | 3517 | __builtin_ia32_scatterdiv4di((void *)(addr), (__mmask8)-1, \ |
| 3423 | (__v4di)(__m256i)(index), \ | 3518 | (__v4di)(__m256i)(index), \ |
| 3424 | (__v4di)(__m256i)(v1), (int)(scale)) | 3519 | (__v4di)(__m256i)(v1), (int)(scale)) |
| 3425 | 3520 | ||
| 3426 | #define _mm256_mask_i64scatter_epi64(addr, mask, index, v1, scale) \ | 3521 | #define _mm256_mask_i64scatter_epi64(addr, mask, index, v1, scale) \ |
| 3427 | __builtin_ia32_scatterdiv4di((long long *)(addr), (__mmask8)(mask), \ | 3522 | __builtin_ia32_scatterdiv4di((void *)(addr), (__mmask8)(mask), \ |
| 3428 | (__v4di)(__m256i)(index), \ | 3523 | (__v4di)(__m256i)(index), \ |
| 3429 | (__v4di)(__m256i)(v1), (int)(scale)) | 3524 | (__v4di)(__m256i)(v1), (int)(scale)) |
| 3430 | 3525 | ||
| 3431 | #define _mm_i64scatter_ps(addr, index, v1, scale) \ | 3526 | #define _mm_i64scatter_ps(addr, index, v1, scale) \ |
| 3432 | __builtin_ia32_scatterdiv4sf((float *)(addr), (__mmask8)-1, \ | 3527 | __builtin_ia32_scatterdiv4sf((void *)(addr), (__mmask8)-1, \ |
| 3433 | (__v2di)(__m128i)(index), (__v4sf)(__m128)(v1), \ | 3528 | (__v2di)(__m128i)(index), (__v4sf)(__m128)(v1), \ |
| 3434 | (int)(scale)) | 3529 | (int)(scale)) |
| 3435 | 3530 | ||
| 3436 | #define _mm_mask_i64scatter_ps(addr, mask, index, v1, scale) \ | 3531 | #define _mm_mask_i64scatter_ps(addr, mask, index, v1, scale) \ |
| 3437 | __builtin_ia32_scatterdiv4sf((float *)(addr), (__mmask8)(mask), \ | 3532 | __builtin_ia32_scatterdiv4sf((void *)(addr), (__mmask8)(mask), \ |
| 3438 | (__v2di)(__m128i)(index), (__v4sf)(__m128)(v1), \ | 3533 | (__v2di)(__m128i)(index), (__v4sf)(__m128)(v1), \ |
| 3439 | (int)(scale)) | 3534 | (int)(scale)) |
| 3440 | 3535 | ||
| 3441 | #define _mm_i64scatter_epi32(addr, index, v1, scale) \ | 3536 | #define _mm_i64scatter_epi32(addr, index, v1, scale) \ |
| 3442 | __builtin_ia32_scatterdiv4si((int *)(addr), (__mmask8)-1, \ | 3537 | __builtin_ia32_scatterdiv4si((void *)(addr), (__mmask8)-1, \ |
| 3443 | (__v2di)(__m128i)(index), \ | 3538 | (__v2di)(__m128i)(index), \ |
| 3444 | (__v4si)(__m128i)(v1), (int)(scale)) | 3539 | (__v4si)(__m128i)(v1), (int)(scale)) |
| 3445 | 3540 | ||
| 3446 | #define _mm_mask_i64scatter_epi32(addr, mask, index, v1, scale) \ | 3541 | #define _mm_mask_i64scatter_epi32(addr, mask, index, v1, scale) \ |
| 3447 | __builtin_ia32_scatterdiv4si((int *)(addr), (__mmask8)(mask), \ | 3542 | __builtin_ia32_scatterdiv4si((void *)(addr), (__mmask8)(mask), \ |
| 3448 | (__v2di)(__m128i)(index), \ | 3543 | (__v2di)(__m128i)(index), \ |
| 3449 | (__v4si)(__m128i)(v1), (int)(scale)) | 3544 | (__v4si)(__m128i)(v1), (int)(scale)) |
| 3450 | 3545 | ||
| 3451 | #define _mm256_i64scatter_ps(addr, index, v1, scale) \ | 3546 | #define _mm256_i64scatter_ps(addr, index, v1, scale) \ |
| 3452 | __builtin_ia32_scatterdiv8sf((float *)(addr), (__mmask8)-1, \ | 3547 | __builtin_ia32_scatterdiv8sf((void *)(addr), (__mmask8)-1, \ |
| 3453 | (__v4di)(__m256i)(index), (__v4sf)(__m128)(v1), \ | 3548 | (__v4di)(__m256i)(index), (__v4sf)(__m128)(v1), \ |
| 3454 | (int)(scale)) | 3549 | (int)(scale)) |
| 3455 | 3550 | ||
| 3456 | #define _mm256_mask_i64scatter_ps(addr, mask, index, v1, scale) \ | 3551 | #define _mm256_mask_i64scatter_ps(addr, mask, index, v1, scale) \ |
| 3457 | __builtin_ia32_scatterdiv8sf((float *)(addr), (__mmask8)(mask), \ | 3552 | __builtin_ia32_scatterdiv8sf((void *)(addr), (__mmask8)(mask), \ |
| 3458 | (__v4di)(__m256i)(index), (__v4sf)(__m128)(v1), \ | 3553 | (__v4di)(__m256i)(index), (__v4sf)(__m128)(v1), \ |
| 3459 | (int)(scale)) | 3554 | (int)(scale)) |
| 3460 | 3555 | ||
| 3461 | #define _mm256_i64scatter_epi32(addr, index, v1, scale) \ | 3556 | #define _mm256_i64scatter_epi32(addr, index, v1, scale) \ |
| 3462 | __builtin_ia32_scatterdiv8si((int *)(addr), (__mmask8)-1, \ | 3557 | __builtin_ia32_scatterdiv8si((void *)(addr), (__mmask8)-1, \ |
| 3463 | (__v4di)(__m256i)(index), \ | 3558 | (__v4di)(__m256i)(index), \ |
| 3464 | (__v4si)(__m128i)(v1), (int)(scale)) | 3559 | (__v4si)(__m128i)(v1), (int)(scale)) |
| 3465 | 3560 | ||
| 3466 | #define _mm256_mask_i64scatter_epi32(addr, mask, index, v1, scale) \ | 3561 | #define _mm256_mask_i64scatter_epi32(addr, mask, index, v1, scale) \ |
| 3467 | __builtin_ia32_scatterdiv8si((int *)(addr), (__mmask8)(mask), \ | 3562 | __builtin_ia32_scatterdiv8si((void *)(addr), (__mmask8)(mask), \ |
| 3468 | (__v4di)(__m256i)(index), \ | 3563 | (__v4di)(__m256i)(index), \ |
| 3469 | (__v4si)(__m128i)(v1), (int)(scale)) | 3564 | (__v4si)(__m128i)(v1), (int)(scale)) |
| 3470 | 3565 | ||
| 3471 | #define _mm_i32scatter_pd(addr, index, v1, scale) \ | 3566 | #define _mm_i32scatter_pd(addr, index, v1, scale) \ |
| 3472 | __builtin_ia32_scattersiv2df((double *)(addr), (__mmask8)-1, \ | 3567 | __builtin_ia32_scattersiv2df((void *)(addr), (__mmask8)-1, \ |
| 3473 | (__v4si)(__m128i)(index), \ | 3568 | (__v4si)(__m128i)(index), \ |
| 3474 | (__v2df)(__m128d)(v1), (int)(scale)) | 3569 | (__v2df)(__m128d)(v1), (int)(scale)) |
| 3475 | 3570 | ||
| 3476 | #define _mm_mask_i32scatter_pd(addr, mask, index, v1, scale) \ | 3571 | #define _mm_mask_i32scatter_pd(addr, mask, index, v1, scale) \ |
| 3477 | __builtin_ia32_scattersiv2df((double *)(addr), (__mmask8)(mask), \ | 3572 | __builtin_ia32_scattersiv2df((void *)(addr), (__mmask8)(mask), \ |
| 3478 | (__v4si)(__m128i)(index), \ | 3573 | (__v4si)(__m128i)(index), \ |
| 3479 | (__v2df)(__m128d)(v1), (int)(scale)) | 3574 | (__v2df)(__m128d)(v1), (int)(scale)) |
| 3480 | 3575 | ||
| 3481 | #define _mm_i32scatter_epi64(addr, index, v1, scale) \ | 3576 | #define _mm_i32scatter_epi64(addr, index, v1, scale) \ |
| 3482 | __builtin_ia32_scattersiv2di((long long *)(addr), (__mmask8)-1, \ | 3577 | __builtin_ia32_scattersiv2di((void *)(addr), (__mmask8)-1, \ |
| 3483 | (__v4si)(__m128i)(index), \ | 3578 | (__v4si)(__m128i)(index), \ |
| 3484 | (__v2di)(__m128i)(v1), (int)(scale)) | 3579 | (__v2di)(__m128i)(v1), (int)(scale)) |
| 3485 | 3580 | ||
| 3486 | #define _mm_mask_i32scatter_epi64(addr, mask, index, v1, scale) \ | 3581 | #define _mm_mask_i32scatter_epi64(addr, mask, index, v1, scale) \ |
| 3487 | __builtin_ia32_scattersiv2di((long long *)(addr), (__mmask8)(mask), \ | 3582 | __builtin_ia32_scattersiv2di((void *)(addr), (__mmask8)(mask), \ |
| 3488 | (__v4si)(__m128i)(index), \ | 3583 | (__v4si)(__m128i)(index), \ |
| 3489 | (__v2di)(__m128i)(v1), (int)(scale)) | 3584 | (__v2di)(__m128i)(v1), (int)(scale)) |
| 3490 | 3585 | ||
| 3491 | #define _mm256_i32scatter_pd(addr, index, v1, scale) \ | 3586 | #define _mm256_i32scatter_pd(addr, index, v1, scale) \ |
| 3492 | __builtin_ia32_scattersiv4df((double *)(addr), (__mmask8)-1, \ | 3587 | __builtin_ia32_scattersiv4df((void *)(addr), (__mmask8)-1, \ |
| 3493 | (__v4si)(__m128i)(index), \ | 3588 | (__v4si)(__m128i)(index), \ |
| 3494 | (__v4df)(__m256d)(v1), (int)(scale)) | 3589 | (__v4df)(__m256d)(v1), (int)(scale)) |
| 3495 | 3590 | ||
| 3496 | #define _mm256_mask_i32scatter_pd(addr, mask, index, v1, scale) \ | 3591 | #define _mm256_mask_i32scatter_pd(addr, mask, index, v1, scale) \ |
| 3497 | __builtin_ia32_scattersiv4df((double *)(addr), (__mmask8)(mask), \ | 3592 | __builtin_ia32_scattersiv4df((void *)(addr), (__mmask8)(mask), \ |
| 3498 | (__v4si)(__m128i)(index), \ | 3593 | (__v4si)(__m128i)(index), \ |
| 3499 | (__v4df)(__m256d)(v1), (int)(scale)) | 3594 | (__v4df)(__m256d)(v1), (int)(scale)) |
| 3500 | 3595 | ||
| 3501 | #define _mm256_i32scatter_epi64(addr, index, v1, scale) \ | 3596 | #define _mm256_i32scatter_epi64(addr, index, v1, scale) \ |
| 3502 | __builtin_ia32_scattersiv4di((long long *)(addr), (__mmask8)-1, \ | 3597 | __builtin_ia32_scattersiv4di((void *)(addr), (__mmask8)-1, \ |
| 3503 | (__v4si)(__m128i)(index), \ | 3598 | (__v4si)(__m128i)(index), \ |
| 3504 | (__v4di)(__m256i)(v1), (int)(scale)) | 3599 | (__v4di)(__m256i)(v1), (int)(scale)) |
| 3505 | 3600 | ||
| 3506 | #define _mm256_mask_i32scatter_epi64(addr, mask, index, v1, scale) \ | 3601 | #define _mm256_mask_i32scatter_epi64(addr, mask, index, v1, scale) \ |
| 3507 | __builtin_ia32_scattersiv4di((long long *)(addr), (__mmask8)(mask), \ | 3602 | __builtin_ia32_scattersiv4di((void *)(addr), (__mmask8)(mask), \ |
| 3508 | (__v4si)(__m128i)(index), \ | 3603 | (__v4si)(__m128i)(index), \ |
| 3509 | (__v4di)(__m256i)(v1), (int)(scale)) | 3604 | (__v4di)(__m256i)(v1), (int)(scale)) |
| 3510 | 3605 | ||
| 3511 | #define _mm_i32scatter_ps(addr, index, v1, scale) \ | 3606 | #define _mm_i32scatter_ps(addr, index, v1, scale) \ |
| 3512 | __builtin_ia32_scattersiv4sf((float *)(addr), (__mmask8)-1, \ | 3607 | __builtin_ia32_scattersiv4sf((void *)(addr), (__mmask8)-1, \ |
| 3513 | (__v4si)(__m128i)(index), (__v4sf)(__m128)(v1), \ | 3608 | (__v4si)(__m128i)(index), (__v4sf)(__m128)(v1), \ |
| 3514 | (int)(scale)) | 3609 | (int)(scale)) |
| 3515 | 3610 | ||
| 3516 | #define _mm_mask_i32scatter_ps(addr, mask, index, v1, scale) \ | 3611 | #define _mm_mask_i32scatter_ps(addr, mask, index, v1, scale) \ |
| 3517 | __builtin_ia32_scattersiv4sf((float *)(addr), (__mmask8)(mask), \ | 3612 | __builtin_ia32_scattersiv4sf((void *)(addr), (__mmask8)(mask), \ |
| 3518 | (__v4si)(__m128i)(index), (__v4sf)(__m128)(v1), \ | 3613 | (__v4si)(__m128i)(index), (__v4sf)(__m128)(v1), \ |
| 3519 | (int)(scale)) | 3614 | (int)(scale)) |
| 3520 | 3615 | ||
| 3521 | #define _mm_i32scatter_epi32(addr, index, v1, scale) \ | 3616 | #define _mm_i32scatter_epi32(addr, index, v1, scale) \ |
| 3522 | __builtin_ia32_scattersiv4si((int *)(addr), (__mmask8)-1, \ | 3617 | __builtin_ia32_scattersiv4si((void *)(addr), (__mmask8)-1, \ |
| 3523 | (__v4si)(__m128i)(index), \ | 3618 | (__v4si)(__m128i)(index), \ |
| 3524 | (__v4si)(__m128i)(v1), (int)(scale)) | 3619 | (__v4si)(__m128i)(v1), (int)(scale)) |
| 3525 | 3620 | ||
| 3526 | #define _mm_mask_i32scatter_epi32(addr, mask, index, v1, scale) \ | 3621 | #define _mm_mask_i32scatter_epi32(addr, mask, index, v1, scale) \ |
| 3527 | __builtin_ia32_scattersiv4si((int *)(addr), (__mmask8)(mask), \ | 3622 | __builtin_ia32_scattersiv4si((void *)(addr), (__mmask8)(mask), \ |
| 3528 | (__v4si)(__m128i)(index), \ | 3623 | (__v4si)(__m128i)(index), \ |
| 3529 | (__v4si)(__m128i)(v1), (int)(scale)) | 3624 | (__v4si)(__m128i)(v1), (int)(scale)) |
| 3530 | 3625 | ||
| 3531 | #define _mm256_i32scatter_ps(addr, index, v1, scale) \ | 3626 | #define _mm256_i32scatter_ps(addr, index, v1, scale) \ |
| 3532 | __builtin_ia32_scattersiv8sf((float *)(addr), (__mmask8)-1, \ | 3627 | __builtin_ia32_scattersiv8sf((void *)(addr), (__mmask8)-1, \ |
| 3533 | (__v8si)(__m256i)(index), (__v8sf)(__m256)(v1), \ | 3628 | (__v8si)(__m256i)(index), (__v8sf)(__m256)(v1), \ |
| 3534 | (int)(scale)) | 3629 | (int)(scale)) |
| 3535 | 3630 | ||
| 3536 | #define _mm256_mask_i32scatter_ps(addr, mask, index, v1, scale) \ | 3631 | #define _mm256_mask_i32scatter_ps(addr, mask, index, v1, scale) \ |
| 3537 | __builtin_ia32_scattersiv8sf((float *)(addr), (__mmask8)(mask), \ | 3632 | __builtin_ia32_scattersiv8sf((void *)(addr), (__mmask8)(mask), \ |
| 3538 | (__v8si)(__m256i)(index), (__v8sf)(__m256)(v1), \ | 3633 | (__v8si)(__m256i)(index), (__v8sf)(__m256)(v1), \ |
| 3539 | (int)(scale)) | 3634 | (int)(scale)) |
| 3540 | 3635 | ||
| 3541 | #define _mm256_i32scatter_epi32(addr, index, v1, scale) \ | 3636 | #define _mm256_i32scatter_epi32(addr, index, v1, scale) \ |
| 3542 | __builtin_ia32_scattersiv8si((int *)(addr), (__mmask8)-1, \ | 3637 | __builtin_ia32_scattersiv8si((void *)(addr), (__mmask8)-1, \ |
| 3543 | (__v8si)(__m256i)(index), \ | 3638 | (__v8si)(__m256i)(index), \ |
| 3544 | (__v8si)(__m256i)(v1), (int)(scale)) | 3639 | (__v8si)(__m256i)(v1), (int)(scale)) |
| 3545 | 3640 | ||
| 3546 | #define _mm256_mask_i32scatter_epi32(addr, mask, index, v1, scale) \ | 3641 | #define _mm256_mask_i32scatter_epi32(addr, mask, index, v1, scale) \ |
| 3547 | __builtin_ia32_scattersiv8si((int *)(addr), (__mmask8)(mask), \ | 3642 | __builtin_ia32_scattersiv8si((void *)(addr), (__mmask8)(mask), \ |
| 3548 | (__v8si)(__m256i)(index), \ | 3643 | (__v8si)(__m256i)(index), \ |
| 3549 | (__v8si)(__m256i)(v1), (int)(scale)) | 3644 | (__v8si)(__m256i)(v1), (int)(scale)) |
| 3550 | 3645 | ||
| ... | @@ -4989,6 +5084,12 @@ _mm256_maskz_mov_epi32 (__mmask8 __U, __m256i __A) | ... | @@ -4989,6 +5084,12 @@ _mm256_maskz_mov_epi32 (__mmask8 __U, __m256i __A) |
| 4989 | (__v8si) _mm256_setzero_si256 ()); | 5084 | (__v8si) _mm256_setzero_si256 ()); |
| 4990 | } | 5085 | } |
| 4991 | 5086 | ||
| 5087 | static __inline __m128i __DEFAULT_FN_ATTRS128 | ||
| 5088 | _mm_load_epi32 (void const *__P) | ||
| 5089 | { | ||
| 5090 | return *(__m128i *) __P; | ||
| 5091 | } | ||
| 5092 | |||
| 4992 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 5093 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 4993 | _mm_mask_load_epi32 (__m128i __W, __mmask8 __U, void const *__P) | 5094 | _mm_mask_load_epi32 (__m128i __W, __mmask8 __U, void const *__P) |
| 4994 | { | 5095 | { |
| ... | @@ -5008,6 +5109,12 @@ _mm_maskz_load_epi32 (__mmask8 __U, void const *__P) | ... | @@ -5008,6 +5109,12 @@ _mm_maskz_load_epi32 (__mmask8 __U, void const *__P) |
| 5008 | __U); | 5109 | __U); |
| 5009 | } | 5110 | } |
| 5010 | 5111 | ||
| 5112 | static __inline __m256i __DEFAULT_FN_ATTRS256 | ||
| 5113 | _mm256_load_epi32 (void const *__P) | ||
| 5114 | { | ||
| 5115 | return *(__m256i *) __P; | ||
| 5116 | } | ||
| 5117 | |||
| 5011 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 5118 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 5012 | _mm256_mask_load_epi32 (__m256i __W, __mmask8 __U, void const *__P) | 5119 | _mm256_mask_load_epi32 (__m256i __W, __mmask8 __U, void const *__P) |
| 5013 | { | 5120 | { |
| ... | @@ -5027,6 +5134,12 @@ _mm256_maskz_load_epi32 (__mmask8 __U, void const *__P) | ... | @@ -5027,6 +5134,12 @@ _mm256_maskz_load_epi32 (__mmask8 __U, void const *__P) |
| 5027 | __U); | 5134 | __U); |
| 5028 | } | 5135 | } |
| 5029 | 5136 | ||
| 5137 | static __inline void __DEFAULT_FN_ATTRS128 | ||
| 5138 | _mm_store_epi32 (void *__P, __m128i __A) | ||
| 5139 | { | ||
| 5140 | *(__m128i *) __P = __A; | ||
| 5141 | } | ||
| 5142 | |||
| 5030 | static __inline__ void __DEFAULT_FN_ATTRS128 | 5143 | static __inline__ void __DEFAULT_FN_ATTRS128 |
| 5031 | _mm_mask_store_epi32 (void *__P, __mmask8 __U, __m128i __A) | 5144 | _mm_mask_store_epi32 (void *__P, __mmask8 __U, __m128i __A) |
| 5032 | { | 5145 | { |
| ... | @@ -5035,6 +5148,12 @@ _mm_mask_store_epi32 (void *__P, __mmask8 __U, __m128i __A) | ... | @@ -5035,6 +5148,12 @@ _mm_mask_store_epi32 (void *__P, __mmask8 __U, __m128i __A) |
| 5035 | (__mmask8) __U); | 5148 | (__mmask8) __U); |
| 5036 | } | 5149 | } |
| 5037 | 5150 | ||
| 5151 | static __inline void __DEFAULT_FN_ATTRS256 | ||
| 5152 | _mm256_store_epi32 (void *__P, __m256i __A) | ||
| 5153 | { | ||
| 5154 | *(__m256i *) __P = __A; | ||
| 5155 | } | ||
| 5156 | |||
| 5038 | static __inline__ void __DEFAULT_FN_ATTRS256 | 5157 | static __inline__ void __DEFAULT_FN_ATTRS256 |
| 5039 | _mm256_mask_store_epi32 (void *__P, __mmask8 __U, __m256i __A) | 5158 | _mm256_mask_store_epi32 (void *__P, __mmask8 __U, __m256i __A) |
| 5040 | { | 5159 | { |
| ... | @@ -5075,6 +5194,12 @@ _mm256_maskz_mov_epi64 (__mmask8 __U, __m256i __A) | ... | @@ -5075,6 +5194,12 @@ _mm256_maskz_mov_epi64 (__mmask8 __U, __m256i __A) |
| 5075 | (__v4di) _mm256_setzero_si256 ()); | 5194 | (__v4di) _mm256_setzero_si256 ()); |
| 5076 | } | 5195 | } |
| 5077 | 5196 | ||
| 5197 | static __inline __m128i __DEFAULT_FN_ATTRS128 | ||
| 5198 | _mm_load_epi64 (void const *__P) | ||
| 5199 | { | ||
| 5200 | return *(__m128i *) __P; | ||
| 5201 | } | ||
| 5202 | |||
| 5078 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 5203 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 5079 | _mm_mask_load_epi64 (__m128i __W, __mmask8 __U, void const *__P) | 5204 | _mm_mask_load_epi64 (__m128i __W, __mmask8 __U, void const *__P) |
| 5080 | { | 5205 | { |
| ... | @@ -5094,6 +5219,12 @@ _mm_maskz_load_epi64 (__mmask8 __U, void const *__P) | ... | @@ -5094,6 +5219,12 @@ _mm_maskz_load_epi64 (__mmask8 __U, void const *__P) |
| 5094 | __U); | 5219 | __U); |
| 5095 | } | 5220 | } |
| 5096 | 5221 | ||
| 5222 | static __inline __m256i __DEFAULT_FN_ATTRS256 | ||
| 5223 | _mm256_load_epi64 (void const *__P) | ||
| 5224 | { | ||
| 5225 | return *(__m256i *) __P; | ||
| 5226 | } | ||
| 5227 | |||
| 5097 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 5228 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 5098 | _mm256_mask_load_epi64 (__m256i __W, __mmask8 __U, void const *__P) | 5229 | _mm256_mask_load_epi64 (__m256i __W, __mmask8 __U, void const *__P) |
| 5099 | { | 5230 | { |
| ... | @@ -5113,6 +5244,12 @@ _mm256_maskz_load_epi64 (__mmask8 __U, void const *__P) | ... | @@ -5113,6 +5244,12 @@ _mm256_maskz_load_epi64 (__mmask8 __U, void const *__P) |
| 5113 | __U); | 5244 | __U); |
| 5114 | } | 5245 | } |
| 5115 | 5246 | ||
| 5247 | static __inline void __DEFAULT_FN_ATTRS128 | ||
| 5248 | _mm_store_epi64 (void *__P, __m128i __A) | ||
| 5249 | { | ||
| 5250 | *(__m128i *) __P = __A; | ||
| 5251 | } | ||
| 5252 | |||
| 5116 | static __inline__ void __DEFAULT_FN_ATTRS128 | 5253 | static __inline__ void __DEFAULT_FN_ATTRS128 |
| 5117 | _mm_mask_store_epi64 (void *__P, __mmask8 __U, __m128i __A) | 5254 | _mm_mask_store_epi64 (void *__P, __mmask8 __U, __m128i __A) |
| 5118 | { | 5255 | { |
| ... | @@ -5121,6 +5258,12 @@ _mm_mask_store_epi64 (void *__P, __mmask8 __U, __m128i __A) | ... | @@ -5121,6 +5258,12 @@ _mm_mask_store_epi64 (void *__P, __mmask8 __U, __m128i __A) |
| 5121 | (__mmask8) __U); | 5258 | (__mmask8) __U); |
| 5122 | } | 5259 | } |
| 5123 | 5260 | ||
| 5261 | static __inline void __DEFAULT_FN_ATTRS256 | ||
| 5262 | _mm256_store_epi64 (void *__P, __m256i __A) | ||
| 5263 | { | ||
| 5264 | *(__m256i *) __P = __A; | ||
| 5265 | } | ||
| 5266 | |||
| 5124 | static __inline__ void __DEFAULT_FN_ATTRS256 | 5267 | static __inline__ void __DEFAULT_FN_ATTRS256 |
| 5125 | _mm256_mask_store_epi64 (void *__P, __mmask8 __U, __m256i __A) | 5268 | _mm256_mask_store_epi64 (void *__P, __mmask8 __U, __m256i __A) |
| 5126 | { | 5269 | { |
| ... | @@ -5366,6 +5509,15 @@ _mm256_maskz_load_ps (__mmask8 __U, void const *__P) | ... | @@ -5366,6 +5509,15 @@ _mm256_maskz_load_ps (__mmask8 __U, void const *__P) |
| 5366 | (__mmask8) __U); | 5509 | (__mmask8) __U); |
| 5367 | } | 5510 | } |
| 5368 | 5511 | ||
| 5512 | static __inline __m128i __DEFAULT_FN_ATTRS128 | ||
| 5513 | _mm_loadu_epi64 (void const *__P) | ||
| 5514 | { | ||
| 5515 | struct __loadu_epi64 { | ||
| 5516 | __m128i __v; | ||
| 5517 | } __attribute__((__packed__, __may_alias__)); | ||
| 5518 | return ((struct __loadu_epi64*)__P)->__v; | ||
| 5519 | } | ||
| 5520 | |||
| 5369 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 5521 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 5370 | _mm_mask_loadu_epi64 (__m128i __W, __mmask8 __U, void const *__P) | 5522 | _mm_mask_loadu_epi64 (__m128i __W, __mmask8 __U, void const *__P) |
| 5371 | { | 5523 | { |
| ... | @@ -5383,6 +5535,15 @@ _mm_maskz_loadu_epi64 (__mmask8 __U, void const *__P) | ... | @@ -5383,6 +5535,15 @@ _mm_maskz_loadu_epi64 (__mmask8 __U, void const *__P) |
| 5383 | (__mmask8) __U); | 5535 | (__mmask8) __U); |
| 5384 | } | 5536 | } |
| 5385 | 5537 | ||
| 5538 | static __inline __m256i __DEFAULT_FN_ATTRS256 | ||
| 5539 | _mm256_loadu_epi64 (void const *__P) | ||
| 5540 | { | ||
| 5541 | struct __loadu_epi64 { | ||
| 5542 | __m256i __v; | ||
| 5543 | } __attribute__((__packed__, __may_alias__)); | ||
| 5544 | return ((struct __loadu_epi64*)__P)->__v; | ||
| 5545 | } | ||
| 5546 | |||
| 5386 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 5547 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 5387 | _mm256_mask_loadu_epi64 (__m256i __W, __mmask8 __U, void const *__P) | 5548 | _mm256_mask_loadu_epi64 (__m256i __W, __mmask8 __U, void const *__P) |
| 5388 | { | 5549 | { |
| ... | @@ -5400,6 +5561,15 @@ _mm256_maskz_loadu_epi64 (__mmask8 __U, void const *__P) | ... | @@ -5400,6 +5561,15 @@ _mm256_maskz_loadu_epi64 (__mmask8 __U, void const *__P) |
| 5400 | (__mmask8) __U); | 5561 | (__mmask8) __U); |
| 5401 | } | 5562 | } |
| 5402 | 5563 | ||
| 5564 | static __inline __m128i __DEFAULT_FN_ATTRS128 | ||
| 5565 | _mm_loadu_epi32 (void const *__P) | ||
| 5566 | { | ||
| 5567 | struct __loadu_epi32 { | ||
| 5568 | __m128i __v; | ||
| 5569 | } __attribute__((__packed__, __may_alias__)); | ||
| 5570 | return ((struct __loadu_epi32*)__P)->__v; | ||
| 5571 | } | ||
| 5572 | |||
| 5403 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 5573 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 5404 | _mm_mask_loadu_epi32 (__m128i __W, __mmask8 __U, void const *__P) | 5574 | _mm_mask_loadu_epi32 (__m128i __W, __mmask8 __U, void const *__P) |
| 5405 | { | 5575 | { |
| ... | @@ -5417,6 +5587,15 @@ _mm_maskz_loadu_epi32 (__mmask8 __U, void const *__P) | ... | @@ -5417,6 +5587,15 @@ _mm_maskz_loadu_epi32 (__mmask8 __U, void const *__P) |
| 5417 | (__mmask8) __U); | 5587 | (__mmask8) __U); |
| 5418 | } | 5588 | } |
| 5419 | 5589 | ||
| 5590 | static __inline __m256i __DEFAULT_FN_ATTRS256 | ||
| 5591 | _mm256_loadu_epi32 (void const *__P) | ||
| 5592 | { | ||
| 5593 | struct __loadu_epi32 { | ||
| 5594 | __m256i __v; | ||
| 5595 | } __attribute__((__packed__, __may_alias__)); | ||
| 5596 | return ((struct __loadu_epi32*)__P)->__v; | ||
| 5597 | } | ||
| 5598 | |||
| 5420 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 5599 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 5421 | _mm256_mask_loadu_epi32 (__m256i __W, __mmask8 __U, void const *__P) | 5600 | _mm256_mask_loadu_epi32 (__m256i __W, __mmask8 __U, void const *__P) |
| 5422 | { | 5601 | { |
| ... | @@ -5534,6 +5713,15 @@ _mm256_mask_store_ps (void *__P, __mmask8 __U, __m256 __A) | ... | @@ -5534,6 +5713,15 @@ _mm256_mask_store_ps (void *__P, __mmask8 __U, __m256 __A) |
| 5534 | (__mmask8) __U); | 5713 | (__mmask8) __U); |
| 5535 | } | 5714 | } |
| 5536 | 5715 | ||
| 5716 | static __inline void __DEFAULT_FN_ATTRS128 | ||
| 5717 | _mm_storeu_epi64 (void *__P, __m128i __A) | ||
| 5718 | { | ||
| 5719 | struct __storeu_epi64 { | ||
| 5720 | __m128i __v; | ||
| 5721 | } __attribute__((__packed__, __may_alias__)); | ||
| 5722 | ((struct __storeu_epi64*)__P)->__v = __A; | ||
| 5723 | } | ||
| 5724 | |||
| 5537 | static __inline__ void __DEFAULT_FN_ATTRS128 | 5725 | static __inline__ void __DEFAULT_FN_ATTRS128 |
| 5538 | _mm_mask_storeu_epi64 (void *__P, __mmask8 __U, __m128i __A) | 5726 | _mm_mask_storeu_epi64 (void *__P, __mmask8 __U, __m128i __A) |
| 5539 | { | 5727 | { |
| ... | @@ -5542,6 +5730,15 @@ _mm_mask_storeu_epi64 (void *__P, __mmask8 __U, __m128i __A) | ... | @@ -5542,6 +5730,15 @@ _mm_mask_storeu_epi64 (void *__P, __mmask8 __U, __m128i __A) |
| 5542 | (__mmask8) __U); | 5730 | (__mmask8) __U); |
| 5543 | } | 5731 | } |
| 5544 | 5732 | ||
| 5733 | static __inline void __DEFAULT_FN_ATTRS256 | ||
| 5734 | _mm256_storeu_epi64 (void *__P, __m256i __A) | ||
| 5735 | { | ||
| 5736 | struct __storeu_epi64 { | ||
| 5737 | __m256i __v; | ||
| 5738 | } __attribute__((__packed__, __may_alias__)); | ||
| 5739 | ((struct __storeu_epi64*)__P)->__v = __A; | ||
| 5740 | } | ||
| 5741 | |||
| 5545 | static __inline__ void __DEFAULT_FN_ATTRS256 | 5742 | static __inline__ void __DEFAULT_FN_ATTRS256 |
| 5546 | _mm256_mask_storeu_epi64 (void *__P, __mmask8 __U, __m256i __A) | 5743 | _mm256_mask_storeu_epi64 (void *__P, __mmask8 __U, __m256i __A) |
| 5547 | { | 5744 | { |
| ... | @@ -5550,6 +5747,15 @@ _mm256_mask_storeu_epi64 (void *__P, __mmask8 __U, __m256i __A) | ... | @@ -5550,6 +5747,15 @@ _mm256_mask_storeu_epi64 (void *__P, __mmask8 __U, __m256i __A) |
| 5550 | (__mmask8) __U); | 5747 | (__mmask8) __U); |
| 5551 | } | 5748 | } |
| 5552 | 5749 | ||
| 5750 | static __inline void __DEFAULT_FN_ATTRS128 | ||
| 5751 | _mm_storeu_epi32 (void *__P, __m128i __A) | ||
| 5752 | { | ||
| 5753 | struct __storeu_epi32 { | ||
| 5754 | __m128i __v; | ||
| 5755 | } __attribute__((__packed__, __may_alias__)); | ||
| 5756 | ((struct __storeu_epi32*)__P)->__v = __A; | ||
| 5757 | } | ||
| 5758 | |||
| 5553 | static __inline__ void __DEFAULT_FN_ATTRS128 | 5759 | static __inline__ void __DEFAULT_FN_ATTRS128 |
| 5554 | _mm_mask_storeu_epi32 (void *__P, __mmask8 __U, __m128i __A) | 5760 | _mm_mask_storeu_epi32 (void *__P, __mmask8 __U, __m128i __A) |
| 5555 | { | 5761 | { |
| ... | @@ -5558,6 +5764,15 @@ _mm_mask_storeu_epi32 (void *__P, __mmask8 __U, __m128i __A) | ... | @@ -5558,6 +5764,15 @@ _mm_mask_storeu_epi32 (void *__P, __mmask8 __U, __m128i __A) |
| 5558 | (__mmask8) __U); | 5764 | (__mmask8) __U); |
| 5559 | } | 5765 | } |
| 5560 | 5766 | ||
| 5767 | static __inline void __DEFAULT_FN_ATTRS256 | ||
| 5768 | _mm256_storeu_epi32 (void *__P, __m256i __A) | ||
| 5769 | { | ||
| 5770 | struct __storeu_epi32 { | ||
| 5771 | __m256i __v; | ||
| 5772 | } __attribute__((__packed__, __may_alias__)); | ||
| 5773 | ((struct __storeu_epi32*)__P)->__v = __A; | ||
| 5774 | } | ||
| 5775 | |||
| 5561 | static __inline__ void __DEFAULT_FN_ATTRS256 | 5776 | static __inline__ void __DEFAULT_FN_ATTRS256 |
| 5562 | _mm256_mask_storeu_epi32 (void *__P, __mmask8 __U, __m256i __A) | 5777 | _mm256_mask_storeu_epi32 (void *__P, __mmask8 __U, __m256i __A) |
| 5563 | { | 5778 | { |
| ... | @@ -7769,97 +7984,97 @@ _mm256_mask_cvtepi64_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A) | ... | @@ -7769,97 +7984,97 @@ _mm256_mask_cvtepi64_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A) |
| 7769 | 7984 | ||
| 7770 | #define _mm_mmask_i64gather_pd(v1_old, mask, index, addr, scale) \ | 7985 | #define _mm_mmask_i64gather_pd(v1_old, mask, index, addr, scale) \ |
| 7771 | (__m128d)__builtin_ia32_gather3div2df((__v2df)(__m128d)(v1_old), \ | 7986 | (__m128d)__builtin_ia32_gather3div2df((__v2df)(__m128d)(v1_old), \ |
| 7772 | (double const *)(addr), \ | 7987 | (void const *)(addr), \ |
| 7773 | (__v2di)(__m128i)(index), \ | 7988 | (__v2di)(__m128i)(index), \ |
| 7774 | (__mmask8)(mask), (int)(scale)) | 7989 | (__mmask8)(mask), (int)(scale)) |
| 7775 | 7990 | ||
| 7776 | #define _mm_mmask_i64gather_epi64(v1_old, mask, index, addr, scale) \ | 7991 | #define _mm_mmask_i64gather_epi64(v1_old, mask, index, addr, scale) \ |
| 7777 | (__m128i)__builtin_ia32_gather3div2di((__v2di)(__m128i)(v1_old), \ | 7992 | (__m128i)__builtin_ia32_gather3div2di((__v2di)(__m128i)(v1_old), \ |
| 7778 | (long long const *)(addr), \ | 7993 | (void const *)(addr), \ |
| 7779 | (__v2di)(__m128i)(index), \ | 7994 | (__v2di)(__m128i)(index), \ |
| 7780 | (__mmask8)(mask), (int)(scale)) | 7995 | (__mmask8)(mask), (int)(scale)) |
| 7781 | 7996 | ||
| 7782 | #define _mm256_mmask_i64gather_pd(v1_old, mask, index, addr, scale) \ | 7997 | #define _mm256_mmask_i64gather_pd(v1_old, mask, index, addr, scale) \ |
| 7783 | (__m256d)__builtin_ia32_gather3div4df((__v4df)(__m256d)(v1_old), \ | 7998 | (__m256d)__builtin_ia32_gather3div4df((__v4df)(__m256d)(v1_old), \ |
| 7784 | (double const *)(addr), \ | 7999 | (void const *)(addr), \ |
| 7785 | (__v4di)(__m256i)(index), \ | 8000 | (__v4di)(__m256i)(index), \ |
| 7786 | (__mmask8)(mask), (int)(scale)) | 8001 | (__mmask8)(mask), (int)(scale)) |
| 7787 | 8002 | ||
| 7788 | #define _mm256_mmask_i64gather_epi64(v1_old, mask, index, addr, scale) \ | 8003 | #define _mm256_mmask_i64gather_epi64(v1_old, mask, index, addr, scale) \ |
| 7789 | (__m256i)__builtin_ia32_gather3div4di((__v4di)(__m256i)(v1_old), \ | 8004 | (__m256i)__builtin_ia32_gather3div4di((__v4di)(__m256i)(v1_old), \ |
| 7790 | (long long const *)(addr), \ | 8005 | (void const *)(addr), \ |
| 7791 | (__v4di)(__m256i)(index), \ | 8006 | (__v4di)(__m256i)(index), \ |
| 7792 | (__mmask8)(mask), (int)(scale)) | 8007 | (__mmask8)(mask), (int)(scale)) |
| 7793 | 8008 | ||
| 7794 | #define _mm_mmask_i64gather_ps(v1_old, mask, index, addr, scale) \ | 8009 | #define _mm_mmask_i64gather_ps(v1_old, mask, index, addr, scale) \ |
| 7795 | (__m128)__builtin_ia32_gather3div4sf((__v4sf)(__m128)(v1_old), \ | 8010 | (__m128)__builtin_ia32_gather3div4sf((__v4sf)(__m128)(v1_old), \ |
| 7796 | (float const *)(addr), \ | 8011 | (void const *)(addr), \ |
| 7797 | (__v2di)(__m128i)(index), \ | 8012 | (__v2di)(__m128i)(index), \ |
| 7798 | (__mmask8)(mask), (int)(scale)) | 8013 | (__mmask8)(mask), (int)(scale)) |
| 7799 | 8014 | ||
| 7800 | #define _mm_mmask_i64gather_epi32(v1_old, mask, index, addr, scale) \ | 8015 | #define _mm_mmask_i64gather_epi32(v1_old, mask, index, addr, scale) \ |
| 7801 | (__m128i)__builtin_ia32_gather3div4si((__v4si)(__m128i)(v1_old), \ | 8016 | (__m128i)__builtin_ia32_gather3div4si((__v4si)(__m128i)(v1_old), \ |
| 7802 | (int const *)(addr), \ | 8017 | (void const *)(addr), \ |
| 7803 | (__v2di)(__m128i)(index), \ | 8018 | (__v2di)(__m128i)(index), \ |
| 7804 | (__mmask8)(mask), (int)(scale)) | 8019 | (__mmask8)(mask), (int)(scale)) |
| 7805 | 8020 | ||
| 7806 | #define _mm256_mmask_i64gather_ps(v1_old, mask, index, addr, scale) \ | 8021 | #define _mm256_mmask_i64gather_ps(v1_old, mask, index, addr, scale) \ |
| 7807 | (__m128)__builtin_ia32_gather3div8sf((__v4sf)(__m128)(v1_old), \ | 8022 | (__m128)__builtin_ia32_gather3div8sf((__v4sf)(__m128)(v1_old), \ |
| 7808 | (float const *)(addr), \ | 8023 | (void const *)(addr), \ |
| 7809 | (__v4di)(__m256i)(index), \ | 8024 | (__v4di)(__m256i)(index), \ |
| 7810 | (__mmask8)(mask), (int)(scale)) | 8025 | (__mmask8)(mask), (int)(scale)) |
| 7811 | 8026 | ||
| 7812 | #define _mm256_mmask_i64gather_epi32(v1_old, mask, index, addr, scale) \ | 8027 | #define _mm256_mmask_i64gather_epi32(v1_old, mask, index, addr, scale) \ |
| 7813 | (__m128i)__builtin_ia32_gather3div8si((__v4si)(__m128i)(v1_old), \ | 8028 | (__m128i)__builtin_ia32_gather3div8si((__v4si)(__m128i)(v1_old), \ |
| 7814 | (int const *)(addr), \ | 8029 | (void const *)(addr), \ |
| 7815 | (__v4di)(__m256i)(index), \ | 8030 | (__v4di)(__m256i)(index), \ |
| 7816 | (__mmask8)(mask), (int)(scale)) | 8031 | (__mmask8)(mask), (int)(scale)) |
| 7817 | 8032 | ||
| 7818 | #define _mm_mmask_i32gather_pd(v1_old, mask, index, addr, scale) \ | 8033 | #define _mm_mmask_i32gather_pd(v1_old, mask, index, addr, scale) \ |
| 7819 | (__m128d)__builtin_ia32_gather3siv2df((__v2df)(__m128d)(v1_old), \ | 8034 | (__m128d)__builtin_ia32_gather3siv2df((__v2df)(__m128d)(v1_old), \ |
| 7820 | (double const *)(addr), \ | 8035 | (void const *)(addr), \ |
| 7821 | (__v4si)(__m128i)(index), \ | 8036 | (__v4si)(__m128i)(index), \ |
| 7822 | (__mmask8)(mask), (int)(scale)) | 8037 | (__mmask8)(mask), (int)(scale)) |
| 7823 | 8038 | ||
| 7824 | #define _mm_mmask_i32gather_epi64(v1_old, mask, index, addr, scale) \ | 8039 | #define _mm_mmask_i32gather_epi64(v1_old, mask, index, addr, scale) \ |
| 7825 | (__m128i)__builtin_ia32_gather3siv2di((__v2di)(__m128i)(v1_old), \ | 8040 | (__m128i)__builtin_ia32_gather3siv2di((__v2di)(__m128i)(v1_old), \ |
| 7826 | (long long const *)(addr), \ | 8041 | (void const *)(addr), \ |
| 7827 | (__v4si)(__m128i)(index), \ | 8042 | (__v4si)(__m128i)(index), \ |
| 7828 | (__mmask8)(mask), (int)(scale)) | 8043 | (__mmask8)(mask), (int)(scale)) |
| 7829 | 8044 | ||
| 7830 | #define _mm256_mmask_i32gather_pd(v1_old, mask, index, addr, scale) \ | 8045 | #define _mm256_mmask_i32gather_pd(v1_old, mask, index, addr, scale) \ |
| 7831 | (__m256d)__builtin_ia32_gather3siv4df((__v4df)(__m256d)(v1_old), \ | 8046 | (__m256d)__builtin_ia32_gather3siv4df((__v4df)(__m256d)(v1_old), \ |
| 7832 | (double const *)(addr), \ | 8047 | (void const *)(addr), \ |
| 7833 | (__v4si)(__m128i)(index), \ | 8048 | (__v4si)(__m128i)(index), \ |
| 7834 | (__mmask8)(mask), (int)(scale)) | 8049 | (__mmask8)(mask), (int)(scale)) |
| 7835 | 8050 | ||
| 7836 | #define _mm256_mmask_i32gather_epi64(v1_old, mask, index, addr, scale) \ | 8051 | #define _mm256_mmask_i32gather_epi64(v1_old, mask, index, addr, scale) \ |
| 7837 | (__m256i)__builtin_ia32_gather3siv4di((__v4di)(__m256i)(v1_old), \ | 8052 | (__m256i)__builtin_ia32_gather3siv4di((__v4di)(__m256i)(v1_old), \ |
| 7838 | (long long const *)(addr), \ | 8053 | (void const *)(addr), \ |
| 7839 | (__v4si)(__m128i)(index), \ | 8054 | (__v4si)(__m128i)(index), \ |
| 7840 | (__mmask8)(mask), (int)(scale)) | 8055 | (__mmask8)(mask), (int)(scale)) |
| 7841 | 8056 | ||
| 7842 | #define _mm_mmask_i32gather_ps(v1_old, mask, index, addr, scale) \ | 8057 | #define _mm_mmask_i32gather_ps(v1_old, mask, index, addr, scale) \ |
| 7843 | (__m128)__builtin_ia32_gather3siv4sf((__v4sf)(__m128)(v1_old), \ | 8058 | (__m128)__builtin_ia32_gather3siv4sf((__v4sf)(__m128)(v1_old), \ |
| 7844 | (float const *)(addr), \ | 8059 | (void const *)(addr), \ |
| 7845 | (__v4si)(__m128i)(index), \ | 8060 | (__v4si)(__m128i)(index), \ |
| 7846 | (__mmask8)(mask), (int)(scale)) | 8061 | (__mmask8)(mask), (int)(scale)) |
| 7847 | 8062 | ||
| 7848 | #define _mm_mmask_i32gather_epi32(v1_old, mask, index, addr, scale) \ | 8063 | #define _mm_mmask_i32gather_epi32(v1_old, mask, index, addr, scale) \ |
| 7849 | (__m128i)__builtin_ia32_gather3siv4si((__v4si)(__m128i)(v1_old), \ | 8064 | (__m128i)__builtin_ia32_gather3siv4si((__v4si)(__m128i)(v1_old), \ |
| 7850 | (int const *)(addr), \ | 8065 | (void const *)(addr), \ |
| 7851 | (__v4si)(__m128i)(index), \ | 8066 | (__v4si)(__m128i)(index), \ |
| 7852 | (__mmask8)(mask), (int)(scale)) | 8067 | (__mmask8)(mask), (int)(scale)) |
| 7853 | 8068 | ||
| 7854 | #define _mm256_mmask_i32gather_ps(v1_old, mask, index, addr, scale) \ | 8069 | #define _mm256_mmask_i32gather_ps(v1_old, mask, index, addr, scale) \ |
| 7855 | (__m256)__builtin_ia32_gather3siv8sf((__v8sf)(__m256)(v1_old), \ | 8070 | (__m256)__builtin_ia32_gather3siv8sf((__v8sf)(__m256)(v1_old), \ |
| 7856 | (float const *)(addr), \ | 8071 | (void const *)(addr), \ |
| 7857 | (__v8si)(__m256i)(index), \ | 8072 | (__v8si)(__m256i)(index), \ |
| 7858 | (__mmask8)(mask), (int)(scale)) | 8073 | (__mmask8)(mask), (int)(scale)) |
| 7859 | 8074 | ||
| 7860 | #define _mm256_mmask_i32gather_epi32(v1_old, mask, index, addr, scale) \ | 8075 | #define _mm256_mmask_i32gather_epi32(v1_old, mask, index, addr, scale) \ |
| 7861 | (__m256i)__builtin_ia32_gather3siv8si((__v8si)(__m256i)(v1_old), \ | 8076 | (__m256i)__builtin_ia32_gather3siv8si((__v8si)(__m256i)(v1_old), \ |
| 7862 | (int const *)(addr), \ | 8077 | (void const *)(addr), \ |
| 7863 | (__v8si)(__m256i)(index), \ | 8078 | (__v8si)(__m256i)(index), \ |
| 7864 | (__mmask8)(mask), (int)(scale)) | 8079 | (__mmask8)(mask), (int)(scale)) |
| 7865 | 8080 |
c_headers/avx512vlvbmi2intrin.h+132-180
| ... | @@ -421,327 +421,279 @@ _mm256_maskz_expandloadu_epi8(__mmask32 __U, void const *__P) | ... | @@ -421,327 +421,279 @@ _mm256_maskz_expandloadu_epi8(__mmask32 __U, void const *__P) |
| 421 | (__v8hi)_mm_setzero_si128()) | 421 | (__v8hi)_mm_setzero_si128()) |
| 422 | 422 | ||
| 423 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 423 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 424 | _mm256_mask_shldv_epi64(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) | 424 | _mm256_shldv_epi64(__m256i __A, __m256i __B, __m256i __C) |
| 425 | { | 425 | { |
| 426 | return (__m256i) __builtin_ia32_vpshldvq256_mask ((__v4di) __S, | 426 | return (__m256i)__builtin_ia32_vpshldvq256((__v4di)__A, (__v4di)__B, |
| 427 | (__v4di) __A, | 427 | (__v4di)__C); |
| 428 | (__v4di) __B, | ||
| 429 | __U); | ||
| 430 | } | 428 | } |
| 431 | 429 | ||
| 432 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 430 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 433 | _mm256_maskz_shldv_epi64(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) | 431 | _mm256_mask_shldv_epi64(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) |
| 434 | { | 432 | { |
| 435 | return (__m256i) __builtin_ia32_vpshldvq256_maskz ((__v4di) __S, | 433 | return (__m256i)__builtin_ia32_selectq_256(__U, |
| 436 | (__v4di) __A, | 434 | (__v4di)_mm256_shldv_epi64(__A, __B, __C), |
| 437 | (__v4di) __B, | 435 | (__v4di)__A); |
| 438 | __U); | ||
| 439 | } | 436 | } |
| 440 | 437 | ||
| 441 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 438 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 442 | _mm256_shldv_epi64(__m256i __S, __m256i __A, __m256i __B) | 439 | _mm256_maskz_shldv_epi64(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) |
| 443 | { | 440 | { |
| 444 | return (__m256i) __builtin_ia32_vpshldvq256_mask ((__v4di) __S, | 441 | return (__m256i)__builtin_ia32_selectq_256(__U, |
| 445 | (__v4di) __A, | 442 | (__v4di)_mm256_shldv_epi64(__A, __B, __C), |
| 446 | (__v4di) __B, | 443 | (__v4di)_mm256_setzero_si256()); |
| 447 | (__mmask8) -1); | ||
| 448 | } | 444 | } |
| 449 | 445 | ||
| 450 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 446 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 451 | _mm_mask_shldv_epi64(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) | 447 | _mm_shldv_epi64(__m128i __A, __m128i __B, __m128i __C) |
| 452 | { | 448 | { |
| 453 | return (__m128i) __builtin_ia32_vpshldvq128_mask ((__v2di) __S, | 449 | return (__m128i)__builtin_ia32_vpshldvq128((__v2di)__A, (__v2di)__B, |
| 454 | (__v2di) __A, | 450 | (__v2di)__C); |
| 455 | (__v2di) __B, | ||
| 456 | __U); | ||
| 457 | } | 451 | } |
| 458 | 452 | ||
| 459 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 453 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 460 | _mm_maskz_shldv_epi64(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) | 454 | _mm_mask_shldv_epi64(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) |
| 461 | { | 455 | { |
| 462 | return (__m128i) __builtin_ia32_vpshldvq128_maskz ((__v2di) __S, | 456 | return (__m128i)__builtin_ia32_selectq_128(__U, |
| 463 | (__v2di) __A, | 457 | (__v2di)_mm_shldv_epi64(__A, __B, __C), |
| 464 | (__v2di) __B, | 458 | (__v2di)__A); |
| 465 | __U); | ||
| 466 | } | 459 | } |
| 467 | 460 | ||
| 468 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 461 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 469 | _mm_shldv_epi64(__m128i __S, __m128i __A, __m128i __B) | 462 | _mm_maskz_shldv_epi64(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) |
| 470 | { | 463 | { |
| 471 | return (__m128i) __builtin_ia32_vpshldvq128_mask ((__v2di) __S, | 464 | return (__m128i)__builtin_ia32_selectq_128(__U, |
| 472 | (__v2di) __A, | 465 | (__v2di)_mm_shldv_epi64(__A, __B, __C), |
| 473 | (__v2di) __B, | 466 | (__v2di)_mm_setzero_si128()); |
| 474 | (__mmask8) -1); | ||
| 475 | } | 467 | } |
| 476 | 468 | ||
| 477 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 469 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 478 | _mm256_mask_shldv_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) | 470 | _mm256_shldv_epi32(__m256i __A, __m256i __B, __m256i __C) |
| 479 | { | 471 | { |
| 480 | return (__m256i) __builtin_ia32_vpshldvd256_mask ((__v8si) __S, | 472 | return (__m256i)__builtin_ia32_vpshldvd256((__v8si)__A, (__v8si)__B, |
| 481 | (__v8si) __A, | 473 | (__v8si)__C); |
| 482 | (__v8si) __B, | ||
| 483 | __U); | ||
| 484 | } | 474 | } |
| 485 | 475 | ||
| 486 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 476 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 487 | _mm256_maskz_shldv_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) | 477 | _mm256_mask_shldv_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) |
| 488 | { | 478 | { |
| 489 | return (__m256i) __builtin_ia32_vpshldvd256_maskz ((__v8si) __S, | 479 | return (__m256i)__builtin_ia32_selectd_256(__U, |
| 490 | (__v8si) __A, | 480 | (__v8si)_mm256_shldv_epi32(__A, __B, __C), |
| 491 | (__v8si) __B, | 481 | (__v8si)__A); |
| 492 | __U); | ||
| 493 | } | 482 | } |
| 494 | 483 | ||
| 495 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 484 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 496 | _mm256_shldv_epi32(__m256i __S, __m256i __A, __m256i __B) | 485 | _mm256_maskz_shldv_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) |
| 497 | { | 486 | { |
| 498 | return (__m256i) __builtin_ia32_vpshldvd256_mask ((__v8si) __S, | 487 | return (__m256i)__builtin_ia32_selectd_256(__U, |
| 499 | (__v8si) __A, | 488 | (__v8si)_mm256_shldv_epi32(__A, __B, __C), |
| 500 | (__v8si) __B, | 489 | (__v8si)_mm256_setzero_si256()); |
| 501 | (__mmask8) -1); | ||
| 502 | } | 490 | } |
| 503 | 491 | ||
| 504 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 492 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 505 | _mm_mask_shldv_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) | 493 | _mm_shldv_epi32(__m128i __A, __m128i __B, __m128i __C) |
| 506 | { | 494 | { |
| 507 | return (__m128i) __builtin_ia32_vpshldvd128_mask ((__v4si) __S, | 495 | return (__m128i)__builtin_ia32_vpshldvd128((__v4si)__A, (__v4si)__B, |
| 508 | (__v4si) __A, | 496 | (__v4si)__C); |
| 509 | (__v4si) __B, | ||
| 510 | __U); | ||
| 511 | } | 497 | } |
| 512 | 498 | ||
| 513 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 499 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 514 | _mm_maskz_shldv_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) | 500 | _mm_mask_shldv_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) |
| 515 | { | 501 | { |
| 516 | return (__m128i) __builtin_ia32_vpshldvd128_maskz ((__v4si) __S, | 502 | return (__m128i)__builtin_ia32_selectd_128(__U, |
| 517 | (__v4si) __A, | 503 | (__v4si)_mm_shldv_epi32(__A, __B, __C), |
| 518 | (__v4si) __B, | 504 | (__v4si)__A); |
| 519 | __U); | ||
| 520 | } | 505 | } |
| 521 | 506 | ||
| 522 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 507 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 523 | _mm_shldv_epi32(__m128i __S, __m128i __A, __m128i __B) | 508 | _mm_maskz_shldv_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) |
| 524 | { | 509 | { |
| 525 | return (__m128i) __builtin_ia32_vpshldvd128_mask ((__v4si) __S, | 510 | return (__m128i)__builtin_ia32_selectd_128(__U, |
| 526 | (__v4si) __A, | 511 | (__v4si)_mm_shldv_epi32(__A, __B, __C), |
| 527 | (__v4si) __B, | 512 | (__v4si)_mm_setzero_si128()); |
| 528 | (__mmask8) -1); | ||
| 529 | } | 513 | } |
| 530 | 514 | ||
| 531 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 515 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 532 | _mm256_mask_shldv_epi16(__m256i __S, __mmask16 __U, __m256i __A, __m256i __B) | 516 | _mm256_shldv_epi16(__m256i __A, __m256i __B, __m256i __C) |
| 533 | { | 517 | { |
| 534 | return (__m256i) __builtin_ia32_vpshldvw256_mask ((__v16hi) __S, | 518 | return (__m256i)__builtin_ia32_vpshldvw256((__v16hi)__A, (__v16hi)__B, |
| 535 | (__v16hi) __A, | 519 | (__v16hi)__C); |
| 536 | (__v16hi) __B, | ||
| 537 | __U); | ||
| 538 | } | 520 | } |
| 539 | 521 | ||
| 540 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 522 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 541 | _mm256_maskz_shldv_epi16(__mmask16 __U, __m256i __S, __m256i __A, __m256i __B) | 523 | _mm256_mask_shldv_epi16(__m256i __A, __mmask16 __U, __m256i __B, __m256i __C) |
| 542 | { | 524 | { |
| 543 | return (__m256i) __builtin_ia32_vpshldvw256_maskz ((__v16hi) __S, | 525 | return (__m256i)__builtin_ia32_selectw_256(__U, |
| 544 | (__v16hi) __A, | 526 | (__v16hi)_mm256_shldv_epi16(__A, __B, __C), |
| 545 | (__v16hi) __B, | 527 | (__v16hi)__A); |
| 546 | __U); | ||
| 547 | } | 528 | } |
| 548 | 529 | ||
| 549 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 530 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 550 | _mm256_shldv_epi16(__m256i __S, __m256i __A, __m256i __B) | 531 | _mm256_maskz_shldv_epi16(__mmask16 __U, __m256i __A, __m256i __B, __m256i __C) |
| 551 | { | 532 | { |
| 552 | return (__m256i) __builtin_ia32_vpshldvw256_mask ((__v16hi) __S, | 533 | return (__m256i)__builtin_ia32_selectw_256(__U, |
| 553 | (__v16hi) __A, | 534 | (__v16hi)_mm256_shldv_epi16(__A, __B, __C), |
| 554 | (__v16hi) __B, | 535 | (__v16hi)_mm256_setzero_si256()); |
| 555 | (__mmask16) -1); | ||
| 556 | } | 536 | } |
| 557 | 537 | ||
| 558 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 538 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 559 | _mm_mask_shldv_epi16(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) | 539 | _mm_shldv_epi16(__m128i __A, __m128i __B, __m128i __C) |
| 560 | { | 540 | { |
| 561 | return (__m128i) __builtin_ia32_vpshldvw128_mask ((__v8hi) __S, | 541 | return (__m128i)__builtin_ia32_vpshldvw128((__v8hi)__A, (__v8hi)__B, |
| 562 | (__v8hi) __A, | 542 | (__v8hi)__C); |
| 563 | (__v8hi) __B, | ||
| 564 | __U); | ||
| 565 | } | 543 | } |
| 566 | 544 | ||
| 567 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 545 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 568 | _mm_maskz_shldv_epi16(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) | 546 | _mm_mask_shldv_epi16(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) |
| 569 | { | 547 | { |
| 570 | return (__m128i) __builtin_ia32_vpshldvw128_maskz ((__v8hi) __S, | 548 | return (__m128i)__builtin_ia32_selectw_128(__U, |
| 571 | (__v8hi) __A, | 549 | (__v8hi)_mm_shldv_epi16(__A, __B, __C), |
| 572 | (__v8hi) __B, | 550 | (__v8hi)__A); |
| 573 | __U); | ||
| 574 | } | 551 | } |
| 575 | 552 | ||
| 576 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 553 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 577 | _mm_shldv_epi16(__m128i __S, __m128i __A, __m128i __B) | 554 | _mm_maskz_shldv_epi16(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) |
| 578 | { | 555 | { |
| 579 | return (__m128i) __builtin_ia32_vpshldvw128_mask ((__v8hi) __S, | 556 | return (__m128i)__builtin_ia32_selectw_128(__U, |
| 580 | (__v8hi) __A, | 557 | (__v8hi)_mm_shldv_epi16(__A, __B, __C), |
| 581 | (__v8hi) __B, | 558 | (__v8hi)_mm_setzero_si128()); |
| 582 | (__mmask8) -1); | ||
| 583 | } | 559 | } |
| 584 | 560 | ||
| 585 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 561 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 586 | _mm256_mask_shrdv_epi64(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) | 562 | _mm256_shrdv_epi64(__m256i __A, __m256i __B, __m256i __C) |
| 587 | { | 563 | { |
| 588 | return (__m256i) __builtin_ia32_vpshrdvq256_mask ((__v4di) __S, | 564 | return (__m256i)__builtin_ia32_vpshrdvq256((__v4di)__A, (__v4di)__B, |
| 589 | (__v4di) __A, | 565 | (__v4di)__C); |
| 590 | (__v4di) __B, | ||
| 591 | __U); | ||
| 592 | } | 566 | } |
| 593 | 567 | ||
| 594 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 568 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 595 | _mm256_maskz_shrdv_epi64(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) | 569 | _mm256_mask_shrdv_epi64(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) |
| 596 | { | 570 | { |
| 597 | return (__m256i) __builtin_ia32_vpshrdvq256_maskz ((__v4di) __S, | 571 | return (__m256i)__builtin_ia32_selectq_256(__U, |
| 598 | (__v4di) __A, | 572 | (__v4di)_mm256_shrdv_epi64(__A, __B, __C), |
| 599 | (__v4di) __B, | 573 | (__v4di)__A); |
| 600 | __U); | ||
| 601 | } | 574 | } |
| 602 | 575 | ||
| 603 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 576 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 604 | _mm256_shrdv_epi64(__m256i __S, __m256i __A, __m256i __B) | 577 | _mm256_maskz_shrdv_epi64(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) |
| 605 | { | 578 | { |
| 606 | return (__m256i) __builtin_ia32_vpshrdvq256_mask ((__v4di) __S, | 579 | return (__m256i)__builtin_ia32_selectq_256(__U, |
| 607 | (__v4di) __A, | 580 | (__v4di)_mm256_shrdv_epi64(__A, __B, __C), |
| 608 | (__v4di) __B, | 581 | (__v4di)_mm256_setzero_si256()); |
| 609 | (__mmask8) -1); | ||
| 610 | } | 582 | } |
| 611 | 583 | ||
| 612 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 584 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 613 | _mm_mask_shrdv_epi64(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) | 585 | _mm_shrdv_epi64(__m128i __A, __m128i __B, __m128i __C) |
| 614 | { | 586 | { |
| 615 | return (__m128i) __builtin_ia32_vpshrdvq128_mask ((__v2di) __S, | 587 | return (__m128i)__builtin_ia32_vpshrdvq128((__v2di)__A, (__v2di)__B, |
| 616 | (__v2di) __A, | 588 | (__v2di)__C); |
| 617 | (__v2di) __B, | ||
| 618 | __U); | ||
| 619 | } | 589 | } |
| 620 | 590 | ||
| 621 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 591 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 622 | _mm_maskz_shrdv_epi64(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) | 592 | _mm_mask_shrdv_epi64(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) |
| 623 | { | 593 | { |
| 624 | return (__m128i) __builtin_ia32_vpshrdvq128_maskz ((__v2di) __S, | 594 | return (__m128i)__builtin_ia32_selectq_128(__U, |
| 625 | (__v2di) __A, | 595 | (__v2di)_mm_shrdv_epi64(__A, __B, __C), |
| 626 | (__v2di) __B, | 596 | (__v2di)__A); |
| 627 | __U); | ||
| 628 | } | 597 | } |
| 629 | 598 | ||
| 630 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 599 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 631 | _mm_shrdv_epi64(__m128i __S, __m128i __A, __m128i __B) | 600 | _mm_maskz_shrdv_epi64(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) |
| 632 | { | 601 | { |
| 633 | return (__m128i) __builtin_ia32_vpshrdvq128_mask ((__v2di) __S, | 602 | return (__m128i)__builtin_ia32_selectq_128(__U, |
| 634 | (__v2di) __A, | 603 | (__v2di)_mm_shrdv_epi64(__A, __B, __C), |
| 635 | (__v2di) __B, | 604 | (__v2di)_mm_setzero_si128()); |
| 636 | (__mmask8) -1); | ||
| 637 | } | 605 | } |
| 638 | 606 | ||
| 639 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 607 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 640 | _mm256_mask_shrdv_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) | 608 | _mm256_shrdv_epi32(__m256i __A, __m256i __B, __m256i __C) |
| 641 | { | 609 | { |
| 642 | return (__m256i) __builtin_ia32_vpshrdvd256_mask ((__v8si) __S, | 610 | return (__m256i)__builtin_ia32_vpshrdvd256((__v8si)__A, (__v8si)__B, |
| 643 | (__v8si) __A, | 611 | (__v8si)__C); |
| 644 | (__v8si) __B, | ||
| 645 | __U); | ||
| 646 | } | 612 | } |
| 647 | 613 | ||
| 648 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 614 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 649 | _mm256_maskz_shrdv_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) | 615 | _mm256_mask_shrdv_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) |
| 650 | { | 616 | { |
| 651 | return (__m256i) __builtin_ia32_vpshrdvd256_maskz ((__v8si) __S, | 617 | return (__m256i)__builtin_ia32_selectd_256(__U, |
| 652 | (__v8si) __A, | 618 | (__v8si)_mm256_shrdv_epi32(__A, __B, __C), |
| 653 | (__v8si) __B, | 619 | (__v8si)__A); |
| 654 | __U); | ||
| 655 | } | 620 | } |
| 656 | 621 | ||
| 657 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 622 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 658 | _mm256_shrdv_epi32(__m256i __S, __m256i __A, __m256i __B) | 623 | _mm256_maskz_shrdv_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) |
| 659 | { | 624 | { |
| 660 | return (__m256i) __builtin_ia32_vpshrdvd256_mask ((__v8si) __S, | 625 | return (__m256i)__builtin_ia32_selectd_256(__U, |
| 661 | (__v8si) __A, | 626 | (__v8si)_mm256_shrdv_epi32(__A, __B, __C), |
| 662 | (__v8si) __B, | 627 | (__v8si)_mm256_setzero_si256()); |
| 663 | (__mmask8) -1); | ||
| 664 | } | 628 | } |
| 665 | 629 | ||
| 666 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 630 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 667 | _mm_mask_shrdv_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) | 631 | _mm_shrdv_epi32(__m128i __A, __m128i __B, __m128i __C) |
| 668 | { | 632 | { |
| 669 | return (__m128i) __builtin_ia32_vpshrdvd128_mask ((__v4si) __S, | 633 | return (__m128i)__builtin_ia32_vpshrdvd128((__v4si)__A, (__v4si)__B, |
| 670 | (__v4si) __A, | 634 | (__v4si)__C); |
| 671 | (__v4si) __B, | ||
| 672 | __U); | ||
| 673 | } | 635 | } |
| 674 | 636 | ||
| 675 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 637 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 676 | _mm_maskz_shrdv_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) | 638 | _mm_mask_shrdv_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) |
| 677 | { | 639 | { |
| 678 | return (__m128i) __builtin_ia32_vpshrdvd128_maskz ((__v4si) __S, | 640 | return (__m128i)__builtin_ia32_selectd_128(__U, |
| 679 | (__v4si) __A, | 641 | (__v4si)_mm_shrdv_epi32(__A, __B, __C), |
| 680 | (__v4si) __B, | 642 | (__v4si)__A); |
| 681 | __U); | ||
| 682 | } | 643 | } |
| 683 | 644 | ||
| 684 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 645 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 685 | _mm_shrdv_epi32(__m128i __S, __m128i __A, __m128i __B) | 646 | _mm_maskz_shrdv_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) |
| 686 | { | 647 | { |
| 687 | return (__m128i) __builtin_ia32_vpshrdvd128_mask ((__v4si) __S, | 648 | return (__m128i)__builtin_ia32_selectd_128(__U, |
| 688 | (__v4si) __A, | 649 | (__v4si)_mm_shrdv_epi32(__A, __B, __C), |
| 689 | (__v4si) __B, | 650 | (__v4si)_mm_setzero_si128()); |
| 690 | (__mmask8) -1); | ||
| 691 | } | 651 | } |
| 692 | 652 | ||
| 693 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 653 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 694 | _mm256_mask_shrdv_epi16(__m256i __S, __mmask16 __U, __m256i __A, __m256i __B) | 654 | _mm256_shrdv_epi16(__m256i __A, __m256i __B, __m256i __C) |
| 695 | { | 655 | { |
| 696 | return (__m256i) __builtin_ia32_vpshrdvw256_mask ((__v16hi) __S, | 656 | return (__m256i)__builtin_ia32_vpshrdvw256((__v16hi)__A, (__v16hi)__B, |
| 697 | (__v16hi) __A, | 657 | (__v16hi)__C); |
| 698 | (__v16hi) __B, | ||
| 699 | __U); | ||
| 700 | } | 658 | } |
| 701 | 659 | ||
| 702 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 660 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 703 | _mm256_maskz_shrdv_epi16(__mmask16 __U, __m256i __S, __m256i __A, __m256i __B) | 661 | _mm256_mask_shrdv_epi16(__m256i __A, __mmask16 __U, __m256i __B, __m256i __C) |
| 704 | { | 662 | { |
| 705 | return (__m256i) __builtin_ia32_vpshrdvw256_maskz ((__v16hi) __S, | 663 | return (__m256i)__builtin_ia32_selectw_256(__U, |
| 706 | (__v16hi) __A, | 664 | (__v16hi)_mm256_shrdv_epi16(__A, __B, __C), |
| 707 | (__v16hi) __B, | 665 | (__v16hi)__A); |
| 708 | __U); | ||
| 709 | } | 666 | } |
| 710 | 667 | ||
| 711 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 668 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 712 | _mm256_shrdv_epi16(__m256i __S, __m256i __A, __m256i __B) | 669 | _mm256_maskz_shrdv_epi16(__mmask16 __U, __m256i __A, __m256i __B, __m256i __C) |
| 713 | { | 670 | { |
| 714 | return (__m256i) __builtin_ia32_vpshrdvw256_mask ((__v16hi) __S, | 671 | return (__m256i)__builtin_ia32_selectw_256(__U, |
| 715 | (__v16hi) __A, | 672 | (__v16hi)_mm256_shrdv_epi16(__A, __B, __C), |
| 716 | (__v16hi) __B, | 673 | (__v16hi)_mm256_setzero_si256()); |
| 717 | (__mmask16) -1); | ||
| 718 | } | 674 | } |
| 719 | 675 | ||
| 720 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 676 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 721 | _mm_mask_shrdv_epi16(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) | 677 | _mm_shrdv_epi16(__m128i __A, __m128i __B, __m128i __C) |
| 722 | { | 678 | { |
| 723 | return (__m128i) __builtin_ia32_vpshrdvw128_mask ((__v8hi) __S, | 679 | return (__m128i)__builtin_ia32_vpshrdvw128((__v8hi)__A, (__v8hi)__B, |
| 724 | (__v8hi) __A, | 680 | (__v8hi)__C); |
| 725 | (__v8hi) __B, | ||
| 726 | __U); | ||
| 727 | } | 681 | } |
| 728 | 682 | ||
| 729 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 683 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 730 | _mm_maskz_shrdv_epi16(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) | 684 | _mm_mask_shrdv_epi16(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) |
| 731 | { | 685 | { |
| 732 | return (__m128i) __builtin_ia32_vpshrdvw128_maskz ((__v8hi) __S, | 686 | return (__m128i)__builtin_ia32_selectw_128(__U, |
| 733 | (__v8hi) __A, | 687 | (__v8hi)_mm_shrdv_epi16(__A, __B, __C), |
| 734 | (__v8hi) __B, | 688 | (__v8hi)__A); |
| 735 | __U); | ||
| 736 | } | 689 | } |
| 737 | 690 | ||
| 738 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 691 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 739 | _mm_shrdv_epi16(__m128i __S, __m128i __A, __m128i __B) | 692 | _mm_maskz_shrdv_epi16(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) |
| 740 | { | 693 | { |
| 741 | return (__m128i) __builtin_ia32_vpshrdvw128_mask ((__v8hi) __S, | 694 | return (__m128i)__builtin_ia32_selectw_128(__U, |
| 742 | (__v8hi) __A, | 695 | (__v8hi)_mm_shrdv_epi16(__A, __B, __C), |
| 743 | (__v8hi) __B, | 696 | (__v8hi)_mm_setzero_si128()); |
| 744 | (__mmask8) -1); | ||
| 745 | } | 697 | } |
| 746 | 698 | ||
| 747 | 699 |
c_headers/bmiintrin.h+5-5
| ... | @@ -62,7 +62,7 @@ | ... | @@ -62,7 +62,7 @@ |
| 62 | static __inline__ unsigned short __RELAXED_FN_ATTRS | 62 | static __inline__ unsigned short __RELAXED_FN_ATTRS |
| 63 | __tzcnt_u16(unsigned short __X) | 63 | __tzcnt_u16(unsigned short __X) |
| 64 | { | 64 | { |
| 65 | return __X ? __builtin_ctzs(__X) : 16; | 65 | return __builtin_ia32_tzcnt_u16(__X); |
| 66 | } | 66 | } |
| 67 | 67 | ||
| 68 | /// Performs a bitwise AND of the second operand with the one's | 68 | /// Performs a bitwise AND of the second operand with the one's |
| ... | @@ -196,7 +196,7 @@ __blsr_u32(unsigned int __X) | ... | @@ -196,7 +196,7 @@ __blsr_u32(unsigned int __X) |
| 196 | static __inline__ unsigned int __RELAXED_FN_ATTRS | 196 | static __inline__ unsigned int __RELAXED_FN_ATTRS |
| 197 | __tzcnt_u32(unsigned int __X) | 197 | __tzcnt_u32(unsigned int __X) |
| 198 | { | 198 | { |
| 199 | return __X ? __builtin_ctz(__X) : 32; | 199 | return __builtin_ia32_tzcnt_u32(__X); |
| 200 | } | 200 | } |
| 201 | 201 | ||
| 202 | /// Counts the number of trailing zero bits in the operand. | 202 | /// Counts the number of trailing zero bits in the operand. |
| ... | @@ -212,7 +212,7 @@ __tzcnt_u32(unsigned int __X) | ... | @@ -212,7 +212,7 @@ __tzcnt_u32(unsigned int __X) |
| 212 | static __inline__ int __RELAXED_FN_ATTRS | 212 | static __inline__ int __RELAXED_FN_ATTRS |
| 213 | _mm_tzcnt_32(unsigned int __X) | 213 | _mm_tzcnt_32(unsigned int __X) |
| 214 | { | 214 | { |
| 215 | return __X ? __builtin_ctz(__X) : 32; | 215 | return __builtin_ia32_tzcnt_u32(__X); |
| 216 | } | 216 | } |
| 217 | 217 | ||
| 218 | #ifdef __x86_64__ | 218 | #ifdef __x86_64__ |
| ... | @@ -359,7 +359,7 @@ __blsr_u64(unsigned long long __X) | ... | @@ -359,7 +359,7 @@ __blsr_u64(unsigned long long __X) |
| 359 | static __inline__ unsigned long long __RELAXED_FN_ATTRS | 359 | static __inline__ unsigned long long __RELAXED_FN_ATTRS |
| 360 | __tzcnt_u64(unsigned long long __X) | 360 | __tzcnt_u64(unsigned long long __X) |
| 361 | { | 361 | { |
| 362 | return __X ? __builtin_ctzll(__X) : 64; | 362 | return __builtin_ia32_tzcnt_u64(__X); |
| 363 | } | 363 | } |
| 364 | 364 | ||
| 365 | /// Counts the number of trailing zero bits in the operand. | 365 | /// Counts the number of trailing zero bits in the operand. |
| ... | @@ -375,7 +375,7 @@ __tzcnt_u64(unsigned long long __X) | ... | @@ -375,7 +375,7 @@ __tzcnt_u64(unsigned long long __X) |
| 375 | static __inline__ long long __RELAXED_FN_ATTRS | 375 | static __inline__ long long __RELAXED_FN_ATTRS |
| 376 | _mm_tzcnt_64(unsigned long long __X) | 376 | _mm_tzcnt_64(unsigned long long __X) |
| 377 | { | 377 | { |
| 378 | return __X ? __builtin_ctzll(__X) : 64; | 378 | return __builtin_ia32_tzcnt_u64(__X); |
| 379 | } | 379 | } |
| 380 | 380 | ||
| 381 | #endif /* __x86_64__ */ | 381 | #endif /* __x86_64__ */ |
c_headers/cuda_wrappers/new+4-2
| ... | @@ -73,10 +73,12 @@ __device__ inline void operator delete[](void *ptr, | ... | @@ -73,10 +73,12 @@ __device__ inline void operator delete[](void *ptr, |
| 73 | 73 | ||
| 74 | // Sized delete, C++14 only. | 74 | // Sized delete, C++14 only. |
| 75 | #if __cplusplus >= 201402L | 75 | #if __cplusplus >= 201402L |
| 76 | __device__ void operator delete(void *ptr, __SIZE_TYPE__ size) CUDA_NOEXCEPT { | 76 | __device__ inline void operator delete(void *ptr, |
| 77 | __SIZE_TYPE__ size) CUDA_NOEXCEPT { | ||
| 77 | ::operator delete(ptr); | 78 | ::operator delete(ptr); |
| 78 | } | 79 | } |
| 79 | __device__ void operator delete[](void *ptr, __SIZE_TYPE__ size) CUDA_NOEXCEPT { | 80 | __device__ inline void operator delete[](void *ptr, |
| 81 | __SIZE_TYPE__ size) CUDA_NOEXCEPT { | ||
| 80 | ::operator delete(ptr); | 82 | ::operator delete(ptr); |
| 81 | } | 83 | } |
| 82 | #endif | 84 | #endif |
c_headers/emmintrin.h+106-1
| ... | @@ -1675,7 +1675,49 @@ _mm_loadu_si64(void const *__a) | ... | @@ -1675,7 +1675,49 @@ _mm_loadu_si64(void const *__a) |
| 1675 | long long __v; | 1675 | long long __v; |
| 1676 | } __attribute__((__packed__, __may_alias__)); | 1676 | } __attribute__((__packed__, __may_alias__)); |
| 1677 | long long __u = ((struct __loadu_si64*)__a)->__v; | 1677 | long long __u = ((struct __loadu_si64*)__a)->__v; |
| 1678 | return __extension__ (__m128i)(__v2di){__u, 0L}; | 1678 | return __extension__ (__m128i)(__v2di){__u, 0LL}; |
| 1679 | } | ||
| 1680 | |||
| 1681 | /// Loads a 32-bit integer value to the low element of a 128-bit integer | ||
| 1682 | /// vector and clears the upper element. | ||
| 1683 | /// | ||
| 1684 | /// \headerfile <x86intrin.h> | ||
| 1685 | /// | ||
| 1686 | /// This intrinsic corresponds to the <c> VMOVD / MOVD </c> instruction. | ||
| 1687 | /// | ||
| 1688 | /// \param __a | ||
| 1689 | /// A pointer to a 32-bit memory location. The address of the memory | ||
| 1690 | /// location does not have to be aligned. | ||
| 1691 | /// \returns A 128-bit vector of [4 x i32] containing the loaded value. | ||
| 1692 | static __inline__ __m128i __DEFAULT_FN_ATTRS | ||
| 1693 | _mm_loadu_si32(void const *__a) | ||
| 1694 | { | ||
| 1695 | struct __loadu_si32 { | ||
| 1696 | int __v; | ||
| 1697 | } __attribute__((__packed__, __may_alias__)); | ||
| 1698 | int __u = ((struct __loadu_si32*)__a)->__v; | ||
| 1699 | return __extension__ (__m128i)(__v4si){__u, 0, 0, 0}; | ||
| 1700 | } | ||
| 1701 | |||
| 1702 | /// Loads a 16-bit integer value to the low element of a 128-bit integer | ||
| 1703 | /// vector and clears the upper element. | ||
| 1704 | /// | ||
| 1705 | /// \headerfile <x86intrin.h> | ||
| 1706 | /// | ||
| 1707 | /// This intrinsic does not correspond to a specific instruction. | ||
| 1708 | /// | ||
| 1709 | /// \param __a | ||
| 1710 | /// A pointer to a 16-bit memory location. The address of the memory | ||
| 1711 | /// location does not have to be aligned. | ||
| 1712 | /// \returns A 128-bit vector of [8 x i16] containing the loaded value. | ||
| 1713 | static __inline__ __m128i __DEFAULT_FN_ATTRS | ||
| 1714 | _mm_loadu_si16(void const *__a) | ||
| 1715 | { | ||
| 1716 | struct __loadu_si16 { | ||
| 1717 | short __v; | ||
| 1718 | } __attribute__((__packed__, __may_alias__)); | ||
| 1719 | short __u = ((struct __loadu_si16*)__a)->__v; | ||
| 1720 | return __extension__ (__m128i)(__v8hi){__u, 0, 0, 0, 0, 0, 0, 0}; | ||
| 1679 | } | 1721 | } |
| 1680 | 1722 | ||
| 1681 | /// Loads a 64-bit double-precision value to the low element of a | 1723 | /// Loads a 64-bit double-precision value to the low element of a |
| ... | @@ -3993,6 +4035,69 @@ _mm_storeu_si128(__m128i *__p, __m128i __b) | ... | @@ -3993,6 +4035,69 @@ _mm_storeu_si128(__m128i *__p, __m128i __b) |
| 3993 | ((struct __storeu_si128*)__p)->__v = __b; | 4035 | ((struct __storeu_si128*)__p)->__v = __b; |
| 3994 | } | 4036 | } |
| 3995 | 4037 | ||
| 4038 | /// Stores a 64-bit integer value from the low element of a 128-bit integer | ||
| 4039 | /// vector. | ||
| 4040 | /// | ||
| 4041 | /// \headerfile <x86intrin.h> | ||
| 4042 | /// | ||
| 4043 | /// This intrinsic corresponds to the <c> VMOVQ / MOVQ </c> instruction. | ||
| 4044 | /// | ||
| 4045 | /// \param __p | ||
| 4046 | /// A pointer to a 64-bit memory location. The address of the memory | ||
| 4047 | /// location does not have to be algned. | ||
| 4048 | /// \param __b | ||
| 4049 | /// A 128-bit integer vector containing the value to be stored. | ||
| 4050 | static __inline__ void __DEFAULT_FN_ATTRS | ||
| 4051 | _mm_storeu_si64(void const *__p, __m128i __b) | ||
| 4052 | { | ||
| 4053 | struct __storeu_si64 { | ||
| 4054 | long long __v; | ||
| 4055 | } __attribute__((__packed__, __may_alias__)); | ||
| 4056 | ((struct __storeu_si64*)__p)->__v = ((__v2di)__b)[0]; | ||
| 4057 | } | ||
| 4058 | |||
| 4059 | /// Stores a 32-bit integer value from the low element of a 128-bit integer | ||
| 4060 | /// vector. | ||
| 4061 | /// | ||
| 4062 | /// \headerfile <x86intrin.h> | ||
| 4063 | /// | ||
| 4064 | /// This intrinsic corresponds to the <c> VMOVD / MOVD </c> instruction. | ||
| 4065 | /// | ||
| 4066 | /// \param __p | ||
| 4067 | /// A pointer to a 32-bit memory location. The address of the memory | ||
| 4068 | /// location does not have to be aligned. | ||
| 4069 | /// \param __b | ||
| 4070 | /// A 128-bit integer vector containing the value to be stored. | ||
| 4071 | static __inline__ void __DEFAULT_FN_ATTRS | ||
| 4072 | _mm_storeu_si32(void const *__p, __m128i __b) | ||
| 4073 | { | ||
| 4074 | struct __storeu_si32 { | ||
| 4075 | int __v; | ||
| 4076 | } __attribute__((__packed__, __may_alias__)); | ||
| 4077 | ((struct __storeu_si32*)__p)->__v = ((__v4si)__b)[0]; | ||
| 4078 | } | ||
| 4079 | |||
| 4080 | /// Stores a 16-bit integer value from the low element of a 128-bit integer | ||
| 4081 | /// vector. | ||
| 4082 | /// | ||
| 4083 | /// \headerfile <x86intrin.h> | ||
| 4084 | /// | ||
| 4085 | /// This intrinsic does not correspond to a specific instruction. | ||
| 4086 | /// | ||
| 4087 | /// \param __p | ||
| 4088 | /// A pointer to a 16-bit memory location. The address of the memory | ||
| 4089 | /// location does not have to be aligned. | ||
| 4090 | /// \param __b | ||
| 4091 | /// A 128-bit integer vector containing the value to be stored. | ||
| 4092 | static __inline__ void __DEFAULT_FN_ATTRS | ||
| 4093 | _mm_storeu_si16(void const *__p, __m128i __b) | ||
| 4094 | { | ||
| 4095 | struct __storeu_si16 { | ||
| 4096 | short __v; | ||
| 4097 | } __attribute__((__packed__, __may_alias__)); | ||
| 4098 | ((struct __storeu_si16*)__p)->__v = ((__v8hi)__b)[0]; | ||
| 4099 | } | ||
| 4100 | |||
| 3996 | /// Moves bytes selected by the mask from the first operand to the | 4101 | /// Moves bytes selected by the mask from the first operand to the |
| 3997 | /// specified unaligned memory location. When a mask bit is 1, the | 4102 | /// specified unaligned memory location. When a mask bit is 1, the |
| 3998 | /// corresponding byte is written, otherwise it is not written. | 4103 | /// corresponding byte is written, otherwise it is not written. |
c_headers/float.h+9-3
| ... | @@ -21,8 +21,8 @@ | ... | @@ -21,8 +21,8 @@ |
| 21 | *===-----------------------------------------------------------------------=== | 21 | *===-----------------------------------------------------------------------=== |
| 22 | */ | 22 | */ |
| 23 | 23 | ||
| 24 | #ifndef __FLOAT_H | 24 | #ifndef __CLANG_FLOAT_H |
| 25 | #define __FLOAT_H | 25 | #define __CLANG_FLOAT_H |
| 26 | 26 | ||
| 27 | /* If we're on MinGW, fall back to the system's float.h, which might have | 27 | /* If we're on MinGW, fall back to the system's float.h, which might have |
| 28 | * additional definitions provided for Windows. | 28 | * additional definitions provided for Windows. |
| ... | @@ -85,6 +85,9 @@ | ... | @@ -85,6 +85,9 @@ |
| 85 | # undef FLT_DECIMAL_DIG | 85 | # undef FLT_DECIMAL_DIG |
| 86 | # undef DBL_DECIMAL_DIG | 86 | # undef DBL_DECIMAL_DIG |
| 87 | # undef LDBL_DECIMAL_DIG | 87 | # undef LDBL_DECIMAL_DIG |
| 88 | # undef FLT_HAS_SUBNORM | ||
| 89 | # undef DBL_HAS_SUBNORM | ||
| 90 | # undef LDBL_HAS_SUBNORM | ||
| 88 | # endif | 91 | # endif |
| 89 | #endif | 92 | #endif |
| 90 | 93 | ||
| ... | @@ -141,6 +144,9 @@ | ... | @@ -141,6 +144,9 @@ |
| 141 | # define FLT_DECIMAL_DIG __FLT_DECIMAL_DIG__ | 144 | # define FLT_DECIMAL_DIG __FLT_DECIMAL_DIG__ |
| 142 | # define DBL_DECIMAL_DIG __DBL_DECIMAL_DIG__ | 145 | # define DBL_DECIMAL_DIG __DBL_DECIMAL_DIG__ |
| 143 | # define LDBL_DECIMAL_DIG __LDBL_DECIMAL_DIG__ | 146 | # define LDBL_DECIMAL_DIG __LDBL_DECIMAL_DIG__ |
| 147 | # define FLT_HAS_SUBNORM __FLT_HAS_DENORM__ | ||
| 148 | # define DBL_HAS_SUBNORM __DBL_HAS_DENORM__ | ||
| 149 | # define LDBL_HAS_SUBNORM __LDBL_HAS_DENORM__ | ||
| 144 | #endif | 150 | #endif |
| 145 | 151 | ||
| 146 | #ifdef __STDC_WANT_IEC_60559_TYPES_EXT__ | 152 | #ifdef __STDC_WANT_IEC_60559_TYPES_EXT__ |
| ... | @@ -157,4 +163,4 @@ | ... | @@ -157,4 +163,4 @@ |
| 157 | # define FLT16_TRUE_MIN __FLT16_TRUE_MIN__ | 163 | # define FLT16_TRUE_MIN __FLT16_TRUE_MIN__ |
| 158 | #endif /* __STDC_WANT_IEC_60559_TYPES_EXT__ */ | 164 | #endif /* __STDC_WANT_IEC_60559_TYPES_EXT__ */ |
| 159 | 165 | ||
| 160 | #endif /* __FLOAT_H */ | 166 | #endif /* __CLANG_FLOAT_H */ |
c_headers/immintrin.h+59
| ... | @@ -306,6 +306,65 @@ _writegsbase_u64(unsigned long long __V) | ... | @@ -306,6 +306,65 @@ _writegsbase_u64(unsigned long long __V) |
| 306 | #endif | 306 | #endif |
| 307 | #endif /* __FSGSBASE__ */ | 307 | #endif /* __FSGSBASE__ */ |
| 308 | 308 | ||
| 309 | #if !defined(_MSC_VER) || __has_feature(modules) || defined(__MOVBE__) | ||
| 310 | |||
| 311 | /* The structs used below are to force the load/store to be unaligned. This | ||
| 312 | * is accomplished with the __packed__ attribute. The __may_alias__ prevents | ||
| 313 | * tbaa metadata from being generated based on the struct and the type of the | ||
| 314 | * field inside of it. | ||
| 315 | */ | ||
| 316 | |||
| 317 | static __inline__ short __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) | ||
| 318 | _loadbe_i16(void const * __P) { | ||
| 319 | struct __loadu_i16 { | ||
| 320 | short __v; | ||
| 321 | } __attribute__((__packed__, __may_alias__)); | ||
| 322 | return __builtin_bswap16(((struct __loadu_i16*)__P)->__v); | ||
| 323 | } | ||
| 324 | |||
| 325 | static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) | ||
| 326 | _storebe_i16(void * __P, short __D) { | ||
| 327 | struct __storeu_i16 { | ||
| 328 | short __v; | ||
| 329 | } __attribute__((__packed__, __may_alias__)); | ||
| 330 | ((struct __storeu_i16*)__P)->__v = __builtin_bswap16(__D); | ||
| 331 | } | ||
| 332 | |||
| 333 | static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) | ||
| 334 | _loadbe_i32(void const * __P) { | ||
| 335 | struct __loadu_i32 { | ||
| 336 | int __v; | ||
| 337 | } __attribute__((__packed__, __may_alias__)); | ||
| 338 | return __builtin_bswap32(((struct __loadu_i32*)__P)->__v); | ||
| 339 | } | ||
| 340 | |||
| 341 | static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) | ||
| 342 | _storebe_i32(void * __P, int __D) { | ||
| 343 | struct __storeu_i32 { | ||
| 344 | int __v; | ||
| 345 | } __attribute__((__packed__, __may_alias__)); | ||
| 346 | ((struct __storeu_i32*)__P)->__v = __builtin_bswap32(__D); | ||
| 347 | } | ||
| 348 | |||
| 349 | #ifdef __x86_64__ | ||
| 350 | static __inline__ long long __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) | ||
| 351 | _loadbe_i64(void const * __P) { | ||
| 352 | struct __loadu_i64 { | ||
| 353 | long long __v; | ||
| 354 | } __attribute__((__packed__, __may_alias__)); | ||
| 355 | return __builtin_bswap64(((struct __loadu_i64*)__P)->__v); | ||
| 356 | } | ||
| 357 | |||
| 358 | static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) | ||
| 359 | _storebe_i64(void * __P, long long __D) { | ||
| 360 | struct __storeu_i64 { | ||
| 361 | long long __v; | ||
| 362 | } __attribute__((__packed__, __may_alias__)); | ||
| 363 | ((struct __storeu_i64*)__P)->__v = __builtin_bswap64(__D); | ||
| 364 | } | ||
| 365 | #endif | ||
| 366 | #endif /* __MOVBE */ | ||
| 367 | |||
| 309 | #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RTM__) | 368 | #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RTM__) |
| 310 | #include <rtmintrin.h> | 369 | #include <rtmintrin.h> |
| 311 | #include <xtestintrin.h> | 370 | #include <xtestintrin.h> |
c_headers/intrin.h+126-412
| ... | @@ -90,8 +90,6 @@ void __inwordstring(unsigned short, unsigned short *, unsigned long); | ... | @@ -90,8 +90,6 @@ void __inwordstring(unsigned short, unsigned short *, unsigned long); |
| 90 | void __lidt(void *); | 90 | void __lidt(void *); |
| 91 | unsigned __int64 __ll_lshift(unsigned __int64, int); | 91 | unsigned __int64 __ll_lshift(unsigned __int64, int); |
| 92 | __int64 __ll_rshift(__int64, int); | 92 | __int64 __ll_rshift(__int64, int); |
| 93 | unsigned int __lzcnt(unsigned int); | ||
| 94 | unsigned short __lzcnt16(unsigned short); | ||
| 95 | static __inline__ | 93 | static __inline__ |
| 96 | void __movsb(unsigned char *, unsigned char const *, size_t); | 94 | void __movsb(unsigned char *, unsigned char const *, size_t); |
| 97 | static __inline__ | 95 | static __inline__ |
| ... | @@ -219,7 +217,6 @@ void __incgsbyte(unsigned long); | ... | @@ -219,7 +217,6 @@ void __incgsbyte(unsigned long); |
| 219 | void __incgsdword(unsigned long); | 217 | void __incgsdword(unsigned long); |
| 220 | void __incgsqword(unsigned long); | 218 | void __incgsqword(unsigned long); |
| 221 | void __incgsword(unsigned long); | 219 | void __incgsword(unsigned long); |
| 222 | unsigned __int64 __lzcnt64(unsigned __int64); | ||
| 223 | static __inline__ | 220 | static __inline__ |
| 224 | void __movsq(unsigned long long *, unsigned long long const *, size_t); | 221 | void __movsq(unsigned long long *, unsigned long long const *, size_t); |
| 225 | static __inline__ | 222 | static __inline__ |
| ... | @@ -329,189 +326,63 @@ __int64 _InterlockedAnd64(__int64 volatile *_Value, __int64 _Mask); | ... | @@ -329,189 +326,63 @@ __int64 _InterlockedAnd64(__int64 volatile *_Value, __int64 _Mask); |
| 329 | |* Interlocked Exchange Add | 326 | |* Interlocked Exchange Add |
| 330 | \*----------------------------------------------------------------------------*/ | 327 | \*----------------------------------------------------------------------------*/ |
| 331 | #if defined(__arm__) || defined(__aarch64__) | 328 | #if defined(__arm__) || defined(__aarch64__) |
| 332 | static __inline__ char __DEFAULT_FN_ATTRS | 329 | char _InterlockedExchangeAdd8_acq(char volatile *_Addend, char _Value); |
| 333 | _InterlockedExchangeAdd8_acq(char volatile *_Addend, char _Value) { | 330 | char _InterlockedExchangeAdd8_nf(char volatile *_Addend, char _Value); |
| 334 | return __atomic_fetch_add(_Addend, _Value, __ATOMIC_ACQUIRE); | 331 | char _InterlockedExchangeAdd8_rel(char volatile *_Addend, char _Value); |
| 335 | } | 332 | short _InterlockedExchangeAdd16_acq(short volatile *_Addend, short _Value); |
| 336 | static __inline__ char __DEFAULT_FN_ATTRS | 333 | short _InterlockedExchangeAdd16_nf(short volatile *_Addend, short _Value); |
| 337 | _InterlockedExchangeAdd8_nf(char volatile *_Addend, char _Value) { | 334 | short _InterlockedExchangeAdd16_rel(short volatile *_Addend, short _Value); |
| 338 | return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELAXED); | 335 | long _InterlockedExchangeAdd_acq(long volatile *_Addend, long _Value); |
| 339 | } | 336 | long _InterlockedExchangeAdd_nf(long volatile *_Addend, long _Value); |
| 340 | static __inline__ char __DEFAULT_FN_ATTRS | 337 | long _InterlockedExchangeAdd_rel(long volatile *_Addend, long _Value); |
| 341 | _InterlockedExchangeAdd8_rel(char volatile *_Addend, char _Value) { | 338 | __int64 _InterlockedExchangeAdd64_acq(__int64 volatile *_Addend, __int64 _Value); |
| 342 | return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELAXED); | 339 | __int64 _InterlockedExchangeAdd64_nf(__int64 volatile *_Addend, __int64 _Value); |
| 343 | } | 340 | __int64 _InterlockedExchangeAdd64_rel(__int64 volatile *_Addend, __int64 _Value); |
| 344 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 345 | _InterlockedExchangeAdd16_acq(short volatile *_Addend, short _Value) { | ||
| 346 | return __atomic_fetch_add(_Addend, _Value, __ATOMIC_ACQUIRE); | ||
| 347 | } | ||
| 348 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 349 | _InterlockedExchangeAdd16_nf(short volatile *_Addend, short _Value) { | ||
| 350 | return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELAXED); | ||
| 351 | } | ||
| 352 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 353 | _InterlockedExchangeAdd16_rel(short volatile *_Addend, short _Value) { | ||
| 354 | return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELEASE); | ||
| 355 | } | ||
| 356 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 357 | _InterlockedExchangeAdd_acq(long volatile *_Addend, long _Value) { | ||
| 358 | return __atomic_fetch_add(_Addend, _Value, __ATOMIC_ACQUIRE); | ||
| 359 | } | ||
| 360 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 361 | _InterlockedExchangeAdd_nf(long volatile *_Addend, long _Value) { | ||
| 362 | return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELAXED); | ||
| 363 | } | ||
| 364 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 365 | _InterlockedExchangeAdd_rel(long volatile *_Addend, long _Value) { | ||
| 366 | return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELEASE); | ||
| 367 | } | ||
| 368 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 369 | _InterlockedExchangeAdd64_acq(__int64 volatile *_Addend, __int64 _Value) { | ||
| 370 | return __atomic_fetch_add(_Addend, _Value, __ATOMIC_ACQUIRE); | ||
| 371 | } | ||
| 372 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 373 | _InterlockedExchangeAdd64_nf(__int64 volatile *_Addend, __int64 _Value) { | ||
| 374 | return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELAXED); | ||
| 375 | } | ||
| 376 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 377 | _InterlockedExchangeAdd64_rel(__int64 volatile *_Addend, __int64 _Value) { | ||
| 378 | return __atomic_fetch_add(_Addend, _Value, __ATOMIC_RELEASE); | ||
| 379 | } | ||
| 380 | #endif | 341 | #endif |
| 381 | /*----------------------------------------------------------------------------*\ | 342 | /*----------------------------------------------------------------------------*\ |
| 382 | |* Interlocked Increment | 343 | |* Interlocked Increment |
| 383 | \*----------------------------------------------------------------------------*/ | 344 | \*----------------------------------------------------------------------------*/ |
| 384 | #if defined(__arm__) || defined(__aarch64__) | 345 | #if defined(__arm__) || defined(__aarch64__) |
| 385 | static __inline__ short __DEFAULT_FN_ATTRS | 346 | short _InterlockedIncrement16_acq(short volatile *_Value); |
| 386 | _InterlockedIncrement16_acq(short volatile *_Value) { | 347 | short _InterlockedIncrement16_nf(short volatile *_Value); |
| 387 | return __atomic_add_fetch(_Value, 1, __ATOMIC_ACQUIRE); | 348 | short _InterlockedIncrement16_rel(short volatile *_Value); |
| 388 | } | 349 | long _InterlockedIncrement_acq(long volatile *_Value); |
| 389 | static __inline__ short __DEFAULT_FN_ATTRS | 350 | long _InterlockedIncrement_nf(long volatile *_Value); |
| 390 | _InterlockedIncrement16_nf(short volatile *_Value) { | 351 | long _InterlockedIncrement_rel(long volatile *_Value); |
| 391 | return __atomic_add_fetch(_Value, 1, __ATOMIC_RELAXED); | 352 | __int64 _InterlockedIncrement64_acq(__int64 volatile *_Value); |
| 392 | } | 353 | __int64 _InterlockedIncrement64_nf(__int64 volatile *_Value); |
| 393 | static __inline__ short __DEFAULT_FN_ATTRS | 354 | __int64 _InterlockedIncrement64_rel(__int64 volatile *_Value); |
| 394 | _InterlockedIncrement16_rel(short volatile *_Value) { | ||
| 395 | return __atomic_add_fetch(_Value, 1, __ATOMIC_RELEASE); | ||
| 396 | } | ||
| 397 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 398 | _InterlockedIncrement_acq(long volatile *_Value) { | ||
| 399 | return __atomic_add_fetch(_Value, 1, __ATOMIC_ACQUIRE); | ||
| 400 | } | ||
| 401 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 402 | _InterlockedIncrement_nf(long volatile *_Value) { | ||
| 403 | return __atomic_add_fetch(_Value, 1, __ATOMIC_RELAXED); | ||
| 404 | } | ||
| 405 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 406 | _InterlockedIncrement_rel(long volatile *_Value) { | ||
| 407 | return __atomic_add_fetch(_Value, 1, __ATOMIC_RELEASE); | ||
| 408 | } | ||
| 409 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 410 | _InterlockedIncrement64_acq(__int64 volatile *_Value) { | ||
| 411 | return __atomic_add_fetch(_Value, 1, __ATOMIC_ACQUIRE); | ||
| 412 | } | ||
| 413 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 414 | _InterlockedIncrement64_nf(__int64 volatile *_Value) { | ||
| 415 | return __atomic_add_fetch(_Value, 1, __ATOMIC_RELAXED); | ||
| 416 | } | ||
| 417 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 418 | _InterlockedIncrement64_rel(__int64 volatile *_Value) { | ||
| 419 | return __atomic_add_fetch(_Value, 1, __ATOMIC_RELEASE); | ||
| 420 | } | ||
| 421 | #endif | 355 | #endif |
| 422 | /*----------------------------------------------------------------------------*\ | 356 | /*----------------------------------------------------------------------------*\ |
| 423 | |* Interlocked Decrement | 357 | |* Interlocked Decrement |
| 424 | \*----------------------------------------------------------------------------*/ | 358 | \*----------------------------------------------------------------------------*/ |
| 425 | #if defined(__arm__) || defined(__aarch64__) | 359 | #if defined(__arm__) || defined(__aarch64__) |
| 426 | static __inline__ short __DEFAULT_FN_ATTRS | 360 | short _InterlockedDecrement16_acq(short volatile *_Value); |
| 427 | _InterlockedDecrement16_acq(short volatile *_Value) { | 361 | short _InterlockedDecrement16_nf(short volatile *_Value); |
| 428 | return __atomic_sub_fetch(_Value, 1, __ATOMIC_ACQUIRE); | 362 | short _InterlockedDecrement16_rel(short volatile *_Value); |
| 429 | } | 363 | long _InterlockedDecrement_acq(long volatile *_Value); |
| 430 | static __inline__ short __DEFAULT_FN_ATTRS | 364 | long _InterlockedDecrement_nf(long volatile *_Value); |
| 431 | _InterlockedDecrement16_nf(short volatile *_Value) { | 365 | long _InterlockedDecrement_rel(long volatile *_Value); |
| 432 | return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELAXED); | 366 | __int64 _InterlockedDecrement64_acq(__int64 volatile *_Value); |
| 433 | } | 367 | __int64 _InterlockedDecrement64_nf(__int64 volatile *_Value); |
| 434 | static __inline__ short __DEFAULT_FN_ATTRS | 368 | __int64 _InterlockedDecrement64_rel(__int64 volatile *_Value); |
| 435 | _InterlockedDecrement16_rel(short volatile *_Value) { | ||
| 436 | return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELEASE); | ||
| 437 | } | ||
| 438 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 439 | _InterlockedDecrement_acq(long volatile *_Value) { | ||
| 440 | return __atomic_sub_fetch(_Value, 1, __ATOMIC_ACQUIRE); | ||
| 441 | } | ||
| 442 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 443 | _InterlockedDecrement_nf(long volatile *_Value) { | ||
| 444 | return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELAXED); | ||
| 445 | } | ||
| 446 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 447 | _InterlockedDecrement_rel(long volatile *_Value) { | ||
| 448 | return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELEASE); | ||
| 449 | } | ||
| 450 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 451 | _InterlockedDecrement64_acq(__int64 volatile *_Value) { | ||
| 452 | return __atomic_sub_fetch(_Value, 1, __ATOMIC_ACQUIRE); | ||
| 453 | } | ||
| 454 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 455 | _InterlockedDecrement64_nf(__int64 volatile *_Value) { | ||
| 456 | return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELAXED); | ||
| 457 | } | ||
| 458 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 459 | _InterlockedDecrement64_rel(__int64 volatile *_Value) { | ||
| 460 | return __atomic_sub_fetch(_Value, 1, __ATOMIC_RELEASE); | ||
| 461 | } | ||
| 462 | #endif | 369 | #endif |
| 463 | /*----------------------------------------------------------------------------*\ | 370 | /*----------------------------------------------------------------------------*\ |
| 464 | |* Interlocked And | 371 | |* Interlocked And |
| 465 | \*----------------------------------------------------------------------------*/ | 372 | \*----------------------------------------------------------------------------*/ |
| 466 | #if defined(__arm__) || defined(__aarch64__) | 373 | #if defined(__arm__) || defined(__aarch64__) |
| 467 | static __inline__ char __DEFAULT_FN_ATTRS | 374 | char _InterlockedAnd8_acq(char volatile *_Value, char _Mask); |
| 468 | _InterlockedAnd8_acq(char volatile *_Value, char _Mask) { | 375 | char _InterlockedAnd8_nf(char volatile *_Value, char _Mask); |
| 469 | return __atomic_fetch_and(_Value, _Mask, __ATOMIC_ACQUIRE); | 376 | char _InterlockedAnd8_rel(char volatile *_Value, char _Mask); |
| 470 | } | 377 | short _InterlockedAnd16_acq(short volatile *_Value, short _Mask); |
| 471 | static __inline__ char __DEFAULT_FN_ATTRS | 378 | short _InterlockedAnd16_nf(short volatile *_Value, short _Mask); |
| 472 | _InterlockedAnd8_nf(char volatile *_Value, char _Mask) { | 379 | short _InterlockedAnd16_rel(short volatile *_Value, short _Mask); |
| 473 | return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELAXED); | 380 | long _InterlockedAnd_acq(long volatile *_Value, long _Mask); |
| 474 | } | 381 | long _InterlockedAnd_nf(long volatile *_Value, long _Mask); |
| 475 | static __inline__ char __DEFAULT_FN_ATTRS | 382 | long _InterlockedAnd_rel(long volatile *_Value, long _Mask); |
| 476 | _InterlockedAnd8_rel(char volatile *_Value, char _Mask) { | 383 | __int64 _InterlockedAnd64_acq(__int64 volatile *_Value, __int64 _Mask); |
| 477 | return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELEASE); | 384 | __int64 _InterlockedAnd64_nf(__int64 volatile *_Value, __int64 _Mask); |
| 478 | } | 385 | __int64 _InterlockedAnd64_rel(__int64 volatile *_Value, __int64 _Mask); |
| 479 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 480 | _InterlockedAnd16_acq(short volatile *_Value, short _Mask) { | ||
| 481 | return __atomic_fetch_and(_Value, _Mask, __ATOMIC_ACQUIRE); | ||
| 482 | } | ||
| 483 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 484 | _InterlockedAnd16_nf(short volatile *_Value, short _Mask) { | ||
| 485 | return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELAXED); | ||
| 486 | } | ||
| 487 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 488 | _InterlockedAnd16_rel(short volatile *_Value, short _Mask) { | ||
| 489 | return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELEASE); | ||
| 490 | } | ||
| 491 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 492 | _InterlockedAnd_acq(long volatile *_Value, long _Mask) { | ||
| 493 | return __atomic_fetch_and(_Value, _Mask, __ATOMIC_ACQUIRE); | ||
| 494 | } | ||
| 495 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 496 | _InterlockedAnd_nf(long volatile *_Value, long _Mask) { | ||
| 497 | return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELAXED); | ||
| 498 | } | ||
| 499 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 500 | _InterlockedAnd_rel(long volatile *_Value, long _Mask) { | ||
| 501 | return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELEASE); | ||
| 502 | } | ||
| 503 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 504 | _InterlockedAnd64_acq(__int64 volatile *_Value, __int64 _Mask) { | ||
| 505 | return __atomic_fetch_and(_Value, _Mask, __ATOMIC_ACQUIRE); | ||
| 506 | } | ||
| 507 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 508 | _InterlockedAnd64_nf(__int64 volatile *_Value, __int64 _Mask) { | ||
| 509 | return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELAXED); | ||
| 510 | } | ||
| 511 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 512 | _InterlockedAnd64_rel(__int64 volatile *_Value, __int64 _Mask) { | ||
| 513 | return __atomic_fetch_and(_Value, _Mask, __ATOMIC_RELEASE); | ||
| 514 | } | ||
| 515 | #endif | 386 | #endif |
| 516 | /*----------------------------------------------------------------------------*\ | 387 | /*----------------------------------------------------------------------------*\ |
| 517 | |* Bit Counting and Testing | 388 | |* Bit Counting and Testing |
| ... | @@ -534,261 +405,81 @@ unsigned char _interlockedbittestandreset_rel(long volatile *_BitBase, | ... | @@ -534,261 +405,81 @@ unsigned char _interlockedbittestandreset_rel(long volatile *_BitBase, |
| 534 | |* Interlocked Or | 405 | |* Interlocked Or |
| 535 | \*----------------------------------------------------------------------------*/ | 406 | \*----------------------------------------------------------------------------*/ |
| 536 | #if defined(__arm__) || defined(__aarch64__) | 407 | #if defined(__arm__) || defined(__aarch64__) |
| 537 | static __inline__ char __DEFAULT_FN_ATTRS | 408 | char _InterlockedOr8_acq(char volatile *_Value, char _Mask); |
| 538 | _InterlockedOr8_acq(char volatile *_Value, char _Mask) { | 409 | char _InterlockedOr8_nf(char volatile *_Value, char _Mask); |
| 539 | return __atomic_fetch_or(_Value, _Mask, __ATOMIC_ACQUIRE); | 410 | char _InterlockedOr8_rel(char volatile *_Value, char _Mask); |
| 540 | } | 411 | short _InterlockedOr16_acq(short volatile *_Value, short _Mask); |
| 541 | static __inline__ char __DEFAULT_FN_ATTRS | 412 | short _InterlockedOr16_nf(short volatile *_Value, short _Mask); |
| 542 | _InterlockedOr8_nf(char volatile *_Value, char _Mask) { | 413 | short _InterlockedOr16_rel(short volatile *_Value, short _Mask); |
| 543 | return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELAXED); | 414 | long _InterlockedOr_acq(long volatile *_Value, long _Mask); |
| 544 | } | 415 | long _InterlockedOr_nf(long volatile *_Value, long _Mask); |
| 545 | static __inline__ char __DEFAULT_FN_ATTRS | 416 | long _InterlockedOr_rel(long volatile *_Value, long _Mask); |
| 546 | _InterlockedOr8_rel(char volatile *_Value, char _Mask) { | 417 | __int64 _InterlockedOr64_acq(__int64 volatile *_Value, __int64 _Mask); |
| 547 | return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELEASE); | 418 | __int64 _InterlockedOr64_nf(__int64 volatile *_Value, __int64 _Mask); |
| 548 | } | 419 | __int64 _InterlockedOr64_rel(__int64 volatile *_Value, __int64 _Mask); |
| 549 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 550 | _InterlockedOr16_acq(short volatile *_Value, short _Mask) { | ||
| 551 | return __atomic_fetch_or(_Value, _Mask, __ATOMIC_ACQUIRE); | ||
| 552 | } | ||
| 553 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 554 | _InterlockedOr16_nf(short volatile *_Value, short _Mask) { | ||
| 555 | return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELAXED); | ||
| 556 | } | ||
| 557 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 558 | _InterlockedOr16_rel(short volatile *_Value, short _Mask) { | ||
| 559 | return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELEASE); | ||
| 560 | } | ||
| 561 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 562 | _InterlockedOr_acq(long volatile *_Value, long _Mask) { | ||
| 563 | return __atomic_fetch_or(_Value, _Mask, __ATOMIC_ACQUIRE); | ||
| 564 | } | ||
| 565 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 566 | _InterlockedOr_nf(long volatile *_Value, long _Mask) { | ||
| 567 | return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELAXED); | ||
| 568 | } | ||
| 569 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 570 | _InterlockedOr_rel(long volatile *_Value, long _Mask) { | ||
| 571 | return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELEASE); | ||
| 572 | } | ||
| 573 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 574 | _InterlockedOr64_acq(__int64 volatile *_Value, __int64 _Mask) { | ||
| 575 | return __atomic_fetch_or(_Value, _Mask, __ATOMIC_ACQUIRE); | ||
| 576 | } | ||
| 577 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 578 | _InterlockedOr64_nf(__int64 volatile *_Value, __int64 _Mask) { | ||
| 579 | return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELAXED); | ||
| 580 | } | ||
| 581 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 582 | _InterlockedOr64_rel(__int64 volatile *_Value, __int64 _Mask) { | ||
| 583 | return __atomic_fetch_or(_Value, _Mask, __ATOMIC_RELEASE); | ||
| 584 | } | ||
| 585 | #endif | 420 | #endif |
| 586 | /*----------------------------------------------------------------------------*\ | 421 | /*----------------------------------------------------------------------------*\ |
| 587 | |* Interlocked Xor | 422 | |* Interlocked Xor |
| 588 | \*----------------------------------------------------------------------------*/ | 423 | \*----------------------------------------------------------------------------*/ |
| 589 | #if defined(__arm__) || defined(__aarch64__) | 424 | #if defined(__arm__) || defined(__aarch64__) |
| 590 | static __inline__ char __DEFAULT_FN_ATTRS | 425 | char _InterlockedXor8_acq(char volatile *_Value, char _Mask); |
| 591 | _InterlockedXor8_acq(char volatile *_Value, char _Mask) { | 426 | char _InterlockedXor8_nf(char volatile *_Value, char _Mask); |
| 592 | return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_ACQUIRE); | 427 | char _InterlockedXor8_rel(char volatile *_Value, char _Mask); |
| 593 | } | 428 | short _InterlockedXor16_acq(short volatile *_Value, short _Mask); |
| 594 | static __inline__ char __DEFAULT_FN_ATTRS | 429 | short _InterlockedXor16_nf(short volatile *_Value, short _Mask); |
| 595 | _InterlockedXor8_nf(char volatile *_Value, char _Mask) { | 430 | short _InterlockedXor16_rel(short volatile *_Value, short _Mask); |
| 596 | return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELAXED); | 431 | long _InterlockedXor_acq(long volatile *_Value, long _Mask); |
| 597 | } | 432 | long _InterlockedXor_nf(long volatile *_Value, long _Mask); |
| 598 | static __inline__ char __DEFAULT_FN_ATTRS | 433 | long _InterlockedXor_rel(long volatile *_Value, long _Mask); |
| 599 | _InterlockedXor8_rel(char volatile *_Value, char _Mask) { | 434 | __int64 _InterlockedXor64_acq(__int64 volatile *_Value, __int64 _Mask); |
| 600 | return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELEASE); | 435 | __int64 _InterlockedXor64_nf(__int64 volatile *_Value, __int64 _Mask); |
| 601 | } | 436 | __int64 _InterlockedXor64_rel(__int64 volatile *_Value, __int64 _Mask); |
| 602 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 603 | _InterlockedXor16_acq(short volatile *_Value, short _Mask) { | ||
| 604 | return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_ACQUIRE); | ||
| 605 | } | ||
| 606 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 607 | _InterlockedXor16_nf(short volatile *_Value, short _Mask) { | ||
| 608 | return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELAXED); | ||
| 609 | } | ||
| 610 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 611 | _InterlockedXor16_rel(short volatile *_Value, short _Mask) { | ||
| 612 | return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELEASE); | ||
| 613 | } | ||
| 614 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 615 | _InterlockedXor_acq(long volatile *_Value, long _Mask) { | ||
| 616 | return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_ACQUIRE); | ||
| 617 | } | ||
| 618 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 619 | _InterlockedXor_nf(long volatile *_Value, long _Mask) { | ||
| 620 | return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELAXED); | ||
| 621 | } | ||
| 622 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 623 | _InterlockedXor_rel(long volatile *_Value, long _Mask) { | ||
| 624 | return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELEASE); | ||
| 625 | } | ||
| 626 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 627 | _InterlockedXor64_acq(__int64 volatile *_Value, __int64 _Mask) { | ||
| 628 | return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_ACQUIRE); | ||
| 629 | } | ||
| 630 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 631 | _InterlockedXor64_nf(__int64 volatile *_Value, __int64 _Mask) { | ||
| 632 | return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELAXED); | ||
| 633 | } | ||
| 634 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 635 | _InterlockedXor64_rel(__int64 volatile *_Value, __int64 _Mask) { | ||
| 636 | return __atomic_fetch_xor(_Value, _Mask, __ATOMIC_RELEASE); | ||
| 637 | } | ||
| 638 | #endif | 437 | #endif |
| 639 | /*----------------------------------------------------------------------------*\ | 438 | /*----------------------------------------------------------------------------*\ |
| 640 | |* Interlocked Exchange | 439 | |* Interlocked Exchange |
| 641 | \*----------------------------------------------------------------------------*/ | 440 | \*----------------------------------------------------------------------------*/ |
| 642 | #if defined(__arm__) || defined(__aarch64__) | 441 | #if defined(__arm__) || defined(__aarch64__) |
| 643 | static __inline__ char __DEFAULT_FN_ATTRS | 442 | char _InterlockedExchange8_acq(char volatile *_Target, char _Value); |
| 644 | _InterlockedExchange8_acq(char volatile *_Target, char _Value) { | 443 | char _InterlockedExchange8_nf(char volatile *_Target, char _Value); |
| 645 | __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_ACQUIRE); | 444 | char _InterlockedExchange8_rel(char volatile *_Target, char _Value); |
| 646 | return _Value; | 445 | short _InterlockedExchange16_acq(short volatile *_Target, short _Value); |
| 647 | } | 446 | short _InterlockedExchange16_nf(short volatile *_Target, short _Value); |
| 648 | static __inline__ char __DEFAULT_FN_ATTRS | 447 | short _InterlockedExchange16_rel(short volatile *_Target, short _Value); |
| 649 | _InterlockedExchange8_nf(char volatile *_Target, char _Value) { | 448 | long _InterlockedExchange_acq(long volatile *_Target, long _Value); |
| 650 | __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELAXED); | 449 | long _InterlockedExchange_nf(long volatile *_Target, long _Value); |
| 651 | return _Value; | 450 | long _InterlockedExchange_rel(long volatile *_Target, long _Value); |
| 652 | } | 451 | __int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value); |
| 653 | static __inline__ char __DEFAULT_FN_ATTRS | 452 | __int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value); |
| 654 | _InterlockedExchange8_rel(char volatile *_Target, char _Value) { | 453 | __int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value); |
| 655 | __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELEASE); | ||
| 656 | return _Value; | ||
| 657 | } | ||
| 658 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 659 | _InterlockedExchange16_acq(short volatile *_Target, short _Value) { | ||
| 660 | __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_ACQUIRE); | ||
| 661 | return _Value; | ||
| 662 | } | ||
| 663 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 664 | _InterlockedExchange16_nf(short volatile *_Target, short _Value) { | ||
| 665 | __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELAXED); | ||
| 666 | return _Value; | ||
| 667 | } | ||
| 668 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 669 | _InterlockedExchange16_rel(short volatile *_Target, short _Value) { | ||
| 670 | __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELEASE); | ||
| 671 | return _Value; | ||
| 672 | } | ||
| 673 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 674 | _InterlockedExchange_acq(long volatile *_Target, long _Value) { | ||
| 675 | __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_ACQUIRE); | ||
| 676 | return _Value; | ||
| 677 | } | ||
| 678 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 679 | _InterlockedExchange_nf(long volatile *_Target, long _Value) { | ||
| 680 | __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELAXED); | ||
| 681 | return _Value; | ||
| 682 | } | ||
| 683 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 684 | _InterlockedExchange_rel(long volatile *_Target, long _Value) { | ||
| 685 | __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELEASE); | ||
| 686 | return _Value; | ||
| 687 | } | ||
| 688 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 689 | _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value) { | ||
| 690 | __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_ACQUIRE); | ||
| 691 | return _Value; | ||
| 692 | } | ||
| 693 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 694 | _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value) { | ||
| 695 | __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELAXED); | ||
| 696 | return _Value; | ||
| 697 | } | ||
| 698 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 699 | _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value) { | ||
| 700 | __atomic_exchange(_Target, &_Value, &_Value, __ATOMIC_RELEASE); | ||
| 701 | return _Value; | ||
| 702 | } | ||
| 703 | #endif | 454 | #endif |
| 704 | /*----------------------------------------------------------------------------*\ | 455 | /*----------------------------------------------------------------------------*\ |
| 705 | |* Interlocked Compare Exchange | 456 | |* Interlocked Compare Exchange |
| 706 | \*----------------------------------------------------------------------------*/ | 457 | \*----------------------------------------------------------------------------*/ |
| 707 | #if defined(__arm__) || defined(__aarch64__) | 458 | #if defined(__arm__) || defined(__aarch64__) |
| 708 | static __inline__ char __DEFAULT_FN_ATTRS | 459 | char _InterlockedCompareExchange8_acq(char volatile *_Destination, |
| 709 | _InterlockedCompareExchange8_acq(char volatile *_Destination, | 460 | char _Exchange, char _Comparand); |
| 710 | char _Exchange, char _Comparand) { | 461 | char _InterlockedCompareExchange8_nf(char volatile *_Destination, |
| 711 | __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, | 462 | char _Exchange, char _Comparand); |
| 712 | __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE); | 463 | char _InterlockedCompareExchange8_rel(char volatile *_Destination, |
| 713 | return _Comparand; | 464 | char _Exchange, char _Comparand); |
| 714 | } | 465 | short _InterlockedCompareExchange16_acq(short volatile *_Destination, |
| 715 | static __inline__ char __DEFAULT_FN_ATTRS | 466 | short _Exchange, short _Comparand); |
| 716 | _InterlockedCompareExchange8_nf(char volatile *_Destination, | 467 | short _InterlockedCompareExchange16_nf(short volatile *_Destination, |
| 717 | char _Exchange, char _Comparand) { | 468 | short _Exchange, short _Comparand); |
| 718 | __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, | 469 | short _InterlockedCompareExchange16_rel(short volatile *_Destination, |
| 719 | __ATOMIC_SEQ_CST, __ATOMIC_RELAXED); | 470 | short _Exchange, short _Comparand); |
| 720 | return _Comparand; | 471 | long _InterlockedCompareExchange_acq(long volatile *_Destination, |
| 721 | } | 472 | long _Exchange, long _Comparand); |
| 722 | static __inline__ char __DEFAULT_FN_ATTRS | 473 | long _InterlockedCompareExchange_nf(long volatile *_Destination, |
| 723 | _InterlockedCompareExchange8_rel(char volatile *_Destination, | 474 | long _Exchange, long _Comparand); |
| 724 | char _Exchange, char _Comparand) { | 475 | long _InterlockedCompareExchange_rel(long volatile *_Destination, |
| 725 | __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, | 476 | long _Exchange, long _Comparand); |
| 726 | __ATOMIC_SEQ_CST, __ATOMIC_RELEASE); | 477 | __int64 _InterlockedCompareExchange64_acq(__int64 volatile *_Destination, |
| 727 | return _Comparand; | 478 | __int64 _Exchange, __int64 _Comparand); |
| 728 | } | 479 | __int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination, |
| 729 | static __inline__ short __DEFAULT_FN_ATTRS | 480 | __int64 _Exchange, __int64 _Comparand); |
| 730 | _InterlockedCompareExchange16_acq(short volatile *_Destination, | 481 | __int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination, |
| 731 | short _Exchange, short _Comparand) { | 482 | __int64 _Exchange, __int64 _Comparand); |
| 732 | __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, | ||
| 733 | __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE); | ||
| 734 | return _Comparand; | ||
| 735 | } | ||
| 736 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 737 | _InterlockedCompareExchange16_nf(short volatile *_Destination, | ||
| 738 | short _Exchange, short _Comparand) { | ||
| 739 | __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, | ||
| 740 | __ATOMIC_SEQ_CST, __ATOMIC_RELAXED); | ||
| 741 | return _Comparand; | ||
| 742 | } | ||
| 743 | static __inline__ short __DEFAULT_FN_ATTRS | ||
| 744 | _InterlockedCompareExchange16_rel(short volatile *_Destination, | ||
| 745 | short _Exchange, short _Comparand) { | ||
| 746 | __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, | ||
| 747 | __ATOMIC_SEQ_CST, __ATOMIC_RELEASE); | ||
| 748 | return _Comparand; | ||
| 749 | } | ||
| 750 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 751 | _InterlockedCompareExchange_acq(long volatile *_Destination, | ||
| 752 | long _Exchange, long _Comparand) { | ||
| 753 | __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, | ||
| 754 | __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE); | ||
| 755 | return _Comparand; | ||
| 756 | } | ||
| 757 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 758 | _InterlockedCompareExchange_nf(long volatile *_Destination, | ||
| 759 | long _Exchange, long _Comparand) { | ||
| 760 | __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, | ||
| 761 | __ATOMIC_SEQ_CST, __ATOMIC_RELAXED); | ||
| 762 | return _Comparand; | ||
| 763 | } | ||
| 764 | static __inline__ long __DEFAULT_FN_ATTRS | ||
| 765 | _InterlockedCompareExchange_rel(long volatile *_Destination, | ||
| 766 | long _Exchange, long _Comparand) { | ||
| 767 | __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, | ||
| 768 | __ATOMIC_SEQ_CST, __ATOMIC_RELEASE); | ||
| 769 | return _Comparand; | ||
| 770 | } | ||
| 771 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 772 | _InterlockedCompareExchange64_acq(__int64 volatile *_Destination, | ||
| 773 | __int64 _Exchange, __int64 _Comparand) { | ||
| 774 | __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, | ||
| 775 | __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE); | ||
| 776 | return _Comparand; | ||
| 777 | } | ||
| 778 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 779 | _InterlockedCompareExchange64_nf(__int64 volatile *_Destination, | ||
| 780 | __int64 _Exchange, __int64 _Comparand) { | ||
| 781 | __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, | ||
| 782 | __ATOMIC_SEQ_CST, __ATOMIC_RELAXED); | ||
| 783 | return _Comparand; | ||
| 784 | } | ||
| 785 | static __inline__ __int64 __DEFAULT_FN_ATTRS | ||
| 786 | _InterlockedCompareExchange64_rel(__int64 volatile *_Destination, | ||
| 787 | __int64 _Exchange, __int64 _Comparand) { | ||
| 788 | __atomic_compare_exchange(_Destination, &_Comparand, &_Exchange, 0, | ||
| 789 | __ATOMIC_SEQ_CST, __ATOMIC_RELEASE); | ||
| 790 | return _Comparand; | ||
| 791 | } | ||
| 792 | #endif | 483 | #endif |
| 793 | 484 | ||
| 794 | /*----------------------------------------------------------------------------*\ | 485 | /*----------------------------------------------------------------------------*\ |
| ... | @@ -841,7 +532,7 @@ __stosq(unsigned __int64 *__dst, unsigned __int64 __x, size_t __n) { | ... | @@ -841,7 +532,7 @@ __stosq(unsigned __int64 *__dst, unsigned __int64 __x, size_t __n) { |
| 841 | static __inline__ void __DEFAULT_FN_ATTRS | 532 | static __inline__ void __DEFAULT_FN_ATTRS |
| 842 | __cpuid(int __info[4], int __level) { | 533 | __cpuid(int __info[4], int __level) { |
| 843 | __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3]) | 534 | __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3]) |
| 844 | : "a"(__level)); | 535 | : "a"(__level), "c"(0)); |
| 845 | } | 536 | } |
| 846 | static __inline__ void __DEFAULT_FN_ATTRS | 537 | static __inline__ void __DEFAULT_FN_ATTRS |
| 847 | __cpuidex(int __info[4], int __level, int __ecx) { | 538 | __cpuidex(int __info[4], int __level, int __ecx) { |
| ... | @@ -858,12 +549,35 @@ static __inline__ void __DEFAULT_FN_ATTRS | ... | @@ -858,12 +549,35 @@ static __inline__ void __DEFAULT_FN_ATTRS |
| 858 | __halt(void) { | 549 | __halt(void) { |
| 859 | __asm__ volatile ("hlt"); | 550 | __asm__ volatile ("hlt"); |
| 860 | } | 551 | } |
| 552 | #endif | ||
| 553 | |||
| 554 | #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) | ||
| 861 | static __inline__ void __DEFAULT_FN_ATTRS | 555 | static __inline__ void __DEFAULT_FN_ATTRS |
| 862 | __nop(void) { | 556 | __nop(void) { |
| 863 | __asm__ volatile ("nop"); | 557 | __asm__ volatile ("nop"); |
| 864 | } | 558 | } |
| 865 | #endif | 559 | #endif |
| 866 | 560 | ||
| 561 | /*----------------------------------------------------------------------------*\ | ||
| 562 | |* MS AArch64 specific | ||
| 563 | \*----------------------------------------------------------------------------*/ | ||
| 564 | #if defined(__aarch64__) | ||
| 565 | unsigned __int64 __getReg(int); | ||
| 566 | long _InterlockedAdd(long volatile *Addend, long Value); | ||
| 567 | int _ReadStatusReg(int); | ||
| 568 | void _WriteStatusReg(int, int); | ||
| 569 | |||
| 570 | static inline unsigned short _byteswap_ushort (unsigned short val) { | ||
| 571 | return __builtin_bswap16(val); | ||
| 572 | } | ||
| 573 | static inline unsigned long _byteswap_ulong (unsigned long val) { | ||
| 574 | return __builtin_bswap32(val); | ||
| 575 | } | ||
| 576 | static inline unsigned __int64 _byteswap_uint64 (unsigned __int64 val) { | ||
| 577 | return __builtin_bswap64(val); | ||
| 578 | } | ||
| 579 | #endif | ||
| 580 | |||
| 867 | /*----------------------------------------------------------------------------*\ | 581 | /*----------------------------------------------------------------------------*\ |
| 868 | |* Privileged intrinsics | 582 | |* Privileged intrinsics |
| 869 | \*----------------------------------------------------------------------------*/ | 583 | \*----------------------------------------------------------------------------*/ |
c_headers/lzcntintrin.h+9-13
| ... | @@ -31,6 +31,7 @@ | ... | @@ -31,6 +31,7 @@ |
| 31 | /* Define the default attributes for the functions in this file. */ | 31 | /* Define the default attributes for the functions in this file. */ |
| 32 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("lzcnt"))) | 32 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("lzcnt"))) |
| 33 | 33 | ||
| 34 | #ifndef _MSC_VER | ||
| 34 | /// Counts the number of leading zero bits in the operand. | 35 | /// Counts the number of leading zero bits in the operand. |
| 35 | /// | 36 | /// |
| 36 | /// \headerfile <x86intrin.h> | 37 | /// \headerfile <x86intrin.h> |
| ... | @@ -41,11 +42,8 @@ | ... | @@ -41,11 +42,8 @@ |
| 41 | /// An unsigned 16-bit integer whose leading zeros are to be counted. | 42 | /// An unsigned 16-bit integer whose leading zeros are to be counted. |
| 42 | /// \returns An unsigned 16-bit integer containing the number of leading zero | 43 | /// \returns An unsigned 16-bit integer containing the number of leading zero |
| 43 | /// bits in the operand. | 44 | /// bits in the operand. |
| 44 | static __inline__ unsigned short __DEFAULT_FN_ATTRS | 45 | #define __lzcnt16(X) __builtin_ia32_lzcnt_u16((unsigned short)(X)) |
| 45 | __lzcnt16(unsigned short __X) | 46 | #endif // _MSC_VER |
| 46 | { | ||
| 47 | return __X ? __builtin_clzs(__X) : 16; | ||
| 48 | } | ||
| 49 | 47 | ||
| 50 | /// Counts the number of leading zero bits in the operand. | 48 | /// Counts the number of leading zero bits in the operand. |
| 51 | /// | 49 | /// |
| ... | @@ -61,7 +59,7 @@ __lzcnt16(unsigned short __X) | ... | @@ -61,7 +59,7 @@ __lzcnt16(unsigned short __X) |
| 61 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | 59 | static __inline__ unsigned int __DEFAULT_FN_ATTRS |
| 62 | __lzcnt32(unsigned int __X) | 60 | __lzcnt32(unsigned int __X) |
| 63 | { | 61 | { |
| 64 | return __X ? __builtin_clz(__X) : 32; | 62 | return __builtin_ia32_lzcnt_u32(__X); |
| 65 | } | 63 | } |
| 66 | 64 | ||
| 67 | /// Counts the number of leading zero bits in the operand. | 65 | /// Counts the number of leading zero bits in the operand. |
| ... | @@ -78,10 +76,11 @@ __lzcnt32(unsigned int __X) | ... | @@ -78,10 +76,11 @@ __lzcnt32(unsigned int __X) |
| 78 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | 76 | static __inline__ unsigned int __DEFAULT_FN_ATTRS |
| 79 | _lzcnt_u32(unsigned int __X) | 77 | _lzcnt_u32(unsigned int __X) |
| 80 | { | 78 | { |
| 81 | return __X ? __builtin_clz(__X) : 32; | 79 | return __builtin_ia32_lzcnt_u32(__X); |
| 82 | } | 80 | } |
| 83 | 81 | ||
| 84 | #ifdef __x86_64__ | 82 | #ifdef __x86_64__ |
| 83 | #ifndef _MSC_VER | ||
| 85 | /// Counts the number of leading zero bits in the operand. | 84 | /// Counts the number of leading zero bits in the operand. |
| 86 | /// | 85 | /// |
| 87 | /// \headerfile <x86intrin.h> | 86 | /// \headerfile <x86intrin.h> |
| ... | @@ -93,11 +92,8 @@ _lzcnt_u32(unsigned int __X) | ... | @@ -93,11 +92,8 @@ _lzcnt_u32(unsigned int __X) |
| 93 | /// \returns An unsigned 64-bit integer containing the number of leading zero | 92 | /// \returns An unsigned 64-bit integer containing the number of leading zero |
| 94 | /// bits in the operand. | 93 | /// bits in the operand. |
| 95 | /// \see _lzcnt_u64 | 94 | /// \see _lzcnt_u64 |
| 96 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 95 | #define __lzcnt64(X) __builtin_ia32_lzcnt_u64((unsigned long long)(X)) |
| 97 | __lzcnt64(unsigned long long __X) | 96 | #endif // _MSC_VER |
| 98 | { | ||
| 99 | return __X ? __builtin_clzll(__X) : 64; | ||
| 100 | } | ||
| 101 | 97 | ||
| 102 | /// Counts the number of leading zero bits in the operand. | 98 | /// Counts the number of leading zero bits in the operand. |
| 103 | /// | 99 | /// |
| ... | @@ -113,7 +109,7 @@ __lzcnt64(unsigned long long __X) | ... | @@ -113,7 +109,7 @@ __lzcnt64(unsigned long long __X) |
| 113 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 109 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS |
| 114 | _lzcnt_u64(unsigned long long __X) | 110 | _lzcnt_u64(unsigned long long __X) |
| 115 | { | 111 | { |
| 116 | return __X ? __builtin_clzll(__X) : 64; | 112 | return __builtin_ia32_lzcnt_u64(__X); |
| 117 | } | 113 | } |
| 118 | #endif | 114 | #endif |
| 119 | 115 |
c_headers/opencl-c.h+654-3
| ... | @@ -22,6 +22,14 @@ | ... | @@ -22,6 +22,14 @@ |
| 22 | #endif //cl_khr_3d_image_writes | 22 | #endif //cl_khr_3d_image_writes |
| 23 | #endif //__OPENCL_C_VERSION__ < CL_VERSION_2_0 | 23 | #endif //__OPENCL_C_VERSION__ < CL_VERSION_2_0 |
| 24 | 24 | ||
| 25 | #if __OPENCL_C_VERSION__ >= CL_VERSION_1_2 | ||
| 26 | #ifndef cl_intel_planar_yuv | ||
| 27 | #define cl_intel_planar_yuv | ||
| 28 | #endif // cl_intel_planar_yuv | ||
| 29 | #pragma OPENCL EXTENSION cl_intel_planar_yuv : begin | ||
| 30 | #pragma OPENCL EXTENSION cl_intel_planar_yuv : end | ||
| 31 | #endif // __OPENCL_C_VERSION__ >= CL_VERSION_1_2 | ||
| 32 | |||
| 25 | #define __ovld __attribute__((overloadable)) | 33 | #define __ovld __attribute__((overloadable)) |
| 26 | #define __conv __attribute__((convergent)) | 34 | #define __conv __attribute__((convergent)) |
| 27 | 35 | ||
| ... | @@ -14602,6 +14610,7 @@ int4 __purefn __ovld read_imagei(read_only image3d_t image, sampler_t sampler, f | ... | @@ -14602,6 +14610,7 @@ int4 __purefn __ovld read_imagei(read_only image3d_t image, sampler_t sampler, f |
| 14602 | uint4 __purefn __ovld read_imageui(read_only image3d_t image, sampler_t sampler, int4 coord); | 14610 | uint4 __purefn __ovld read_imageui(read_only image3d_t image, sampler_t sampler, int4 coord); |
| 14603 | uint4 __purefn __ovld read_imageui(read_only image3d_t image, sampler_t sampler, float4 coord); | 14611 | uint4 __purefn __ovld read_imageui(read_only image3d_t image, sampler_t sampler, float4 coord); |
| 14604 | 14612 | ||
| 14613 | #if __OPENCL_C_VERSION__ >= CL_VERSION_1_2 | ||
| 14605 | float4 __purefn __ovld read_imagef(read_only image2d_array_t image_array, sampler_t sampler, int4 coord); | 14614 | float4 __purefn __ovld read_imagef(read_only image2d_array_t image_array, sampler_t sampler, int4 coord); |
| 14606 | float4 __purefn __ovld read_imagef(read_only image2d_array_t image_array, sampler_t sampler, float4 coord); | 14615 | float4 __purefn __ovld read_imagef(read_only image2d_array_t image_array, sampler_t sampler, float4 coord); |
| 14607 | 14616 | ||
| ... | @@ -14609,6 +14618,7 @@ int4 __purefn __ovld read_imagei(read_only image2d_array_t image_array, sampler_ | ... | @@ -14609,6 +14618,7 @@ int4 __purefn __ovld read_imagei(read_only image2d_array_t image_array, sampler_ |
| 14609 | int4 __purefn __ovld read_imagei(read_only image2d_array_t image_array, sampler_t sampler, float4 coord); | 14618 | int4 __purefn __ovld read_imagei(read_only image2d_array_t image_array, sampler_t sampler, float4 coord); |
| 14610 | uint4 __purefn __ovld read_imageui(read_only image2d_array_t image_array, sampler_t sampler, int4 coord); | 14619 | uint4 __purefn __ovld read_imageui(read_only image2d_array_t image_array, sampler_t sampler, int4 coord); |
| 14611 | uint4 __purefn __ovld read_imageui(read_only image2d_array_t image_array, sampler_t sampler, float4 coord); | 14620 | uint4 __purefn __ovld read_imageui(read_only image2d_array_t image_array, sampler_t sampler, float4 coord); |
| 14621 | #endif // __OPENCL_C_VERSION__ >= CL_VERSION_1_2 | ||
| 14612 | 14622 | ||
| 14613 | float4 __purefn __ovld read_imagef(read_only image1d_t image, sampler_t sampler, int coord); | 14623 | float4 __purefn __ovld read_imagef(read_only image1d_t image, sampler_t sampler, int coord); |
| 14614 | float4 __purefn __ovld read_imagef(read_only image1d_t image, sampler_t sampler, float coord); | 14624 | float4 __purefn __ovld read_imagef(read_only image1d_t image, sampler_t sampler, float coord); |
| ... | @@ -14618,6 +14628,7 @@ int4 __purefn __ovld read_imagei(read_only image1d_t image, sampler_t sampler, f | ... | @@ -14618,6 +14628,7 @@ int4 __purefn __ovld read_imagei(read_only image1d_t image, sampler_t sampler, f |
| 14618 | uint4 __purefn __ovld read_imageui(read_only image1d_t image, sampler_t sampler, int coord); | 14628 | uint4 __purefn __ovld read_imageui(read_only image1d_t image, sampler_t sampler, int coord); |
| 14619 | uint4 __purefn __ovld read_imageui(read_only image1d_t image, sampler_t sampler, float coord); | 14629 | uint4 __purefn __ovld read_imageui(read_only image1d_t image, sampler_t sampler, float coord); |
| 14620 | 14630 | ||
| 14631 | #if __OPENCL_C_VERSION__ >= CL_VERSION_1_2 | ||
| 14621 | float4 __purefn __ovld read_imagef(read_only image1d_array_t image_array, sampler_t sampler, int2 coord); | 14632 | float4 __purefn __ovld read_imagef(read_only image1d_array_t image_array, sampler_t sampler, int2 coord); |
| 14622 | float4 __purefn __ovld read_imagef(read_only image1d_array_t image_array, sampler_t sampler, float2 coord); | 14633 | float4 __purefn __ovld read_imagef(read_only image1d_array_t image_array, sampler_t sampler, float2 coord); |
| 14623 | 14634 | ||
| ... | @@ -14625,6 +14636,7 @@ int4 __purefn __ovld read_imagei(read_only image1d_array_t image_array, sampler_ | ... | @@ -14625,6 +14636,7 @@ int4 __purefn __ovld read_imagei(read_only image1d_array_t image_array, sampler_ |
| 14625 | int4 __purefn __ovld read_imagei(read_only image1d_array_t image_array, sampler_t sampler, float2 coord); | 14636 | int4 __purefn __ovld read_imagei(read_only image1d_array_t image_array, sampler_t sampler, float2 coord); |
| 14626 | uint4 __purefn __ovld read_imageui(read_only image1d_array_t image_array, sampler_t sampler, int2 coord); | 14637 | uint4 __purefn __ovld read_imageui(read_only image1d_array_t image_array, sampler_t sampler, int2 coord); |
| 14627 | uint4 __purefn __ovld read_imageui(read_only image1d_array_t image_array, sampler_t sampler, float2 coord); | 14638 | uint4 __purefn __ovld read_imageui(read_only image1d_array_t image_array, sampler_t sampler, float2 coord); |
| 14639 | #endif // __OPENCL_C_VERSION__ >= CL_VERSION_1_2 | ||
| 14628 | 14640 | ||
| 14629 | #ifdef cl_khr_depth_images | 14641 | #ifdef cl_khr_depth_images |
| 14630 | float __purefn __ovld read_imagef(read_only image2d_depth_t image, sampler_t sampler, float2 coord); | 14642 | float __purefn __ovld read_imagef(read_only image2d_depth_t image, sampler_t sampler, float2 coord); |
| ... | @@ -14727,6 +14739,8 @@ uint4 __purefn __ovld read_imageui(read_only image3d_t image, sampler_t sampler, | ... | @@ -14727,6 +14739,8 @@ uint4 __purefn __ovld read_imageui(read_only image3d_t image, sampler_t sampler, |
| 14727 | #endif //cl_khr_mipmap_image | 14739 | #endif //cl_khr_mipmap_image |
| 14728 | #endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0 | 14740 | #endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0 |
| 14729 | 14741 | ||
| 14742 | #if __OPENCL_C_VERSION__ >= CL_VERSION_1_2 | ||
| 14743 | |||
| 14730 | /** | 14744 | /** |
| 14731 | * Sampler-less Image Access | 14745 | * Sampler-less Image Access |
| 14732 | */ | 14746 | */ |
| ... | @@ -14760,24 +14774,31 @@ float4 __purefn __ovld read_imagef(read_only image3d_t image, int4 coord); | ... | @@ -14760,24 +14774,31 @@ float4 __purefn __ovld read_imagef(read_only image3d_t image, int4 coord); |
| 14760 | int4 __purefn __ovld read_imagei(read_only image3d_t image, int4 coord); | 14774 | int4 __purefn __ovld read_imagei(read_only image3d_t image, int4 coord); |
| 14761 | uint4 __purefn __ovld read_imageui(read_only image3d_t image, int4 coord); | 14775 | uint4 __purefn __ovld read_imageui(read_only image3d_t image, int4 coord); |
| 14762 | 14776 | ||
| 14777 | #endif // __OPENCL_C_VERSION__ >= CL_VERSION_1_2 | ||
| 14778 | |||
| 14763 | // Image read functions returning half4 type | 14779 | // Image read functions returning half4 type |
| 14764 | #ifdef cl_khr_fp16 | 14780 | #ifdef cl_khr_fp16 |
| 14765 | half4 __purefn __ovld read_imageh(read_only image1d_t image, sampler_t sampler, int coord); | 14781 | half4 __purefn __ovld read_imageh(read_only image1d_t image, sampler_t sampler, int coord); |
| 14766 | half4 __purefn __ovld read_imageh(read_only image1d_t image, sampler_t sampler, float coord); | 14782 | half4 __purefn __ovld read_imageh(read_only image1d_t image, sampler_t sampler, float coord); |
| 14767 | half4 __purefn __ovld read_imageh(read_only image1d_array_t image, sampler_t sampler, int2 coord); | ||
| 14768 | half4 __purefn __ovld read_imageh(read_only image1d_array_t image, sampler_t sampler, float2 coord); | ||
| 14769 | half4 __purefn __ovld read_imageh(read_only image2d_t image, sampler_t sampler, int2 coord); | 14783 | half4 __purefn __ovld read_imageh(read_only image2d_t image, sampler_t sampler, int2 coord); |
| 14770 | half4 __purefn __ovld read_imageh(read_only image2d_t image, sampler_t sampler, float2 coord); | 14784 | half4 __purefn __ovld read_imageh(read_only image2d_t image, sampler_t sampler, float2 coord); |
| 14771 | half4 __purefn __ovld read_imageh(read_only image3d_t image, sampler_t sampler, int4 coord); | 14785 | half4 __purefn __ovld read_imageh(read_only image3d_t image, sampler_t sampler, int4 coord); |
| 14772 | half4 __purefn __ovld read_imageh(read_only image3d_t image, sampler_t sampler, float4 coord); | 14786 | half4 __purefn __ovld read_imageh(read_only image3d_t image, sampler_t sampler, float4 coord); |
| 14787 | #if __OPENCL_C_VERSION__ >= CL_VERSION_1_2 | ||
| 14788 | half4 __purefn __ovld read_imageh(read_only image1d_array_t image, sampler_t sampler, int2 coord); | ||
| 14789 | half4 __purefn __ovld read_imageh(read_only image1d_array_t image, sampler_t sampler, float2 coord); | ||
| 14773 | half4 __purefn __ovld read_imageh(read_only image2d_array_t image, sampler_t sampler, int4 coord); | 14790 | half4 __purefn __ovld read_imageh(read_only image2d_array_t image, sampler_t sampler, int4 coord); |
| 14774 | half4 __purefn __ovld read_imageh(read_only image2d_array_t image, sampler_t sampler, float4 coord); | 14791 | half4 __purefn __ovld read_imageh(read_only image2d_array_t image, sampler_t sampler, float4 coord); |
| 14792 | /** | ||
| 14793 | * Sampler-less Image Access | ||
| 14794 | */ | ||
| 14775 | half4 __purefn __ovld read_imageh(read_only image1d_t image, int coord); | 14795 | half4 __purefn __ovld read_imageh(read_only image1d_t image, int coord); |
| 14776 | half4 __purefn __ovld read_imageh(read_only image2d_t image, int2 coord); | 14796 | half4 __purefn __ovld read_imageh(read_only image2d_t image, int2 coord); |
| 14777 | half4 __purefn __ovld read_imageh(read_only image3d_t image, int4 coord); | 14797 | half4 __purefn __ovld read_imageh(read_only image3d_t image, int4 coord); |
| 14778 | half4 __purefn __ovld read_imageh(read_only image1d_array_t image, int2 coord); | 14798 | half4 __purefn __ovld read_imageh(read_only image1d_array_t image, int2 coord); |
| 14779 | half4 __purefn __ovld read_imageh(read_only image2d_array_t image, int4 coord); | 14799 | half4 __purefn __ovld read_imageh(read_only image2d_array_t image, int4 coord); |
| 14780 | half4 __purefn __ovld read_imageh(read_only image1d_buffer_t image, int coord); | 14800 | half4 __purefn __ovld read_imageh(read_only image1d_buffer_t image, int coord); |
| 14801 | #endif // __OPENCL_C_VERSION__ >= CL_VERSION_1_2 | ||
| 14781 | #endif //cl_khr_fp16 | 14802 | #endif //cl_khr_fp16 |
| 14782 | 14803 | ||
| 14783 | // Image read functions for read_write images | 14804 | // Image read functions for read_write images |
| ... | @@ -15707,7 +15728,6 @@ double __ovld __conv work_group_scan_inclusive_max(double x); | ... | @@ -15707,7 +15728,6 @@ double __ovld __conv work_group_scan_inclusive_max(double x); |
| 15707 | 15728 | ||
| 15708 | // OpenCL v2.0 s6.13.16 - Pipe Functions | 15729 | // OpenCL v2.0 s6.13.16 - Pipe Functions |
| 15709 | #if __OPENCL_C_VERSION__ >= CL_VERSION_2_0 | 15730 | #if __OPENCL_C_VERSION__ >= CL_VERSION_2_0 |
| 15710 | #define PIPE_RESERVE_ID_VALID_BIT (1U << 30) | ||
| 15711 | #define CLK_NULL_RESERVE_ID (__builtin_astype(((void*)(__SIZE_MAX__)), reserve_id_t)) | 15731 | #define CLK_NULL_RESERVE_ID (__builtin_astype(((void*)(__SIZE_MAX__)), reserve_id_t)) |
| 15712 | bool __ovld is_valid_reserve_id(reserve_id_t reserve_id); | 15732 | bool __ovld is_valid_reserve_id(reserve_id_t reserve_id); |
| 15713 | #endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0 | 15733 | #endif //__OPENCL_C_VERSION__ >= CL_VERSION_2_0 |
| ... | @@ -16193,6 +16213,637 @@ void __ovld __conv intel_sub_group_block_write_us4( __global ushort* p, u | ... | @@ -16193,6 +16213,637 @@ void __ovld __conv intel_sub_group_block_write_us4( __global ushort* p, u |
| 16193 | void __ovld __conv intel_sub_group_block_write_us8( __global ushort* p, ushort8 data ); | 16213 | void __ovld __conv intel_sub_group_block_write_us8( __global ushort* p, ushort8 data ); |
| 16194 | #endif // cl_intel_subgroups_short | 16214 | #endif // cl_intel_subgroups_short |
| 16195 | 16215 | ||
| 16216 | #ifdef cl_intel_device_side_avc_motion_estimation | ||
| 16217 | #pragma OPENCL EXTENSION cl_intel_device_side_avc_motion_estimation : begin | ||
| 16218 | |||
| 16219 | #define CLK_AVC_ME_MAJOR_16x16_INTEL 0x0 | ||
| 16220 | #define CLK_AVC_ME_MAJOR_16x8_INTEL 0x1 | ||
| 16221 | #define CLK_AVC_ME_MAJOR_8x16_INTEL 0x2 | ||
| 16222 | #define CLK_AVC_ME_MAJOR_8x8_INTEL 0x3 | ||
| 16223 | |||
| 16224 | #define CLK_AVC_ME_MINOR_8x8_INTEL 0x0 | ||
| 16225 | #define CLK_AVC_ME_MINOR_8x4_INTEL 0x1 | ||
| 16226 | #define CLK_AVC_ME_MINOR_4x8_INTEL 0x2 | ||
| 16227 | #define CLK_AVC_ME_MINOR_4x4_INTEL 0x3 | ||
| 16228 | |||
| 16229 | #define CLK_AVC_ME_MAJOR_FORWARD_INTEL 0x0 | ||
| 16230 | #define CLK_AVC_ME_MAJOR_BACKWARD_INTEL 0x1 | ||
| 16231 | #define CLK_AVC_ME_MAJOR_BIDIRECTIONAL_INTEL 0x2 | ||
| 16232 | |||
| 16233 | #define CLK_AVC_ME_PARTITION_MASK_ALL_INTEL 0x0 | ||
| 16234 | #define CLK_AVC_ME_PARTITION_MASK_16x16_INTEL 0x7E | ||
| 16235 | #define CLK_AVC_ME_PARTITION_MASK_16x8_INTEL 0x7D | ||
| 16236 | #define CLK_AVC_ME_PARTITION_MASK_8x16_INTEL 0x7B | ||
| 16237 | #define CLK_AVC_ME_PARTITION_MASK_8x8_INTEL 0x77 | ||
| 16238 | #define CLK_AVC_ME_PARTITION_MASK_8x4_INTEL 0x6F | ||
| 16239 | #define CLK_AVC_ME_PARTITION_MASK_4x8_INTEL 0x5F | ||
| 16240 | #define CLK_AVC_ME_PARTITION_MASK_4x4_INTEL 0x3F | ||
| 16241 | |||
| 16242 | #define CLK_AVC_ME_SLICE_TYPE_PRED_INTEL 0x0 | ||
| 16243 | #define CLK_AVC_ME_SLICE_TYPE_BPRED_INTEL 0x1 | ||
| 16244 | #define CLK_AVC_ME_SLICE_TYPE_INTRA_INTEL 0x2 | ||
| 16245 | |||
| 16246 | #define CLK_AVC_ME_SEARCH_WINDOW_EXHAUSTIVE_INTEL 0x0 | ||
| 16247 | #define CLK_AVC_ME_SEARCH_WINDOW_SMALL_INTEL 0x1 | ||
| 16248 | #define CLK_AVC_ME_SEARCH_WINDOW_TINY_INTEL 0x2 | ||
| 16249 | #define CLK_AVC_ME_SEARCH_WINDOW_EXTRA_TINY_INTEL 0x3 | ||
| 16250 | #define CLK_AVC_ME_SEARCH_WINDOW_DIAMOND_INTEL 0x4 | ||
| 16251 | #define CLK_AVC_ME_SEARCH_WINDOW_LARGE_DIAMOND_INTEL 0x5 | ||
| 16252 | #define CLK_AVC_ME_SEARCH_WINDOW_RESERVED0_INTEL 0x6 | ||
| 16253 | #define CLK_AVC_ME_SEARCH_WINDOW_RESERVED1_INTEL 0x7 | ||
| 16254 | #define CLK_AVC_ME_SEARCH_WINDOW_CUSTOM_INTEL 0x8 | ||
| 16255 | |||
| 16256 | #define CLK_AVC_ME_SAD_ADJUST_MODE_NONE_INTEL 0x0 | ||
| 16257 | #define CLK_AVC_ME_SAD_ADJUST_MODE_HAAR_INTEL 0x2 | ||
| 16258 | |||
| 16259 | #define CLK_AVC_ME_SUBPIXEL_MODE_INTEGER_INTEL 0x0 | ||
| 16260 | #define CLK_AVC_ME_SUBPIXEL_MODE_HPEL_INTEL 0x1 | ||
| 16261 | #define CLK_AVC_ME_SUBPIXEL_MODE_QPEL_INTEL 0x3 | ||
| 16262 | |||
| 16263 | #define CLK_AVC_ME_COST_PRECISION_QPEL_INTEL 0x0 | ||
| 16264 | #define CLK_AVC_ME_COST_PRECISION_HPEL_INTEL 0x1 | ||
| 16265 | #define CLK_AVC_ME_COST_PRECISION_PEL_INTEL 0x2 | ||
| 16266 | #define CLK_AVC_ME_COST_PRECISION_DPEL_INTEL 0x3 | ||
| 16267 | |||
| 16268 | #define CLK_AVC_ME_BIDIR_WEIGHT_QUARTER_INTEL 0x10 | ||
| 16269 | #define CLK_AVC_ME_BIDIR_WEIGHT_THIRD_INTEL 0x15 | ||
| 16270 | #define CLK_AVC_ME_BIDIR_WEIGHT_HALF_INTEL 0x20 | ||
| 16271 | #define CLK_AVC_ME_BIDIR_WEIGHT_TWO_THIRD_INTEL 0x2B | ||
| 16272 | #define CLK_AVC_ME_BIDIR_WEIGHT_THREE_QUARTER_INTEL 0x30 | ||
| 16273 | |||
| 16274 | #define CLK_AVC_ME_BORDER_REACHED_LEFT_INTEL 0x0 | ||
| 16275 | #define CLK_AVC_ME_BORDER_REACHED_RIGHT_INTEL 0x2 | ||
| 16276 | #define CLK_AVC_ME_BORDER_REACHED_TOP_INTEL 0x4 | ||
| 16277 | #define CLK_AVC_ME_BORDER_REACHED_BOTTOM_INTEL 0x8 | ||
| 16278 | |||
| 16279 | #define CLK_AVC_ME_INTRA_16x16_INTEL 0x0 | ||
| 16280 | #define CLK_AVC_ME_INTRA_8x8_INTEL 0x1 | ||
| 16281 | #define CLK_AVC_ME_INTRA_4x4_INTEL 0x2 | ||
| 16282 | |||
| 16283 | #define CLK_AVC_ME_SKIP_BLOCK_PARTITION_16x16_INTEL 0x0 | ||
| 16284 | #define CLK_AVC_ME_SKIP_BLOCK_PARTITION_8x8_INTEL 0x4000 | ||
| 16285 | |||
| 16286 | #define CLK_AVC_ME_SKIP_BLOCK_16x16_FORWARD_ENABLE_INTEL (0x1 << 24) | ||
| 16287 | #define CLK_AVC_ME_SKIP_BLOCK_16x16_BACKWARD_ENABLE_INTEL (0x2 << 24) | ||
| 16288 | #define CLK_AVC_ME_SKIP_BLOCK_16x16_DUAL_ENABLE_INTEL (0x3 << 24) | ||
| 16289 | #define CLK_AVC_ME_SKIP_BLOCK_8x8_FORWARD_ENABLE_INTEL (0x55 << 24) | ||
| 16290 | #define CLK_AVC_ME_SKIP_BLOCK_8x8_BACKWARD_ENABLE_INTEL (0xAA << 24) | ||
| 16291 | #define CLK_AVC_ME_SKIP_BLOCK_8x8_DUAL_ENABLE_INTEL (0xFF << 24) | ||
| 16292 | #define CLK_AVC_ME_SKIP_BLOCK_8x8_0_FORWARD_ENABLE_INTEL (0x1 << 24) | ||
| 16293 | #define CLK_AVC_ME_SKIP_BLOCK_8x8_0_BACKWARD_ENABLE_INTEL (0x2 << 24) | ||
| 16294 | #define CLK_AVC_ME_SKIP_BLOCK_8x8_1_FORWARD_ENABLE_INTEL (0x1 << 26) | ||
| 16295 | #define CLK_AVC_ME_SKIP_BLOCK_8x8_1_BACKWARD_ENABLE_INTEL (0x2 << 26) | ||
| 16296 | #define CLK_AVC_ME_SKIP_BLOCK_8x8_2_FORWARD_ENABLE_INTEL (0x1 << 28) | ||
| 16297 | #define CLK_AVC_ME_SKIP_BLOCK_8x8_2_BACKWARD_ENABLE_INTEL (0x2 << 28) | ||
| 16298 | #define CLK_AVC_ME_SKIP_BLOCK_8x8_3_FORWARD_ENABLE_INTEL (0x1 << 30) | ||
| 16299 | #define CLK_AVC_ME_SKIP_BLOCK_8x8_3_BACKWARD_ENABLE_INTEL (0x2 << 30) | ||
| 16300 | |||
| 16301 | #define CLK_AVC_ME_BLOCK_BASED_SKIP_4x4_INTEL 0x00 | ||
| 16302 | #define CLK_AVC_ME_BLOCK_BASED_SKIP_8x8_INTEL 0x80 | ||
| 16303 | |||
| 16304 | #define CLK_AVC_ME_INTRA_LUMA_PARTITION_MASK_ALL_INTEL 0x0 | ||
| 16305 | #define CLK_AVC_ME_INTRA_LUMA_PARTITION_MASK_16x16_INTEL 0x6 | ||
| 16306 | #define CLK_AVC_ME_INTRA_LUMA_PARTITION_MASK_8x8_INTEL 0x5 | ||
| 16307 | #define CLK_AVC_ME_INTRA_LUMA_PARTITION_MASK_4x4_INTEL 0x3 | ||
| 16308 | |||
| 16309 | #define CLK_AVC_ME_INTRA_NEIGHBOR_LEFT_MASK_ENABLE_INTEL 0x60 | ||
| 16310 | #define CLK_AVC_ME_INTRA_NEIGHBOR_UPPER_MASK_ENABLE_INTEL 0x10 | ||
| 16311 | #define CLK_AVC_ME_INTRA_NEIGHBOR_UPPER_RIGHT_MASK_ENABLE_INTEL 0x8 | ||
| 16312 | #define CLK_AVC_ME_INTRA_NEIGHBOR_UPPER_LEFT_MASK_ENABLE_INTEL 0x4 | ||
| 16313 | |||
| 16314 | #define CLK_AVC_ME_LUMA_PREDICTOR_MODE_VERTICAL_INTEL 0x0 | ||
| 16315 | #define CLK_AVC_ME_LUMA_PREDICTOR_MODE_HORIZONTAL_INTEL 0x1 | ||
| 16316 | #define CLK_AVC_ME_LUMA_PREDICTOR_MODE_DC_INTEL 0x2 | ||
| 16317 | #define CLK_AVC_ME_LUMA_PREDICTOR_MODE_DIAGONAL_DOWN_LEFT_INTEL 0x3 | ||
| 16318 | #define CLK_AVC_ME_LUMA_PREDICTOR_MODE_DIAGONAL_DOWN_RIGHT_INTEL 0x4 | ||
| 16319 | #define CLK_AVC_ME_LUMA_PREDICTOR_MODE_PLANE_INTEL 0x4 | ||
| 16320 | #define CLK_AVC_ME_LUMA_PREDICTOR_MODE_VERTICAL_RIGHT_INTEL 0x5 | ||
| 16321 | #define CLK_AVC_ME_LUMA_PREDICTOR_MODE_HORIZONTAL_DOWN_INTEL 0x6 | ||
| 16322 | #define CLK_AVC_ME_LUMA_PREDICTOR_MODE_VERTICAL_LEFT_INTEL 0x7 | ||
| 16323 | #define CLK_AVC_ME_LUMA_PREDICTOR_MODE_HORIZONTAL_UP_INTEL 0x8 | ||
| 16324 | #define CLK_AVC_ME_CHROMA_PREDICTOR_MODE_DC_INTEL 0x0 | ||
| 16325 | #define CLK_AVC_ME_CHROMA_PREDICTOR_MODE_HORIZONTAL_INTEL 0x1 | ||
| 16326 | #define CLK_AVC_ME_CHROMA_PREDICTOR_MODE_VERTICAL_INTEL 0x2 | ||
| 16327 | #define CLK_AVC_ME_CHROMA_PREDICTOR_MODE_PLANE_INTEL 0x3 | ||
| 16328 | |||
| 16329 | #define CLK_AVC_ME_FRAME_FORWARD_INTEL 0x1 | ||
| 16330 | #define CLK_AVC_ME_FRAME_BACKWARD_INTEL 0x2 | ||
| 16331 | #define CLK_AVC_ME_FRAME_DUAL_INTEL 0x3 | ||
| 16332 | |||
| 16333 | #define CLK_AVC_ME_INTERLACED_SCAN_TOP_FIELD_INTEL 0x0 | ||
| 16334 | #define CLK_AVC_ME_INTERLACED_SCAN_BOTTOM_FIELD_INTEL 0x1 | ||
| 16335 | |||
| 16336 | #define CLK_AVC_ME_INITIALIZE_INTEL 0x0 | ||
| 16337 | |||
| 16338 | #define CLK_AVC_IME_PAYLOAD_INITIALIZE_INTEL 0x0 | ||
| 16339 | #define CLK_AVC_REF_PAYLOAD_INITIALIZE_INTEL 0x0 | ||
| 16340 | #define CLK_AVC_SIC_PAYLOAD_INITIALIZE_INTEL 0x0 | ||
| 16341 | |||
| 16342 | #define CLK_AVC_IME_RESULT_INITIALIZE_INTEL 0x0 | ||
| 16343 | #define CLK_AVC_REF_RESULT_INITIALIZE_INTEL 0x0 | ||
| 16344 | #define CLK_AVC_SIC_RESULT_INITIALIZE_INTEL 0x0 | ||
| 16345 | |||
| 16346 | #define CLK_AVC_IME_RESULT_SINGLE_REFERENCE_STREAMOUT_INITIALIZE_INTEL 0x0 | ||
| 16347 | #define CLK_AVC_IME_RESULT_SINGLE_REFERENCE_STREAMIN_INITIALIZE_INTEL 0x0 | ||
| 16348 | #define CLK_AVC_IME_RESULT_DUAL_REFERENCE_STREAMOUT_INITIALIZE_INTEL 0x0 | ||
| 16349 | #define CLK_AVC_IME_RESULT_DUAL_REFERENCE_STREAMIN_INITIALIZE_INTEL 0x0 | ||
| 16350 | |||
| 16351 | // MCE built-in functions | ||
| 16352 | uchar __ovld | ||
| 16353 | intel_sub_group_avc_mce_get_default_inter_base_multi_reference_penalty( | ||
| 16354 | uchar slice_type, uchar qp); | ||
| 16355 | ulong __ovld intel_sub_group_avc_mce_get_default_inter_shape_penalty( | ||
| 16356 | uchar slice_type, uchar qp); | ||
| 16357 | uchar __ovld intel_sub_group_avc_mce_get_default_inter_direction_penalty( | ||
| 16358 | uchar slice_type, uchar qp); | ||
| 16359 | uint __ovld intel_sub_group_avc_mce_get_default_intra_luma_shape_penalty( | ||
| 16360 | uchar slice_type, uchar qp); | ||
| 16361 | uint2 __ovld | ||
| 16362 | intel_sub_group_avc_mce_get_default_inter_motion_vector_cost_table( | ||
| 16363 | uchar slice_type, uchar qp); | ||
| 16364 | uchar __ovld intel_sub_group_avc_mce_get_default_intra_luma_mode_penalty( | ||
| 16365 | uchar slice_type, uchar qp); | ||
| 16366 | |||
| 16367 | uint2 __ovld intel_sub_group_avc_mce_get_default_high_penalty_cost_table(); | ||
| 16368 | uint2 __ovld intel_sub_group_avc_mce_get_default_medium_penalty_cost_table(); | ||
| 16369 | uint2 __ovld intel_sub_group_avc_mce_get_default_low_penalty_cost_table(); | ||
| 16370 | uint __ovld intel_sub_group_avc_mce_get_default_non_dc_luma_intra_penalty(); | ||
| 16371 | uchar __ovld | ||
| 16372 | intel_sub_group_avc_mce_get_default_intra_chroma_mode_base_penalty(); | ||
| 16373 | |||
| 16374 | intel_sub_group_avc_mce_payload_t __ovld | ||
| 16375 | intel_sub_group_avc_mce_set_inter_base_multi_reference_penalty( | ||
| 16376 | uchar reference_base_penalty, intel_sub_group_avc_mce_payload_t payload); | ||
| 16377 | intel_sub_group_avc_mce_payload_t __ovld | ||
| 16378 | intel_sub_group_avc_mce_set_inter_shape_penalty( | ||
| 16379 | ulong packed_shape_penalty, intel_sub_group_avc_mce_payload_t payload); | ||
| 16380 | intel_sub_group_avc_mce_payload_t __ovld | ||
| 16381 | intel_sub_group_avc_mce_set_inter_direction_penalty( | ||
| 16382 | uchar direction_cost, intel_sub_group_avc_mce_payload_t payload); | ||
| 16383 | intel_sub_group_avc_mce_payload_t __ovld | ||
| 16384 | intel_sub_group_avc_mce_set_motion_vector_cost_function( | ||
| 16385 | ulong packed_cost_center_delta, uint2 packed_cost_table, | ||
| 16386 | uchar cost_precision, intel_sub_group_avc_mce_payload_t payload); | ||
| 16387 | intel_sub_group_avc_mce_payload_t __ovld | ||
| 16388 | intel_sub_group_avc_mce_set_ac_only_haar( | ||
| 16389 | intel_sub_group_avc_mce_payload_t payload); | ||
| 16390 | intel_sub_group_avc_mce_payload_t __ovld | ||
| 16391 | intel_sub_group_avc_mce_set_source_interlaced_field_polarity( | ||
| 16392 | uchar src_field_polarity, intel_sub_group_avc_mce_payload_t payload); | ||
| 16393 | intel_sub_group_avc_mce_payload_t __ovld | ||
| 16394 | intel_sub_group_avc_mce_set_single_reference_interlaced_field_polarity( | ||
| 16395 | uchar ref_field_polarity, intel_sub_group_avc_mce_payload_t payload); | ||
| 16396 | intel_sub_group_avc_mce_payload_t __ovld | ||
| 16397 | intel_sub_group_avc_mce_set_dual_reference_interlaced_field_polarities( | ||
| 16398 | uchar fwd_ref_field_polarity, uchar bwd_ref_field_polarity, | ||
| 16399 | intel_sub_group_avc_mce_payload_t payload); | ||
| 16400 | |||
| 16401 | ulong __ovld intel_sub_group_avc_mce_get_motion_vectors( | ||
| 16402 | intel_sub_group_avc_mce_result_t result); | ||
| 16403 | ushort __ovld intel_sub_group_avc_mce_get_inter_distortions( | ||
| 16404 | intel_sub_group_avc_mce_result_t result); | ||
| 16405 | ushort __ovld intel_sub_group_avc_mce_get_best_inter_distortion( | ||
| 16406 | intel_sub_group_avc_mce_result_t result); | ||
| 16407 | uchar __ovld intel_sub_group_avc_mce_get_inter_major_shape( | ||
| 16408 | intel_sub_group_avc_mce_result_t result); | ||
| 16409 | uchar __ovld intel_sub_group_avc_mce_get_inter_minor_shapes( | ||
| 16410 | intel_sub_group_avc_mce_result_t result); | ||
| 16411 | uchar __ovld intel_sub_group_avc_mce_get_inter_directions( | ||
| 16412 | intel_sub_group_avc_mce_result_t result); | ||
| 16413 | uchar __ovld intel_sub_group_avc_mce_get_inter_motion_vector_count( | ||
| 16414 | intel_sub_group_avc_mce_result_t result); | ||
| 16415 | uint __ovld intel_sub_group_avc_mce_get_inter_reference_ids( | ||
| 16416 | intel_sub_group_avc_mce_result_t result); | ||
| 16417 | uchar __ovld | ||
| 16418 | intel_sub_group_avc_mce_get_inter_reference_interlaced_field_polarities( | ||
| 16419 | uint packed_reference_ids, uint packed_reference_parameter_field_polarities, | ||
| 16420 | intel_sub_group_avc_mce_result_t result); | ||
| 16421 | |||
| 16422 | // IME built-in functions | ||
| 16423 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16424 | intel_sub_group_avc_ime_initialize( | ||
| 16425 | ushort2 src_coord, uchar partition_mask, uchar sad_adjustment); | ||
| 16426 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16427 | intel_sub_group_avc_ime_set_single_reference( | ||
| 16428 | short2 ref_offset, uchar search_window_config, | ||
| 16429 | intel_sub_group_avc_ime_payload_t payload); | ||
| 16430 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16431 | intel_sub_group_avc_ime_set_dual_reference( | ||
| 16432 | short2 fwd_ref_offset, short2 bwd_ref_offset, uchar search_window_config, | ||
| 16433 | intel_sub_group_avc_ime_payload_t payload); | ||
| 16434 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16435 | intel_sub_group_avc_ime_set_max_motion_vector_count( | ||
| 16436 | uchar max_motion_vector_count, intel_sub_group_avc_ime_payload_t payload); | ||
| 16437 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16438 | intel_sub_group_avc_ime_set_unidirectional_mix_disable( | ||
| 16439 | intel_sub_group_avc_ime_payload_t payload); | ||
| 16440 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16441 | intel_sub_group_avc_ime_set_early_search_termination_threshold( | ||
| 16442 | uchar threshold, intel_sub_group_avc_ime_payload_t payload); | ||
| 16443 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16444 | intel_sub_group_avc_ime_set_weighted_sad( | ||
| 16445 | uint packed_sad_weights, intel_sub_group_avc_ime_payload_t payload); | ||
| 16446 | |||
| 16447 | __attribute__((deprecated("If you use the latest Intel driver, please use " | ||
| 16448 | "intel_sub_group_avc_ime_ref_window_size instead", | ||
| 16449 | "intel_sub_group_avc_ime_ref_window_size"))) | ||
| 16450 | ushort2 __ovld | ||
| 16451 | intel_sub_group_ime_ref_window_size(uchar search_window_config, char dual_ref); | ||
| 16452 | ushort2 __ovld intel_sub_group_avc_ime_ref_window_size( | ||
| 16453 | uchar search_window_config, char dual_ref); | ||
| 16454 | short2 __ovld intel_sub_group_avc_ime_adjust_ref_offset( | ||
| 16455 | short2 ref_offset, ushort2 src_coord, ushort2 ref_window_size, | ||
| 16456 | ushort2 image_size); | ||
| 16457 | |||
| 16458 | intel_sub_group_avc_ime_result_t __ovld | ||
| 16459 | intel_sub_group_avc_ime_evaluate_with_single_reference( | ||
| 16460 | read_only image2d_t src_image, read_only image2d_t ref_image, | ||
| 16461 | sampler_t vme_media_sampler, intel_sub_group_avc_ime_payload_t payload); | ||
| 16462 | intel_sub_group_avc_ime_result_t __ovld | ||
| 16463 | intel_sub_group_avc_ime_evaluate_with_dual_reference( | ||
| 16464 | read_only image2d_t src_image, read_only image2d_t fwd_ref_image, | ||
| 16465 | read_only image2d_t bwd_ref_image, sampler_t vme_media_sampler, | ||
| 16466 | intel_sub_group_avc_ime_payload_t payload); | ||
| 16467 | intel_sub_group_avc_ime_result_single_reference_streamout_t __ovld | ||
| 16468 | intel_sub_group_avc_ime_evaluate_with_single_reference_streamout( | ||
| 16469 | read_only image2d_t src_image, read_only image2d_t ref_image, | ||
| 16470 | sampler_t vme_media_sampler, intel_sub_group_avc_ime_payload_t payload); | ||
| 16471 | intel_sub_group_avc_ime_result_dual_reference_streamout_t __ovld | ||
| 16472 | intel_sub_group_avc_ime_evaluate_with_dual_reference_streamout( | ||
| 16473 | read_only image2d_t src_image, read_only image2d_t fwd_ref_image, | ||
| 16474 | read_only image2d_t bwd_ref_image, sampler_t vme_media_sampler, | ||
| 16475 | intel_sub_group_avc_ime_payload_t payload); | ||
| 16476 | intel_sub_group_avc_ime_result_t __ovld | ||
| 16477 | intel_sub_group_avc_ime_evaluate_with_single_reference_streamin( | ||
| 16478 | read_only image2d_t src_image, read_only image2d_t ref_image, | ||
| 16479 | sampler_t vme_media_sampler, intel_sub_group_avc_ime_payload_t payload, | ||
| 16480 | intel_sub_group_avc_ime_single_reference_streamin_t streamin_components); | ||
| 16481 | intel_sub_group_avc_ime_result_t __ovld | ||
| 16482 | intel_sub_group_avc_ime_evaluate_with_dual_reference_streamin( | ||
| 16483 | read_only image2d_t src_image, read_only image2d_t fwd_ref_image, | ||
| 16484 | read_only image2d_t bwd_ref_image, sampler_t vme_media_sampler, | ||
| 16485 | intel_sub_group_avc_ime_payload_t payload, | ||
| 16486 | intel_sub_group_avc_ime_dual_reference_streamin_t streamin_components); | ||
| 16487 | intel_sub_group_avc_ime_result_single_reference_streamout_t __ovld | ||
| 16488 | intel_sub_group_avc_ime_evaluate_with_single_reference_streaminout( | ||
| 16489 | read_only image2d_t src_image, read_only image2d_t ref_image, | ||
| 16490 | sampler_t vme_media_sampler, intel_sub_group_avc_ime_payload_t payload, | ||
| 16491 | intel_sub_group_avc_ime_single_reference_streamin_t streamin_components); | ||
| 16492 | intel_sub_group_avc_ime_result_dual_reference_streamout_t __ovld | ||
| 16493 | intel_sub_group_avc_ime_evaluate_with_dual_reference_streaminout( | ||
| 16494 | read_only image2d_t src_image, read_only image2d_t fwd_ref_image, | ||
| 16495 | read_only image2d_t bwd_ref_image, sampler_t vme_media_sampler, | ||
| 16496 | intel_sub_group_avc_ime_payload_t payload, | ||
| 16497 | intel_sub_group_avc_ime_dual_reference_streamin_t streamin_components); | ||
| 16498 | |||
| 16499 | intel_sub_group_avc_ime_single_reference_streamin_t __ovld | ||
| 16500 | intel_sub_group_avc_ime_get_single_reference_streamin( | ||
| 16501 | intel_sub_group_avc_ime_result_single_reference_streamout_t result); | ||
| 16502 | intel_sub_group_avc_ime_dual_reference_streamin_t __ovld | ||
| 16503 | intel_sub_group_avc_ime_get_dual_reference_streamin( | ||
| 16504 | intel_sub_group_avc_ime_result_dual_reference_streamout_t result); | ||
| 16505 | intel_sub_group_avc_ime_result_t __ovld | ||
| 16506 | intel_sub_group_avc_ime_strip_single_reference_streamout( | ||
| 16507 | intel_sub_group_avc_ime_result_single_reference_streamout_t result); | ||
| 16508 | intel_sub_group_avc_ime_result_t __ovld | ||
| 16509 | intel_sub_group_avc_ime_strip_dual_reference_streamout( | ||
| 16510 | intel_sub_group_avc_ime_result_dual_reference_streamout_t result); | ||
| 16511 | |||
| 16512 | uint __ovld intel_sub_group_avc_ime_get_streamout_major_shape_motion_vectors( | ||
| 16513 | intel_sub_group_avc_ime_result_single_reference_streamout_t result, | ||
| 16514 | uchar major_shape); | ||
| 16515 | ushort __ovld intel_sub_group_avc_ime_get_streamout_major_shape_distortions( | ||
| 16516 | intel_sub_group_avc_ime_result_single_reference_streamout_t result, | ||
| 16517 | uchar major_shape); | ||
| 16518 | uchar __ovld intel_sub_group_avc_ime_get_streamout_major_shape_reference_ids( | ||
| 16519 | intel_sub_group_avc_ime_result_single_reference_streamout_t result, | ||
| 16520 | uchar major_shape); | ||
| 16521 | uint __ovld intel_sub_group_avc_ime_get_streamout_major_shape_motion_vectors( | ||
| 16522 | intel_sub_group_avc_ime_result_dual_reference_streamout_t result, | ||
| 16523 | uchar major_shape, uchar direction); | ||
| 16524 | ushort __ovld intel_sub_group_avc_ime_get_streamout_major_shape_distortions( | ||
| 16525 | intel_sub_group_avc_ime_result_dual_reference_streamout_t result, | ||
| 16526 | uchar major_shape, uchar direction); | ||
| 16527 | uchar __ovld intel_sub_group_avc_ime_get_streamout_major_shape_reference_ids( | ||
| 16528 | intel_sub_group_avc_ime_result_dual_reference_streamout_t result, | ||
| 16529 | uchar major_shape, uchar direction); | ||
| 16530 | |||
| 16531 | uchar __ovld intel_sub_group_avc_ime_get_border_reached( | ||
| 16532 | uchar image_select, intel_sub_group_avc_ime_result_t result); | ||
| 16533 | uchar __ovld intel_sub_group_avc_ime_get_truncated_search_indication( | ||
| 16534 | intel_sub_group_avc_ime_result_t result); | ||
| 16535 | uchar __ovld | ||
| 16536 | intel_sub_group_avc_ime_get_unidirectional_early_search_termination( | ||
| 16537 | intel_sub_group_avc_ime_result_t result); | ||
| 16538 | uint __ovld intel_sub_group_avc_ime_get_weighting_pattern_minimum_motion_vector( | ||
| 16539 | intel_sub_group_avc_ime_result_t result); | ||
| 16540 | ushort __ovld intel_sub_group_avc_ime_get_weighting_pattern_minimum_distortion( | ||
| 16541 | intel_sub_group_avc_ime_result_t result); | ||
| 16542 | |||
| 16543 | // REF built-in functions | ||
| 16544 | intel_sub_group_avc_ref_payload_t __ovld | ||
| 16545 | intel_sub_group_avc_fme_initialize( | ||
| 16546 | ushort2 src_coord, ulong motion_vectors, uchar major_shapes, | ||
| 16547 | uchar minor_shapes, uchar directions, uchar pixel_resolution, | ||
| 16548 | uchar sad_adjustment); | ||
| 16549 | intel_sub_group_avc_ref_payload_t __ovld | ||
| 16550 | intel_sub_group_avc_bme_initialize( | ||
| 16551 | ushort2 src_coord, ulong motion_vectors, uchar major_shapes, | ||
| 16552 | uchar minor_shapes, uchar directions, uchar pixel_resolution, | ||
| 16553 | uchar bidirectional_weight, uchar sad_adjustment); | ||
| 16554 | |||
| 16555 | intel_sub_group_avc_ref_payload_t __ovld | ||
| 16556 | intel_sub_group_avc_ref_set_bidirectional_mix_disable( | ||
| 16557 | intel_sub_group_avc_ref_payload_t payload); | ||
| 16558 | intel_sub_group_avc_ref_payload_t __ovld | ||
| 16559 | intel_sub_group_avc_ref_set_bilinear_filter_enable( | ||
| 16560 | intel_sub_group_avc_ref_payload_t payload); | ||
| 16561 | |||
| 16562 | intel_sub_group_avc_ref_result_t __ovld | ||
| 16563 | intel_sub_group_avc_ref_evaluate_with_single_reference( | ||
| 16564 | read_only image2d_t src_image, read_only image2d_t ref_image, | ||
| 16565 | sampler_t vme_media_sampler, intel_sub_group_avc_ref_payload_t payload); | ||
| 16566 | intel_sub_group_avc_ref_result_t __ovld | ||
| 16567 | intel_sub_group_avc_ref_evaluate_with_dual_reference( | ||
| 16568 | read_only image2d_t src_image, read_only image2d_t fwd_ref_image, | ||
| 16569 | read_only image2d_t bwd_ref_image, sampler_t vme_media_sampler, | ||
| 16570 | intel_sub_group_avc_ref_payload_t payload); | ||
| 16571 | intel_sub_group_avc_ref_result_t __ovld | ||
| 16572 | intel_sub_group_avc_ref_evaluate_with_multi_reference( | ||
| 16573 | read_only image2d_t src_image, uint packed_reference_ids, | ||
| 16574 | sampler_t vme_media_sampler, intel_sub_group_avc_ref_payload_t payload); | ||
| 16575 | intel_sub_group_avc_ref_result_t __ovld | ||
| 16576 | intel_sub_group_avc_ref_evaluate_with_multi_reference( | ||
| 16577 | read_only image2d_t src_image, uint packed_reference_ids, | ||
| 16578 | uchar packed_reference_field_polarities, sampler_t vme_media_sampler, | ||
| 16579 | intel_sub_group_avc_ref_payload_t payload); | ||
| 16580 | |||
| 16581 | // SIC built-in functions | ||
| 16582 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16583 | intel_sub_group_avc_sic_initialize( | ||
| 16584 | ushort2 src_coord); | ||
| 16585 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16586 | intel_sub_group_avc_sic_configure_skc( | ||
| 16587 | uint skip_block_partition_type, uint skip_motion_vector_mask, | ||
| 16588 | ulong motion_vectors, uchar bidirectional_weight, uchar skip_sad_adjustment, | ||
| 16589 | intel_sub_group_avc_sic_payload_t payload); | ||
| 16590 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16591 | intel_sub_group_avc_sic_configure_ipe( | ||
| 16592 | uchar luma_intra_partition_mask, uchar intra_neighbour_availabilty, | ||
| 16593 | uchar left_edge_luma_pixels, uchar upper_left_corner_luma_pixel, | ||
| 16594 | uchar upper_edge_luma_pixels, uchar upper_right_edge_luma_pixels, | ||
| 16595 | uchar intra_sad_adjustment, intel_sub_group_avc_sic_payload_t payload); | ||
| 16596 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16597 | intel_sub_group_avc_sic_configure_ipe( | ||
| 16598 | uchar luma_intra_partition_mask, uchar intra_neighbour_availabilty, | ||
| 16599 | uchar left_edge_luma_pixels, uchar upper_left_corner_luma_pixel, | ||
| 16600 | uchar upper_edge_luma_pixels, uchar upper_right_edge_luma_pixels, | ||
| 16601 | ushort left_edge_chroma_pixels, ushort upper_left_corner_chroma_pixel, | ||
| 16602 | ushort upper_edge_chroma_pixels, uchar intra_sad_adjustment, | ||
| 16603 | intel_sub_group_avc_sic_payload_t payload); | ||
| 16604 | uint __ovld | ||
| 16605 | intel_sub_group_avc_sic_get_motion_vector_mask( | ||
| 16606 | uint skip_block_partition_type, uchar direction); | ||
| 16607 | |||
| 16608 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16609 | intel_sub_group_avc_sic_set_intra_luma_shape_penalty( | ||
| 16610 | uint packed_shape_cost, intel_sub_group_avc_sic_payload_t payload); | ||
| 16611 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16612 | intel_sub_group_avc_sic_set_intra_luma_mode_cost_function( | ||
| 16613 | uchar luma_mode_penalty, uint luma_packed_neighbor_modes, | ||
| 16614 | uint luma_packed_non_dc_penalty, intel_sub_group_avc_sic_payload_t payload); | ||
| 16615 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16616 | intel_sub_group_avc_sic_set_intra_chroma_mode_cost_function( | ||
| 16617 | uchar chroma_mode_penalty, intel_sub_group_avc_sic_payload_t payload); | ||
| 16618 | |||
| 16619 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16620 | intel_sub_group_avc_sic_set_skc_bilinear_filter_enable( | ||
| 16621 | intel_sub_group_avc_sic_payload_t payload); | ||
| 16622 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16623 | intel_sub_group_avc_sic_set_skc_forward_transform_enable( | ||
| 16624 | ulong packed_sad_coefficients, intel_sub_group_avc_sic_payload_t payload); | ||
| 16625 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16626 | intel_sub_group_avc_sic_set_block_based_raw_skip_sad( | ||
| 16627 | uchar block_based_skip_type, | ||
| 16628 | intel_sub_group_avc_sic_payload_t payload); | ||
| 16629 | |||
| 16630 | intel_sub_group_avc_sic_result_t __ovld | ||
| 16631 | intel_sub_group_avc_sic_evaluate_ipe( | ||
| 16632 | read_only image2d_t src_image, sampler_t vme_media_sampler, | ||
| 16633 | intel_sub_group_avc_sic_payload_t payload); | ||
| 16634 | intel_sub_group_avc_sic_result_t __ovld | ||
| 16635 | intel_sub_group_avc_sic_evaluate_with_single_reference( | ||
| 16636 | read_only image2d_t src_image, read_only image2d_t ref_image, | ||
| 16637 | sampler_t vme_media_sampler, intel_sub_group_avc_sic_payload_t payload); | ||
| 16638 | intel_sub_group_avc_sic_result_t __ovld | ||
| 16639 | intel_sub_group_avc_sic_evaluate_with_dual_reference( | ||
| 16640 | read_only image2d_t src_image, read_only image2d_t fwd_ref_image, | ||
| 16641 | read_only image2d_t bwd_ref_image, sampler_t vme_media_sampler, | ||
| 16642 | intel_sub_group_avc_sic_payload_t payload); | ||
| 16643 | intel_sub_group_avc_sic_result_t __ovld | ||
| 16644 | intel_sub_group_avc_sic_evaluate_with_multi_reference( | ||
| 16645 | read_only image2d_t src_image, uint packed_reference_ids, | ||
| 16646 | sampler_t vme_media_sampler, intel_sub_group_avc_sic_payload_t payload); | ||
| 16647 | intel_sub_group_avc_sic_result_t __ovld | ||
| 16648 | intel_sub_group_avc_sic_evaluate_with_multi_reference( | ||
| 16649 | read_only image2d_t src_image, uint packed_reference_ids, | ||
| 16650 | uchar packed_reference_field_polarities, sampler_t vme_media_sampler, | ||
| 16651 | intel_sub_group_avc_sic_payload_t payload); | ||
| 16652 | |||
| 16653 | uchar __ovld intel_sub_group_avc_sic_get_ipe_luma_shape( | ||
| 16654 | intel_sub_group_avc_sic_result_t result); | ||
| 16655 | ushort __ovld intel_sub_group_avc_sic_get_best_ipe_luma_distortion( | ||
| 16656 | intel_sub_group_avc_sic_result_t result); | ||
| 16657 | ushort __ovld intel_sub_group_avc_sic_get_best_ipe_chroma_distortion( | ||
| 16658 | intel_sub_group_avc_sic_result_t result); | ||
| 16659 | ulong __ovld intel_sub_group_avc_sic_get_packed_ipe_luma_modes( | ||
| 16660 | intel_sub_group_avc_sic_result_t result); | ||
| 16661 | uchar __ovld intel_sub_group_avc_sic_get_ipe_chroma_mode( | ||
| 16662 | intel_sub_group_avc_sic_result_t result); | ||
| 16663 | uint __ovld intel_sub_group_avc_sic_get_packed_skc_luma_count_threshold( | ||
| 16664 | intel_sub_group_avc_sic_result_t result); | ||
| 16665 | ulong __ovld intel_sub_group_avc_sic_get_packed_skc_luma_sum_threshold( | ||
| 16666 | intel_sub_group_avc_sic_result_t result); | ||
| 16667 | ushort __ovld intel_sub_group_avc_sic_get_inter_raw_sads( | ||
| 16668 | intel_sub_group_avc_sic_result_t result); | ||
| 16669 | |||
| 16670 | // Wrappers | ||
| 16671 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16672 | intel_sub_group_avc_ime_set_inter_base_multi_reference_penalty( | ||
| 16673 | uchar reference_base_penalty, intel_sub_group_avc_ime_payload_t payload); | ||
| 16674 | intel_sub_group_avc_ref_payload_t __ovld | ||
| 16675 | intel_sub_group_avc_ref_set_inter_base_multi_reference_penalty( | ||
| 16676 | uchar reference_base_penalty, intel_sub_group_avc_ref_payload_t payload); | ||
| 16677 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16678 | intel_sub_group_avc_sic_set_inter_base_multi_reference_penalty( | ||
| 16679 | uchar reference_base_penalty, intel_sub_group_avc_sic_payload_t payload); | ||
| 16680 | |||
| 16681 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16682 | intel_sub_group_avc_ime_set_inter_shape_penalty( | ||
| 16683 | ulong packed_shape_cost, intel_sub_group_avc_ime_payload_t payload); | ||
| 16684 | intel_sub_group_avc_ref_payload_t __ovld | ||
| 16685 | intel_sub_group_avc_ref_set_inter_shape_penalty( | ||
| 16686 | ulong packed_shape_cost, intel_sub_group_avc_ref_payload_t payload); | ||
| 16687 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16688 | intel_sub_group_avc_sic_set_inter_shape_penalty( | ||
| 16689 | ulong packed_shape_cost, intel_sub_group_avc_sic_payload_t payload); | ||
| 16690 | |||
| 16691 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16692 | intel_sub_group_avc_ime_set_inter_direction_penalty( | ||
| 16693 | uchar direction_cost, intel_sub_group_avc_ime_payload_t payload); | ||
| 16694 | intel_sub_group_avc_ref_payload_t __ovld | ||
| 16695 | intel_sub_group_avc_ref_set_inter_direction_penalty( | ||
| 16696 | uchar direction_cost, intel_sub_group_avc_ref_payload_t payload); | ||
| 16697 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16698 | intel_sub_group_avc_sic_set_inter_direction_penalty( | ||
| 16699 | uchar direction_cost, intel_sub_group_avc_sic_payload_t payload); | ||
| 16700 | |||
| 16701 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16702 | intel_sub_group_avc_ime_set_motion_vector_cost_function( | ||
| 16703 | ulong packed_cost_center_delta, uint2 packed_cost_table, | ||
| 16704 | uchar cost_precision, intel_sub_group_avc_ime_payload_t payload); | ||
| 16705 | intel_sub_group_avc_ref_payload_t __ovld | ||
| 16706 | intel_sub_group_avc_ref_set_motion_vector_cost_function( | ||
| 16707 | ulong packed_cost_center_delta, uint2 packed_cost_table, | ||
| 16708 | uchar cost_precision, intel_sub_group_avc_ref_payload_t payload); | ||
| 16709 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16710 | intel_sub_group_avc_sic_set_motion_vector_cost_function( | ||
| 16711 | ulong packed_cost_center_delta, uint2 packed_cost_table, | ||
| 16712 | uchar cost_precision, intel_sub_group_avc_sic_payload_t payload); | ||
| 16713 | |||
| 16714 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16715 | intel_sub_group_avc_ime_set_source_interlaced_field_polarity( | ||
| 16716 | uchar src_field_polarity, intel_sub_group_avc_ime_payload_t payload); | ||
| 16717 | intel_sub_group_avc_ref_payload_t __ovld | ||
| 16718 | intel_sub_group_avc_ref_set_source_interlaced_field_polarity( | ||
| 16719 | uchar src_field_polarity, intel_sub_group_avc_ref_payload_t payload); | ||
| 16720 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16721 | intel_sub_group_avc_sic_set_source_interlaced_field_polarity( | ||
| 16722 | uchar src_field_polarity, intel_sub_group_avc_sic_payload_t payload); | ||
| 16723 | |||
| 16724 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16725 | intel_sub_group_avc_ime_set_single_reference_interlaced_field_polarity( | ||
| 16726 | uchar ref_field_polarity, intel_sub_group_avc_ime_payload_t payload); | ||
| 16727 | intel_sub_group_avc_ref_payload_t __ovld | ||
| 16728 | intel_sub_group_avc_ref_set_single_reference_interlaced_field_polarity( | ||
| 16729 | uchar ref_field_polarity, intel_sub_group_avc_ref_payload_t payload); | ||
| 16730 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16731 | intel_sub_group_avc_sic_set_single_reference_interlaced_field_polarity( | ||
| 16732 | uchar ref_field_polarity, intel_sub_group_avc_sic_payload_t payload); | ||
| 16733 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16734 | intel_sub_group_avc_ime_set_dual_reference_interlaced_field_polarities( | ||
| 16735 | uchar fwd_ref_field_polarity, uchar bwd_ref_field_polarity, | ||
| 16736 | intel_sub_group_avc_ime_payload_t payload); | ||
| 16737 | intel_sub_group_avc_ref_payload_t __ovld | ||
| 16738 | intel_sub_group_avc_ref_set_dual_reference_interlaced_field_polarities( | ||
| 16739 | uchar fwd_ref_field_polarity, uchar bwd_ref_field_polarity, | ||
| 16740 | intel_sub_group_avc_ref_payload_t payload); | ||
| 16741 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16742 | intel_sub_group_avc_sic_set_dual_reference_interlaced_field_polarities( | ||
| 16743 | uchar fwd_ref_field_polarity, uchar bwd_ref_field_polarity, | ||
| 16744 | intel_sub_group_avc_sic_payload_t payload); | ||
| 16745 | |||
| 16746 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16747 | intel_sub_group_avc_ime_set_ac_only_haar( | ||
| 16748 | intel_sub_group_avc_ime_payload_t payload); | ||
| 16749 | intel_sub_group_avc_ref_payload_t __ovld | ||
| 16750 | intel_sub_group_avc_ref_set_ac_only_haar( | ||
| 16751 | intel_sub_group_avc_ref_payload_t payload); | ||
| 16752 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16753 | intel_sub_group_avc_sic_set_ac_only_haar( | ||
| 16754 | intel_sub_group_avc_sic_payload_t payload); | ||
| 16755 | |||
| 16756 | ulong __ovld intel_sub_group_avc_ime_get_motion_vectors( | ||
| 16757 | intel_sub_group_avc_ime_result_t result); | ||
| 16758 | ulong __ovld intel_sub_group_avc_ref_get_motion_vectors( | ||
| 16759 | intel_sub_group_avc_ref_result_t result); | ||
| 16760 | |||
| 16761 | ushort __ovld intel_sub_group_avc_ime_get_inter_distortions( | ||
| 16762 | intel_sub_group_avc_ime_result_t result); | ||
| 16763 | ushort __ovld intel_sub_group_avc_ref_get_inter_distortions( | ||
| 16764 | intel_sub_group_avc_ref_result_t result); | ||
| 16765 | ushort __ovld intel_sub_group_avc_sic_get_inter_distortions( | ||
| 16766 | intel_sub_group_avc_sic_result_t result); | ||
| 16767 | |||
| 16768 | ushort __ovld intel_sub_group_avc_ime_get_best_inter_distortion( | ||
| 16769 | intel_sub_group_avc_ime_result_t result); | ||
| 16770 | ushort __ovld intel_sub_group_avc_ref_get_best_inter_distortion( | ||
| 16771 | intel_sub_group_avc_ref_result_t result); | ||
| 16772 | |||
| 16773 | uchar __ovld intel_sub_group_avc_ime_get_inter_major_shape( | ||
| 16774 | intel_sub_group_avc_ime_result_t result); | ||
| 16775 | uchar __ovld intel_sub_group_avc_ref_get_inter_major_shape( | ||
| 16776 | intel_sub_group_avc_ref_result_t result); | ||
| 16777 | uchar __ovld intel_sub_group_avc_ime_get_inter_minor_shapes( | ||
| 16778 | intel_sub_group_avc_ime_result_t result); | ||
| 16779 | uchar __ovld intel_sub_group_avc_ref_get_inter_minor_shapes( | ||
| 16780 | intel_sub_group_avc_ref_result_t result); | ||
| 16781 | |||
| 16782 | uchar __ovld intel_sub_group_avc_ime_get_inter_directions( | ||
| 16783 | intel_sub_group_avc_ime_result_t result); | ||
| 16784 | uchar __ovld intel_sub_group_avc_ref_get_inter_directions( | ||
| 16785 | intel_sub_group_avc_ref_result_t result); | ||
| 16786 | |||
| 16787 | uchar __ovld intel_sub_group_avc_ime_get_inter_motion_vector_count( | ||
| 16788 | intel_sub_group_avc_ime_result_t result); | ||
| 16789 | uchar __ovld intel_sub_group_avc_ref_get_inter_motion_vector_count( | ||
| 16790 | intel_sub_group_avc_ref_result_t result); | ||
| 16791 | |||
| 16792 | uint __ovld intel_sub_group_avc_ime_get_inter_reference_ids( | ||
| 16793 | intel_sub_group_avc_ime_result_t result); | ||
| 16794 | uint __ovld intel_sub_group_avc_ref_get_inter_reference_ids( | ||
| 16795 | intel_sub_group_avc_ref_result_t result); | ||
| 16796 | |||
| 16797 | uchar __ovld | ||
| 16798 | intel_sub_group_avc_ime_get_inter_reference_interlaced_field_polarities( | ||
| 16799 | uint packed_reference_ids, uint packed_reference_parameter_field_polarities, | ||
| 16800 | intel_sub_group_avc_ime_result_t result); | ||
| 16801 | uchar __ovld | ||
| 16802 | intel_sub_group_avc_ref_get_inter_reference_interlaced_field_polarities( | ||
| 16803 | uint packed_reference_ids, uint packed_reference_parameter_field_polarities, | ||
| 16804 | intel_sub_group_avc_ref_result_t result); | ||
| 16805 | |||
| 16806 | // Type conversion functions | ||
| 16807 | intel_sub_group_avc_mce_payload_t __ovld | ||
| 16808 | intel_sub_group_avc_ime_convert_to_mce_payload( | ||
| 16809 | intel_sub_group_avc_ime_payload_t payload); | ||
| 16810 | intel_sub_group_avc_ime_payload_t __ovld | ||
| 16811 | intel_sub_group_avc_mce_convert_to_ime_payload( | ||
| 16812 | intel_sub_group_avc_mce_payload_t payload); | ||
| 16813 | intel_sub_group_avc_mce_payload_t __ovld | ||
| 16814 | intel_sub_group_avc_ref_convert_to_mce_payload( | ||
| 16815 | intel_sub_group_avc_ref_payload_t payload); | ||
| 16816 | intel_sub_group_avc_ref_payload_t __ovld | ||
| 16817 | intel_sub_group_avc_mce_convert_to_ref_payload( | ||
| 16818 | intel_sub_group_avc_mce_payload_t payload); | ||
| 16819 | intel_sub_group_avc_mce_payload_t __ovld | ||
| 16820 | intel_sub_group_avc_sic_convert_to_mce_payload( | ||
| 16821 | intel_sub_group_avc_sic_payload_t payload); | ||
| 16822 | intel_sub_group_avc_sic_payload_t __ovld | ||
| 16823 | intel_sub_group_avc_mce_convert_to_sic_payload( | ||
| 16824 | intel_sub_group_avc_mce_payload_t payload); | ||
| 16825 | |||
| 16826 | intel_sub_group_avc_mce_result_t __ovld | ||
| 16827 | intel_sub_group_avc_ime_convert_to_mce_result( | ||
| 16828 | intel_sub_group_avc_ime_result_t result); | ||
| 16829 | intel_sub_group_avc_ime_result_t __ovld | ||
| 16830 | intel_sub_group_avc_mce_convert_to_ime_result( | ||
| 16831 | intel_sub_group_avc_mce_result_t result); | ||
| 16832 | intel_sub_group_avc_mce_result_t __ovld | ||
| 16833 | intel_sub_group_avc_ref_convert_to_mce_result( | ||
| 16834 | intel_sub_group_avc_ref_result_t result); | ||
| 16835 | intel_sub_group_avc_ref_result_t __ovld | ||
| 16836 | intel_sub_group_avc_mce_convert_to_ref_result( | ||
| 16837 | intel_sub_group_avc_mce_result_t result); | ||
| 16838 | intel_sub_group_avc_mce_result_t __ovld | ||
| 16839 | intel_sub_group_avc_sic_convert_to_mce_result( | ||
| 16840 | intel_sub_group_avc_sic_result_t result); | ||
| 16841 | intel_sub_group_avc_sic_result_t __ovld | ||
| 16842 | intel_sub_group_avc_mce_convert_to_sic_result( | ||
| 16843 | intel_sub_group_avc_mce_result_t result); | ||
| 16844 | #pragma OPENCL EXTENSION cl_intel_device_side_avc_motion_estimation : end | ||
| 16845 | #endif // cl_intel_device_side_avc_motion_estimation | ||
| 16846 | |||
| 16196 | #ifdef cl_amd_media_ops | 16847 | #ifdef cl_amd_media_ops |
| 16197 | uint __ovld amd_bitalign(uint a, uint b, uint c); | 16848 | uint __ovld amd_bitalign(uint a, uint b, uint c); |
| 16198 | uint2 __ovld amd_bitalign(uint2 a, uint2 b, uint2 c); | 16849 | uint2 __ovld amd_bitalign(uint2 a, uint2 b, uint2 c); |
c_headers/vecintrin.h+3-3
| ... | @@ -381,7 +381,7 @@ vec_insert_and_zero(const unsigned long long *__ptr) { | ... | @@ -381,7 +381,7 @@ vec_insert_and_zero(const unsigned long long *__ptr) { |
| 381 | static inline __ATTRS_o_ai vector float | 381 | static inline __ATTRS_o_ai vector float |
| 382 | vec_insert_and_zero(const float *__ptr) { | 382 | vec_insert_and_zero(const float *__ptr) { |
| 383 | vector float __vec = (vector float)0; | 383 | vector float __vec = (vector float)0; |
| 384 | __vec[0] = *__ptr; | 384 | __vec[1] = *__ptr; |
| 385 | return __vec; | 385 | return __vec; |
| 386 | } | 386 | } |
| 387 | #endif | 387 | #endif |
| ... | @@ -5942,13 +5942,13 @@ vec_orc(vector unsigned long long __a, vector unsigned long long __b) { | ... | @@ -5942,13 +5942,13 @@ vec_orc(vector unsigned long long __a, vector unsigned long long __b) { |
| 5942 | 5942 | ||
| 5943 | static inline __ATTRS_o_ai vector float | 5943 | static inline __ATTRS_o_ai vector float |
| 5944 | vec_orc(vector float __a, vector float __b) { | 5944 | vec_orc(vector float __a, vector float __b) { |
| 5945 | return (vector float)((vector unsigned int)__a & | 5945 | return (vector float)((vector unsigned int)__a | |
| 5946 | ~(vector unsigned int)__b); | 5946 | ~(vector unsigned int)__b); |
| 5947 | } | 5947 | } |
| 5948 | 5948 | ||
| 5949 | static inline __ATTRS_o_ai vector double | 5949 | static inline __ATTRS_o_ai vector double |
| 5950 | vec_orc(vector double __a, vector double __b) { | 5950 | vec_orc(vector double __a, vector double __b) { |
| 5951 | return (vector double)((vector unsigned long long)__a & | 5951 | return (vector double)((vector unsigned long long)__a | |
| 5952 | ~(vector unsigned long long)__b); | 5952 | ~(vector unsigned long long)__b); |
| 5953 | } | 5953 | } |
| 5954 | #endif | 5954 | #endif |