| author |  Andrew Kelley <andrew@ziglang.org> 2023-08-11 16:20:14-07:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2023-09-19 09:37:31-07:00 |
| log | 1861036f3b61828d0b7641dcce65f37e516fd3f6 |
| tree | 30fb5a5dfec4e83c176d10296ee78d97318c29eb |
| parent | 3ed40b114020ff774279a5af993851d74da64b52 |
update C language headers to LLVM 17
release/17.x branch, commit 8f4dd44097c9ae25dd203d5ac87f3b48f854bba849 files changed, 8865 insertions(+), 401 deletions(-)
lib- include
- __clang_cuda_intrinsics.h
- __clang_hip_cmath.h
- __clang_hip_libdevice_declares.h
- __clang_hip_math.h
- __clang_hip_runtime_wrapper.h
- adxintrin.h
- altivec.h
- amxcomplexintrin.h
- arm_acle.h
- arm_neon.h
- arm_sme_draft_spec_subject_to_change.h
- arm_sve.h
- avx2intrin.h
- avx512fintrin.h
- avx512fp16intrin.h
- avxintrin.h
- avxvnniint16intrin.h
- bmi2intrin.h
- clflushoptintrin.h
- clzerointrin.h
- cuda_wrappers
- fmaintrin.h
- immintrin.h
- limits.h
- llvm_libc_wrappers
- mwaitxintrin.h
- opencl-c-base.h
- openmp_wrappers
- pmmintrin.h
- ppc_wrappers
- rdseedintrin.h
- riscv_ntlh.h
- riscv_vector.h
- sha512intrin.h
- shaintrin.h
- sifive_vector.h
- sm3intrin.h
- sm4intrin.h
- stdalign.h
- stdatomic.h
- stddef.h
- wasm_simd128.h
- xsavecintrin.h
lib/include/__clang_cuda_intrinsics.h+191
| ... | @@ -513,6 +513,197 @@ __device__ inline cuuint32_t __nvvm_get_smem_pointer(void *__ptr) { | ... | @@ -513,6 +513,197 @@ __device__ inline cuuint32_t __nvvm_get_smem_pointer(void *__ptr) { |
| 513 | return __nv_cvta_generic_to_shared_impl(__ptr); | 513 | return __nv_cvta_generic_to_shared_impl(__ptr); |
| 514 | } | 514 | } |
| 515 | } // extern "C" | 515 | } // extern "C" |
| 516 | |||
| 517 | #if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800 | ||
| 518 | __device__ inline unsigned __reduce_add_sync(unsigned __mask, | ||
| 519 | unsigned __value) { | ||
| 520 | return __nvvm_redux_sync_add(__mask, __value); | ||
| 521 | } | ||
| 522 | __device__ inline unsigned __reduce_min_sync(unsigned __mask, | ||
| 523 | unsigned __value) { | ||
| 524 | return __nvvm_redux_sync_umin(__mask, __value); | ||
| 525 | } | ||
| 526 | __device__ inline unsigned __reduce_max_sync(unsigned __mask, | ||
| 527 | unsigned __value) { | ||
| 528 | return __nvvm_redux_sync_umax(__mask, __value); | ||
| 529 | } | ||
| 530 | __device__ inline int __reduce_min_sync(unsigned __mask, int __value) { | ||
| 531 | return __nvvm_redux_sync_min(__mask, __value); | ||
| 532 | } | ||
| 533 | __device__ inline int __reduce_max_sync(unsigned __mask, int __value) { | ||
| 534 | return __nvvm_redux_sync_max(__mask, __value); | ||
| 535 | } | ||
| 536 | __device__ inline unsigned __reduce_or_sync(unsigned __mask, unsigned __value) { | ||
| 537 | return __nvvm_redux_sync_or(__mask, __value); | ||
| 538 | } | ||
| 539 | __device__ inline unsigned __reduce_and_sync(unsigned __mask, | ||
| 540 | unsigned __value) { | ||
| 541 | return __nvvm_redux_sync_and(__mask, __value); | ||
| 542 | } | ||
| 543 | __device__ inline unsigned __reduce_xor_sync(unsigned __mask, | ||
| 544 | unsigned __value) { | ||
| 545 | return __nvvm_redux_sync_xor(__mask, __value); | ||
| 546 | } | ||
| 547 | |||
| 548 | __device__ inline void __nv_memcpy_async_shared_global_4(void *__dst, | ||
| 549 | const void *__src, | ||
| 550 | unsigned __src_size) { | ||
| 551 | __nvvm_cp_async_ca_shared_global_4( | ||
| 552 | (void __attribute__((address_space(3))) *)__dst, | ||
| 553 | (const void __attribute__((address_space(1))) *)__src, __src_size); | ||
| 554 | } | ||
| 555 | __device__ inline void __nv_memcpy_async_shared_global_8(void *__dst, | ||
| 556 | const void *__src, | ||
| 557 | unsigned __src_size) { | ||
| 558 | __nvvm_cp_async_ca_shared_global_8( | ||
| 559 | (void __attribute__((address_space(3))) *)__dst, | ||
| 560 | (const void __attribute__((address_space(1))) *)__src, __src_size); | ||
| 561 | } | ||
| 562 | __device__ inline void __nv_memcpy_async_shared_global_16(void *__dst, | ||
| 563 | const void *__src, | ||
| 564 | unsigned __src_size) { | ||
| 565 | __nvvm_cp_async_ca_shared_global_16( | ||
| 566 | (void __attribute__((address_space(3))) *)__dst, | ||
| 567 | (const void __attribute__((address_space(1))) *)__src, __src_size); | ||
| 568 | } | ||
| 569 | |||
| 570 | __device__ inline void * | ||
| 571 | __nv_associate_access_property(const void *__ptr, unsigned long long __prop) { | ||
| 572 | // TODO: it appears to provide compiler with some sort of a hint. We do not | ||
| 573 | // know what exactly it is supposed to do. However, CUDA headers suggest that | ||
| 574 | // just passing through __ptr should not affect correctness. They do so on | ||
| 575 | // pre-sm80 GPUs where this builtin is not available. | ||
| 576 | return (void*)__ptr; | ||
| 577 | } | ||
| 578 | #endif // !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800 | ||
| 579 | |||
| 580 | #if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 900 | ||
| 581 | __device__ inline unsigned __isCtaShared(const void *ptr) { | ||
| 582 | return __isShared(ptr); | ||
| 583 | } | ||
| 584 | |||
| 585 | __device__ inline unsigned __isClusterShared(const void *__ptr) { | ||
| 586 | return __nvvm_isspacep_shared_cluster(__ptr); | ||
| 587 | } | ||
| 588 | |||
| 589 | __device__ inline void *__cluster_map_shared_rank(const void *__ptr, | ||
| 590 | unsigned __rank) { | ||
| 591 | return __nvvm_mapa((void *)__ptr, __rank); | ||
| 592 | } | ||
| 593 | |||
| 594 | __device__ inline unsigned __cluster_query_shared_rank(const void *__ptr) { | ||
| 595 | return __nvvm_getctarank((void *)__ptr); | ||
| 596 | } | ||
| 597 | |||
| 598 | __device__ inline uint2 | ||
| 599 | __cluster_map_shared_multicast(const void *__ptr, | ||
| 600 | unsigned int __cluster_cta_mask) { | ||
| 601 | return make_uint2((unsigned)__cvta_generic_to_shared(__ptr), | ||
| 602 | __cluster_cta_mask); | ||
| 603 | } | ||
| 604 | |||
| 605 | __device__ inline unsigned __clusterDimIsSpecified() { | ||
| 606 | return __nvvm_is_explicit_cluster(); | ||
| 607 | } | ||
| 608 | |||
| 609 | __device__ inline dim3 __clusterDim() { | ||
| 610 | return dim3(__nvvm_read_ptx_sreg_cluster_nctaid_x(), | ||
| 611 | __nvvm_read_ptx_sreg_cluster_nctaid_y(), | ||
| 612 | __nvvm_read_ptx_sreg_cluster_nctaid_z()); | ||
| 613 | } | ||
| 614 | |||
| 615 | __device__ inline dim3 __clusterRelativeBlockIdx() { | ||
| 616 | return dim3(__nvvm_read_ptx_sreg_cluster_ctaid_x(), | ||
| 617 | __nvvm_read_ptx_sreg_cluster_ctaid_y(), | ||
| 618 | __nvvm_read_ptx_sreg_cluster_ctaid_z()); | ||
| 619 | } | ||
| 620 | |||
| 621 | __device__ inline dim3 __clusterGridDimInClusters() { | ||
| 622 | return dim3(__nvvm_read_ptx_sreg_nclusterid_x(), | ||
| 623 | __nvvm_read_ptx_sreg_nclusterid_y(), | ||
| 624 | __nvvm_read_ptx_sreg_nclusterid_z()); | ||
| 625 | } | ||
| 626 | |||
| 627 | __device__ inline dim3 __clusterIdx() { | ||
| 628 | return dim3(__nvvm_read_ptx_sreg_clusterid_x(), | ||
| 629 | __nvvm_read_ptx_sreg_clusterid_y(), | ||
| 630 | __nvvm_read_ptx_sreg_clusterid_z()); | ||
| 631 | } | ||
| 632 | |||
| 633 | __device__ inline unsigned __clusterRelativeBlockRank() { | ||
| 634 | return __nvvm_read_ptx_sreg_cluster_ctarank(); | ||
| 635 | } | ||
| 636 | |||
| 637 | __device__ inline unsigned __clusterSizeInBlocks() { | ||
| 638 | return __nvvm_read_ptx_sreg_cluster_nctarank(); | ||
| 639 | } | ||
| 640 | |||
| 641 | __device__ inline void __cluster_barrier_arrive() { | ||
| 642 | __nvvm_barrier_cluster_arrive(); | ||
| 643 | } | ||
| 644 | |||
| 645 | __device__ inline void __cluster_barrier_arrive_relaxed() { | ||
| 646 | __nvvm_barrier_cluster_arrive_relaxed(); | ||
| 647 | } | ||
| 648 | |||
| 649 | __device__ inline void __cluster_barrier_wait() { | ||
| 650 | __nvvm_barrier_cluster_wait(); | ||
| 651 | } | ||
| 652 | |||
| 653 | __device__ inline void __threadfence_cluster() { __nvvm_fence_sc_cluster(); } | ||
| 654 | |||
| 655 | __device__ inline float2 atomicAdd(float2 *__ptr, float2 __val) { | ||
| 656 | float2 __ret; | ||
| 657 | __asm__("atom.add.v2.f32 {%0, %1}, [%2], {%3, %4};" | ||
| 658 | : "=f"(__ret.x), "=f"(__ret.y) | ||
| 659 | : "l"(__ptr), "f"(__val.x), "f"(__val.y)); | ||
| 660 | return __ret; | ||
| 661 | } | ||
| 662 | |||
| 663 | __device__ inline float2 atomicAdd_block(float2 *__ptr, float2 __val) { | ||
| 664 | float2 __ret; | ||
| 665 | __asm__("atom.cta.add.v2.f32 {%0, %1}, [%2], {%3, %4};" | ||
| 666 | : "=f"(__ret.x), "=f"(__ret.y) | ||
| 667 | : "l"(__ptr), "f"(__val.x), "f"(__val.y)); | ||
| 668 | return __ret; | ||
| 669 | } | ||
| 670 | |||
| 671 | __device__ inline float2 atomicAdd_system(float2 *__ptr, float2 __val) { | ||
| 672 | float2 __ret; | ||
| 673 | __asm__("atom.sys.add.v2.f32 {%0, %1}, [%2], {%3, %4};" | ||
| 674 | : "=f"(__ret.x), "=f"(__ret.y) | ||
| 675 | : "l"(__ptr), "f"(__val.x), "f"(__val.y)); | ||
| 676 | return __ret; | ||
| 677 | } | ||
| 678 | |||
| 679 | __device__ inline float4 atomicAdd(float4 *__ptr, float4 __val) { | ||
| 680 | float4 __ret; | ||
| 681 | __asm__("atom.add.v4.f32 {%0, %1, %2, %3}, [%4], {%5, %6, %7, %8};" | ||
| 682 | : "=f"(__ret.x), "=f"(__ret.y), "=f"(__ret.z), "=f"(__ret.w) | ||
| 683 | : "l"(__ptr), "f"(__val.x), "f"(__val.y), "f"(__val.z), "f"(__val.w)); | ||
| 684 | return __ret; | ||
| 685 | } | ||
| 686 | |||
| 687 | __device__ inline float4 atomicAdd_block(float4 *__ptr, float4 __val) { | ||
| 688 | float4 __ret; | ||
| 689 | __asm__( | ||
| 690 | "atom.cta.add.v4.f32 {%0, %1, %2, %3}, [%4], {%5, %6, %7, %8};" | ||
| 691 | : "=f"(__ret.x), "=f"(__ret.y), "=f"(__ret.z), "=f"(__ret.w) | ||
| 692 | : "l"(__ptr), "f"(__val.x), "f"(__val.y), "f"(__val.z), "f"(__val.w)); | ||
| 693 | return __ret; | ||
| 694 | } | ||
| 695 | |||
| 696 | __device__ inline float4 atomicAdd_system(float4 *__ptr, float4 __val) { | ||
| 697 | float4 __ret; | ||
| 698 | __asm__( | ||
| 699 | "atom.sys.add.v4.f32 {%0, %1, %2, %3}, [%4], {%5, %6, %7, %8};" | ||
| 700 | : "=f"(__ret.x), "=f"(__ret.y), "=f"(__ret.z), "=f"(__ret.w) | ||
| 701 | : "l"(__ptr), "f"(__val.x), "f"(__val.y), "f"(__val.z), "f"(__val.w) | ||
| 702 | :); | ||
| 703 | return __ret; | ||
| 704 | } | ||
| 705 | |||
| 706 | #endif // !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 900 | ||
| 516 | #endif // CUDA_VERSION >= 11000 | 707 | #endif // CUDA_VERSION >= 11000 |
| 517 | 708 | ||
| 518 | #endif // defined(__CLANG_CUDA_INTRINSICS_H__) | 709 | #endif // defined(__CLANG_CUDA_INTRINSICS_H__) |
lib/include/__clang_hip_cmath.h+1-1
| ... | @@ -171,7 +171,7 @@ __DEVICE__ __CONSTEXPR__ bool signbit(double __x) { return ::__signbit(__x); } | ... | @@ -171,7 +171,7 @@ __DEVICE__ __CONSTEXPR__ bool signbit(double __x) { return ::__signbit(__x); } |
| 171 | // Other functions. | 171 | // Other functions. |
| 172 | __DEVICE__ __CONSTEXPR__ _Float16 fma(_Float16 __x, _Float16 __y, | 172 | __DEVICE__ __CONSTEXPR__ _Float16 fma(_Float16 __x, _Float16 __y, |
| 173 | _Float16 __z) { | 173 | _Float16 __z) { |
| 174 | return __ocml_fma_f16(__x, __y, __z); | 174 | return __builtin_fmaf16(__x, __y, __z); |
| 175 | } | 175 | } |
| 176 | __DEVICE__ __CONSTEXPR__ _Float16 pow(_Float16 __base, int __iexp) { | 176 | __DEVICE__ __CONSTEXPR__ _Float16 pow(_Float16 __base, int __iexp) { |
| 177 | return __ocml_pown_f16(__base, __iexp); | 177 | return __ocml_pown_f16(__base, __iexp); |
lib/include/__clang_hip_libdevice_declares.h+32-30
| ... | @@ -10,6 +10,10 @@ | ... | @@ -10,6 +10,10 @@ |
| 10 | #ifndef __CLANG_HIP_LIBDEVICE_DECLARES_H__ | 10 | #ifndef __CLANG_HIP_LIBDEVICE_DECLARES_H__ |
| 11 | #define __CLANG_HIP_LIBDEVICE_DECLARES_H__ | 11 | #define __CLANG_HIP_LIBDEVICE_DECLARES_H__ |
| 12 | 12 | ||
| 13 | #if !defined(__HIPCC_RTC__) && __has_include("hip/hip_version.h") | ||
| 14 | #include "hip/hip_version.h" | ||
| 15 | #endif // __has_include("hip/hip_version.h") | ||
| 16 | |||
| 13 | #ifdef __cplusplus | 17 | #ifdef __cplusplus |
| 14 | extern "C" { | 18 | extern "C" { |
| 15 | #endif | 19 | #endif |
| ... | @@ -137,23 +141,6 @@ __device__ __attribute__((const)) float __ocml_fma_rte_f32(float, float, float); | ... | @@ -137,23 +141,6 @@ __device__ __attribute__((const)) float __ocml_fma_rte_f32(float, float, float); |
| 137 | __device__ __attribute__((const)) float __ocml_fma_rtn_f32(float, float, float); | 141 | __device__ __attribute__((const)) float __ocml_fma_rtn_f32(float, float, float); |
| 138 | __device__ __attribute__((const)) float __ocml_fma_rtp_f32(float, float, float); | 142 | __device__ __attribute__((const)) float __ocml_fma_rtp_f32(float, float, float); |
| 139 | __device__ __attribute__((const)) float __ocml_fma_rtz_f32(float, float, float); | 143 | __device__ __attribute__((const)) float __ocml_fma_rtz_f32(float, float, float); |
| 140 | |||
| 141 | __device__ inline __attribute__((const)) float | ||
| 142 | __llvm_amdgcn_cos_f32(float __x) { | ||
| 143 | return __builtin_amdgcn_cosf(__x); | ||
| 144 | } | ||
| 145 | __device__ inline __attribute__((const)) float | ||
| 146 | __llvm_amdgcn_rcp_f32(float __x) { | ||
| 147 | return __builtin_amdgcn_rcpf(__x); | ||
| 148 | } | ||
| 149 | __device__ inline __attribute__((const)) float | ||
| 150 | __llvm_amdgcn_rsq_f32(float __x) { | ||
| 151 | return __builtin_amdgcn_rsqf(__x); | ||
| 152 | } | ||
| 153 | __device__ inline __attribute__((const)) float | ||
| 154 | __llvm_amdgcn_sin_f32(float __x) { | ||
| 155 | return __builtin_amdgcn_sinf(__x); | ||
| 156 | } | ||
| 157 | // END INTRINSICS | 144 | // END INTRINSICS |
| 158 | // END FLOAT | 145 | // END FLOAT |
| 159 | 146 | ||
| ... | @@ -277,15 +264,6 @@ __device__ __attribute__((const)) double __ocml_fma_rtp_f64(double, double, | ... | @@ -277,15 +264,6 @@ __device__ __attribute__((const)) double __ocml_fma_rtp_f64(double, double, |
| 277 | __device__ __attribute__((const)) double __ocml_fma_rtz_f64(double, double, | 264 | __device__ __attribute__((const)) double __ocml_fma_rtz_f64(double, double, |
| 278 | double); | 265 | double); |
| 279 | 266 | ||
| 280 | __device__ inline __attribute__((const)) double | ||
| 281 | __llvm_amdgcn_rcp_f64(double __x) { | ||
| 282 | return __builtin_amdgcn_rcp(__x); | ||
| 283 | } | ||
| 284 | __device__ inline __attribute__((const)) double | ||
| 285 | __llvm_amdgcn_rsq_f64(double __x) { | ||
| 286 | return __builtin_amdgcn_rsq(__x); | ||
| 287 | } | ||
| 288 | |||
| 289 | __device__ __attribute__((const)) _Float16 __ocml_ceil_f16(_Float16); | 267 | __device__ __attribute__((const)) _Float16 __ocml_ceil_f16(_Float16); |
| 290 | __device__ _Float16 __ocml_cos_f16(_Float16); | 268 | __device__ _Float16 __ocml_cos_f16(_Float16); |
| 291 | __device__ __attribute__((const)) _Float16 __ocml_cvtrtn_f16_f32(float); | 269 | __device__ __attribute__((const)) _Float16 __ocml_cvtrtn_f16_f32(float); |
| ... | @@ -305,7 +283,6 @@ __device__ __attribute__((const)) int __ocml_isnan_f16(_Float16); | ... | @@ -305,7 +283,6 @@ __device__ __attribute__((const)) int __ocml_isnan_f16(_Float16); |
| 305 | __device__ __attribute__((pure)) _Float16 __ocml_log_f16(_Float16); | 283 | __device__ __attribute__((pure)) _Float16 __ocml_log_f16(_Float16); |
| 306 | __device__ __attribute__((pure)) _Float16 __ocml_log10_f16(_Float16); | 284 | __device__ __attribute__((pure)) _Float16 __ocml_log10_f16(_Float16); |
| 307 | __device__ __attribute__((pure)) _Float16 __ocml_log2_f16(_Float16); | 285 | __device__ __attribute__((pure)) _Float16 __ocml_log2_f16(_Float16); |
| 308 | __device__ __attribute__((const)) _Float16 __llvm_amdgcn_rcp_f16(_Float16); | ||
| 309 | __device__ __attribute__((const)) _Float16 __ocml_rint_f16(_Float16); | 286 | __device__ __attribute__((const)) _Float16 __ocml_rint_f16(_Float16); |
| 310 | __device__ __attribute__((const)) _Float16 __ocml_rsqrt_f16(_Float16); | 287 | __device__ __attribute__((const)) _Float16 __ocml_rsqrt_f16(_Float16); |
| 311 | __device__ _Float16 __ocml_sin_f16(_Float16); | 288 | __device__ _Float16 __ocml_sin_f16(_Float16); |
| ... | @@ -316,8 +293,15 @@ __device__ __attribute__((pure)) _Float16 __ocml_pown_f16(_Float16, int); | ... | @@ -316,8 +293,15 @@ __device__ __attribute__((pure)) _Float16 __ocml_pown_f16(_Float16, int); |
| 316 | typedef _Float16 __2f16 __attribute__((ext_vector_type(2))); | 293 | typedef _Float16 __2f16 __attribute__((ext_vector_type(2))); |
| 317 | typedef short __2i16 __attribute__((ext_vector_type(2))); | 294 | typedef short __2i16 __attribute__((ext_vector_type(2))); |
| 318 | 295 | ||
| 296 | // We need to match C99's bool and get an i1 in the IR. | ||
| 297 | #ifdef __cplusplus | ||
| 298 | typedef bool __ockl_bool; | ||
| 299 | #else | ||
| 300 | typedef _Bool __ockl_bool; | ||
| 301 | #endif | ||
| 302 | |||
| 319 | __device__ __attribute__((const)) float __ockl_fdot2(__2f16 a, __2f16 b, | 303 | __device__ __attribute__((const)) float __ockl_fdot2(__2f16 a, __2f16 b, |
| 320 | float c, bool s); | 304 | float c, __ockl_bool s); |
| 321 | __device__ __attribute__((const)) __2f16 __ocml_ceil_2f16(__2f16); | 305 | __device__ __attribute__((const)) __2f16 __ocml_ceil_2f16(__2f16); |
| 322 | __device__ __attribute__((const)) __2f16 __ocml_fabs_2f16(__2f16); | 306 | __device__ __attribute__((const)) __2f16 __ocml_fabs_2f16(__2f16); |
| 323 | __device__ __2f16 __ocml_cos_2f16(__2f16); | 307 | __device__ __2f16 __ocml_cos_2f16(__2f16); |
| ... | @@ -332,11 +316,29 @@ __device__ __attribute__((const)) __2i16 __ocml_isnan_2f16(__2f16); | ... | @@ -332,11 +316,29 @@ __device__ __attribute__((const)) __2i16 __ocml_isnan_2f16(__2f16); |
| 332 | __device__ __attribute__((pure)) __2f16 __ocml_log_2f16(__2f16); | 316 | __device__ __attribute__((pure)) __2f16 __ocml_log_2f16(__2f16); |
| 333 | __device__ __attribute__((pure)) __2f16 __ocml_log10_2f16(__2f16); | 317 | __device__ __attribute__((pure)) __2f16 __ocml_log10_2f16(__2f16); |
| 334 | __device__ __attribute__((pure)) __2f16 __ocml_log2_2f16(__2f16); | 318 | __device__ __attribute__((pure)) __2f16 __ocml_log2_2f16(__2f16); |
| 319 | |||
| 320 | #if HIP_VERSION_MAJOR * 100 + HIP_VERSION_MINOR >= 560 | ||
| 321 | #define __DEPRECATED_SINCE_HIP_560(X) __attribute__((deprecated(X))) | ||
| 322 | #else | ||
| 323 | #define __DEPRECATED_SINCE_HIP_560(X) | ||
| 324 | #endif | ||
| 325 | |||
| 326 | // Deprecated, should be removed when rocm releases using it are no longer | ||
| 327 | // relevant. | ||
| 328 | __DEPRECATED_SINCE_HIP_560("use ((_Float16)1.0) / ") | ||
| 329 | __device__ inline _Float16 __llvm_amdgcn_rcp_f16(_Float16 x) { | ||
| 330 | return ((_Float16)1.0f) / x; | ||
| 331 | } | ||
| 332 | |||
| 333 | __DEPRECATED_SINCE_HIP_560("use ((__2f16)1.0) / ") | ||
| 335 | __device__ inline __2f16 | 334 | __device__ inline __2f16 |
| 336 | __llvm_amdgcn_rcp_2f16(__2f16 __x) // Not currently exposed by ROCDL. | 335 | __llvm_amdgcn_rcp_2f16(__2f16 __x) |
| 337 | { | 336 | { |
| 338 | return (__2f16)(__llvm_amdgcn_rcp_f16(__x.x), __llvm_amdgcn_rcp_f16(__x.y)); | 337 | return ((__2f16)1.0f) / __x; |
| 339 | } | 338 | } |
| 339 | |||
| 340 | #undef __DEPRECATED_SINCE_HIP_560 | ||
| 341 | |||
| 340 | __device__ __attribute__((const)) __2f16 __ocml_rint_2f16(__2f16); | 342 | __device__ __attribute__((const)) __2f16 __ocml_rint_2f16(__2f16); |
| 341 | __device__ __attribute__((const)) __2f16 __ocml_rsqrt_2f16(__2f16); | 343 | __device__ __attribute__((const)) __2f16 __ocml_rsqrt_2f16(__2f16); |
| 342 | __device__ __2f16 __ocml_sin_2f16(__2f16); | 344 | __device__ __2f16 __ocml_sin_2f16(__2f16); |
lib/include/__clang_hip_math.h+56-71
| ... | @@ -182,10 +182,10 @@ __DEVICE__ | ... | @@ -182,10 +182,10 @@ __DEVICE__ |
| 182 | float cbrtf(float __x) { return __ocml_cbrt_f32(__x); } | 182 | float cbrtf(float __x) { return __ocml_cbrt_f32(__x); } |
| 183 | 183 | ||
| 184 | __DEVICE__ | 184 | __DEVICE__ |
| 185 | float ceilf(float __x) { return __ocml_ceil_f32(__x); } | 185 | float ceilf(float __x) { return __builtin_ceilf(__x); } |
| 186 | 186 | ||
| 187 | __DEVICE__ | 187 | __DEVICE__ |
| 188 | float copysignf(float __x, float __y) { return __ocml_copysign_f32(__x, __y); } | 188 | float copysignf(float __x, float __y) { return __builtin_copysignf(__x, __y); } |
| 189 | 189 | ||
| 190 | __DEVICE__ | 190 | __DEVICE__ |
| 191 | float cosf(float __x) { return __ocml_cos_f32(__x); } | 191 | float cosf(float __x) { return __ocml_cos_f32(__x); } |
| ... | @@ -221,10 +221,10 @@ __DEVICE__ | ... | @@ -221,10 +221,10 @@ __DEVICE__ |
| 221 | float exp10f(float __x) { return __ocml_exp10_f32(__x); } | 221 | float exp10f(float __x) { return __ocml_exp10_f32(__x); } |
| 222 | 222 | ||
| 223 | __DEVICE__ | 223 | __DEVICE__ |
| 224 | float exp2f(float __x) { return __ocml_exp2_f32(__x); } | 224 | float exp2f(float __x) { return __builtin_exp2f(__x); } |
| 225 | 225 | ||
| 226 | __DEVICE__ | 226 | __DEVICE__ |
| 227 | float expf(float __x) { return __ocml_exp_f32(__x); } | 227 | float expf(float __x) { return __builtin_expf(__x); } |
| 228 | 228 | ||
| 229 | __DEVICE__ | 229 | __DEVICE__ |
| 230 | float expm1f(float __x) { return __ocml_expm1_f32(__x); } | 230 | float expm1f(float __x) { return __ocml_expm1_f32(__x); } |
| ... | @@ -239,33 +239,25 @@ __DEVICE__ | ... | @@ -239,33 +239,25 @@ __DEVICE__ |
| 239 | float fdividef(float __x, float __y) { return __x / __y; } | 239 | float fdividef(float __x, float __y) { return __x / __y; } |
| 240 | 240 | ||
| 241 | __DEVICE__ | 241 | __DEVICE__ |
| 242 | float floorf(float __x) { return __ocml_floor_f32(__x); } | 242 | float floorf(float __x) { return __builtin_floorf(__x); } |
| 243 | 243 | ||
| 244 | __DEVICE__ | 244 | __DEVICE__ |
| 245 | float fmaf(float __x, float __y, float __z) { | 245 | float fmaf(float __x, float __y, float __z) { |
| 246 | return __ocml_fma_f32(__x, __y, __z); | 246 | return __builtin_fmaf(__x, __y, __z); |
| 247 | } | 247 | } |
| 248 | 248 | ||
| 249 | __DEVICE__ | 249 | __DEVICE__ |
| 250 | float fmaxf(float __x, float __y) { return __ocml_fmax_f32(__x, __y); } | 250 | float fmaxf(float __x, float __y) { return __builtin_fmaxf(__x, __y); } |
| 251 | 251 | ||
| 252 | __DEVICE__ | 252 | __DEVICE__ |
| 253 | float fminf(float __x, float __y) { return __ocml_fmin_f32(__x, __y); } | 253 | float fminf(float __x, float __y) { return __builtin_fminf(__x, __y); } |
| 254 | 254 | ||
| 255 | __DEVICE__ | 255 | __DEVICE__ |
| 256 | float fmodf(float __x, float __y) { return __ocml_fmod_f32(__x, __y); } | 256 | float fmodf(float __x, float __y) { return __ocml_fmod_f32(__x, __y); } |
| 257 | 257 | ||
| 258 | __DEVICE__ | 258 | __DEVICE__ |
| 259 | float frexpf(float __x, int *__nptr) { | 259 | float frexpf(float __x, int *__nptr) { |
| 260 | int __tmp; | 260 | return __builtin_frexpf(__x, __nptr); |
| 261 | #ifdef __OPENMP_AMDGCN__ | ||
| 262 | #pragma omp allocate(__tmp) allocator(omp_thread_mem_alloc) | ||
| 263 | #endif | ||
| 264 | float __r = | ||
| 265 | __ocml_frexp_f32(__x, (__attribute__((address_space(5))) int *)&__tmp); | ||
| 266 | *__nptr = __tmp; | ||
| 267 | |||
| 268 | return __r; | ||
| 269 | } | 261 | } |
| 270 | 262 | ||
| 271 | __DEVICE__ | 263 | __DEVICE__ |
| ... | @@ -275,13 +267,13 @@ __DEVICE__ | ... | @@ -275,13 +267,13 @@ __DEVICE__ |
| 275 | int ilogbf(float __x) { return __ocml_ilogb_f32(__x); } | 267 | int ilogbf(float __x) { return __ocml_ilogb_f32(__x); } |
| 276 | 268 | ||
| 277 | __DEVICE__ | 269 | __DEVICE__ |
| 278 | __RETURN_TYPE __finitef(float __x) { return __ocml_isfinite_f32(__x); } | 270 | __RETURN_TYPE __finitef(float __x) { return __builtin_isfinite(__x); } |
| 279 | 271 | ||
| 280 | __DEVICE__ | 272 | __DEVICE__ |
| 281 | __RETURN_TYPE __isinff(float __x) { return __ocml_isinf_f32(__x); } | 273 | __RETURN_TYPE __isinff(float __x) { return __builtin_isinf(__x); } |
| 282 | 274 | ||
| 283 | __DEVICE__ | 275 | __DEVICE__ |
| 284 | __RETURN_TYPE __isnanf(float __x) { return __ocml_isnan_f32(__x); } | 276 | __RETURN_TYPE __isnanf(float __x) { return __builtin_isnan(__x); } |
| 285 | 277 | ||
| 286 | __DEVICE__ | 278 | __DEVICE__ |
| 287 | float j0f(float __x) { return __ocml_j0_f32(__x); } | 279 | float j0f(float __x) { return __ocml_j0_f32(__x); } |
| ... | @@ -311,37 +303,37 @@ float jnf(int __n, float __x) { // TODO: we could use Ahmes multiplication | ... | @@ -311,37 +303,37 @@ float jnf(int __n, float __x) { // TODO: we could use Ahmes multiplication |
| 311 | } | 303 | } |
| 312 | 304 | ||
| 313 | __DEVICE__ | 305 | __DEVICE__ |
| 314 | float ldexpf(float __x, int __e) { return __ocml_ldexp_f32(__x, __e); } | 306 | float ldexpf(float __x, int __e) { return __builtin_amdgcn_ldexpf(__x, __e); } |
| 315 | 307 | ||
| 316 | __DEVICE__ | 308 | __DEVICE__ |
| 317 | float lgammaf(float __x) { return __ocml_lgamma_f32(__x); } | 309 | float lgammaf(float __x) { return __ocml_lgamma_f32(__x); } |
| 318 | 310 | ||
| 319 | __DEVICE__ | 311 | __DEVICE__ |
| 320 | long long int llrintf(float __x) { return __ocml_rint_f32(__x); } | 312 | long long int llrintf(float __x) { return __builtin_rintf(__x); } |
| 321 | 313 | ||
| 322 | __DEVICE__ | 314 | __DEVICE__ |
| 323 | long long int llroundf(float __x) { return __ocml_round_f32(__x); } | 315 | long long int llroundf(float __x) { return __builtin_roundf(__x); } |
| 324 | 316 | ||
| 325 | __DEVICE__ | 317 | __DEVICE__ |
| 326 | float log10f(float __x) { return __ocml_log10_f32(__x); } | 318 | float log10f(float __x) { return __builtin_log10f(__x); } |
| 327 | 319 | ||
| 328 | __DEVICE__ | 320 | __DEVICE__ |
| 329 | float log1pf(float __x) { return __ocml_log1p_f32(__x); } | 321 | float log1pf(float __x) { return __ocml_log1p_f32(__x); } |
| 330 | 322 | ||
| 331 | __DEVICE__ | 323 | __DEVICE__ |
| 332 | float log2f(float __x) { return __ocml_log2_f32(__x); } | 324 | float log2f(float __x) { return __builtin_log2f(__x); } |
| 333 | 325 | ||
| 334 | __DEVICE__ | 326 | __DEVICE__ |
| 335 | float logbf(float __x) { return __ocml_logb_f32(__x); } | 327 | float logbf(float __x) { return __ocml_logb_f32(__x); } |
| 336 | 328 | ||
| 337 | __DEVICE__ | 329 | __DEVICE__ |
| 338 | float logf(float __x) { return __ocml_log_f32(__x); } | 330 | float logf(float __x) { return __builtin_logf(__x); } |
| 339 | 331 | ||
| 340 | __DEVICE__ | 332 | __DEVICE__ |
| 341 | long int lrintf(float __x) { return __ocml_rint_f32(__x); } | 333 | long int lrintf(float __x) { return __builtin_rintf(__x); } |
| 342 | 334 | ||
| 343 | __DEVICE__ | 335 | __DEVICE__ |
| 344 | long int lroundf(float __x) { return __ocml_round_f32(__x); } | 336 | long int lroundf(float __x) { return __builtin_roundf(__x); } |
| 345 | 337 | ||
| 346 | __DEVICE__ | 338 | __DEVICE__ |
| 347 | float modff(float __x, float *__iptr) { | 339 | float modff(float __x, float *__iptr) { |
| ... | @@ -377,7 +369,7 @@ float nanf(const char *__tagp __attribute__((nonnull))) { | ... | @@ -377,7 +369,7 @@ float nanf(const char *__tagp __attribute__((nonnull))) { |
| 377 | } | 369 | } |
| 378 | 370 | ||
| 379 | __DEVICE__ | 371 | __DEVICE__ |
| 380 | float nearbyintf(float __x) { return __ocml_nearbyint_f32(__x); } | 372 | float nearbyintf(float __x) { return __builtin_nearbyintf(__x); } |
| 381 | 373 | ||
| 382 | __DEVICE__ | 374 | __DEVICE__ |
| 383 | float nextafterf(float __x, float __y) { | 375 | float nextafterf(float __x, float __y) { |
| ... | @@ -443,7 +435,7 @@ __DEVICE__ | ... | @@ -443,7 +435,7 @@ __DEVICE__ |
| 443 | float rhypotf(float __x, float __y) { return __ocml_rhypot_f32(__x, __y); } | 435 | float rhypotf(float __x, float __y) { return __ocml_rhypot_f32(__x, __y); } |
| 444 | 436 | ||
| 445 | __DEVICE__ | 437 | __DEVICE__ |
| 446 | float rintf(float __x) { return __ocml_rint_f32(__x); } | 438 | float rintf(float __x) { return __builtin_rintf(__x); } |
| 447 | 439 | ||
| 448 | __DEVICE__ | 440 | __DEVICE__ |
| 449 | float rnorm3df(float __x, float __y, float __z) { | 441 | float rnorm3df(float __x, float __y, float __z) { |
| ... | @@ -468,22 +460,22 @@ float rnormf(int __dim, | ... | @@ -468,22 +460,22 @@ float rnormf(int __dim, |
| 468 | } | 460 | } |
| 469 | 461 | ||
| 470 | __DEVICE__ | 462 | __DEVICE__ |
| 471 | float roundf(float __x) { return __ocml_round_f32(__x); } | 463 | float roundf(float __x) { return __builtin_roundf(__x); } |
| 472 | 464 | ||
| 473 | __DEVICE__ | 465 | __DEVICE__ |
| 474 | float rsqrtf(float __x) { return __ocml_rsqrt_f32(__x); } | 466 | float rsqrtf(float __x) { return __ocml_rsqrt_f32(__x); } |
| 475 | 467 | ||
| 476 | __DEVICE__ | 468 | __DEVICE__ |
| 477 | float scalblnf(float __x, long int __n) { | 469 | float scalblnf(float __x, long int __n) { |
| 478 | return (__n < INT_MAX) ? __ocml_scalbn_f32(__x, __n) | 470 | return (__n < INT_MAX) ? __builtin_amdgcn_ldexpf(__x, __n) |
| 479 | : __ocml_scalb_f32(__x, __n); | 471 | : __ocml_scalb_f32(__x, __n); |
| 480 | } | 472 | } |
| 481 | 473 | ||
| 482 | __DEVICE__ | 474 | __DEVICE__ |
| 483 | float scalbnf(float __x, int __n) { return __ocml_scalbn_f32(__x, __n); } | 475 | float scalbnf(float __x, int __n) { return __builtin_amdgcn_ldexpf(__x, __n); } |
| 484 | 476 | ||
| 485 | __DEVICE__ | 477 | __DEVICE__ |
| 486 | __RETURN_TYPE __signbitf(float __x) { return __ocml_signbit_f32(__x); } | 478 | __RETURN_TYPE __signbitf(float __x) { return __builtin_signbitf(__x); } |
| 487 | 479 | ||
| 488 | __DEVICE__ | 480 | __DEVICE__ |
| 489 | void sincosf(float __x, float *__sinptr, float *__cosptr) { | 481 | void sincosf(float __x, float *__sinptr, float *__cosptr) { |
| ... | @@ -529,7 +521,7 @@ __DEVICE__ | ... | @@ -529,7 +521,7 @@ __DEVICE__ |
| 529 | float tgammaf(float __x) { return __ocml_tgamma_f32(__x); } | 521 | float tgammaf(float __x) { return __ocml_tgamma_f32(__x); } |
| 530 | 522 | ||
| 531 | __DEVICE__ | 523 | __DEVICE__ |
| 532 | float truncf(float __x) { return __ocml_trunc_f32(__x); } | 524 | float truncf(float __x) { return __builtin_truncf(__x); } |
| 533 | 525 | ||
| 534 | __DEVICE__ | 526 | __DEVICE__ |
| 535 | float y0f(float __x) { return __ocml_y0_f32(__x); } | 527 | float y0f(float __x) { return __ocml_y0_f32(__x); } |
| ... | @@ -621,7 +613,7 @@ float __fmaf_rz(float __x, float __y, float __z) { | ... | @@ -621,7 +613,7 @@ float __fmaf_rz(float __x, float __y, float __z) { |
| 621 | #else | 613 | #else |
| 622 | __DEVICE__ | 614 | __DEVICE__ |
| 623 | float __fmaf_rn(float __x, float __y, float __z) { | 615 | float __fmaf_rn(float __x, float __y, float __z) { |
| 624 | return __ocml_fma_f32(__x, __y, __z); | 616 | return __builtin_fmaf(__x, __y, __z); |
| 625 | } | 617 | } |
| 626 | #endif | 618 | #endif |
| 627 | 619 | ||
| ... | @@ -654,7 +646,7 @@ float __frcp_rn(float __x) { return 1.0f / __x; } | ... | @@ -654,7 +646,7 @@ float __frcp_rn(float __x) { return 1.0f / __x; } |
| 654 | #endif | 646 | #endif |
| 655 | 647 | ||
| 656 | __DEVICE__ | 648 | __DEVICE__ |
| 657 | float __frsqrt_rn(float __x) { return __llvm_amdgcn_rsq_f32(__x); } | 649 | float __frsqrt_rn(float __x) { return __builtin_amdgcn_rsqf(__x); } |
| 658 | 650 | ||
| 659 | #if defined OCML_BASIC_ROUNDED_OPERATIONS | 651 | #if defined OCML_BASIC_ROUNDED_OPERATIONS |
| 660 | __DEVICE__ | 652 | __DEVICE__ |
| ... | @@ -739,11 +731,11 @@ __DEVICE__ | ... | @@ -739,11 +731,11 @@ __DEVICE__ |
| 739 | double cbrt(double __x) { return __ocml_cbrt_f64(__x); } | 731 | double cbrt(double __x) { return __ocml_cbrt_f64(__x); } |
| 740 | 732 | ||
| 741 | __DEVICE__ | 733 | __DEVICE__ |
| 742 | double ceil(double __x) { return __ocml_ceil_f64(__x); } | 734 | double ceil(double __x) { return __builtin_ceil(__x); } |
| 743 | 735 | ||
| 744 | __DEVICE__ | 736 | __DEVICE__ |
| 745 | double copysign(double __x, double __y) { | 737 | double copysign(double __x, double __y) { |
| 746 | return __ocml_copysign_f64(__x, __y); | 738 | return __builtin_copysign(__x, __y); |
| 747 | } | 739 | } |
| 748 | 740 | ||
| 749 | __DEVICE__ | 741 | __DEVICE__ |
| ... | @@ -795,32 +787,25 @@ __DEVICE__ | ... | @@ -795,32 +787,25 @@ __DEVICE__ |
| 795 | double fdim(double __x, double __y) { return __ocml_fdim_f64(__x, __y); } | 787 | double fdim(double __x, double __y) { return __ocml_fdim_f64(__x, __y); } |
| 796 | 788 | ||
| 797 | __DEVICE__ | 789 | __DEVICE__ |
| 798 | double floor(double __x) { return __ocml_floor_f64(__x); } | 790 | double floor(double __x) { return __builtin_floor(__x); } |
| 799 | 791 | ||
| 800 | __DEVICE__ | 792 | __DEVICE__ |
| 801 | double fma(double __x, double __y, double __z) { | 793 | double fma(double __x, double __y, double __z) { |
| 802 | return __ocml_fma_f64(__x, __y, __z); | 794 | return __builtin_fma(__x, __y, __z); |
| 803 | } | 795 | } |
| 804 | 796 | ||
| 805 | __DEVICE__ | 797 | __DEVICE__ |
| 806 | double fmax(double __x, double __y) { return __ocml_fmax_f64(__x, __y); } | 798 | double fmax(double __x, double __y) { return __builtin_fmax(__x, __y); } |
| 807 | 799 | ||
| 808 | __DEVICE__ | 800 | __DEVICE__ |
| 809 | double fmin(double __x, double __y) { return __ocml_fmin_f64(__x, __y); } | 801 | double fmin(double __x, double __y) { return __builtin_fmin(__x, __y); } |
| 810 | 802 | ||
| 811 | __DEVICE__ | 803 | __DEVICE__ |
| 812 | double fmod(double __x, double __y) { return __ocml_fmod_f64(__x, __y); } | 804 | double fmod(double __x, double __y) { return __ocml_fmod_f64(__x, __y); } |
| 813 | 805 | ||
| 814 | __DEVICE__ | 806 | __DEVICE__ |
| 815 | double frexp(double __x, int *__nptr) { | 807 | double frexp(double __x, int *__nptr) { |
| 816 | int __tmp; | 808 | return __builtin_frexp(__x, __nptr); |
| 817 | #ifdef __OPENMP_AMDGCN__ | ||
| 818 | #pragma omp allocate(__tmp) allocator(omp_thread_mem_alloc) | ||
| 819 | #endif | ||
| 820 | double __r = | ||
| 821 | __ocml_frexp_f64(__x, (__attribute__((address_space(5))) int *)&__tmp); | ||
| 822 | *__nptr = __tmp; | ||
| 823 | return __r; | ||
| 824 | } | 809 | } |
| 825 | 810 | ||
| 826 | __DEVICE__ | 811 | __DEVICE__ |
| ... | @@ -830,13 +815,13 @@ __DEVICE__ | ... | @@ -830,13 +815,13 @@ __DEVICE__ |
| 830 | int ilogb(double __x) { return __ocml_ilogb_f64(__x); } | 815 | int ilogb(double __x) { return __ocml_ilogb_f64(__x); } |
| 831 | 816 | ||
| 832 | __DEVICE__ | 817 | __DEVICE__ |
| 833 | __RETURN_TYPE __finite(double __x) { return __ocml_isfinite_f64(__x); } | 818 | __RETURN_TYPE __finite(double __x) { return __builtin_isfinite(__x); } |
| 834 | 819 | ||
| 835 | __DEVICE__ | 820 | __DEVICE__ |
| 836 | __RETURN_TYPE __isinf(double __x) { return __ocml_isinf_f64(__x); } | 821 | __RETURN_TYPE __isinf(double __x) { return __builtin_isinf(__x); } |
| 837 | 822 | ||
| 838 | __DEVICE__ | 823 | __DEVICE__ |
| 839 | __RETURN_TYPE __isnan(double __x) { return __ocml_isnan_f64(__x); } | 824 | __RETURN_TYPE __isnan(double __x) { return __builtin_isnan(__x); } |
| 840 | 825 | ||
| 841 | __DEVICE__ | 826 | __DEVICE__ |
| 842 | double j0(double __x) { return __ocml_j0_f64(__x); } | 827 | double j0(double __x) { return __ocml_j0_f64(__x); } |
| ... | @@ -866,16 +851,16 @@ double jn(int __n, double __x) { // TODO: we could use Ahmes multiplication | ... | @@ -866,16 +851,16 @@ double jn(int __n, double __x) { // TODO: we could use Ahmes multiplication |
| 866 | } | 851 | } |
| 867 | 852 | ||
| 868 | __DEVICE__ | 853 | __DEVICE__ |
| 869 | double ldexp(double __x, int __e) { return __ocml_ldexp_f64(__x, __e); } | 854 | double ldexp(double __x, int __e) { return __builtin_amdgcn_ldexp(__x, __e); } |
| 870 | 855 | ||
| 871 | __DEVICE__ | 856 | __DEVICE__ |
| 872 | double lgamma(double __x) { return __ocml_lgamma_f64(__x); } | 857 | double lgamma(double __x) { return __ocml_lgamma_f64(__x); } |
| 873 | 858 | ||
| 874 | __DEVICE__ | 859 | __DEVICE__ |
| 875 | long long int llrint(double __x) { return __ocml_rint_f64(__x); } | 860 | long long int llrint(double __x) { return __builtin_rint(__x); } |
| 876 | 861 | ||
| 877 | __DEVICE__ | 862 | __DEVICE__ |
| 878 | long long int llround(double __x) { return __ocml_round_f64(__x); } | 863 | long long int llround(double __x) { return __builtin_round(__x); } |
| 879 | 864 | ||
| 880 | __DEVICE__ | 865 | __DEVICE__ |
| 881 | double log(double __x) { return __ocml_log_f64(__x); } | 866 | double log(double __x) { return __ocml_log_f64(__x); } |
| ... | @@ -893,10 +878,10 @@ __DEVICE__ | ... | @@ -893,10 +878,10 @@ __DEVICE__ |
| 893 | double logb(double __x) { return __ocml_logb_f64(__x); } | 878 | double logb(double __x) { return __ocml_logb_f64(__x); } |
| 894 | 879 | ||
| 895 | __DEVICE__ | 880 | __DEVICE__ |
| 896 | long int lrint(double __x) { return __ocml_rint_f64(__x); } | 881 | long int lrint(double __x) { return __builtin_rint(__x); } |
| 897 | 882 | ||
| 898 | __DEVICE__ | 883 | __DEVICE__ |
| 899 | long int lround(double __x) { return __ocml_round_f64(__x); } | 884 | long int lround(double __x) { return __builtin_round(__x); } |
| 900 | 885 | ||
| 901 | __DEVICE__ | 886 | __DEVICE__ |
| 902 | double modf(double __x, double *__iptr) { | 887 | double modf(double __x, double *__iptr) { |
| ... | @@ -940,7 +925,7 @@ double nan(const char *__tagp) { | ... | @@ -940,7 +925,7 @@ double nan(const char *__tagp) { |
| 940 | } | 925 | } |
| 941 | 926 | ||
| 942 | __DEVICE__ | 927 | __DEVICE__ |
| 943 | double nearbyint(double __x) { return __ocml_nearbyint_f64(__x); } | 928 | double nearbyint(double __x) { return __builtin_nearbyint(__x); } |
| 944 | 929 | ||
| 945 | __DEVICE__ | 930 | __DEVICE__ |
| 946 | double nextafter(double __x, double __y) { | 931 | double nextafter(double __x, double __y) { |
| ... | @@ -1006,7 +991,7 @@ __DEVICE__ | ... | @@ -1006,7 +991,7 @@ __DEVICE__ |
| 1006 | double rhypot(double __x, double __y) { return __ocml_rhypot_f64(__x, __y); } | 991 | double rhypot(double __x, double __y) { return __ocml_rhypot_f64(__x, __y); } |
| 1007 | 992 | ||
| 1008 | __DEVICE__ | 993 | __DEVICE__ |
| 1009 | double rint(double __x) { return __ocml_rint_f64(__x); } | 994 | double rint(double __x) { return __builtin_rint(__x); } |
| 1010 | 995 | ||
| 1011 | __DEVICE__ | 996 | __DEVICE__ |
| 1012 | double rnorm(int __dim, | 997 | double rnorm(int __dim, |
| ... | @@ -1031,21 +1016,21 @@ double rnorm4d(double __x, double __y, double __z, double __w) { | ... | @@ -1031,21 +1016,21 @@ double rnorm4d(double __x, double __y, double __z, double __w) { |
| 1031 | } | 1016 | } |
| 1032 | 1017 | ||
| 1033 | __DEVICE__ | 1018 | __DEVICE__ |
| 1034 | double round(double __x) { return __ocml_round_f64(__x); } | 1019 | double round(double __x) { return __builtin_round(__x); } |
| 1035 | 1020 | ||
| 1036 | __DEVICE__ | 1021 | __DEVICE__ |
| 1037 | double rsqrt(double __x) { return __ocml_rsqrt_f64(__x); } | 1022 | double rsqrt(double __x) { return __ocml_rsqrt_f64(__x); } |
| 1038 | 1023 | ||
| 1039 | __DEVICE__ | 1024 | __DEVICE__ |
| 1040 | double scalbln(double __x, long int __n) { | 1025 | double scalbln(double __x, long int __n) { |
| 1041 | return (__n < INT_MAX) ? __ocml_scalbn_f64(__x, __n) | 1026 | return (__n < INT_MAX) ? __builtin_amdgcn_ldexp(__x, __n) |
| 1042 | : __ocml_scalb_f64(__x, __n); | 1027 | : __ocml_scalb_f64(__x, __n); |
| 1043 | } | 1028 | } |
| 1044 | __DEVICE__ | 1029 | __DEVICE__ |
| 1045 | double scalbn(double __x, int __n) { return __ocml_scalbn_f64(__x, __n); } | 1030 | double scalbn(double __x, int __n) { return __builtin_amdgcn_ldexp(__x, __n); } |
| 1046 | 1031 | ||
| 1047 | __DEVICE__ | 1032 | __DEVICE__ |
| 1048 | __RETURN_TYPE __signbit(double __x) { return __ocml_signbit_f64(__x); } | 1033 | __RETURN_TYPE __signbit(double __x) { return __builtin_signbit(__x); } |
| 1049 | 1034 | ||
| 1050 | __DEVICE__ | 1035 | __DEVICE__ |
| 1051 | double sin(double __x) { return __ocml_sin_f64(__x); } | 1036 | double sin(double __x) { return __ocml_sin_f64(__x); } |
| ... | @@ -1091,7 +1076,7 @@ __DEVICE__ | ... | @@ -1091,7 +1076,7 @@ __DEVICE__ |
| 1091 | double tgamma(double __x) { return __ocml_tgamma_f64(__x); } | 1076 | double tgamma(double __x) { return __ocml_tgamma_f64(__x); } |
| 1092 | 1077 | ||
| 1093 | __DEVICE__ | 1078 | __DEVICE__ |
| 1094 | double trunc(double __x) { return __ocml_trunc_f64(__x); } | 1079 | double trunc(double __x) { return __builtin_trunc(__x); } |
| 1095 | 1080 | ||
| 1096 | __DEVICE__ | 1081 | __DEVICE__ |
| 1097 | double y0(double __x) { return __ocml_y0_f64(__x); } | 1082 | double y0(double __x) { return __ocml_y0_f64(__x); } |
| ... | @@ -1258,7 +1243,7 @@ double __fma_rz(double __x, double __y, double __z) { | ... | @@ -1258,7 +1243,7 @@ double __fma_rz(double __x, double __y, double __z) { |
| 1258 | #else | 1243 | #else |
| 1259 | __DEVICE__ | 1244 | __DEVICE__ |
| 1260 | double __fma_rn(double __x, double __y, double __z) { | 1245 | double __fma_rn(double __x, double __y, double __z) { |
| 1261 | return __ocml_fma_f64(__x, __y, __z); | 1246 | return __builtin_fma(__x, __y, __z); |
| 1262 | } | 1247 | } |
| 1263 | #endif | 1248 | #endif |
| 1264 | // END INTRINSICS | 1249 | // END INTRINSICS |
| ... | @@ -1290,16 +1275,16 @@ __DEVICE__ int max(int __arg1, int __arg2) { | ... | @@ -1290,16 +1275,16 @@ __DEVICE__ int max(int __arg1, int __arg2) { |
| 1290 | } | 1275 | } |
| 1291 | 1276 | ||
| 1292 | __DEVICE__ | 1277 | __DEVICE__ |
| 1293 | float max(float __x, float __y) { return fmaxf(__x, __y); } | 1278 | float max(float __x, float __y) { return __builtin_fmaxf(__x, __y); } |
| 1294 | 1279 | ||
| 1295 | __DEVICE__ | 1280 | __DEVICE__ |
| 1296 | double max(double __x, double __y) { return fmax(__x, __y); } | 1281 | double max(double __x, double __y) { return __builtin_fmax(__x, __y); } |
| 1297 | 1282 | ||
| 1298 | __DEVICE__ | 1283 | __DEVICE__ |
| 1299 | float min(float __x, float __y) { return fminf(__x, __y); } | 1284 | float min(float __x, float __y) { return __builtin_fminf(__x, __y); } |
| 1300 | 1285 | ||
| 1301 | __DEVICE__ | 1286 | __DEVICE__ |
| 1302 | double min(double __x, double __y) { return fmin(__x, __y); } | 1287 | double min(double __x, double __y) { return __builtin_fmin(__x, __y); } |
| 1303 | 1288 | ||
| 1304 | #if !defined(__HIPCC_RTC__) && !defined(__OPENMP_AMDGCN__) | 1289 | #if !defined(__HIPCC_RTC__) && !defined(__OPENMP_AMDGCN__) |
| 1305 | __host__ inline static int min(int __arg1, int __arg2) { | 1290 | __host__ inline static int min(int __arg1, int __arg2) { |
lib/include/__clang_hip_runtime_wrapper.h+13
| ... | @@ -80,12 +80,25 @@ extern "C" { | ... | @@ -80,12 +80,25 @@ extern "C" { |
| 80 | #if HIP_VERSION_MAJOR * 100 + HIP_VERSION_MINOR >= 405 | 80 | #if HIP_VERSION_MAJOR * 100 + HIP_VERSION_MINOR >= 405 |
| 81 | extern "C" __device__ unsigned long long __ockl_dm_alloc(unsigned long long __size); | 81 | extern "C" __device__ unsigned long long __ockl_dm_alloc(unsigned long long __size); |
| 82 | extern "C" __device__ void __ockl_dm_dealloc(unsigned long long __addr); | 82 | extern "C" __device__ void __ockl_dm_dealloc(unsigned long long __addr); |
| 83 | #if __has_feature(address_sanitizer) | ||
| 84 | extern "C" __device__ unsigned long long __asan_malloc_impl(unsigned long long __size, unsigned long long __pc); | ||
| 85 | extern "C" __device__ void __asan_free_impl(unsigned long long __addr, unsigned long long __pc); | ||
| 86 | __attribute__((noinline, weak)) __device__ void *malloc(__hip_size_t __size) { | ||
| 87 | unsigned long long __pc = (unsigned long long)__builtin_return_address(0); | ||
| 88 | return (void *)__asan_malloc_impl(__size, __pc); | ||
| 89 | } | ||
| 90 | __attribute__((noinline, weak)) __device__ void free(void *__ptr) { | ||
| 91 | unsigned long long __pc = (unsigned long long)__builtin_return_address(0); | ||
| 92 | __asan_free_impl((unsigned long long)__ptr, __pc); | ||
| 93 | } | ||
| 94 | #else | ||
| 83 | __attribute__((weak)) inline __device__ void *malloc(__hip_size_t __size) { | 95 | __attribute__((weak)) inline __device__ void *malloc(__hip_size_t __size) { |
| 84 | return (void *) __ockl_dm_alloc(__size); | 96 | return (void *) __ockl_dm_alloc(__size); |
| 85 | } | 97 | } |
| 86 | __attribute__((weak)) inline __device__ void free(void *__ptr) { | 98 | __attribute__((weak)) inline __device__ void free(void *__ptr) { |
| 87 | __ockl_dm_dealloc((unsigned long long)__ptr); | 99 | __ockl_dm_dealloc((unsigned long long)__ptr); |
| 88 | } | 100 | } |
| 101 | #endif // __has_feature(address_sanitizer) | ||
| 89 | #else // HIP version check | 102 | #else // HIP version check |
| 90 | #if __HIP_ENABLE_DEVICE_MALLOC__ | 103 | #if __HIP_ENABLE_DEVICE_MALLOC__ |
| 91 | __device__ void *__hip_malloc(__hip_size_t __size); | 104 | __device__ void *__hip_malloc(__hip_size_t __size); |
lib/include/adxintrin.h+179-24
| ... | @@ -17,56 +17,211 @@ | ... | @@ -17,56 +17,211 @@ |
| 17 | /* Define the default attributes for the functions in this file. */ | 17 | /* Define the default attributes for the functions in this file. */ |
| 18 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) | 18 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) |
| 19 | 19 | ||
| 20 | /* Intrinsics that are available only if __ADX__ defined */ | 20 | /* Use C++ inline semantics in C++, GNU inline for C mode. */ |
| 21 | static __inline unsigned char __attribute__((__always_inline__, __nodebug__, __target__("adx"))) | 21 | #if defined(__cplusplus) |
| 22 | _addcarryx_u32(unsigned char __cf, unsigned int __x, unsigned int __y, | 22 | #define __INLINE __inline |
| 23 | unsigned int *__p) | 23 | #else |
| 24 | { | 24 | #define __INLINE static __inline |
| 25 | #endif | ||
| 26 | |||
| 27 | #if defined(__cplusplus) | ||
| 28 | extern "C" { | ||
| 29 | #endif | ||
| 30 | |||
| 31 | /* Intrinsics that are available only if __ADX__ is defined. */ | ||
| 32 | |||
| 33 | /// Adds unsigned 32-bit integers \a __x and \a __y, plus 0 or 1 as indicated | ||
| 34 | /// by the carry flag \a __cf. Stores the unsigned 32-bit sum in the memory | ||
| 35 | /// at \a __p, and returns the 8-bit carry-out (carry flag). | ||
| 36 | /// | ||
| 37 | /// \code{.operation} | ||
| 38 | /// temp := (__cf == 0) ? 0 : 1 | ||
| 39 | /// Store32(__p, __x + __y + temp) | ||
| 40 | /// result := CF | ||
| 41 | /// \endcode | ||
| 42 | /// | ||
| 43 | /// \headerfile <immintrin.h> | ||
| 44 | /// | ||
| 45 | /// This intrinsic corresponds to the \c ADCX instruction. | ||
| 46 | /// | ||
| 47 | /// \param __cf | ||
| 48 | /// The 8-bit unsigned carry flag; any non-zero value indicates carry. | ||
| 49 | /// \param __x | ||
| 50 | /// A 32-bit unsigned addend. | ||
| 51 | /// \param __y | ||
| 52 | /// A 32-bit unsigned addend. | ||
| 53 | /// \param __p | ||
| 54 | /// Pointer to memory for storing the sum. | ||
| 55 | /// \returns The 8-bit unsigned carry-out value. | ||
| 56 | __INLINE unsigned char | ||
| 57 | __attribute__((__always_inline__, __nodebug__, __target__("adx"))) | ||
| 58 | _addcarryx_u32(unsigned char __cf, unsigned int __x, unsigned int __y, | ||
| 59 | unsigned int *__p) { | ||
| 25 | return __builtin_ia32_addcarryx_u32(__cf, __x, __y, __p); | 60 | return __builtin_ia32_addcarryx_u32(__cf, __x, __y, __p); |
| 26 | } | 61 | } |
| 27 | 62 | ||
| 28 | #ifdef __x86_64__ | 63 | #ifdef __x86_64__ |
| 29 | static __inline unsigned char __attribute__((__always_inline__, __nodebug__, __target__("adx"))) | 64 | /// Adds unsigned 64-bit integers \a __x and \a __y, plus 0 or 1 as indicated |
| 30 | _addcarryx_u64(unsigned char __cf, unsigned long long __x, | 65 | /// by the carry flag \a __cf. Stores the unsigned 64-bit sum in the memory |
| 31 | unsigned long long __y, unsigned long long *__p) | 66 | /// at \a __p, and returns the 8-bit carry-out (carry flag). |
| 32 | { | 67 | /// |
| 68 | /// \code{.operation} | ||
| 69 | /// temp := (__cf == 0) ? 0 : 1 | ||
| 70 | /// Store64(__p, __x + __y + temp) | ||
| 71 | /// result := CF | ||
| 72 | /// \endcode | ||
| 73 | /// | ||
| 74 | /// \headerfile <immintrin.h> | ||
| 75 | /// | ||
| 76 | /// This intrinsic corresponds to the \c ADCX instruction. | ||
| 77 | /// | ||
| 78 | /// \param __cf | ||
| 79 | /// The 8-bit unsigned carry flag; any non-zero value indicates carry. | ||
| 80 | /// \param __x | ||
| 81 | /// A 64-bit unsigned addend. | ||
| 82 | /// \param __y | ||
| 83 | /// A 64-bit unsigned addend. | ||
| 84 | /// \param __p | ||
| 85 | /// Pointer to memory for storing the sum. | ||
| 86 | /// \returns The 8-bit unsigned carry-out value. | ||
| 87 | __INLINE unsigned char | ||
| 88 | __attribute__((__always_inline__, __nodebug__, __target__("adx"))) | ||
| 89 | _addcarryx_u64(unsigned char __cf, unsigned long long __x, | ||
| 90 | unsigned long long __y, unsigned long long *__p) { | ||
| 33 | return __builtin_ia32_addcarryx_u64(__cf, __x, __y, __p); | 91 | return __builtin_ia32_addcarryx_u64(__cf, __x, __y, __p); |
| 34 | } | 92 | } |
| 35 | #endif | 93 | #endif |
| 36 | 94 | ||
| 37 | /* Intrinsics that are also available if __ADX__ undefined */ | 95 | /* Intrinsics that are also available if __ADX__ is undefined. */ |
| 38 | static __inline unsigned char __DEFAULT_FN_ATTRS | 96 | |
| 39 | _addcarry_u32(unsigned char __cf, unsigned int __x, unsigned int __y, | 97 | /// Adds unsigned 32-bit integers \a __x and \a __y, plus 0 or 1 as indicated |
| 40 | unsigned int *__p) | 98 | /// by the carry flag \a __cf. Stores the unsigned 32-bit sum in the memory |
| 41 | { | 99 | /// at \a __p, and returns the 8-bit carry-out (carry flag). |
| 100 | /// | ||
| 101 | /// \code{.operation} | ||
| 102 | /// temp := (__cf == 0) ? 0 : 1 | ||
| 103 | /// Store32(__p, __x + __y + temp) | ||
| 104 | /// result := CF | ||
| 105 | /// \endcode | ||
| 106 | /// | ||
| 107 | /// \headerfile <immintrin.h> | ||
| 108 | /// | ||
| 109 | /// This intrinsic corresponds to the \c ADC instruction. | ||
| 110 | /// | ||
| 111 | /// \param __cf | ||
| 112 | /// The 8-bit unsigned carry flag; any non-zero value indicates carry. | ||
| 113 | /// \param __x | ||
| 114 | /// A 32-bit unsigned addend. | ||
| 115 | /// \param __y | ||
| 116 | /// A 32-bit unsigned addend. | ||
| 117 | /// \param __p | ||
| 118 | /// Pointer to memory for storing the sum. | ||
| 119 | /// \returns The 8-bit unsigned carry-out value. | ||
| 120 | __INLINE unsigned char __DEFAULT_FN_ATTRS _addcarry_u32(unsigned char __cf, | ||
| 121 | unsigned int __x, | ||
| 122 | unsigned int __y, | ||
| 123 | unsigned int *__p) { | ||
| 42 | return __builtin_ia32_addcarryx_u32(__cf, __x, __y, __p); | 124 | return __builtin_ia32_addcarryx_u32(__cf, __x, __y, __p); |
| 43 | } | 125 | } |
| 44 | 126 | ||
| 45 | #ifdef __x86_64__ | 127 | #ifdef __x86_64__ |
| 46 | static __inline unsigned char __DEFAULT_FN_ATTRS | 128 | /// Adds unsigned 64-bit integers \a __x and \a __y, plus 0 or 1 as indicated |
| 129 | /// by the carry flag \a __cf. Stores the unsigned 64-bit sum in the memory | ||
| 130 | /// at \a __p, and returns the 8-bit carry-out (carry flag). | ||
| 131 | /// | ||
| 132 | /// \code{.operation} | ||
| 133 | /// temp := (__cf == 0) ? 0 : 1 | ||
| 134 | /// Store64(__p, __x + __y + temp) | ||
| 135 | /// result := CF | ||
| 136 | /// \endcode | ||
| 137 | /// | ||
| 138 | /// \headerfile <immintrin.h> | ||
| 139 | /// | ||
| 140 | /// This intrinsic corresponds to the \c ADC instruction. | ||
| 141 | /// | ||
| 142 | /// \param __cf | ||
| 143 | /// The 8-bit unsigned carry flag; any non-zero value indicates carry. | ||
| 144 | /// \param __x | ||
| 145 | /// A 64-bit unsigned addend. | ||
| 146 | /// \param __y | ||
| 147 | /// A 64-bit unsigned addend. | ||
| 148 | /// \param __p | ||
| 149 | /// Pointer to memory for storing the sum. | ||
| 150 | /// \returns The 8-bit unsigned carry-out value. | ||
| 151 | __INLINE unsigned char __DEFAULT_FN_ATTRS | ||
| 47 | _addcarry_u64(unsigned char __cf, unsigned long long __x, | 152 | _addcarry_u64(unsigned char __cf, unsigned long long __x, |
| 48 | unsigned long long __y, unsigned long long *__p) | 153 | unsigned long long __y, unsigned long long *__p) { |
| 49 | { | ||
| 50 | return __builtin_ia32_addcarryx_u64(__cf, __x, __y, __p); | 154 | return __builtin_ia32_addcarryx_u64(__cf, __x, __y, __p); |
| 51 | } | 155 | } |
| 52 | #endif | 156 | #endif |
| 53 | 157 | ||
| 54 | static __inline unsigned char __DEFAULT_FN_ATTRS | 158 | /// Adds unsigned 32-bit integer \a __y to 0 or 1 as indicated by the carry |
| 55 | _subborrow_u32(unsigned char __cf, unsigned int __x, unsigned int __y, | 159 | /// flag \a __cf, and subtracts the result from unsigned 32-bit integer |
| 56 | unsigned int *__p) | 160 | /// \a __x. Stores the unsigned 32-bit difference in the memory at \a __p, |
| 57 | { | 161 | /// and returns the 8-bit carry-out (carry or overflow flag). |
| 162 | /// | ||
| 163 | /// \code{.operation} | ||
| 164 | /// temp := (__cf == 0) ? 0 : 1 | ||
| 165 | /// Store32(__p, __x - (__y + temp)) | ||
| 166 | /// result := CF | ||
| 167 | /// \endcode | ||
| 168 | /// | ||
| 169 | /// \headerfile <immintrin.h> | ||
| 170 | /// | ||
| 171 | /// This intrinsic corresponds to the \c SBB instruction. | ||
| 172 | /// | ||
| 173 | /// \param __cf | ||
| 174 | /// The 8-bit unsigned carry flag; any non-zero value indicates carry. | ||
| 175 | /// \param __x | ||
| 176 | /// The 32-bit unsigned minuend. | ||
| 177 | /// \param __y | ||
| 178 | /// The 32-bit unsigned subtrahend. | ||
| 179 | /// \param __p | ||
| 180 | /// Pointer to memory for storing the difference. | ||
| 181 | /// \returns The 8-bit unsigned carry-out value. | ||
| 182 | __INLINE unsigned char __DEFAULT_FN_ATTRS _subborrow_u32(unsigned char __cf, | ||
| 183 | unsigned int __x, | ||
| 184 | unsigned int __y, | ||
| 185 | unsigned int *__p) { | ||
| 58 | return __builtin_ia32_subborrow_u32(__cf, __x, __y, __p); | 186 | return __builtin_ia32_subborrow_u32(__cf, __x, __y, __p); |
| 59 | } | 187 | } |
| 60 | 188 | ||
| 61 | #ifdef __x86_64__ | 189 | #ifdef __x86_64__ |
| 62 | static __inline unsigned char __DEFAULT_FN_ATTRS | 190 | /// Adds unsigned 64-bit integer \a __y to 0 or 1 as indicated by the carry |
| 191 | /// flag \a __cf, and subtracts the result from unsigned 64-bit integer | ||
| 192 | /// \a __x. Stores the unsigned 64-bit difference in the memory at \a __p, | ||
| 193 | /// and returns the 8-bit carry-out (carry or overflow flag). | ||
| 194 | /// | ||
| 195 | /// \code{.operation} | ||
| 196 | /// temp := (__cf == 0) ? 0 : 1 | ||
| 197 | /// Store64(__p, __x - (__y + temp)) | ||
| 198 | /// result := CF | ||
| 199 | /// \endcode | ||
| 200 | /// | ||
| 201 | /// \headerfile <immintrin.h> | ||
| 202 | /// | ||
| 203 | /// This intrinsic corresponds to the \c ADC instruction. | ||
| 204 | /// | ||
| 205 | /// \param __cf | ||
| 206 | /// The 8-bit unsigned carry flag; any non-zero value indicates carry. | ||
| 207 | /// \param __x | ||
| 208 | /// The 64-bit unsigned minuend. | ||
| 209 | /// \param __y | ||
| 210 | /// The 64-bit unsigned subtrahend. | ||
| 211 | /// \param __p | ||
| 212 | /// Pointer to memory for storing the difference. | ||
| 213 | /// \returns The 8-bit unsigned carry-out value. | ||
| 214 | __INLINE unsigned char __DEFAULT_FN_ATTRS | ||
| 63 | _subborrow_u64(unsigned char __cf, unsigned long long __x, | 215 | _subborrow_u64(unsigned char __cf, unsigned long long __x, |
| 64 | unsigned long long __y, unsigned long long *__p) | 216 | unsigned long long __y, unsigned long long *__p) { |
| 65 | { | ||
| 66 | return __builtin_ia32_subborrow_u64(__cf, __x, __y, __p); | 217 | return __builtin_ia32_subborrow_u64(__cf, __x, __y, __p); |
| 67 | } | 218 | } |
| 68 | #endif | 219 | #endif |
| 69 | 220 | ||
| 221 | #if defined(__cplusplus) | ||
| 222 | } | ||
| 223 | #endif | ||
| 224 | |||
| 70 | #undef __DEFAULT_FN_ATTRS | 225 | #undef __DEFAULT_FN_ATTRS |
| 71 | 226 | ||
| 72 | #endif /* __ADXINTRIN_H */ | 227 | #endif /* __ADXINTRIN_H */ |
lib/include/altivec.h+136-124
| ... | @@ -3202,71 +3202,79 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a, | ... | @@ -3202,71 +3202,79 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a, |
| 3202 | // the XL-compatible signatures are used for those functions. | 3202 | // the XL-compatible signatures are used for those functions. |
| 3203 | #ifdef __XL_COMPAT_ALTIVEC__ | 3203 | #ifdef __XL_COMPAT_ALTIVEC__ |
| 3204 | #define vec_ctf(__a, __b) \ | 3204 | #define vec_ctf(__a, __b) \ |
| 3205 | _Generic( \ | 3205 | _Generic((__a), \ |
| 3206 | (__a), vector int \ | 3206 | vector int: (vector float)__builtin_altivec_vcfsx((vector int)(__a), \ |
| 3207 | : (vector float)__builtin_altivec_vcfsx((vector int)(__a), (__b)), \ | 3207 | ((__b)&0x1F)), \ |
| 3208 | vector unsigned int \ | 3208 | vector unsigned int: (vector float)__builtin_altivec_vcfux( \ |
| 3209 | : (vector float)__builtin_altivec_vcfux((vector unsigned int)(__a), \ | 3209 | (vector unsigned int)(__a), ((__b)&0x1F)), \ |
| 3210 | (__b)), \ | 3210 | vector unsigned long long: ( \ |
| 3211 | vector unsigned long long \ | 3211 | vector float)(__builtin_vsx_xvcvuxdsp( \ |
| 3212 | : (vector float)(__builtin_vsx_xvcvuxdsp( \ | 3212 | (vector unsigned long long)(__a)) * \ |
| 3213 | (vector unsigned long long)(__a)) * \ | 3213 | (vector float)(vector unsigned)((0x7f - \ |
| 3214 | (vector float)(vector unsigned)((0x7f - (__b)) << 23)), \ | 3214 | ((__b)&0x1F)) \ |
| 3215 | vector signed long long \ | 3215 | << 23)), \ |
| 3216 | : (vector float)(__builtin_vsx_xvcvsxdsp( \ | 3216 | vector signed long long: ( \ |
| 3217 | (vector signed long long)(__a)) * \ | 3217 | vector float)(__builtin_vsx_xvcvsxdsp( \ |
| 3218 | (vector float)(vector unsigned)((0x7f - (__b)) << 23))) | 3218 | (vector signed long long)(__a)) * \ |
| 3219 | (vector float)(vector unsigned)((0x7f - \ | ||
| 3220 | ((__b)&0x1F)) \ | ||
| 3221 | << 23))) | ||
| 3219 | #else // __XL_COMPAT_ALTIVEC__ | 3222 | #else // __XL_COMPAT_ALTIVEC__ |
| 3220 | #define vec_ctf(__a, __b) \ | 3223 | #define vec_ctf(__a, __b) \ |
| 3221 | _Generic( \ | 3224 | _Generic( \ |
| 3222 | (__a), vector int \ | 3225 | (__a), \ |
| 3223 | : (vector float)__builtin_altivec_vcfsx((vector int)(__a), (__b)), \ | 3226 | vector int: (vector float)__builtin_altivec_vcfsx((vector int)(__a), \ |
| 3224 | vector unsigned int \ | 3227 | ((__b)&0x1F)), \ |
| 3225 | : (vector float)__builtin_altivec_vcfux((vector unsigned int)(__a), \ | 3228 | vector unsigned int: (vector float)__builtin_altivec_vcfux( \ |
| 3226 | (__b)), \ | 3229 | (vector unsigned int)(__a), ((__b)&0x1F)), \ |
| 3227 | vector unsigned long long \ | 3230 | vector unsigned long long: ( \ |
| 3228 | : (vector float)(__builtin_convertvector( \ | 3231 | vector float)(__builtin_convertvector( \ |
| 3229 | (vector unsigned long long)(__a), vector double) * \ | 3232 | (vector unsigned long long)(__a), vector double) * \ |
| 3230 | (vector double)(vector unsigned long long)((0x3ffULL - \ | 3233 | (vector double)(vector unsigned long long)((0x3ffULL - \ |
| 3231 | (__b)) \ | 3234 | ((__b)&0x1F)) \ |
| 3232 | << 52)), \ | 3235 | << 52)), \ |
| 3233 | vector signed long long \ | 3236 | vector signed long long: ( \ |
| 3234 | : (vector float)(__builtin_convertvector((vector signed long long)(__a), \ | 3237 | vector float)(__builtin_convertvector( \ |
| 3235 | vector double) * \ | 3238 | (vector signed long long)(__a), vector double) * \ |
| 3236 | (vector double)(vector unsigned long long)((0x3ffULL - \ | 3239 | (vector double)(vector unsigned long long)((0x3ffULL - \ |
| 3237 | (__b)) \ | 3240 | ((__b)&0x1F)) \ |
| 3238 | << 52))) | 3241 | << 52))) |
| 3239 | #endif // __XL_COMPAT_ALTIVEC__ | 3242 | #endif // __XL_COMPAT_ALTIVEC__ |
| 3240 | #else | 3243 | #else |
| 3241 | #define vec_ctf(__a, __b) \ | 3244 | #define vec_ctf(__a, __b) \ |
| 3242 | _Generic((__a), vector int \ | 3245 | _Generic((__a), \ |
| 3243 | : (vector float)__builtin_altivec_vcfsx((vector int)(__a), (__b)), \ | 3246 | vector int: (vector float)__builtin_altivec_vcfsx((vector int)(__a), \ |
| 3244 | vector unsigned int \ | 3247 | ((__b)&0x1F)), \ |
| 3245 | : (vector float)__builtin_altivec_vcfux((vector unsigned int)(__a), \ | 3248 | vector unsigned int: (vector float)__builtin_altivec_vcfux( \ |
| 3246 | (__b))) | 3249 | (vector unsigned int)(__a), ((__b)&0x1F))) |
| 3247 | #endif | 3250 | #endif |
| 3248 | 3251 | ||
| 3249 | /* vec_ctd */ | 3252 | /* vec_ctd */ |
| 3250 | #ifdef __VSX__ | 3253 | #ifdef __VSX__ |
| 3251 | #define vec_ctd(__a, __b) \ | 3254 | #define vec_ctd(__a, __b) \ |
| 3252 | _Generic((__a), vector signed int \ | 3255 | _Generic((__a), \ |
| 3253 | : (vec_doublee((vector signed int)(__a)) * \ | 3256 | vector signed int: ( \ |
| 3254 | (vector double)(vector unsigned long long)((0x3ffULL - (__b)) \ | 3257 | vec_doublee((vector signed int)(__a)) * \ |
| 3255 | << 52)), \ | 3258 | (vector double)(vector unsigned long long)((0x3ffULL - \ |
| 3256 | vector unsigned int \ | 3259 | ((__b)&0x1F)) \ |
| 3257 | : (vec_doublee((vector unsigned int)(__a)) * \ | 3260 | << 52)), \ |
| 3258 | (vector double)(vector unsigned long long)((0x3ffULL - (__b)) \ | 3261 | vector unsigned int: ( \ |
| 3259 | << 52)), \ | 3262 | vec_doublee((vector unsigned int)(__a)) * \ |
| 3260 | vector unsigned long long \ | 3263 | (vector double)(vector unsigned long long)((0x3ffULL - \ |
| 3261 | : (__builtin_convertvector((vector unsigned long long)(__a), \ | 3264 | ((__b)&0x1F)) \ |
| 3262 | vector double) * \ | 3265 | << 52)), \ |
| 3263 | (vector double)(vector unsigned long long)((0x3ffULL - (__b)) \ | 3266 | vector unsigned long long: ( \ |
| 3264 | << 52)), \ | 3267 | __builtin_convertvector((vector unsigned long long)(__a), \ |
| 3265 | vector signed long long \ | 3268 | vector double) * \ |
| 3266 | : (__builtin_convertvector((vector signed long long)(__a), \ | 3269 | (vector double)(vector unsigned long long)((0x3ffULL - \ |
| 3267 | vector double) * \ | 3270 | ((__b)&0x1F)) \ |
| 3268 | (vector double)(vector unsigned long long)((0x3ffULL - (__b)) \ | 3271 | << 52)), \ |
| 3269 | << 52))) | 3272 | vector signed long long: ( \ |
| 3273 | __builtin_convertvector((vector signed long long)(__a), \ | ||
| 3274 | vector double) * \ | ||
| 3275 | (vector double)(vector unsigned long long)((0x3ffULL - \ | ||
| 3276 | ((__b)&0x1F)) \ | ||
| 3277 | << 52))) | ||
| 3270 | #endif // __VSX__ | 3278 | #endif // __VSX__ |
| 3271 | 3279 | ||
| 3272 | /* vec_vcfsx */ | 3280 | /* vec_vcfsx */ |
| ... | @@ -3281,27 +3289,27 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a, | ... | @@ -3281,27 +3289,27 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a, |
| 3281 | #ifdef __VSX__ | 3289 | #ifdef __VSX__ |
| 3282 | #ifdef __XL_COMPAT_ALTIVEC__ | 3290 | #ifdef __XL_COMPAT_ALTIVEC__ |
| 3283 | #define vec_cts(__a, __b) \ | 3291 | #define vec_cts(__a, __b) \ |
| 3284 | _Generic((__a), vector float \ | 3292 | _Generic((__a), \ |
| 3285 | : (vector signed int)__builtin_altivec_vctsxs((vector float)(__a), \ | 3293 | vector float: (vector signed int)__builtin_altivec_vctsxs( \ |
| 3286 | (__b)), \ | 3294 | (vector float)(__a), ((__b)&0x1F)), \ |
| 3287 | vector double \ | 3295 | vector double: __extension__({ \ |
| 3288 | : __extension__({ \ | ||
| 3289 | vector double __ret = \ | 3296 | vector double __ret = \ |
| 3290 | (vector double)(__a) * \ | 3297 | (vector double)(__a) * \ |
| 3291 | (vector double)(vector unsigned long long)((0x3ffULL + (__b)) \ | 3298 | (vector double)(vector unsigned long long)((0x3ffULL + \ |
| 3299 | ((__b)&0x1F)) \ | ||
| 3292 | << 52); \ | 3300 | << 52); \ |
| 3293 | (vector signed long long)__builtin_vsx_xvcvdpsxws(__ret); \ | 3301 | (vector signed long long)__builtin_vsx_xvcvdpsxws(__ret); \ |
| 3294 | })) | 3302 | })) |
| 3295 | #else // __XL_COMPAT_ALTIVEC__ | 3303 | #else // __XL_COMPAT_ALTIVEC__ |
| 3296 | #define vec_cts(__a, __b) \ | 3304 | #define vec_cts(__a, __b) \ |
| 3297 | _Generic((__a), vector float \ | 3305 | _Generic((__a), \ |
| 3298 | : (vector signed int)__builtin_altivec_vctsxs((vector float)(__a), \ | 3306 | vector float: (vector signed int)__builtin_altivec_vctsxs( \ |
| 3299 | (__b)), \ | 3307 | (vector float)(__a), ((__b)&0x1F)), \ |
| 3300 | vector double \ | 3308 | vector double: __extension__({ \ |
| 3301 | : __extension__({ \ | ||
| 3302 | vector double __ret = \ | 3309 | vector double __ret = \ |
| 3303 | (vector double)(__a) * \ | 3310 | (vector double)(__a) * \ |
| 3304 | (vector double)(vector unsigned long long)((0x3ffULL + (__b)) \ | 3311 | (vector double)(vector unsigned long long)((0x3ffULL + \ |
| 3312 | ((__b)&0x1F)) \ | ||
| 3305 | << 52); \ | 3313 | << 52); \ |
| 3306 | (vector signed long long)__builtin_convertvector( \ | 3314 | (vector signed long long)__builtin_convertvector( \ |
| 3307 | __ret, vector signed long long); \ | 3315 | __ret, vector signed long long); \ |
| ... | @@ -3320,27 +3328,27 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a, | ... | @@ -3320,27 +3328,27 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a, |
| 3320 | #ifdef __VSX__ | 3328 | #ifdef __VSX__ |
| 3321 | #ifdef __XL_COMPAT_ALTIVEC__ | 3329 | #ifdef __XL_COMPAT_ALTIVEC__ |
| 3322 | #define vec_ctu(__a, __b) \ | 3330 | #define vec_ctu(__a, __b) \ |
| 3323 | _Generic((__a), vector float \ | 3331 | _Generic((__a), \ |
| 3324 | : (vector unsigned int)__builtin_altivec_vctuxs( \ | 3332 | vector float: (vector unsigned int)__builtin_altivec_vctuxs( \ |
| 3325 | (vector float)(__a), (__b)), \ | 3333 | (vector float)(__a), ((__b)&0x1F)), \ |
| 3326 | vector double \ | 3334 | vector double: __extension__({ \ |
| 3327 | : __extension__({ \ | ||
| 3328 | vector double __ret = \ | 3335 | vector double __ret = \ |
| 3329 | (vector double)(__a) * \ | 3336 | (vector double)(__a) * \ |
| 3330 | (vector double)(vector unsigned long long)((0x3ffULL + __b) \ | 3337 | (vector double)(vector unsigned long long)((0x3ffULL + \ |
| 3338 | ((__b)&0x1F)) \ | ||
| 3331 | << 52); \ | 3339 | << 52); \ |
| 3332 | (vector unsigned long long)__builtin_vsx_xvcvdpuxws(__ret); \ | 3340 | (vector unsigned long long)__builtin_vsx_xvcvdpuxws(__ret); \ |
| 3333 | })) | 3341 | })) |
| 3334 | #else // __XL_COMPAT_ALTIVEC__ | 3342 | #else // __XL_COMPAT_ALTIVEC__ |
| 3335 | #define vec_ctu(__a, __b) \ | 3343 | #define vec_ctu(__a, __b) \ |
| 3336 | _Generic((__a), vector float \ | 3344 | _Generic((__a), \ |
| 3337 | : (vector unsigned int)__builtin_altivec_vctuxs( \ | 3345 | vector float: (vector unsigned int)__builtin_altivec_vctuxs( \ |
| 3338 | (vector float)(__a), (__b)), \ | 3346 | (vector float)(__a), ((__b)&0x1F)), \ |
| 3339 | vector double \ | 3347 | vector double: __extension__({ \ |
| 3340 | : __extension__({ \ | ||
| 3341 | vector double __ret = \ | 3348 | vector double __ret = \ |
| 3342 | (vector double)(__a) * \ | 3349 | (vector double)(__a) * \ |
| 3343 | (vector double)(vector unsigned long long)((0x3ffULL + __b) \ | 3350 | (vector double)(vector unsigned long long)((0x3ffULL + \ |
| 3351 | ((__b)&0x1F)) \ | ||
| 3344 | << 52); \ | 3352 | << 52); \ |
| 3345 | (vector unsigned long long)__builtin_convertvector( \ | 3353 | (vector unsigned long long)__builtin_convertvector( \ |
| 3346 | __ret, vector unsigned long long); \ | 3354 | __ret, vector unsigned long long); \ |
| ... | @@ -3355,60 +3363,62 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a, | ... | @@ -3355,60 +3363,62 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a, |
| 3355 | 3363 | ||
| 3356 | #ifdef __VSX__ | 3364 | #ifdef __VSX__ |
| 3357 | #define vec_ctsl(__a, __b) \ | 3365 | #define vec_ctsl(__a, __b) \ |
| 3358 | _Generic((__a), vector float \ | 3366 | _Generic( \ |
| 3359 | : __extension__({ \ | 3367 | (__a), vector float \ |
| 3360 | vector float __ret = \ | 3368 | : __extension__({ \ |
| 3361 | (vector float)(__a) * \ | 3369 | vector float __ret = \ |
| 3362 | (vector float)(vector unsigned)((0x7f + (__b)) << 23); \ | 3370 | (vector float)(__a) * \ |
| 3363 | __builtin_vsx_xvcvspsxds( \ | 3371 | (vector float)(vector unsigned)((0x7f + ((__b)&0x1F)) << 23); \ |
| 3364 | __builtin_vsx_xxsldwi(__ret, __ret, 1)); \ | 3372 | __builtin_vsx_xvcvspsxds(__builtin_vsx_xxsldwi(__ret, __ret, 1)); \ |
| 3365 | }), \ | 3373 | }), \ |
| 3366 | vector double \ | 3374 | vector double \ |
| 3367 | : __extension__({ \ | 3375 | : __extension__({ \ |
| 3368 | vector double __ret = \ | 3376 | vector double __ret = \ |
| 3369 | (vector double)(__a) * \ | 3377 | (vector double)(__a) * \ |
| 3370 | (vector double)(vector unsigned long long)((0x3ffULL + __b) \ | 3378 | (vector double)(vector unsigned long long)((0x3ffULL + \ |
| 3371 | << 52); \ | 3379 | ((__b)&0x1F)) \ |
| 3372 | __builtin_convertvector(__ret, vector signed long long); \ | 3380 | << 52); \ |
| 3373 | })) | 3381 | __builtin_convertvector(__ret, vector signed long long); \ |
| 3382 | })) | ||
| 3374 | 3383 | ||
| 3375 | /* vec_ctul */ | 3384 | /* vec_ctul */ |
| 3376 | 3385 | ||
| 3377 | #define vec_ctul(__a, __b) \ | 3386 | #define vec_ctul(__a, __b) \ |
| 3378 | _Generic((__a), vector float \ | 3387 | _Generic( \ |
| 3379 | : __extension__({ \ | 3388 | (__a), vector float \ |
| 3380 | vector float __ret = \ | 3389 | : __extension__({ \ |
| 3381 | (vector float)(__a) * \ | 3390 | vector float __ret = \ |
| 3382 | (vector float)(vector unsigned)((0x7f + (__b)) << 23); \ | 3391 | (vector float)(__a) * \ |
| 3383 | __builtin_vsx_xvcvspuxds( \ | 3392 | (vector float)(vector unsigned)((0x7f + ((__b)&0x1F)) << 23); \ |
| 3384 | __builtin_vsx_xxsldwi(__ret, __ret, 1)); \ | 3393 | __builtin_vsx_xvcvspuxds(__builtin_vsx_xxsldwi(__ret, __ret, 1)); \ |
| 3385 | }), \ | 3394 | }), \ |
| 3386 | vector double \ | 3395 | vector double \ |
| 3387 | : __extension__({ \ | 3396 | : __extension__({ \ |
| 3388 | vector double __ret = \ | 3397 | vector double __ret = \ |
| 3389 | (vector double)(__a) * \ | 3398 | (vector double)(__a) * \ |
| 3390 | (vector double)(vector unsigned long long)((0x3ffULL + __b) \ | 3399 | (vector double)(vector unsigned long long)((0x3ffULL + \ |
| 3391 | << 52); \ | 3400 | ((__b)&0x1F)) \ |
| 3392 | __builtin_convertvector(__ret, vector unsigned long long); \ | 3401 | << 52); \ |
| 3393 | })) | 3402 | __builtin_convertvector(__ret, vector unsigned long long); \ |
| 3403 | })) | ||
| 3394 | #endif | 3404 | #endif |
| 3395 | #else // __LITTLE_ENDIAN__ | 3405 | #else // __LITTLE_ENDIAN__ |
| 3396 | /* vec_ctsl */ | 3406 | /* vec_ctsl */ |
| 3397 | 3407 | ||
| 3398 | #ifdef __VSX__ | 3408 | #ifdef __VSX__ |
| 3399 | #define vec_ctsl(__a, __b) \ | 3409 | #define vec_ctsl(__a, __b) \ |
| 3400 | _Generic((__a), vector float \ | 3410 | _Generic((__a), \ |
| 3401 | : __extension__({ \ | 3411 | vector float: __extension__({ \ |
| 3402 | vector float __ret = \ | 3412 | vector float __ret = \ |
| 3403 | (vector float)(__a) * \ | 3413 | (vector float)(__a) * \ |
| 3404 | (vector float)(vector unsigned)((0x7f + (__b)) << 23); \ | 3414 | (vector float)(vector unsigned)((0x7f + ((__b)&0x1F)) << 23); \ |
| 3405 | __builtin_vsx_xvcvspsxds(__ret); \ | 3415 | __builtin_vsx_xvcvspsxds(__ret); \ |
| 3406 | }), \ | 3416 | }), \ |
| 3407 | vector double \ | 3417 | vector double: __extension__({ \ |
| 3408 | : __extension__({ \ | ||
| 3409 | vector double __ret = \ | 3418 | vector double __ret = \ |
| 3410 | (vector double)(__a) * \ | 3419 | (vector double)(__a) * \ |
| 3411 | (vector double)(vector unsigned long long)((0x3ffULL + __b) \ | 3420 | (vector double)(vector unsigned long long)((0x3ffULL + \ |
| 3421 | ((__b)&0x1F)) \ | ||
| 3412 | << 52); \ | 3422 | << 52); \ |
| 3413 | __builtin_convertvector(__ret, vector signed long long); \ | 3423 | __builtin_convertvector(__ret, vector signed long long); \ |
| 3414 | })) | 3424 | })) |
| ... | @@ -3420,14 +3430,16 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a, | ... | @@ -3420,14 +3430,16 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a, |
| 3420 | : __extension__({ \ | 3430 | : __extension__({ \ |
| 3421 | vector float __ret = \ | 3431 | vector float __ret = \ |
| 3422 | (vector float)(__a) * \ | 3432 | (vector float)(__a) * \ |
| 3423 | (vector float)(vector unsigned)((0x7f + (__b)) << 23); \ | 3433 | (vector float)(vector unsigned)((0x7f + ((__b)&0x1F)) \ |
| 3434 | << 23); \ | ||
| 3424 | __builtin_vsx_xvcvspuxds(__ret); \ | 3435 | __builtin_vsx_xvcvspuxds(__ret); \ |
| 3425 | }), \ | 3436 | }), \ |
| 3426 | vector double \ | 3437 | vector double \ |
| 3427 | : __extension__({ \ | 3438 | : __extension__({ \ |
| 3428 | vector double __ret = \ | 3439 | vector double __ret = \ |
| 3429 | (vector double)(__a) * \ | 3440 | (vector double)(__a) * \ |
| 3430 | (vector double)(vector unsigned long long)((0x3ffULL + __b) \ | 3441 | (vector double)(vector unsigned long long)((0x3ffULL + \ |
| 3442 | ((__b)&0x1F)) \ | ||
| 3431 | << 52); \ | 3443 | << 52); \ |
| 3432 | __builtin_convertvector(__ret, vector unsigned long long); \ | 3444 | __builtin_convertvector(__ret, vector unsigned long long); \ |
| 3433 | })) | 3445 | })) |
lib/include/amxcomplexintrin.h created+169
| ... | @@ -0,0 +1,169 @@ | ||
| 1 | /*===--------- amxcomplexintrin.h - AMXCOMPLEX intrinsics -*- C++ -*---------=== | ||
| 2 | * | ||
| 3 | * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| 4 | * See https://llvm.org/LICENSE.txt for license information. | ||
| 5 | * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| 6 | * | ||
| 7 | *===------------------------------------------------------------------------=== | ||
| 8 | */ | ||
| 9 | |||
| 10 | #ifndef __IMMINTRIN_H | ||
| 11 | #error "Never use <amxcomplexintrin.h> directly; include <immintrin.h> instead." | ||
| 12 | #endif // __IMMINTRIN_H | ||
| 13 | |||
| 14 | #ifndef __AMX_COMPLEXINTRIN_H | ||
| 15 | #define __AMX_COMPLEXINTRIN_H | ||
| 16 | #ifdef __x86_64__ | ||
| 17 | |||
| 18 | #define __DEFAULT_FN_ATTRS_COMPLEX \ | ||
| 19 | __attribute__((__always_inline__, __nodebug__, __target__("amx-complex"))) | ||
| 20 | |||
| 21 | /// Perform matrix multiplication of two tiles containing complex elements and | ||
| 22 | /// accumulate the results into a packed single precision tile. Each dword | ||
| 23 | /// element in input tiles \a a and \a b is interpreted as a complex number | ||
| 24 | /// with FP16 real part and FP16 imaginary part. | ||
| 25 | /// Calculates the imaginary part of the result. For each possible combination | ||
| 26 | /// of (row of \a a, column of \a b), it performs a set of multiplication | ||
| 27 | /// and accumulations on all corresponding complex numbers (one from \a a | ||
| 28 | /// and one from \a b). The imaginary part of the \a a element is multiplied | ||
| 29 | /// with the real part of the corresponding \a b element, and the real part | ||
| 30 | /// of the \a a element is multiplied with the imaginary part of the | ||
| 31 | /// corresponding \a b elements. The two accumulated results are added, and | ||
| 32 | /// then accumulated into the corresponding row and column of \a dst. | ||
| 33 | /// | ||
| 34 | /// \headerfile <x86intrin.h> | ||
| 35 | /// | ||
| 36 | /// \code | ||
| 37 | /// void _tile_cmmimfp16ps(__tile dst, __tile a, __tile b); | ||
| 38 | /// \endcode | ||
| 39 | /// | ||
| 40 | /// \code{.operation} | ||
| 41 | /// FOR m := 0 TO dst.rows - 1 | ||
| 42 | /// tmp := dst.row[m] | ||
| 43 | /// FOR k := 0 TO (a.colsb / 4) - 1 | ||
| 44 | /// FOR n := 0 TO (dst.colsb / 4) - 1 | ||
| 45 | /// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+0]) * FP32(b.row[k].fp16[2*n+1]) | ||
| 46 | /// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+1]) * FP32(b.row[k].fp16[2*n+0]) | ||
| 47 | /// ENDFOR | ||
| 48 | /// ENDFOR | ||
| 49 | /// write_row_and_zero(dst, m, tmp, dst.colsb) | ||
| 50 | /// ENDFOR | ||
| 51 | /// zero_upper_rows(dst, dst.rows) | ||
| 52 | /// zero_tileconfig_start() | ||
| 53 | /// \endcode | ||
| 54 | /// | ||
| 55 | /// This intrinsic corresponds to the \c TCMMIMFP16PS instruction. | ||
| 56 | /// | ||
| 57 | /// \param dst | ||
| 58 | /// The destination tile. Max size is 1024 Bytes. | ||
| 59 | /// \param a | ||
| 60 | /// The 1st source tile. Max size is 1024 Bytes. | ||
| 61 | /// \param b | ||
| 62 | /// The 2nd source tile. Max size is 1024 Bytes. | ||
| 63 | #define _tile_cmmimfp16ps(dst, a, b) __builtin_ia32_tcmmimfp16ps(dst, a, b) | ||
| 64 | |||
| 65 | /// Perform matrix multiplication of two tiles containing complex elements and | ||
| 66 | /// accumulate the results into a packed single precision tile. Each dword | ||
| 67 | /// element in input tiles \a a and \a b is interpreted as a complex number | ||
| 68 | /// with FP16 real part and FP16 imaginary part. | ||
| 69 | /// Calculates the real part of the result. For each possible combination | ||
| 70 | /// of (row of \a a, column of \a b), it performs a set of multiplication | ||
| 71 | /// and accumulations on all corresponding complex numbers (one from \a a | ||
| 72 | /// and one from \a b). The real part of the \a a element is multiplied | ||
| 73 | /// with the real part of the corresponding \a b element, and the negated | ||
| 74 | /// imaginary part of the \a a element is multiplied with the imaginary | ||
| 75 | /// part of the corresponding \a b elements. The two accumulated results | ||
| 76 | /// are added, and then accumulated into the corresponding row and column | ||
| 77 | /// of \a dst. | ||
| 78 | /// | ||
| 79 | /// \headerfile <x86intrin.h> | ||
| 80 | /// | ||
| 81 | /// \code | ||
| 82 | /// void _tile_cmmrlfp16ps(__tile dst, __tile a, __tile b); | ||
| 83 | /// \endcode | ||
| 84 | /// | ||
| 85 | /// \code{.operation} | ||
| 86 | /// FOR m := 0 TO dst.rows - 1 | ||
| 87 | /// tmp := dst.row[m] | ||
| 88 | /// FOR k := 0 TO (a.colsb / 4) - 1 | ||
| 89 | /// FOR n := 0 TO (dst.colsb / 4) - 1 | ||
| 90 | /// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+0]) * FP32(b.row[k].fp16[2*n+0]) | ||
| 91 | /// tmp.fp32[n] += FP32(-a.row[m].fp16[2*k+1]) * FP32(b.row[k].fp16[2*n+1]) | ||
| 92 | /// ENDFOR | ||
| 93 | /// ENDFOR | ||
| 94 | /// write_row_and_zero(dst, m, tmp, dst.colsb) | ||
| 95 | /// ENDFOR | ||
| 96 | /// zero_upper_rows(dst, dst.rows) | ||
| 97 | /// zero_tileconfig_start() | ||
| 98 | /// \endcode | ||
| 99 | /// | ||
| 100 | /// This intrinsic corresponds to the \c TCMMIMFP16PS instruction. | ||
| 101 | /// | ||
| 102 | /// \param dst | ||
| 103 | /// The destination tile. Max size is 1024 Bytes. | ||
| 104 | /// \param a | ||
| 105 | /// The 1st source tile. Max size is 1024 Bytes. | ||
| 106 | /// \param b | ||
| 107 | /// The 2nd source tile. Max size is 1024 Bytes. | ||
| 108 | #define _tile_cmmrlfp16ps(dst, a, b) __builtin_ia32_tcmmrlfp16ps(dst, a, b) | ||
| 109 | |||
| 110 | static __inline__ _tile1024i __DEFAULT_FN_ATTRS_COMPLEX | ||
| 111 | _tile_cmmimfp16ps_internal(unsigned short m, unsigned short n, unsigned short k, | ||
| 112 | _tile1024i dst, _tile1024i src1, _tile1024i src2) { | ||
| 113 | return __builtin_ia32_tcmmimfp16ps_internal(m, n, k, dst, src1, src2); | ||
| 114 | } | ||
| 115 | |||
| 116 | static __inline__ _tile1024i __DEFAULT_FN_ATTRS_COMPLEX | ||
| 117 | _tile_cmmrlfp16ps_internal(unsigned short m, unsigned short n, unsigned short k, | ||
| 118 | _tile1024i dst, _tile1024i src1, _tile1024i src2) { | ||
| 119 | return __builtin_ia32_tcmmrlfp16ps_internal(m, n, k, dst, src1, src2); | ||
| 120 | } | ||
| 121 | |||
| 122 | /// Perform matrix multiplication of two tiles containing complex elements and | ||
| 123 | /// accumulate the results into a packed single precision tile. Each dword | ||
| 124 | /// element in input tiles src0 and src1 is interpreted as a complex number with | ||
| 125 | /// FP16 real part and FP16 imaginary part. | ||
| 126 | /// This function calculates the imaginary part of the result. | ||
| 127 | /// | ||
| 128 | /// \headerfile <immintrin.h> | ||
| 129 | /// | ||
| 130 | /// This intrinsic corresponds to the <c> TCMMIMFP16PS </c> instruction. | ||
| 131 | /// | ||
| 132 | /// \param dst | ||
| 133 | /// The destination tile. Max size is 1024 Bytes. | ||
| 134 | /// \param src0 | ||
| 135 | /// The 1st source tile. Max size is 1024 Bytes. | ||
| 136 | /// \param src1 | ||
| 137 | /// The 2nd source tile. Max size is 1024 Bytes. | ||
| 138 | __DEFAULT_FN_ATTRS_COMPLEX | ||
| 139 | static void __tile_cmmimfp16ps(__tile1024i *dst, __tile1024i src0, | ||
| 140 | __tile1024i src1) { | ||
| 141 | dst->tile = _tile_cmmimfp16ps_internal(src0.row, src1.col, src0.col, | ||
| 142 | dst->tile, src0.tile, src1.tile); | ||
| 143 | } | ||
| 144 | |||
| 145 | /// Perform matrix multiplication of two tiles containing complex elements and | ||
| 146 | /// accumulate the results into a packed single precision tile. Each dword | ||
| 147 | /// element in input tiles src0 and src1 is interpreted as a complex number with | ||
| 148 | /// FP16 real part and FP16 imaginary part. | ||
| 149 | /// This function calculates the real part of the result. | ||
| 150 | /// | ||
| 151 | /// \headerfile <immintrin.h> | ||
| 152 | /// | ||
| 153 | /// This intrinsic corresponds to the <c> TCMMRLFP16PS </c> instruction. | ||
| 154 | /// | ||
| 155 | /// \param dst | ||
| 156 | /// The destination tile. Max size is 1024 Bytes. | ||
| 157 | /// \param src0 | ||
| 158 | /// The 1st source tile. Max size is 1024 Bytes. | ||
| 159 | /// \param src1 | ||
| 160 | /// The 2nd source tile. Max size is 1024 Bytes. | ||
| 161 | __DEFAULT_FN_ATTRS_COMPLEX | ||
| 162 | static void __tile_cmmrlfp16ps(__tile1024i *dst, __tile1024i src0, | ||
| 163 | __tile1024i src1) { | ||
| 164 | dst->tile = _tile_cmmrlfp16ps_internal(src0.row, src1.col, src0.col, | ||
| 165 | dst->tile, src0.tile, src1.tile); | ||
| 166 | } | ||
| 167 | |||
| 168 | #endif // __x86_64__ | ||
| 169 | #endif // __AMX_COMPLEXINTRIN_H | ||
lib/include/arm_acle.h+13-9
| ... | @@ -138,28 +138,32 @@ __rorl(unsigned long __x, uint32_t __y) { | ... | @@ -138,28 +138,32 @@ __rorl(unsigned long __x, uint32_t __y) { |
| 138 | 138 | ||
| 139 | 139 | ||
| 140 | /* CLZ */ | 140 | /* CLZ */ |
| 141 | static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__)) | 141 | static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) |
| 142 | __clz(uint32_t __t) { | 142 | __clz(uint32_t __t) { |
| 143 | return (uint32_t)__builtin_clz(__t); | 143 | return __builtin_arm_clz(__t); |
| 144 | } | 144 | } |
| 145 | 145 | ||
| 146 | static __inline__ unsigned long __attribute__((__always_inline__, __nodebug__)) | 146 | static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) |
| 147 | __clzl(unsigned long __t) { | 147 | __clzl(unsigned long __t) { |
| 148 | return (unsigned long)__builtin_clzl(__t); | 148 | #if __SIZEOF_LONG__ == 4 |
| 149 | return __builtin_arm_clz(__t); | ||
| 150 | #else | ||
| 151 | return __builtin_arm_clz64(__t); | ||
| 152 | #endif | ||
| 149 | } | 153 | } |
| 150 | 154 | ||
| 151 | static __inline__ uint64_t __attribute__((__always_inline__, __nodebug__)) | 155 | static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) |
| 152 | __clzll(uint64_t __t) { | 156 | __clzll(uint64_t __t) { |
| 153 | return (uint64_t)__builtin_clzll(__t); | 157 | return __builtin_arm_clz64(__t); |
| 154 | } | 158 | } |
| 155 | 159 | ||
| 156 | /* CLS */ | 160 | /* CLS */ |
| 157 | static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__)) | 161 | static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) |
| 158 | __cls(uint32_t __t) { | 162 | __cls(uint32_t __t) { |
| 159 | return __builtin_arm_cls(__t); | 163 | return __builtin_arm_cls(__t); |
| 160 | } | 164 | } |
| 161 | 165 | ||
| 162 | static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__)) | 166 | static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) |
| 163 | __clsl(unsigned long __t) { | 167 | __clsl(unsigned long __t) { |
| 164 | #if __SIZEOF_LONG__ == 4 | 168 | #if __SIZEOF_LONG__ == 4 |
| 165 | return __builtin_arm_cls(__t); | 169 | return __builtin_arm_cls(__t); |
| ... | @@ -168,7 +172,7 @@ __clsl(unsigned long __t) { | ... | @@ -168,7 +172,7 @@ __clsl(unsigned long __t) { |
| 168 | #endif | 172 | #endif |
| 169 | } | 173 | } |
| 170 | 174 | ||
| 171 | static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__)) | 175 | static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__)) |
| 172 | __clsll(uint64_t __t) { | 176 | __clsll(uint64_t __t) { |
| 173 | return __builtin_arm_cls64(__t); | 177 | return __builtin_arm_cls64(__t); |
| 174 | } | 178 | } |
lib/include/arm_neon.h+164-1
| ... | @@ -35,7 +35,6 @@ | ... | @@ -35,7 +35,6 @@ |
| 35 | #include <stdint.h> | 35 | #include <stdint.h> |
| 36 | 36 | ||
| 37 | #include <arm_bf16.h> | 37 | #include <arm_bf16.h> |
| 38 | typedef __bf16 bfloat16_t; | ||
| 39 | typedef float float32_t; | 38 | typedef float float32_t; |
| 40 | typedef __fp16 float16_t; | 39 | typedef __fp16 float16_t; |
| 41 | #ifdef __aarch64__ | 40 | #ifdef __aarch64__ |
| ... | @@ -64938,6 +64937,170 @@ int8x16_t __reint_786 = __rev2_786; \ | ... | @@ -64938,6 +64937,170 @@ int8x16_t __reint_786 = __rev2_786; \ |
| 64938 | }) | 64937 | }) |
| 64939 | #endif | 64938 | #endif |
| 64940 | 64939 | ||
| 64940 | #define vldap1_lane_p64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 64941 | poly64x1_t __ret; \ | ||
| 64942 | poly64x1_t __s1 = __p1; \ | ||
| 64943 | __ret = (poly64x1_t) __builtin_neon_vldap1_lane_p64(__p0, (int8x8_t)__s1, __p2, 6); \ | ||
| 64944 | __ret; \ | ||
| 64945 | }) | ||
| 64946 | #ifdef __LITTLE_ENDIAN__ | ||
| 64947 | #define vldap1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 64948 | poly64x2_t __ret; \ | ||
| 64949 | poly64x2_t __s1 = __p1; \ | ||
| 64950 | __ret = (poly64x2_t) __builtin_neon_vldap1q_lane_p64(__p0, (int8x16_t)__s1, __p2, 38); \ | ||
| 64951 | __ret; \ | ||
| 64952 | }) | ||
| 64953 | #else | ||
| 64954 | #define vldap1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 64955 | poly64x2_t __ret; \ | ||
| 64956 | poly64x2_t __s1 = __p1; \ | ||
| 64957 | poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ | ||
| 64958 | __ret = (poly64x2_t) __builtin_neon_vldap1q_lane_p64(__p0, (int8x16_t)__rev1, __p2, 38); \ | ||
| 64959 | __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ | ||
| 64960 | __ret; \ | ||
| 64961 | }) | ||
| 64962 | #endif | ||
| 64963 | |||
| 64964 | #ifdef __LITTLE_ENDIAN__ | ||
| 64965 | #define vldap1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 64966 | uint64x2_t __ret; \ | ||
| 64967 | uint64x2_t __s1 = __p1; \ | ||
| 64968 | __ret = (uint64x2_t) __builtin_neon_vldap1q_lane_u64(__p0, (int8x16_t)__s1, __p2, 51); \ | ||
| 64969 | __ret; \ | ||
| 64970 | }) | ||
| 64971 | #else | ||
| 64972 | #define vldap1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 64973 | uint64x2_t __ret; \ | ||
| 64974 | uint64x2_t __s1 = __p1; \ | ||
| 64975 | uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ | ||
| 64976 | __ret = (uint64x2_t) __builtin_neon_vldap1q_lane_u64(__p0, (int8x16_t)__rev1, __p2, 51); \ | ||
| 64977 | __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ | ||
| 64978 | __ret; \ | ||
| 64979 | }) | ||
| 64980 | #endif | ||
| 64981 | |||
| 64982 | #ifdef __LITTLE_ENDIAN__ | ||
| 64983 | #define vldap1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 64984 | float64x2_t __ret; \ | ||
| 64985 | float64x2_t __s1 = __p1; \ | ||
| 64986 | __ret = (float64x2_t) __builtin_neon_vldap1q_lane_f64(__p0, (int8x16_t)__s1, __p2, 42); \ | ||
| 64987 | __ret; \ | ||
| 64988 | }) | ||
| 64989 | #else | ||
| 64990 | #define vldap1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 64991 | float64x2_t __ret; \ | ||
| 64992 | float64x2_t __s1 = __p1; \ | ||
| 64993 | float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ | ||
| 64994 | __ret = (float64x2_t) __builtin_neon_vldap1q_lane_f64(__p0, (int8x16_t)__rev1, __p2, 42); \ | ||
| 64995 | __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ | ||
| 64996 | __ret; \ | ||
| 64997 | }) | ||
| 64998 | #endif | ||
| 64999 | |||
| 65000 | #ifdef __LITTLE_ENDIAN__ | ||
| 65001 | #define vldap1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65002 | int64x2_t __ret; \ | ||
| 65003 | int64x2_t __s1 = __p1; \ | ||
| 65004 | __ret = (int64x2_t) __builtin_neon_vldap1q_lane_s64(__p0, (int8x16_t)__s1, __p2, 35); \ | ||
| 65005 | __ret; \ | ||
| 65006 | }) | ||
| 65007 | #else | ||
| 65008 | #define vldap1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65009 | int64x2_t __ret; \ | ||
| 65010 | int64x2_t __s1 = __p1; \ | ||
| 65011 | int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ | ||
| 65012 | __ret = (int64x2_t) __builtin_neon_vldap1q_lane_s64(__p0, (int8x16_t)__rev1, __p2, 35); \ | ||
| 65013 | __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ | ||
| 65014 | __ret; \ | ||
| 65015 | }) | ||
| 65016 | #endif | ||
| 65017 | |||
| 65018 | #define vldap1_lane_u64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65019 | uint64x1_t __ret; \ | ||
| 65020 | uint64x1_t __s1 = __p1; \ | ||
| 65021 | __ret = (uint64x1_t) __builtin_neon_vldap1_lane_u64(__p0, (int8x8_t)__s1, __p2, 19); \ | ||
| 65022 | __ret; \ | ||
| 65023 | }) | ||
| 65024 | #define vldap1_lane_f64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65025 | float64x1_t __ret; \ | ||
| 65026 | float64x1_t __s1 = __p1; \ | ||
| 65027 | __ret = (float64x1_t) __builtin_neon_vldap1_lane_f64(__p0, (int8x8_t)__s1, __p2, 10); \ | ||
| 65028 | __ret; \ | ||
| 65029 | }) | ||
| 65030 | #define vldap1_lane_s64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65031 | int64x1_t __ret; \ | ||
| 65032 | int64x1_t __s1 = __p1; \ | ||
| 65033 | __ret = (int64x1_t) __builtin_neon_vldap1_lane_s64(__p0, (int8x8_t)__s1, __p2, 3); \ | ||
| 65034 | __ret; \ | ||
| 65035 | }) | ||
| 65036 | #define vstl1_lane_p64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65037 | poly64x1_t __s1 = __p1; \ | ||
| 65038 | __builtin_neon_vstl1_lane_p64(__p0, (int8x8_t)__s1, __p2, 6); \ | ||
| 65039 | }) | ||
| 65040 | #ifdef __LITTLE_ENDIAN__ | ||
| 65041 | #define vstl1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65042 | poly64x2_t __s1 = __p1; \ | ||
| 65043 | __builtin_neon_vstl1q_lane_p64(__p0, (int8x16_t)__s1, __p2, 38); \ | ||
| 65044 | }) | ||
| 65045 | #else | ||
| 65046 | #define vstl1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65047 | poly64x2_t __s1 = __p1; \ | ||
| 65048 | poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ | ||
| 65049 | __builtin_neon_vstl1q_lane_p64(__p0, (int8x16_t)__rev1, __p2, 38); \ | ||
| 65050 | }) | ||
| 65051 | #endif | ||
| 65052 | |||
| 65053 | #ifdef __LITTLE_ENDIAN__ | ||
| 65054 | #define vstl1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65055 | uint64x2_t __s1 = __p1; \ | ||
| 65056 | __builtin_neon_vstl1q_lane_u64(__p0, (int8x16_t)__s1, __p2, 51); \ | ||
| 65057 | }) | ||
| 65058 | #else | ||
| 65059 | #define vstl1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65060 | uint64x2_t __s1 = __p1; \ | ||
| 65061 | uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ | ||
| 65062 | __builtin_neon_vstl1q_lane_u64(__p0, (int8x16_t)__rev1, __p2, 51); \ | ||
| 65063 | }) | ||
| 65064 | #endif | ||
| 65065 | |||
| 65066 | #ifdef __LITTLE_ENDIAN__ | ||
| 65067 | #define vstl1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65068 | float64x2_t __s1 = __p1; \ | ||
| 65069 | __builtin_neon_vstl1q_lane_f64(__p0, (int8x16_t)__s1, __p2, 42); \ | ||
| 65070 | }) | ||
| 65071 | #else | ||
| 65072 | #define vstl1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65073 | float64x2_t __s1 = __p1; \ | ||
| 65074 | float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ | ||
| 65075 | __builtin_neon_vstl1q_lane_f64(__p0, (int8x16_t)__rev1, __p2, 42); \ | ||
| 65076 | }) | ||
| 65077 | #endif | ||
| 65078 | |||
| 65079 | #ifdef __LITTLE_ENDIAN__ | ||
| 65080 | #define vstl1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65081 | int64x2_t __s1 = __p1; \ | ||
| 65082 | __builtin_neon_vstl1q_lane_s64(__p0, (int8x16_t)__s1, __p2, 35); \ | ||
| 65083 | }) | ||
| 65084 | #else | ||
| 65085 | #define vstl1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65086 | int64x2_t __s1 = __p1; \ | ||
| 65087 | int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ | ||
| 65088 | __builtin_neon_vstl1q_lane_s64(__p0, (int8x16_t)__rev1, __p2, 35); \ | ||
| 65089 | }) | ||
| 65090 | #endif | ||
| 65091 | |||
| 65092 | #define vstl1_lane_u64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65093 | uint64x1_t __s1 = __p1; \ | ||
| 65094 | __builtin_neon_vstl1_lane_u64(__p0, (int8x8_t)__s1, __p2, 19); \ | ||
| 65095 | }) | ||
| 65096 | #define vstl1_lane_f64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65097 | float64x1_t __s1 = __p1; \ | ||
| 65098 | __builtin_neon_vstl1_lane_f64(__p0, (int8x8_t)__s1, __p2, 10); \ | ||
| 65099 | }) | ||
| 65100 | #define vstl1_lane_s64(__p0, __p1, __p2) __extension__ ({ \ | ||
| 65101 | int64x1_t __s1 = __p1; \ | ||
| 65102 | __builtin_neon_vstl1_lane_s64(__p0, (int8x8_t)__s1, __p2, 3); \ | ||
| 65103 | }) | ||
| 64941 | #ifdef __LITTLE_ENDIAN__ | 65104 | #ifdef __LITTLE_ENDIAN__ |
| 64942 | __ai __attribute__((target("sha3"))) uint8x16_t vbcaxq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { | 65105 | __ai __attribute__((target("sha3"))) uint8x16_t vbcaxq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { |
| 64943 | uint8x16_t __ret; | 65106 | uint8x16_t __ret; |
lib/include/arm_sme_draft_spec_subject_to_change.h created+642
| ... | @@ -0,0 +1,642 @@ | ||
| 1 | /*===---- arm_sme_draft_spec_subject_to_change.h - ARM SME intrinsics ------=== | ||
| 2 | * | ||
| 3 | * | ||
| 4 | * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| 5 | * See https://llvm.org/LICENSE.txt for license information. | ||
| 6 | * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| 7 | * | ||
| 8 | *===-----------------------------------------------------------------------=== | ||
| 9 | */ | ||
| 10 | |||
| 11 | #ifndef __ARM_SME_H | ||
| 12 | #define __ARM_SME_H | ||
| 13 | |||
| 14 | #if !defined(__LITTLE_ENDIAN__) | ||
| 15 | #error "Big endian is currently not supported for arm_sme_draft_spec_subject_to_change.h" | ||
| 16 | #endif | ||
| 17 | #include <arm_sve.h> | ||
| 18 | |||
| 19 | /* Function attributes */ | ||
| 20 | #define __ai static __inline__ __attribute__((__always_inline__, __nodebug__)) | ||
| 21 | |||
| 22 | #define __aio static __inline__ __attribute__((__always_inline__, __nodebug__, __overloadable__)) | ||
| 23 | |||
| 24 | #ifdef __cplusplus | ||
| 25 | extern "C" { | ||
| 26 | #endif | ||
| 27 | |||
| 28 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_u32_m), arm_streaming, arm_shared_za)) | ||
| 29 | void svaddha_za32_u32_m(uint64_t, svbool_t, svbool_t, svuint32_t); | ||
| 30 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_s32_m), arm_streaming, arm_shared_za)) | ||
| 31 | void svaddha_za32_s32_m(uint64_t, svbool_t, svbool_t, svint32_t); | ||
| 32 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za32_u32_m), arm_streaming, arm_shared_za)) | ||
| 33 | void svaddva_za32_u32_m(uint64_t, svbool_t, svbool_t, svuint32_t); | ||
| 34 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za32_s32_m), arm_streaming, arm_shared_za)) | ||
| 35 | void svaddva_za32_s32_m(uint64_t, svbool_t, svbool_t, svint32_t); | ||
| 36 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svcntsb), arm_streaming_compatible, arm_preserves_za)) | ||
| 37 | uint64_t svcntsb(void); | ||
| 38 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svcntsd), arm_streaming_compatible, arm_preserves_za)) | ||
| 39 | uint64_t svcntsd(void); | ||
| 40 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svcntsh), arm_streaming_compatible, arm_preserves_za)) | ||
| 41 | uint64_t svcntsh(void); | ||
| 42 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svcntsw), arm_streaming_compatible, arm_preserves_za)) | ||
| 43 | uint64_t svcntsw(void); | ||
| 44 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za128), arm_streaming, arm_shared_za)) | ||
| 45 | void svld1_hor_vnum_za128(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t); | ||
| 46 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za16), arm_streaming, arm_shared_za)) | ||
| 47 | void svld1_hor_vnum_za16(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t); | ||
| 48 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za32), arm_streaming, arm_shared_za)) | ||
| 49 | void svld1_hor_vnum_za32(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t); | ||
| 50 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za64), arm_streaming, arm_shared_za)) | ||
| 51 | void svld1_hor_vnum_za64(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t); | ||
| 52 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za8), arm_streaming, arm_shared_za)) | ||
| 53 | void svld1_hor_vnum_za8(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t); | ||
| 54 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za128), arm_streaming, arm_shared_za)) | ||
| 55 | void svld1_hor_za128(uint64_t, uint32_t, uint64_t, svbool_t, void const *); | ||
| 56 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za16), arm_streaming, arm_shared_za)) | ||
| 57 | void svld1_hor_za16(uint64_t, uint32_t, uint64_t, svbool_t, void const *); | ||
| 58 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za32), arm_streaming, arm_shared_za)) | ||
| 59 | void svld1_hor_za32(uint64_t, uint32_t, uint64_t, svbool_t, void const *); | ||
| 60 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za64), arm_streaming, arm_shared_za)) | ||
| 61 | void svld1_hor_za64(uint64_t, uint32_t, uint64_t, svbool_t, void const *); | ||
| 62 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za8), arm_streaming, arm_shared_za)) | ||
| 63 | void svld1_hor_za8(uint64_t, uint32_t, uint64_t, svbool_t, void const *); | ||
| 64 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za128), arm_streaming, arm_shared_za)) | ||
| 65 | void svld1_ver_vnum_za128(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t); | ||
| 66 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za16), arm_streaming, arm_shared_za)) | ||
| 67 | void svld1_ver_vnum_za16(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t); | ||
| 68 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za32), arm_streaming, arm_shared_za)) | ||
| 69 | void svld1_ver_vnum_za32(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t); | ||
| 70 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za64), arm_streaming, arm_shared_za)) | ||
| 71 | void svld1_ver_vnum_za64(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t); | ||
| 72 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za8), arm_streaming, arm_shared_za)) | ||
| 73 | void svld1_ver_vnum_za8(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t); | ||
| 74 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za128), arm_streaming, arm_shared_za)) | ||
| 75 | void svld1_ver_za128(uint64_t, uint32_t, uint64_t, svbool_t, void const *); | ||
| 76 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za16), arm_streaming, arm_shared_za)) | ||
| 77 | void svld1_ver_za16(uint64_t, uint32_t, uint64_t, svbool_t, void const *); | ||
| 78 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za32), arm_streaming, arm_shared_za)) | ||
| 79 | void svld1_ver_za32(uint64_t, uint32_t, uint64_t, svbool_t, void const *); | ||
| 80 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za64), arm_streaming, arm_shared_za)) | ||
| 81 | void svld1_ver_za64(uint64_t, uint32_t, uint64_t, svbool_t, void const *); | ||
| 82 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za8), arm_streaming, arm_shared_za)) | ||
| 83 | void svld1_ver_za8(uint64_t, uint32_t, uint64_t, svbool_t, void const *); | ||
| 84 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_f16_m), arm_streaming, arm_shared_za)) | ||
| 85 | void svmopa_za32_f16_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t); | ||
| 86 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_bf16_m), arm_streaming, arm_shared_za)) | ||
| 87 | void svmopa_za32_bf16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t); | ||
| 88 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_f32_m), arm_streaming, arm_shared_za)) | ||
| 89 | void svmopa_za32_f32_m(uint64_t, svbool_t, svbool_t, svfloat32_t, svfloat32_t); | ||
| 90 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_s8_m), arm_streaming, arm_shared_za)) | ||
| 91 | void svmopa_za32_s8_m(uint64_t, svbool_t, svbool_t, svint8_t, svint8_t); | ||
| 92 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_u8_m), arm_streaming, arm_shared_za)) | ||
| 93 | void svmopa_za32_u8_m(uint64_t, svbool_t, svbool_t, svuint8_t, svuint8_t); | ||
| 94 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_f16_m), arm_streaming, arm_shared_za)) | ||
| 95 | void svmops_za32_f16_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t); | ||
| 96 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_bf16_m), arm_streaming, arm_shared_za)) | ||
| 97 | void svmops_za32_bf16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t); | ||
| 98 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_f32_m), arm_streaming, arm_shared_za)) | ||
| 99 | void svmops_za32_f32_m(uint64_t, svbool_t, svbool_t, svfloat32_t, svfloat32_t); | ||
| 100 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_s8_m), arm_streaming, arm_shared_za)) | ||
| 101 | void svmops_za32_s8_m(uint64_t, svbool_t, svbool_t, svint8_t, svint8_t); | ||
| 102 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_u8_m), arm_streaming, arm_shared_za)) | ||
| 103 | void svmops_za32_u8_m(uint64_t, svbool_t, svbool_t, svuint8_t, svuint8_t); | ||
| 104 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 105 | svuint8_t svread_hor_za128_u8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 106 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 107 | svuint32_t svread_hor_za128_u32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 108 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 109 | svuint64_t svread_hor_za128_u64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 110 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 111 | svuint16_t svread_hor_za128_u16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 112 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 113 | svbfloat16_t svread_hor_za128_bf16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 114 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 115 | svint8_t svread_hor_za128_s8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 116 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 117 | svfloat64_t svread_hor_za128_f64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 118 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 119 | svfloat32_t svread_hor_za128_f32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 120 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 121 | svfloat16_t svread_hor_za128_f16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 122 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 123 | svint32_t svread_hor_za128_s32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 124 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 125 | svint64_t svread_hor_za128_s64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 126 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 127 | svint16_t svread_hor_za128_s16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 128 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_u16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 129 | svuint16_t svread_hor_za16_u16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 130 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 131 | svbfloat16_t svread_hor_za16_bf16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 132 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_f16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 133 | svfloat16_t svread_hor_za16_f16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 134 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_s16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 135 | svint16_t svread_hor_za16_s16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 136 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_u32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 137 | svuint32_t svread_hor_za32_u32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 138 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_f32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 139 | svfloat32_t svread_hor_za32_f32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 140 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_s32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 141 | svint32_t svread_hor_za32_s32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 142 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_u64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 143 | svuint64_t svread_hor_za64_u64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 144 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_f64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 145 | svfloat64_t svread_hor_za64_f64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 146 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_s64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 147 | svint64_t svread_hor_za64_s64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 148 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za8_u8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 149 | svuint8_t svread_hor_za8_u8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 150 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za8_s8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 151 | svint8_t svread_hor_za8_s8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 152 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 153 | svuint8_t svread_ver_za128_u8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 154 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 155 | svuint32_t svread_ver_za128_u32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 156 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 157 | svuint64_t svread_ver_za128_u64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 158 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 159 | svuint16_t svread_ver_za128_u16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 160 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 161 | svbfloat16_t svread_ver_za128_bf16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 162 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 163 | svint8_t svread_ver_za128_s8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 164 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 165 | svfloat64_t svread_ver_za128_f64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 166 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 167 | svfloat32_t svread_ver_za128_f32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 168 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 169 | svfloat16_t svread_ver_za128_f16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 170 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 171 | svint32_t svread_ver_za128_s32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 172 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 173 | svint64_t svread_ver_za128_s64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 174 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 175 | svint16_t svread_ver_za128_s16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 176 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_u16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 177 | svuint16_t svread_ver_za16_u16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 178 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 179 | svbfloat16_t svread_ver_za16_bf16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 180 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_f16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 181 | svfloat16_t svread_ver_za16_f16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 182 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_s16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 183 | svint16_t svread_ver_za16_s16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 184 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_u32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 185 | svuint32_t svread_ver_za32_u32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 186 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_f32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 187 | svfloat32_t svread_ver_za32_f32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 188 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_s32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 189 | svint32_t svread_ver_za32_s32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 190 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_u64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 191 | svuint64_t svread_ver_za64_u64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 192 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_f64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 193 | svfloat64_t svread_ver_za64_f64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 194 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_s64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 195 | svint64_t svread_ver_za64_s64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 196 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za8_u8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 197 | svuint8_t svread_ver_za8_u8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 198 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za8_s8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 199 | svint8_t svread_ver_za8_s8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 200 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za128), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 201 | void svst1_hor_vnum_za128(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t); | ||
| 202 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za16), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 203 | void svst1_hor_vnum_za16(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t); | ||
| 204 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za32), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 205 | void svst1_hor_vnum_za32(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t); | ||
| 206 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za64), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 207 | void svst1_hor_vnum_za64(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t); | ||
| 208 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za8), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 209 | void svst1_hor_vnum_za8(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t); | ||
| 210 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za128), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 211 | void svst1_hor_za128(uint64_t, uint32_t, uint64_t, svbool_t, void *); | ||
| 212 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za16), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 213 | void svst1_hor_za16(uint64_t, uint32_t, uint64_t, svbool_t, void *); | ||
| 214 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za32), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 215 | void svst1_hor_za32(uint64_t, uint32_t, uint64_t, svbool_t, void *); | ||
| 216 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za64), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 217 | void svst1_hor_za64(uint64_t, uint32_t, uint64_t, svbool_t, void *); | ||
| 218 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za8), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 219 | void svst1_hor_za8(uint64_t, uint32_t, uint64_t, svbool_t, void *); | ||
| 220 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za128), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 221 | void svst1_ver_vnum_za128(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t); | ||
| 222 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za16), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 223 | void svst1_ver_vnum_za16(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t); | ||
| 224 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za32), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 225 | void svst1_ver_vnum_za32(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t); | ||
| 226 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za64), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 227 | void svst1_ver_vnum_za64(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t); | ||
| 228 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za8), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 229 | void svst1_ver_vnum_za8(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t); | ||
| 230 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za128), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 231 | void svst1_ver_za128(uint64_t, uint32_t, uint64_t, svbool_t, void *); | ||
| 232 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za16), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 233 | void svst1_ver_za16(uint64_t, uint32_t, uint64_t, svbool_t, void *); | ||
| 234 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za32), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 235 | void svst1_ver_za32(uint64_t, uint32_t, uint64_t, svbool_t, void *); | ||
| 236 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za64), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 237 | void svst1_ver_za64(uint64_t, uint32_t, uint64_t, svbool_t, void *); | ||
| 238 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za8), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 239 | void svst1_ver_za8(uint64_t, uint32_t, uint64_t, svbool_t, void *); | ||
| 240 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumopa_za32_s8_m), arm_streaming, arm_shared_za)) | ||
| 241 | void svsumopa_za32_s8_m(uint64_t, svbool_t, svbool_t, svint8_t, svuint8_t); | ||
| 242 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumops_za32_s8_m), arm_streaming, arm_shared_za)) | ||
| 243 | void svsumops_za32_s8_m(uint64_t, svbool_t, svbool_t, svint8_t, svuint8_t); | ||
| 244 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za32_u8_m), arm_streaming, arm_shared_za)) | ||
| 245 | void svusmopa_za32_u8_m(uint64_t, svbool_t, svbool_t, svuint8_t, svint8_t); | ||
| 246 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za32_u8_m), arm_streaming, arm_shared_za)) | ||
| 247 | void svusmops_za32_u8_m(uint64_t, svbool_t, svbool_t, svuint8_t, svint8_t); | ||
| 248 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u8_m), arm_streaming, arm_shared_za)) | ||
| 249 | void svwrite_hor_za128_u8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t); | ||
| 250 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u32_m), arm_streaming, arm_shared_za)) | ||
| 251 | void svwrite_hor_za128_u32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t); | ||
| 252 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u64_m), arm_streaming, arm_shared_za)) | ||
| 253 | void svwrite_hor_za128_u64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t); | ||
| 254 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u16_m), arm_streaming, arm_shared_za)) | ||
| 255 | void svwrite_hor_za128_u16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t); | ||
| 256 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_bf16_m), arm_streaming, arm_shared_za)) | ||
| 257 | void svwrite_hor_za128_bf16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t); | ||
| 258 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s8_m), arm_streaming, arm_shared_za)) | ||
| 259 | void svwrite_hor_za128_s8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t); | ||
| 260 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f64_m), arm_streaming, arm_shared_za)) | ||
| 261 | void svwrite_hor_za128_f64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t); | ||
| 262 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f32_m), arm_streaming, arm_shared_za)) | ||
| 263 | void svwrite_hor_za128_f32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t); | ||
| 264 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f16_m), arm_streaming, arm_shared_za)) | ||
| 265 | void svwrite_hor_za128_f16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t); | ||
| 266 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s32_m), arm_streaming, arm_shared_za)) | ||
| 267 | void svwrite_hor_za128_s32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t); | ||
| 268 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s64_m), arm_streaming, arm_shared_za)) | ||
| 269 | void svwrite_hor_za128_s64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t); | ||
| 270 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s16_m), arm_streaming, arm_shared_za)) | ||
| 271 | void svwrite_hor_za128_s16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t); | ||
| 272 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_u16_m), arm_streaming, arm_shared_za)) | ||
| 273 | void svwrite_hor_za16_u16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t); | ||
| 274 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_bf16_m), arm_streaming, arm_shared_za)) | ||
| 275 | void svwrite_hor_za16_bf16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t); | ||
| 276 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_f16_m), arm_streaming, arm_shared_za)) | ||
| 277 | void svwrite_hor_za16_f16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t); | ||
| 278 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_s16_m), arm_streaming, arm_shared_za)) | ||
| 279 | void svwrite_hor_za16_s16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t); | ||
| 280 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_u32_m), arm_streaming, arm_shared_za)) | ||
| 281 | void svwrite_hor_za32_u32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t); | ||
| 282 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_f32_m), arm_streaming, arm_shared_za)) | ||
| 283 | void svwrite_hor_za32_f32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t); | ||
| 284 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_s32_m), arm_streaming, arm_shared_za)) | ||
| 285 | void svwrite_hor_za32_s32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t); | ||
| 286 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_u64_m), arm_streaming, arm_shared_za)) | ||
| 287 | void svwrite_hor_za64_u64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t); | ||
| 288 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_f64_m), arm_streaming, arm_shared_za)) | ||
| 289 | void svwrite_hor_za64_f64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t); | ||
| 290 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_s64_m), arm_streaming, arm_shared_za)) | ||
| 291 | void svwrite_hor_za64_s64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t); | ||
| 292 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za8_u8_m), arm_streaming, arm_shared_za)) | ||
| 293 | void svwrite_hor_za8_u8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t); | ||
| 294 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za8_s8_m), arm_streaming, arm_shared_za)) | ||
| 295 | void svwrite_hor_za8_s8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t); | ||
| 296 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u8_m), arm_streaming, arm_shared_za)) | ||
| 297 | void svwrite_ver_za128_u8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t); | ||
| 298 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u32_m), arm_streaming, arm_shared_za)) | ||
| 299 | void svwrite_ver_za128_u32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t); | ||
| 300 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u64_m), arm_streaming, arm_shared_za)) | ||
| 301 | void svwrite_ver_za128_u64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t); | ||
| 302 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u16_m), arm_streaming, arm_shared_za)) | ||
| 303 | void svwrite_ver_za128_u16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t); | ||
| 304 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_bf16_m), arm_streaming, arm_shared_za)) | ||
| 305 | void svwrite_ver_za128_bf16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t); | ||
| 306 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s8_m), arm_streaming, arm_shared_za)) | ||
| 307 | void svwrite_ver_za128_s8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t); | ||
| 308 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f64_m), arm_streaming, arm_shared_za)) | ||
| 309 | void svwrite_ver_za128_f64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t); | ||
| 310 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f32_m), arm_streaming, arm_shared_za)) | ||
| 311 | void svwrite_ver_za128_f32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t); | ||
| 312 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f16_m), arm_streaming, arm_shared_za)) | ||
| 313 | void svwrite_ver_za128_f16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t); | ||
| 314 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s32_m), arm_streaming, arm_shared_za)) | ||
| 315 | void svwrite_ver_za128_s32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t); | ||
| 316 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s64_m), arm_streaming, arm_shared_za)) | ||
| 317 | void svwrite_ver_za128_s64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t); | ||
| 318 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s16_m), arm_streaming, arm_shared_za)) | ||
| 319 | void svwrite_ver_za128_s16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t); | ||
| 320 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_u16_m), arm_streaming, arm_shared_za)) | ||
| 321 | void svwrite_ver_za16_u16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t); | ||
| 322 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_bf16_m), arm_streaming, arm_shared_za)) | ||
| 323 | void svwrite_ver_za16_bf16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t); | ||
| 324 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_f16_m), arm_streaming, arm_shared_za)) | ||
| 325 | void svwrite_ver_za16_f16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t); | ||
| 326 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_s16_m), arm_streaming, arm_shared_za)) | ||
| 327 | void svwrite_ver_za16_s16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t); | ||
| 328 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_u32_m), arm_streaming, arm_shared_za)) | ||
| 329 | void svwrite_ver_za32_u32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t); | ||
| 330 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_f32_m), arm_streaming, arm_shared_za)) | ||
| 331 | void svwrite_ver_za32_f32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t); | ||
| 332 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_s32_m), arm_streaming, arm_shared_za)) | ||
| 333 | void svwrite_ver_za32_s32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t); | ||
| 334 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_u64_m), arm_streaming, arm_shared_za)) | ||
| 335 | void svwrite_ver_za64_u64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t); | ||
| 336 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_f64_m), arm_streaming, arm_shared_za)) | ||
| 337 | void svwrite_ver_za64_f64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t); | ||
| 338 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_s64_m), arm_streaming, arm_shared_za)) | ||
| 339 | void svwrite_ver_za64_s64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t); | ||
| 340 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_u8_m), arm_streaming, arm_shared_za)) | ||
| 341 | void svwrite_ver_za8_u8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t); | ||
| 342 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_s8_m), arm_streaming, arm_shared_za)) | ||
| 343 | void svwrite_ver_za8_s8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t); | ||
| 344 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svzero_mask_za), arm_streaming_compatible, arm_shared_za)) | ||
| 345 | void svzero_mask_za(uint64_t); | ||
| 346 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svzero_za), arm_streaming_compatible, arm_shared_za)) | ||
| 347 | void svzero_za(); | ||
| 348 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_u32_m), arm_streaming, arm_shared_za)) | ||
| 349 | void svaddha_za32_m(uint64_t, svbool_t, svbool_t, svuint32_t); | ||
| 350 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_s32_m), arm_streaming, arm_shared_za)) | ||
| 351 | void svaddha_za32_m(uint64_t, svbool_t, svbool_t, svint32_t); | ||
| 352 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za32_u32_m), arm_streaming, arm_shared_za)) | ||
| 353 | void svaddva_za32_m(uint64_t, svbool_t, svbool_t, svuint32_t); | ||
| 354 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za32_s32_m), arm_streaming, arm_shared_za)) | ||
| 355 | void svaddva_za32_m(uint64_t, svbool_t, svbool_t, svint32_t); | ||
| 356 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_f16_m), arm_streaming, arm_shared_za)) | ||
| 357 | void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t); | ||
| 358 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_bf16_m), arm_streaming, arm_shared_za)) | ||
| 359 | void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t); | ||
| 360 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_f32_m), arm_streaming, arm_shared_za)) | ||
| 361 | void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svfloat32_t, svfloat32_t); | ||
| 362 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_s8_m), arm_streaming, arm_shared_za)) | ||
| 363 | void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svint8_t, svint8_t); | ||
| 364 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_u8_m), arm_streaming, arm_shared_za)) | ||
| 365 | void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svuint8_t, svuint8_t); | ||
| 366 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_f16_m), arm_streaming, arm_shared_za)) | ||
| 367 | void svmops_za32_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t); | ||
| 368 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_bf16_m), arm_streaming, arm_shared_za)) | ||
| 369 | void svmops_za32_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t); | ||
| 370 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_f32_m), arm_streaming, arm_shared_za)) | ||
| 371 | void svmops_za32_m(uint64_t, svbool_t, svbool_t, svfloat32_t, svfloat32_t); | ||
| 372 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_s8_m), arm_streaming, arm_shared_za)) | ||
| 373 | void svmops_za32_m(uint64_t, svbool_t, svbool_t, svint8_t, svint8_t); | ||
| 374 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_u8_m), arm_streaming, arm_shared_za)) | ||
| 375 | void svmops_za32_m(uint64_t, svbool_t, svbool_t, svuint8_t, svuint8_t); | ||
| 376 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 377 | svuint8_t svread_hor_za128_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 378 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 379 | svuint32_t svread_hor_za128_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 380 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 381 | svuint64_t svread_hor_za128_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 382 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 383 | svuint16_t svread_hor_za128_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 384 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 385 | svbfloat16_t svread_hor_za128_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 386 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 387 | svint8_t svread_hor_za128_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 388 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 389 | svfloat64_t svread_hor_za128_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 390 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 391 | svfloat32_t svread_hor_za128_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 392 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 393 | svfloat16_t svread_hor_za128_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 394 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 395 | svint32_t svread_hor_za128_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 396 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 397 | svint64_t svread_hor_za128_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 398 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 399 | svint16_t svread_hor_za128_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 400 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_u16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 401 | svuint16_t svread_hor_za16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 402 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 403 | svbfloat16_t svread_hor_za16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 404 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_f16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 405 | svfloat16_t svread_hor_za16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 406 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_s16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 407 | svint16_t svread_hor_za16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 408 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_u32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 409 | svuint32_t svread_hor_za32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 410 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_f32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 411 | svfloat32_t svread_hor_za32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 412 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_s32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 413 | svint32_t svread_hor_za32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 414 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_u64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 415 | svuint64_t svread_hor_za64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 416 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_f64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 417 | svfloat64_t svread_hor_za64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 418 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_s64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 419 | svint64_t svread_hor_za64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 420 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za8_u8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 421 | svuint8_t svread_hor_za8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 422 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za8_s8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 423 | svint8_t svread_hor_za8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 424 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 425 | svuint8_t svread_ver_za128_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 426 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 427 | svuint32_t svread_ver_za128_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 428 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 429 | svuint64_t svread_ver_za128_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 430 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 431 | svuint16_t svread_ver_za128_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 432 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 433 | svbfloat16_t svread_ver_za128_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 434 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 435 | svint8_t svread_ver_za128_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 436 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 437 | svfloat64_t svread_ver_za128_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 438 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 439 | svfloat32_t svread_ver_za128_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 440 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 441 | svfloat16_t svread_ver_za128_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 442 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 443 | svint32_t svread_ver_za128_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 444 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 445 | svint64_t svread_ver_za128_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 446 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 447 | svint16_t svread_ver_za128_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 448 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_u16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 449 | svuint16_t svread_ver_za16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 450 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 451 | svbfloat16_t svread_ver_za16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 452 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_f16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 453 | svfloat16_t svread_ver_za16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 454 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_s16_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 455 | svint16_t svread_ver_za16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 456 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_u32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 457 | svuint32_t svread_ver_za32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 458 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_f32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 459 | svfloat32_t svread_ver_za32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 460 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_s32_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 461 | svint32_t svread_ver_za32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 462 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_u64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 463 | svuint64_t svread_ver_za64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 464 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_f64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 465 | svfloat64_t svread_ver_za64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 466 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_s64_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 467 | svint64_t svread_ver_za64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 468 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za8_u8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 469 | svuint8_t svread_ver_za8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 470 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za8_s8_m), arm_streaming, arm_shared_za, arm_preserves_za)) | ||
| 471 | svint8_t svread_ver_za8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t); | ||
| 472 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumopa_za32_s8_m), arm_streaming, arm_shared_za)) | ||
| 473 | void svsumopa_za32_m(uint64_t, svbool_t, svbool_t, svint8_t, svuint8_t); | ||
| 474 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumops_za32_s8_m), arm_streaming, arm_shared_za)) | ||
| 475 | void svsumops_za32_m(uint64_t, svbool_t, svbool_t, svint8_t, svuint8_t); | ||
| 476 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za32_u8_m), arm_streaming, arm_shared_za)) | ||
| 477 | void svusmopa_za32_m(uint64_t, svbool_t, svbool_t, svuint8_t, svint8_t); | ||
| 478 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za32_u8_m), arm_streaming, arm_shared_za)) | ||
| 479 | void svusmops_za32_m(uint64_t, svbool_t, svbool_t, svuint8_t, svint8_t); | ||
| 480 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u8_m), arm_streaming, arm_shared_za)) | ||
| 481 | void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t); | ||
| 482 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u32_m), arm_streaming, arm_shared_za)) | ||
| 483 | void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t); | ||
| 484 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u64_m), arm_streaming, arm_shared_za)) | ||
| 485 | void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t); | ||
| 486 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u16_m), arm_streaming, arm_shared_za)) | ||
| 487 | void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t); | ||
| 488 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_bf16_m), arm_streaming, arm_shared_za)) | ||
| 489 | void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t); | ||
| 490 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s8_m), arm_streaming, arm_shared_za)) | ||
| 491 | void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t); | ||
| 492 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f64_m), arm_streaming, arm_shared_za)) | ||
| 493 | void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t); | ||
| 494 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f32_m), arm_streaming, arm_shared_za)) | ||
| 495 | void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t); | ||
| 496 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f16_m), arm_streaming, arm_shared_za)) | ||
| 497 | void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t); | ||
| 498 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s32_m), arm_streaming, arm_shared_za)) | ||
| 499 | void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t); | ||
| 500 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s64_m), arm_streaming, arm_shared_za)) | ||
| 501 | void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t); | ||
| 502 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s16_m), arm_streaming, arm_shared_za)) | ||
| 503 | void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t); | ||
| 504 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_u16_m), arm_streaming, arm_shared_za)) | ||
| 505 | void svwrite_hor_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t); | ||
| 506 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_bf16_m), arm_streaming, arm_shared_za)) | ||
| 507 | void svwrite_hor_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t); | ||
| 508 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_f16_m), arm_streaming, arm_shared_za)) | ||
| 509 | void svwrite_hor_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t); | ||
| 510 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_s16_m), arm_streaming, arm_shared_za)) | ||
| 511 | void svwrite_hor_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t); | ||
| 512 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_u32_m), arm_streaming, arm_shared_za)) | ||
| 513 | void svwrite_hor_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t); | ||
| 514 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_f32_m), arm_streaming, arm_shared_za)) | ||
| 515 | void svwrite_hor_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t); | ||
| 516 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_s32_m), arm_streaming, arm_shared_za)) | ||
| 517 | void svwrite_hor_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t); | ||
| 518 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_u64_m), arm_streaming, arm_shared_za)) | ||
| 519 | void svwrite_hor_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t); | ||
| 520 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_f64_m), arm_streaming, arm_shared_za)) | ||
| 521 | void svwrite_hor_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t); | ||
| 522 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_s64_m), arm_streaming, arm_shared_za)) | ||
| 523 | void svwrite_hor_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t); | ||
| 524 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za8_u8_m), arm_streaming, arm_shared_za)) | ||
| 525 | void svwrite_hor_za8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t); | ||
| 526 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za8_s8_m), arm_streaming, arm_shared_za)) | ||
| 527 | void svwrite_hor_za8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t); | ||
| 528 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u8_m), arm_streaming, arm_shared_za)) | ||
| 529 | void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t); | ||
| 530 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u32_m), arm_streaming, arm_shared_za)) | ||
| 531 | void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t); | ||
| 532 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u64_m), arm_streaming, arm_shared_za)) | ||
| 533 | void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t); | ||
| 534 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u16_m), arm_streaming, arm_shared_za)) | ||
| 535 | void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t); | ||
| 536 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_bf16_m), arm_streaming, arm_shared_za)) | ||
| 537 | void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t); | ||
| 538 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s8_m), arm_streaming, arm_shared_za)) | ||
| 539 | void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t); | ||
| 540 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f64_m), arm_streaming, arm_shared_za)) | ||
| 541 | void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t); | ||
| 542 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f32_m), arm_streaming, arm_shared_za)) | ||
| 543 | void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t); | ||
| 544 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f16_m), arm_streaming, arm_shared_za)) | ||
| 545 | void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t); | ||
| 546 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s32_m), arm_streaming, arm_shared_za)) | ||
| 547 | void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t); | ||
| 548 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s64_m), arm_streaming, arm_shared_za)) | ||
| 549 | void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t); | ||
| 550 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s16_m), arm_streaming, arm_shared_za)) | ||
| 551 | void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t); | ||
| 552 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_u16_m), arm_streaming, arm_shared_za)) | ||
| 553 | void svwrite_ver_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t); | ||
| 554 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_bf16_m), arm_streaming, arm_shared_za)) | ||
| 555 | void svwrite_ver_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t); | ||
| 556 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_f16_m), arm_streaming, arm_shared_za)) | ||
| 557 | void svwrite_ver_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t); | ||
| 558 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_s16_m), arm_streaming, arm_shared_za)) | ||
| 559 | void svwrite_ver_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t); | ||
| 560 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_u32_m), arm_streaming, arm_shared_za)) | ||
| 561 | void svwrite_ver_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t); | ||
| 562 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_f32_m), arm_streaming, arm_shared_za)) | ||
| 563 | void svwrite_ver_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t); | ||
| 564 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_s32_m), arm_streaming, arm_shared_za)) | ||
| 565 | void svwrite_ver_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t); | ||
| 566 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_u64_m), arm_streaming, arm_shared_za)) | ||
| 567 | void svwrite_ver_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t); | ||
| 568 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_f64_m), arm_streaming, arm_shared_za)) | ||
| 569 | void svwrite_ver_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t); | ||
| 570 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_s64_m), arm_streaming, arm_shared_za)) | ||
| 571 | void svwrite_ver_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t); | ||
| 572 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_u8_m), arm_streaming, arm_shared_za)) | ||
| 573 | void svwrite_ver_za8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t); | ||
| 574 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_s8_m), arm_streaming, arm_shared_za)) | ||
| 575 | void svwrite_ver_za8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t); | ||
| 576 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_f64_m), arm_streaming, arm_shared_za)) | ||
| 577 | void svmopa_za64_f64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t); | ||
| 578 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_f64_m), arm_streaming, arm_shared_za)) | ||
| 579 | void svmops_za64_f64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t); | ||
| 580 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_f64_m), arm_streaming, arm_shared_za)) | ||
| 581 | void svmopa_za64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t); | ||
| 582 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_f64_m), arm_streaming, arm_shared_za)) | ||
| 583 | void svmops_za64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t); | ||
| 584 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_u64_m), arm_streaming, arm_shared_za)) | ||
| 585 | void svaddha_za64_u64_m(uint64_t, svbool_t, svbool_t, svuint64_t); | ||
| 586 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_s64_m), arm_streaming, arm_shared_za)) | ||
| 587 | void svaddha_za64_s64_m(uint64_t, svbool_t, svbool_t, svint64_t); | ||
| 588 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za64_u64_m), arm_streaming, arm_shared_za)) | ||
| 589 | void svaddva_za64_u64_m(uint64_t, svbool_t, svbool_t, svuint64_t); | ||
| 590 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za64_s64_m), arm_streaming, arm_shared_za)) | ||
| 591 | void svaddva_za64_s64_m(uint64_t, svbool_t, svbool_t, svint64_t); | ||
| 592 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_s16_m), arm_streaming, arm_shared_za)) | ||
| 593 | void svmopa_za64_s16_m(uint64_t, svbool_t, svbool_t, svint16_t, svint16_t); | ||
| 594 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_u16_m), arm_streaming, arm_shared_za)) | ||
| 595 | void svmopa_za64_u16_m(uint64_t, svbool_t, svbool_t, svuint16_t, svuint16_t); | ||
| 596 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_s16_m), arm_streaming, arm_shared_za)) | ||
| 597 | void svmops_za64_s16_m(uint64_t, svbool_t, svbool_t, svint16_t, svint16_t); | ||
| 598 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_u16_m), arm_streaming, arm_shared_za)) | ||
| 599 | void svmops_za64_u16_m(uint64_t, svbool_t, svbool_t, svuint16_t, svuint16_t); | ||
| 600 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumopa_za64_s16_m), arm_streaming, arm_shared_za)) | ||
| 601 | void svsumopa_za64_s16_m(uint64_t, svbool_t, svbool_t, svint16_t, svuint16_t); | ||
| 602 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumops_za64_s16_m), arm_streaming, arm_shared_za)) | ||
| 603 | void svsumops_za64_s16_m(uint64_t, svbool_t, svbool_t, svint16_t, svuint16_t); | ||
| 604 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za64_u16_m), arm_streaming, arm_shared_za)) | ||
| 605 | void svusmopa_za64_u16_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t); | ||
| 606 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za64_u16_m), arm_streaming, arm_shared_za)) | ||
| 607 | void svusmops_za64_u16_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t); | ||
| 608 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_u64_m), arm_streaming, arm_shared_za)) | ||
| 609 | void svaddha_za64_m(uint64_t, svbool_t, svbool_t, svuint64_t); | ||
| 610 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_s64_m), arm_streaming, arm_shared_za)) | ||
| 611 | void svaddha_za64_m(uint64_t, svbool_t, svbool_t, svint64_t); | ||
| 612 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za64_u64_m), arm_streaming, arm_shared_za)) | ||
| 613 | void svaddva_za64_m(uint64_t, svbool_t, svbool_t, svuint64_t); | ||
| 614 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za64_s64_m), arm_streaming, arm_shared_za)) | ||
| 615 | void svaddva_za64_m(uint64_t, svbool_t, svbool_t, svint64_t); | ||
| 616 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_s16_m), arm_streaming, arm_shared_za)) | ||
| 617 | void svmopa_za64_m(uint64_t, svbool_t, svbool_t, svint16_t, svint16_t); | ||
| 618 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_u16_m), arm_streaming, arm_shared_za)) | ||
| 619 | void svmopa_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svuint16_t); | ||
| 620 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_s16_m), arm_streaming, arm_shared_za)) | ||
| 621 | void svmops_za64_m(uint64_t, svbool_t, svbool_t, svint16_t, svint16_t); | ||
| 622 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_u16_m), arm_streaming, arm_shared_za)) | ||
| 623 | void svmops_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svuint16_t); | ||
| 624 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumopa_za64_s16_m), arm_streaming, arm_shared_za)) | ||
| 625 | void svsumopa_za64_m(uint64_t, svbool_t, svbool_t, svint16_t, svuint16_t); | ||
| 626 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumops_za64_s16_m), arm_streaming, arm_shared_za)) | ||
| 627 | void svsumops_za64_m(uint64_t, svbool_t, svbool_t, svint16_t, svuint16_t); | ||
| 628 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za64_u16_m), arm_streaming, arm_shared_za)) | ||
| 629 | void svusmopa_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t); | ||
| 630 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za64_u16_m), arm_streaming, arm_shared_za)) | ||
| 631 | void svusmops_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t); | ||
| 632 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svldr_vnum_za), arm_streaming_compatible, arm_shared_za)) | ||
| 633 | void svldr_vnum_za(uint32_t, uint64_t, void const *); | ||
| 634 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svstr_vnum_za), arm_streaming_compatible, arm_shared_za, arm_preserves_za)) | ||
| 635 | void svstr_vnum_za(uint32_t, uint64_t, void *); | ||
| 636 | #ifdef __cplusplus | ||
| 637 | } // extern "C" | ||
| 638 | #endif | ||
| 639 | |||
| 640 | #undef __ai | ||
| 641 | |||
| 642 | #endif /* __ARM_SME_H */ | ||
lib/include/arm_sve.h+68-13
| ... | @@ -37,7 +37,6 @@ typedef __SVFloat16_t svfloat16_t; | ... | @@ -37,7 +37,6 @@ typedef __SVFloat16_t svfloat16_t; |
| 37 | 37 | ||
| 38 | typedef __SVBFloat16_t svbfloat16_t; | 38 | typedef __SVBFloat16_t svbfloat16_t; |
| 39 | #include <arm_bf16.h> | 39 | #include <arm_bf16.h> |
| 40 | typedef __bf16 bfloat16_t; | ||
| 41 | typedef __SVFloat32_t svfloat32_t; | 40 | typedef __SVFloat32_t svfloat32_t; |
| 42 | typedef __SVFloat64_t svfloat64_t; | 41 | typedef __SVFloat64_t svfloat64_t; |
| 43 | typedef __clang_svint8x2_t svint8x2_t; | 42 | typedef __clang_svint8x2_t svint8x2_t; |
| ... | @@ -74,10 +73,14 @@ typedef __clang_svfloat16x4_t svfloat16x4_t; | ... | @@ -74,10 +73,14 @@ typedef __clang_svfloat16x4_t svfloat16x4_t; |
| 74 | typedef __clang_svfloat32x4_t svfloat32x4_t; | 73 | typedef __clang_svfloat32x4_t svfloat32x4_t; |
| 75 | typedef __clang_svfloat64x4_t svfloat64x4_t; | 74 | typedef __clang_svfloat64x4_t svfloat64x4_t; |
| 76 | typedef __SVBool_t svbool_t; | 75 | typedef __SVBool_t svbool_t; |
| 76 | typedef __clang_svboolx2_t svboolx2_t; | ||
| 77 | typedef __clang_svboolx4_t svboolx4_t; | ||
| 77 | 78 | ||
| 78 | typedef __clang_svbfloat16x2_t svbfloat16x2_t; | 79 | typedef __clang_svbfloat16x2_t svbfloat16x2_t; |
| 79 | typedef __clang_svbfloat16x3_t svbfloat16x3_t; | 80 | typedef __clang_svbfloat16x3_t svbfloat16x3_t; |
| 80 | typedef __clang_svbfloat16x4_t svbfloat16x4_t; | 81 | typedef __clang_svbfloat16x4_t svbfloat16x4_t; |
| 82 | typedef __SVCount_t svcount_t; | ||
| 83 | |||
| 81 | enum svpattern | 84 | enum svpattern |
| 82 | { | 85 | { |
| 83 | SV_POW2 = 0, | 86 | SV_POW2 = 0, |
| ... | @@ -2914,6 +2917,10 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s64))) | ... | @@ -2914,6 +2917,10 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s64))) |
| 2914 | svint64_t svdup_lane_s64(svint64_t, uint64_t); | 2917 | svint64_t svdup_lane_s64(svint64_t, uint64_t); |
| 2915 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s16))) | 2918 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s16))) |
| 2916 | svint16_t svdup_lane_s16(svint16_t, uint16_t); | 2919 | svint16_t svdup_lane_s16(svint16_t, uint16_t); |
| 2920 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u8))) | ||
| 2921 | svuint8_t svdupq_n_u8(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t); | ||
| 2922 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s8))) | ||
| 2923 | svint8_t svdupq_n_s8(int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t); | ||
| 2917 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u16))) | 2924 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u16))) |
| 2918 | svuint16_t svdupq_n_u16(uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t); | 2925 | svuint16_t svdupq_n_u16(uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t); |
| 2919 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f16))) | 2926 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f16))) |
| ... | @@ -2932,18 +2939,14 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f64))) | ... | @@ -2932,18 +2939,14 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f64))) |
| 2932 | svfloat64_t svdupq_n_f64(float64_t, float64_t); | 2939 | svfloat64_t svdupq_n_f64(float64_t, float64_t); |
| 2933 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s64))) | 2940 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s64))) |
| 2934 | svint64_t svdupq_n_s64(int64_t, int64_t); | 2941 | svint64_t svdupq_n_s64(int64_t, int64_t); |
| 2935 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u8))) | 2942 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b8))) |
| 2936 | svuint8_t svdupq_n_u8(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t); | 2943 | svbool_t svdupq_n_b8(bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool); |
| 2937 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s8))) | ||
| 2938 | svint8_t svdupq_n_s8(int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t); | ||
| 2939 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b16))) | 2944 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b16))) |
| 2940 | svbool_t svdupq_n_b16(bool, bool, bool, bool, bool, bool, bool, bool); | 2945 | svbool_t svdupq_n_b16(bool, bool, bool, bool, bool, bool, bool, bool); |
| 2941 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b32))) | 2946 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b32))) |
| 2942 | svbool_t svdupq_n_b32(bool, bool, bool, bool); | 2947 | svbool_t svdupq_n_b32(bool, bool, bool, bool); |
| 2943 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b64))) | 2948 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b64))) |
| 2944 | svbool_t svdupq_n_b64(bool, bool); | 2949 | svbool_t svdupq_n_b64(bool, bool); |
| 2945 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b8))) | ||
| 2946 | svbool_t svdupq_n_b8(bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool); | ||
| 2947 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u8))) | 2950 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u8))) |
| 2948 | svuint8_t svdupq_lane_u8(svuint8_t, uint64_t); | 2951 | svuint8_t svdupq_lane_u8(svuint8_t, uint64_t); |
| 2949 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u32))) | 2952 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u32))) |
| ... | @@ -10528,6 +10531,10 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s64))) | ... | @@ -10528,6 +10531,10 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s64))) |
| 10528 | svint64_t svdup_lane(svint64_t, uint64_t); | 10531 | svint64_t svdup_lane(svint64_t, uint64_t); |
| 10529 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s16))) | 10532 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s16))) |
| 10530 | svint16_t svdup_lane(svint16_t, uint16_t); | 10533 | svint16_t svdup_lane(svint16_t, uint16_t); |
| 10534 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u8))) | ||
| 10535 | svuint8_t svdupq_u8(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t); | ||
| 10536 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s8))) | ||
| 10537 | svint8_t svdupq_s8(int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t); | ||
| 10531 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u16))) | 10538 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u16))) |
| 10532 | svuint16_t svdupq_u16(uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t); | 10539 | svuint16_t svdupq_u16(uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t); |
| 10533 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f16))) | 10540 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f16))) |
| ... | @@ -10546,18 +10553,14 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f64))) | ... | @@ -10546,18 +10553,14 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f64))) |
| 10546 | svfloat64_t svdupq_f64(float64_t, float64_t); | 10553 | svfloat64_t svdupq_f64(float64_t, float64_t); |
| 10547 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s64))) | 10554 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s64))) |
| 10548 | svint64_t svdupq_s64(int64_t, int64_t); | 10555 | svint64_t svdupq_s64(int64_t, int64_t); |
| 10549 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u8))) | 10556 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b8))) |
| 10550 | svuint8_t svdupq_u8(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t); | 10557 | svbool_t svdupq_b8(bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool); |
| 10551 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s8))) | ||
| 10552 | svint8_t svdupq_s8(int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t); | ||
| 10553 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b16))) | 10558 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b16))) |
| 10554 | svbool_t svdupq_b16(bool, bool, bool, bool, bool, bool, bool, bool); | 10559 | svbool_t svdupq_b16(bool, bool, bool, bool, bool, bool, bool, bool); |
| 10555 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b32))) | 10560 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b32))) |
| 10556 | svbool_t svdupq_b32(bool, bool, bool, bool); | 10561 | svbool_t svdupq_b32(bool, bool, bool, bool); |
| 10557 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b64))) | 10562 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b64))) |
| 10558 | svbool_t svdupq_b64(bool, bool); | 10563 | svbool_t svdupq_b64(bool, bool); |
| 10559 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b8))) | ||
| 10560 | svbool_t svdupq_b8(bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool); | ||
| 10561 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u8))) | 10564 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u8))) |
| 10562 | svuint8_t svdupq_lane(svuint8_t, uint64_t); | 10565 | svuint8_t svdupq_lane(svuint8_t, uint64_t); |
| 10563 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u32))) | 10566 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u32))) |
| ... | @@ -23874,6 +23877,58 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4e_u32))) | ... | @@ -23874,6 +23877,58 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4e_u32))) |
| 23874 | svuint32_t svsm4e(svuint32_t, svuint32_t); | 23877 | svuint32_t svsm4e(svuint32_t, svuint32_t); |
| 23875 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4ekey_u32))) | 23878 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4ekey_u32))) |
| 23876 | svuint32_t svsm4ekey(svuint32_t, svuint32_t); | 23879 | svuint32_t svsm4ekey(svuint32_t, svuint32_t); |
| 23880 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f64))) | ||
| 23881 | svfloat64_t svclamp_f64(svfloat64_t, svfloat64_t, svfloat64_t); | ||
| 23882 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f32))) | ||
| 23883 | svfloat32_t svclamp_f32(svfloat32_t, svfloat32_t, svfloat32_t); | ||
| 23884 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f16))) | ||
| 23885 | svfloat16_t svclamp_f16(svfloat16_t, svfloat16_t, svfloat16_t); | ||
| 23886 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s8))) | ||
| 23887 | svint8_t svclamp_s8(svint8_t, svint8_t, svint8_t); | ||
| 23888 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s32))) | ||
| 23889 | svint32_t svclamp_s32(svint32_t, svint32_t, svint32_t); | ||
| 23890 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s64))) | ||
| 23891 | svint64_t svclamp_s64(svint64_t, svint64_t, svint64_t); | ||
| 23892 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s16))) | ||
| 23893 | svint16_t svclamp_s16(svint16_t, svint16_t, svint16_t); | ||
| 23894 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u8))) | ||
| 23895 | svuint8_t svclamp_u8(svuint8_t, svuint8_t, svuint8_t); | ||
| 23896 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u32))) | ||
| 23897 | svuint32_t svclamp_u32(svuint32_t, svuint32_t, svuint32_t); | ||
| 23898 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u64))) | ||
| 23899 | svuint64_t svclamp_u64(svuint64_t, svuint64_t, svuint64_t); | ||
| 23900 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u16))) | ||
| 23901 | svuint16_t svclamp_u16(svuint16_t, svuint16_t, svuint16_t); | ||
| 23902 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c8))) | ||
| 23903 | svcount_t svptrue_c8(void); | ||
| 23904 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c32))) | ||
| 23905 | svcount_t svptrue_c32(void); | ||
| 23906 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c64))) | ||
| 23907 | svcount_t svptrue_c64(void); | ||
| 23908 | __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c16))) | ||
| 23909 | svcount_t svptrue_c16(void); | ||
| 23910 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f64))) | ||
| 23911 | svfloat64_t svclamp(svfloat64_t, svfloat64_t, svfloat64_t); | ||
| 23912 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f32))) | ||
| 23913 | svfloat32_t svclamp(svfloat32_t, svfloat32_t, svfloat32_t); | ||
| 23914 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f16))) | ||
| 23915 | svfloat16_t svclamp(svfloat16_t, svfloat16_t, svfloat16_t); | ||
| 23916 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s8))) | ||
| 23917 | svint8_t svclamp(svint8_t, svint8_t, svint8_t); | ||
| 23918 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s32))) | ||
| 23919 | svint32_t svclamp(svint32_t, svint32_t, svint32_t); | ||
| 23920 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s64))) | ||
| 23921 | svint64_t svclamp(svint64_t, svint64_t, svint64_t); | ||
| 23922 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s16))) | ||
| 23923 | svint16_t svclamp(svint16_t, svint16_t, svint16_t); | ||
| 23924 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u8))) | ||
| 23925 | svuint8_t svclamp(svuint8_t, svuint8_t, svuint8_t); | ||
| 23926 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u32))) | ||
| 23927 | svuint32_t svclamp(svuint32_t, svuint32_t, svuint32_t); | ||
| 23928 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u64))) | ||
| 23929 | svuint64_t svclamp(svuint64_t, svuint64_t, svuint64_t); | ||
| 23930 | __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u16))) | ||
| 23931 | svuint16_t svclamp(svuint16_t, svuint16_t, svuint16_t); | ||
| 23877 | #define svcvtnt_bf16_x svcvtnt_bf16_m | 23932 | #define svcvtnt_bf16_x svcvtnt_bf16_m |
| 23878 | #define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m | 23933 | #define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m |
| 23879 | #define svcvtnt_f16_x svcvtnt_f16_m | 23934 | #define svcvtnt_f16_x svcvtnt_f16_m |
lib/include/avx2intrin.h+4116-1
| ... | @@ -19,128 +19,539 @@ | ... | @@ -19,128 +19,539 @@ |
| 19 | #define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avx2"), __min_vector_width__(128))) | 19 | #define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avx2"), __min_vector_width__(128))) |
| 20 | 20 | ||
| 21 | /* SSE4 Multiple Packed Sums of Absolute Difference. */ | 21 | /* SSE4 Multiple Packed Sums of Absolute Difference. */ |
| 22 | /// Computes sixteen sum of absolute difference (SAD) operations on sets of | ||
| 23 | /// four unsigned 8-bit integers from the 256-bit integer vectors \a X and | ||
| 24 | /// \a Y. | ||
| 25 | /// | ||
| 26 | /// Eight SAD results are computed using the lower half of the input | ||
| 27 | /// vectors, and another eight using the upper half. These 16-bit values | ||
| 28 | /// are returned in the lower and upper halves of the 256-bit result, | ||
| 29 | /// respectively. | ||
| 30 | /// | ||
| 31 | /// A single SAD operation selects four bytes from \a X and four bytes from | ||
| 32 | /// \a Y as input. It computes the differences between each \a X byte and | ||
| 33 | /// the corresponding \a Y byte, takes the absolute value of each | ||
| 34 | /// difference, and sums these four values to form one 16-bit result. The | ||
| 35 | /// intrinsic computes 16 of these results with different sets of input | ||
| 36 | /// bytes. | ||
| 37 | /// | ||
| 38 | /// For each set of eight results, the SAD operations use the same four | ||
| 39 | /// bytes from \a Y; the starting bit position for these four bytes is | ||
| 40 | /// specified by \a M[1:0] times 32. The eight operations use successive | ||
| 41 | /// sets of four bytes from \a X; the starting bit position for the first | ||
| 42 | /// set of four bytes is specified by \a M[2] times 32. These bit positions | ||
| 43 | /// are all relative to the 128-bit lane for each set of eight operations. | ||
| 44 | /// | ||
| 45 | /// \code{.operation} | ||
| 46 | /// r := 0 | ||
| 47 | /// FOR i := 0 TO 1 | ||
| 48 | /// j := i*3 | ||
| 49 | /// Ybase := M[j+1:j]*32 + i*128 | ||
| 50 | /// Xbase := M[j+2]*32 + i*128 | ||
| 51 | /// FOR k := 0 TO 3 | ||
| 52 | /// temp0 := ABS(X[Xbase+7:Xbase] - Y[Ybase+7:Ybase]) | ||
| 53 | /// temp1 := ABS(X[Xbase+15:Xbase+8] - Y[Ybase+15:Ybase+8]) | ||
| 54 | /// temp2 := ABS(X[Xbase+23:Xbase+16] - Y[Ybase+23:Ybase+16]) | ||
| 55 | /// temp3 := ABS(X[Xbase+31:Xbase+24] - Y[Ybase+31:Ybase+24]) | ||
| 56 | /// result[r+15:r] := temp0 + temp1 + temp2 + temp3 | ||
| 57 | /// Xbase := Xbase + 8 | ||
| 58 | /// r := r + 16 | ||
| 59 | /// ENDFOR | ||
| 60 | /// ENDFOR | ||
| 61 | /// \endcode | ||
| 62 | /// | ||
| 63 | /// \headerfile <immintrin.h> | ||
| 64 | /// | ||
| 65 | /// \code | ||
| 66 | /// __m256i _mm256_mpsadbw_epu8(__m256i X, __m256i Y, const int M); | ||
| 67 | /// \endcode | ||
| 68 | /// | ||
| 69 | /// This intrinsic corresponds to the \c VMPSADBW instruction. | ||
| 70 | /// | ||
| 71 | /// \param X | ||
| 72 | /// A 256-bit integer vector containing one of the inputs. | ||
| 73 | /// \param Y | ||
| 74 | /// A 256-bit integer vector containing one of the inputs. | ||
| 75 | /// \param M | ||
| 76 | /// An unsigned immediate value specifying the starting positions of the | ||
| 77 | /// bytes to operate on. | ||
| 78 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 22 | #define _mm256_mpsadbw_epu8(X, Y, M) \ | 79 | #define _mm256_mpsadbw_epu8(X, Y, M) \ |
| 23 | ((__m256i)__builtin_ia32_mpsadbw256((__v32qi)(__m256i)(X), \ | 80 | ((__m256i)__builtin_ia32_mpsadbw256((__v32qi)(__m256i)(X), \ |
| 24 | (__v32qi)(__m256i)(Y), (int)(M))) | 81 | (__v32qi)(__m256i)(Y), (int)(M))) |
| 25 | 82 | ||
| 83 | /// Computes the absolute value of each signed byte in the 256-bit integer | ||
| 84 | /// vector \a __a and returns each value in the corresponding byte of | ||
| 85 | /// the result. | ||
| 86 | /// | ||
| 87 | /// \headerfile <immintrin.h> | ||
| 88 | /// | ||
| 89 | /// This intrinsic corresponds to the \c VPABSB instruction. | ||
| 90 | /// | ||
| 91 | /// \param __a | ||
| 92 | /// A 256-bit integer vector. | ||
| 93 | /// \returns A 256-bit integer vector containing the result. | ||
| 26 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 94 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 27 | _mm256_abs_epi8(__m256i __a) | 95 | _mm256_abs_epi8(__m256i __a) |
| 28 | { | 96 | { |
| 29 | return (__m256i)__builtin_elementwise_abs((__v32qs)__a); | 97 | return (__m256i)__builtin_elementwise_abs((__v32qs)__a); |
| 30 | } | 98 | } |
| 31 | 99 | ||
| 100 | /// Computes the absolute value of each signed 16-bit element in the 256-bit | ||
| 101 | /// vector of [16 x i16] in \a __a and returns each value in the | ||
| 102 | /// corresponding element of the result. | ||
| 103 | /// | ||
| 104 | /// \headerfile <immintrin.h> | ||
| 105 | /// | ||
| 106 | /// This intrinsic corresponds to the \c VPABSW instruction. | ||
| 107 | /// | ||
| 108 | /// \param __a | ||
| 109 | /// A 256-bit vector of [16 x i16]. | ||
| 110 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 32 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 111 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 33 | _mm256_abs_epi16(__m256i __a) | 112 | _mm256_abs_epi16(__m256i __a) |
| 34 | { | 113 | { |
| 35 | return (__m256i)__builtin_elementwise_abs((__v16hi)__a); | 114 | return (__m256i)__builtin_elementwise_abs((__v16hi)__a); |
| 36 | } | 115 | } |
| 37 | 116 | ||
| 117 | /// Computes the absolute value of each signed 32-bit element in the 256-bit | ||
| 118 | /// vector of [8 x i32] in \a __a and returns each value in the | ||
| 119 | /// corresponding element of the result. | ||
| 120 | /// | ||
| 121 | /// \headerfile <immintrin.h> | ||
| 122 | /// | ||
| 123 | /// This intrinsic corresponds to the \c VPABSD instruction. | ||
| 124 | /// | ||
| 125 | /// \param __a | ||
| 126 | /// A 256-bit vector of [8 x i32]. | ||
| 127 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 38 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 128 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 39 | _mm256_abs_epi32(__m256i __a) | 129 | _mm256_abs_epi32(__m256i __a) |
| 40 | { | 130 | { |
| 41 | return (__m256i)__builtin_elementwise_abs((__v8si)__a); | 131 | return (__m256i)__builtin_elementwise_abs((__v8si)__a); |
| 42 | } | 132 | } |
| 43 | 133 | ||
| 134 | /// Converts the elements of two 256-bit vectors of [16 x i16] to 8-bit | ||
| 135 | /// integers using signed saturation, and returns the 256-bit result. | ||
| 136 | /// | ||
| 137 | /// \code{.operation} | ||
| 138 | /// FOR i := 0 TO 7 | ||
| 139 | /// j := i*16 | ||
| 140 | /// k := i*8 | ||
| 141 | /// result[7+k:k] := SATURATE8(__a[15+j:j]) | ||
| 142 | /// result[71+k:64+k] := SATURATE8(__b[15+j:j]) | ||
| 143 | /// result[135+k:128+k] := SATURATE8(__a[143+j:128+j]) | ||
| 144 | /// result[199+k:192+k] := SATURATE8(__b[143+j:128+j]) | ||
| 145 | /// ENDFOR | ||
| 146 | /// \endcode | ||
| 147 | /// | ||
| 148 | /// \headerfile <immintrin.h> | ||
| 149 | /// | ||
| 150 | /// This intrinsic corresponds to the \c VPACKSSWB instruction. | ||
| 151 | /// | ||
| 152 | /// \param __a | ||
| 153 | /// A 256-bit vector of [16 x i16] used to generate result[63:0] and | ||
| 154 | /// result[191:128]. | ||
| 155 | /// \param __b | ||
| 156 | /// A 256-bit vector of [16 x i16] used to generate result[127:64] and | ||
| 157 | /// result[255:192]. | ||
| 158 | /// \returns A 256-bit integer vector containing the result. | ||
| 44 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 159 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 45 | _mm256_packs_epi16(__m256i __a, __m256i __b) | 160 | _mm256_packs_epi16(__m256i __a, __m256i __b) |
| 46 | { | 161 | { |
| 47 | return (__m256i)__builtin_ia32_packsswb256((__v16hi)__a, (__v16hi)__b); | 162 | return (__m256i)__builtin_ia32_packsswb256((__v16hi)__a, (__v16hi)__b); |
| 48 | } | 163 | } |
| 49 | 164 | ||
| 165 | /// Converts the elements of two 256-bit vectors of [8 x i32] to 16-bit | ||
| 166 | /// integers using signed saturation, and returns the resulting 256-bit | ||
| 167 | /// vector of [16 x i16]. | ||
| 168 | /// | ||
| 169 | /// \code{.operation} | ||
| 170 | /// FOR i := 0 TO 3 | ||
| 171 | /// j := i*32 | ||
| 172 | /// k := i*16 | ||
| 173 | /// result[15+k:k] := SATURATE16(__a[31+j:j]) | ||
| 174 | /// result[79+k:64+k] := SATURATE16(__b[31+j:j]) | ||
| 175 | /// result[143+k:128+k] := SATURATE16(__a[159+j:128+j]) | ||
| 176 | /// result[207+k:192+k] := SATURATE16(__b[159+j:128+j]) | ||
| 177 | /// ENDFOR | ||
| 178 | /// \endcode | ||
| 179 | /// | ||
| 180 | /// \headerfile <immintrin.h> | ||
| 181 | /// | ||
| 182 | /// This intrinsic corresponds to the \c VPACKSSDW instruction. | ||
| 183 | /// | ||
| 184 | /// \param __a | ||
| 185 | /// A 256-bit vector of [8 x i32] used to generate result[63:0] and | ||
| 186 | /// result[191:128]. | ||
| 187 | /// \param __b | ||
| 188 | /// A 256-bit vector of [8 x i32] used to generate result[127:64] and | ||
| 189 | /// result[255:192]. | ||
| 190 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 50 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 191 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 51 | _mm256_packs_epi32(__m256i __a, __m256i __b) | 192 | _mm256_packs_epi32(__m256i __a, __m256i __b) |
| 52 | { | 193 | { |
| 53 | return (__m256i)__builtin_ia32_packssdw256((__v8si)__a, (__v8si)__b); | 194 | return (__m256i)__builtin_ia32_packssdw256((__v8si)__a, (__v8si)__b); |
| 54 | } | 195 | } |
| 55 | 196 | ||
| 197 | /// Converts elements from two 256-bit vectors of [16 x i16] to 8-bit integers | ||
| 198 | /// using unsigned saturation, and returns the 256-bit result. | ||
| 199 | /// | ||
| 200 | /// \code{.operation} | ||
| 201 | /// FOR i := 0 TO 7 | ||
| 202 | /// j := i*16 | ||
| 203 | /// k := i*8 | ||
| 204 | /// result[7+k:k] := SATURATE8U(__a[15+j:j]) | ||
| 205 | /// result[71+k:64+k] := SATURATE8U(__b[15+j:j]) | ||
| 206 | /// result[135+k:128+k] := SATURATE8U(__a[143+j:128+j]) | ||
| 207 | /// result[199+k:192+k] := SATURATE8U(__b[143+j:128+j]) | ||
| 208 | /// ENDFOR | ||
| 209 | /// \endcode | ||
| 210 | /// | ||
| 211 | /// \headerfile <immintrin.h> | ||
| 212 | /// | ||
| 213 | /// This intrinsic corresponds to the \c VPACKUSWB instruction. | ||
| 214 | /// | ||
| 215 | /// \param __a | ||
| 216 | /// A 256-bit vector of [16 x i16] used to generate result[63:0] and | ||
| 217 | /// result[191:128]. | ||
| 218 | /// \param __b | ||
| 219 | /// A 256-bit vector of [16 x i16] used to generate result[127:64] and | ||
| 220 | /// result[255:192]. | ||
| 221 | /// \returns A 256-bit integer vector containing the result. | ||
| 56 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 222 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 57 | _mm256_packus_epi16(__m256i __a, __m256i __b) | 223 | _mm256_packus_epi16(__m256i __a, __m256i __b) |
| 58 | { | 224 | { |
| 59 | return (__m256i)__builtin_ia32_packuswb256((__v16hi)__a, (__v16hi)__b); | 225 | return (__m256i)__builtin_ia32_packuswb256((__v16hi)__a, (__v16hi)__b); |
| 60 | } | 226 | } |
| 61 | 227 | ||
| 228 | /// Converts elements from two 256-bit vectors of [8 x i32] to 16-bit integers | ||
| 229 | /// using unsigned saturation, and returns the resulting 256-bit vector of | ||
| 230 | /// [16 x i16]. | ||
| 231 | /// | ||
| 232 | /// \code{.operation} | ||
| 233 | /// FOR i := 0 TO 3 | ||
| 234 | /// j := i*32 | ||
| 235 | /// k := i*16 | ||
| 236 | /// result[15+k:k] := SATURATE16U(__V1[31+j:j]) | ||
| 237 | /// result[79+k:64+k] := SATURATE16U(__V2[31+j:j]) | ||
| 238 | /// result[143+k:128+k] := SATURATE16U(__V1[159+j:128+j]) | ||
| 239 | /// result[207+k:192+k] := SATURATE16U(__V2[159+j:128+j]) | ||
| 240 | /// ENDFOR | ||
| 241 | /// \endcode | ||
| 242 | /// | ||
| 243 | /// \headerfile <immintrin.h> | ||
| 244 | /// | ||
| 245 | /// This intrinsic corresponds to the \c VPACKUSDW instruction. | ||
| 246 | /// | ||
| 247 | /// \param __V1 | ||
| 248 | /// A 256-bit vector of [8 x i32] used to generate result[63:0] and | ||
| 249 | /// result[191:128]. | ||
| 250 | /// \param __V2 | ||
| 251 | /// A 256-bit vector of [8 x i32] used to generate result[127:64] and | ||
| 252 | /// result[255:192]. | ||
| 253 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 62 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 254 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 63 | _mm256_packus_epi32(__m256i __V1, __m256i __V2) | 255 | _mm256_packus_epi32(__m256i __V1, __m256i __V2) |
| 64 | { | 256 | { |
| 65 | return (__m256i) __builtin_ia32_packusdw256((__v8si)__V1, (__v8si)__V2); | 257 | return (__m256i) __builtin_ia32_packusdw256((__v8si)__V1, (__v8si)__V2); |
| 66 | } | 258 | } |
| 67 | 259 | ||
| 260 | /// Adds 8-bit integers from corresponding bytes of two 256-bit integer | ||
| 261 | /// vectors and returns the lower 8 bits of each sum in the corresponding | ||
| 262 | /// byte of the 256-bit integer vector result (overflow is ignored). | ||
| 263 | /// | ||
| 264 | /// \headerfile <immintrin.h> | ||
| 265 | /// | ||
| 266 | /// This intrinsic corresponds to the \c VPADDB instruction. | ||
| 267 | /// | ||
| 268 | /// \param __a | ||
| 269 | /// A 256-bit integer vector containing one of the source operands. | ||
| 270 | /// \param __b | ||
| 271 | /// A 256-bit integer vector containing one of the source operands. | ||
| 272 | /// \returns A 256-bit integer vector containing the sums. | ||
| 68 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 273 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 69 | _mm256_add_epi8(__m256i __a, __m256i __b) | 274 | _mm256_add_epi8(__m256i __a, __m256i __b) |
| 70 | { | 275 | { |
| 71 | return (__m256i)((__v32qu)__a + (__v32qu)__b); | 276 | return (__m256i)((__v32qu)__a + (__v32qu)__b); |
| 72 | } | 277 | } |
| 73 | 278 | ||
| 279 | /// Adds 16-bit integers from corresponding elements of two 256-bit vectors of | ||
| 280 | /// [16 x i16] and returns the lower 16 bits of each sum in the | ||
| 281 | /// corresponding element of the [16 x i16] result (overflow is ignored). | ||
| 282 | /// | ||
| 283 | /// \headerfile <immintrin.h> | ||
| 284 | /// | ||
| 285 | /// This intrinsic corresponds to the \c VPADDW instruction. | ||
| 286 | /// | ||
| 287 | /// \param __a | ||
| 288 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 289 | /// \param __b | ||
| 290 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 291 | /// \returns A 256-bit vector of [16 x i16] containing the sums. | ||
| 74 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 292 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 75 | _mm256_add_epi16(__m256i __a, __m256i __b) | 293 | _mm256_add_epi16(__m256i __a, __m256i __b) |
| 76 | { | 294 | { |
| 77 | return (__m256i)((__v16hu)__a + (__v16hu)__b); | 295 | return (__m256i)((__v16hu)__a + (__v16hu)__b); |
| 78 | } | 296 | } |
| 79 | 297 | ||
| 298 | /// Adds 32-bit integers from corresponding elements of two 256-bit vectors of | ||
| 299 | /// [8 x i32] and returns the lower 32 bits of each sum in the corresponding | ||
| 300 | /// element of the [8 x i32] result (overflow is ignored). | ||
| 301 | /// | ||
| 302 | /// \headerfile <immintrin.h> | ||
| 303 | /// | ||
| 304 | /// This intrinsic corresponds to the \c VPADDD instruction. | ||
| 305 | /// | ||
| 306 | /// \param __a | ||
| 307 | /// A 256-bit vector of [8 x i32] containing one of the source operands. | ||
| 308 | /// \param __b | ||
| 309 | /// A 256-bit vector of [8 x i32] containing one of the source operands. | ||
| 310 | /// \returns A 256-bit vector of [8 x i32] containing the sums. | ||
| 80 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 311 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 81 | _mm256_add_epi32(__m256i __a, __m256i __b) | 312 | _mm256_add_epi32(__m256i __a, __m256i __b) |
| 82 | { | 313 | { |
| 83 | return (__m256i)((__v8su)__a + (__v8su)__b); | 314 | return (__m256i)((__v8su)__a + (__v8su)__b); |
| 84 | } | 315 | } |
| 85 | 316 | ||
| 317 | /// Adds 64-bit integers from corresponding elements of two 256-bit vectors of | ||
| 318 | /// [4 x i64] and returns the lower 64 bits of each sum in the corresponding | ||
| 319 | /// element of the [4 x i64] result (overflow is ignored). | ||
| 320 | /// | ||
| 321 | /// \headerfile <immintrin.h> | ||
| 322 | /// | ||
| 323 | /// This intrinsic corresponds to the \c VPADDQ instruction. | ||
| 324 | /// | ||
| 325 | /// \param __a | ||
| 326 | /// A 256-bit vector of [4 x i64] containing one of the source operands. | ||
| 327 | /// \param __b | ||
| 328 | /// A 256-bit vector of [4 x i64] containing one of the source operands. | ||
| 329 | /// \returns A 256-bit vector of [4 x i64] containing the sums. | ||
| 86 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 330 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 87 | _mm256_add_epi64(__m256i __a, __m256i __b) | 331 | _mm256_add_epi64(__m256i __a, __m256i __b) |
| 88 | { | 332 | { |
| 89 | return (__m256i)((__v4du)__a + (__v4du)__b); | 333 | return (__m256i)((__v4du)__a + (__v4du)__b); |
| 90 | } | 334 | } |
| 91 | 335 | ||
| 336 | /// Adds 8-bit integers from corresponding bytes of two 256-bit integer | ||
| 337 | /// vectors using signed saturation, and returns each sum in the | ||
| 338 | /// corresponding byte of the 256-bit integer vector result. | ||
| 339 | /// | ||
| 340 | /// \headerfile <immintrin.h> | ||
| 341 | /// | ||
| 342 | /// This intrinsic corresponds to the \c VPADDSB instruction. | ||
| 343 | /// | ||
| 344 | /// \param __a | ||
| 345 | /// A 256-bit integer vector containing one of the source operands. | ||
| 346 | /// \param __b | ||
| 347 | /// A 256-bit integer vector containing one of the source operands. | ||
| 348 | /// \returns A 256-bit integer vector containing the sums. | ||
| 92 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 349 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 93 | _mm256_adds_epi8(__m256i __a, __m256i __b) | 350 | _mm256_adds_epi8(__m256i __a, __m256i __b) |
| 94 | { | 351 | { |
| 95 | return (__m256i)__builtin_elementwise_add_sat((__v32qs)__a, (__v32qs)__b); | 352 | return (__m256i)__builtin_elementwise_add_sat((__v32qs)__a, (__v32qs)__b); |
| 96 | } | 353 | } |
| 97 | 354 | ||
| 355 | /// Adds 16-bit integers from corresponding elements of two 256-bit vectors of | ||
| 356 | /// [16 x i16] using signed saturation, and returns the [16 x i16] result. | ||
| 357 | /// | ||
| 358 | /// \headerfile <immintrin.h> | ||
| 359 | /// | ||
| 360 | /// This intrinsic corresponds to the \c VPADDSW instruction. | ||
| 361 | /// | ||
| 362 | /// \param __a | ||
| 363 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 364 | /// \param __b | ||
| 365 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 366 | /// \returns A 256-bit vector of [16 x i16] containing the sums. | ||
| 98 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 367 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 99 | _mm256_adds_epi16(__m256i __a, __m256i __b) | 368 | _mm256_adds_epi16(__m256i __a, __m256i __b) |
| 100 | { | 369 | { |
| 101 | return (__m256i)__builtin_elementwise_add_sat((__v16hi)__a, (__v16hi)__b); | 370 | return (__m256i)__builtin_elementwise_add_sat((__v16hi)__a, (__v16hi)__b); |
| 102 | } | 371 | } |
| 103 | 372 | ||
| 373 | /// Adds 8-bit integers from corresponding bytes of two 256-bit integer | ||
| 374 | /// vectors using unsigned saturation, and returns each sum in the | ||
| 375 | /// corresponding byte of the 256-bit integer vector result. | ||
| 376 | /// | ||
| 377 | /// \headerfile <immintrin.h> | ||
| 378 | /// | ||
| 379 | /// This intrinsic corresponds to the \c VPADDUSB instruction. | ||
| 380 | /// | ||
| 381 | /// \param __a | ||
| 382 | /// A 256-bit integer vector containing one of the source operands. | ||
| 383 | /// \param __b | ||
| 384 | /// A 256-bit integer vector containing one of the source operands. | ||
| 385 | /// \returns A 256-bit integer vector containing the sums. | ||
| 104 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 386 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 105 | _mm256_adds_epu8(__m256i __a, __m256i __b) | 387 | _mm256_adds_epu8(__m256i __a, __m256i __b) |
| 106 | { | 388 | { |
| 107 | return (__m256i)__builtin_elementwise_add_sat((__v32qu)__a, (__v32qu)__b); | 389 | return (__m256i)__builtin_elementwise_add_sat((__v32qu)__a, (__v32qu)__b); |
| 108 | } | 390 | } |
| 109 | 391 | ||
| 392 | /// Adds 16-bit integers from corresponding elements of two 256-bit vectors of | ||
| 393 | /// [16 x i16] using unsigned saturation, and returns the [16 x i16] result. | ||
| 394 | /// | ||
| 395 | /// \headerfile <immintrin.h> | ||
| 396 | /// | ||
| 397 | /// This intrinsic corresponds to the \c VPADDUSW instruction. | ||
| 398 | /// | ||
| 399 | /// \param __a | ||
| 400 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 401 | /// \param __b | ||
| 402 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 403 | /// \returns A 256-bit vector of [16 x i16] containing the sums. | ||
| 110 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 404 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 111 | _mm256_adds_epu16(__m256i __a, __m256i __b) | 405 | _mm256_adds_epu16(__m256i __a, __m256i __b) |
| 112 | { | 406 | { |
| 113 | return (__m256i)__builtin_elementwise_add_sat((__v16hu)__a, (__v16hu)__b); | 407 | return (__m256i)__builtin_elementwise_add_sat((__v16hu)__a, (__v16hu)__b); |
| 114 | } | 408 | } |
| 115 | 409 | ||
| 410 | /// Uses the lower half of the 256-bit vector \a a as the upper half of a | ||
| 411 | /// temporary 256-bit value, and the lower half of the 256-bit vector \a b | ||
| 412 | /// as the lower half of the temporary value. Right-shifts the temporary | ||
| 413 | /// value by \a n bytes, and uses the lower 16 bytes of the shifted value | ||
| 414 | /// as the lower 16 bytes of the result. Uses the upper halves of \a a and | ||
| 415 | /// \a b to make another temporary value, right shifts by \a n, and uses | ||
| 416 | /// the lower 16 bytes of the shifted value as the upper 16 bytes of the | ||
| 417 | /// result. | ||
| 418 | /// | ||
| 419 | /// \headerfile <immintrin.h> | ||
| 420 | /// | ||
| 421 | /// \code | ||
| 422 | /// __m256i _mm256_alignr_epi8(__m256i a, __m256i b, const int n); | ||
| 423 | /// \endcode | ||
| 424 | /// | ||
| 425 | /// This intrinsic corresponds to the \c VPALIGNR instruction. | ||
| 426 | /// | ||
| 427 | /// \param a | ||
| 428 | /// A 256-bit integer vector containing source values. | ||
| 429 | /// \param b | ||
| 430 | /// A 256-bit integer vector containing source values. | ||
| 431 | /// \param n | ||
| 432 | /// An immediate value specifying the number of bytes to shift. | ||
| 433 | /// \returns A 256-bit integer vector containing the result. | ||
| 116 | #define _mm256_alignr_epi8(a, b, n) \ | 434 | #define _mm256_alignr_epi8(a, b, n) \ |
| 117 | ((__m256i)__builtin_ia32_palignr256((__v32qi)(__m256i)(a), \ | 435 | ((__m256i)__builtin_ia32_palignr256((__v32qi)(__m256i)(a), \ |
| 118 | (__v32qi)(__m256i)(b), (n))) | 436 | (__v32qi)(__m256i)(b), (n))) |
| 119 | 437 | ||
| 438 | /// Computes the bitwise AND of the 256-bit integer vectors in \a __a and | ||
| 439 | /// \a __b. | ||
| 440 | /// | ||
| 441 | /// \headerfile <immintrin.h> | ||
| 442 | /// | ||
| 443 | /// This intrinsic corresponds to the \c VPAND instruction. | ||
| 444 | /// | ||
| 445 | /// \param __a | ||
| 446 | /// A 256-bit integer vector. | ||
| 447 | /// \param __b | ||
| 448 | /// A 256-bit integer vector. | ||
| 449 | /// \returns A 256-bit integer vector containing the result. | ||
| 120 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 450 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 121 | _mm256_and_si256(__m256i __a, __m256i __b) | 451 | _mm256_and_si256(__m256i __a, __m256i __b) |
| 122 | { | 452 | { |
| 123 | return (__m256i)((__v4du)__a & (__v4du)__b); | 453 | return (__m256i)((__v4du)__a & (__v4du)__b); |
| 124 | } | 454 | } |
| 125 | 455 | ||
| 456 | /// Computes the bitwise AND of the 256-bit integer vector in \a __b with | ||
| 457 | /// the bitwise NOT of the 256-bit integer vector in \a __a. | ||
| 458 | /// | ||
| 459 | /// \headerfile <immintrin.h> | ||
| 460 | /// | ||
| 461 | /// This intrinsic corresponds to the \c VPANDN instruction. | ||
| 462 | /// | ||
| 463 | /// \param __a | ||
| 464 | /// A 256-bit integer vector. | ||
| 465 | /// \param __b | ||
| 466 | /// A 256-bit integer vector. | ||
| 467 | /// \returns A 256-bit integer vector containing the result. | ||
| 126 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 468 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 127 | _mm256_andnot_si256(__m256i __a, __m256i __b) | 469 | _mm256_andnot_si256(__m256i __a, __m256i __b) |
| 128 | { | 470 | { |
| 129 | return (__m256i)(~(__v4du)__a & (__v4du)__b); | 471 | return (__m256i)(~(__v4du)__a & (__v4du)__b); |
| 130 | } | 472 | } |
| 131 | 473 | ||
| 474 | /// Computes the averages of the corresponding unsigned bytes in the two | ||
| 475 | /// 256-bit integer vectors in \a __a and \a __b and returns each | ||
| 476 | /// average in the corresponding byte of the 256-bit result. | ||
| 477 | /// | ||
| 478 | /// \code{.operation} | ||
| 479 | /// FOR i := 0 TO 31 | ||
| 480 | /// j := i*8 | ||
| 481 | /// result[j+7:j] := (__a[j+7:j] + __b[j+7:j] + 1) >> 1 | ||
| 482 | /// ENDFOR | ||
| 483 | /// \endcode | ||
| 484 | /// | ||
| 485 | /// \headerfile <immintrin.h> | ||
| 486 | /// | ||
| 487 | /// This intrinsic corresponds to the \c VPAVGB instruction. | ||
| 488 | /// | ||
| 489 | /// \param __a | ||
| 490 | /// A 256-bit integer vector. | ||
| 491 | /// \param __b | ||
| 492 | /// A 256-bit integer vector. | ||
| 493 | /// \returns A 256-bit integer vector containing the result. | ||
| 132 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 494 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 133 | _mm256_avg_epu8(__m256i __a, __m256i __b) | 495 | _mm256_avg_epu8(__m256i __a, __m256i __b) |
| 134 | { | 496 | { |
| 135 | return (__m256i)__builtin_ia32_pavgb256((__v32qi)__a, (__v32qi)__b); | 497 | return (__m256i)__builtin_ia32_pavgb256((__v32qi)__a, (__v32qi)__b); |
| 136 | } | 498 | } |
| 137 | 499 | ||
| 500 | /// Computes the averages of the corresponding unsigned 16-bit integers in | ||
| 501 | /// the two 256-bit vectors of [16 x i16] in \a __a and \a __b and returns | ||
| 502 | /// each average in the corresponding element of the 256-bit result. | ||
| 503 | /// | ||
| 504 | /// \code{.operation} | ||
| 505 | /// FOR i := 0 TO 15 | ||
| 506 | /// j := i*16 | ||
| 507 | /// result[j+15:j] := (__a[j+15:j] + __b[j+15:j] + 1) >> 1 | ||
| 508 | /// ENDFOR | ||
| 509 | /// \endcode | ||
| 510 | /// | ||
| 511 | /// \headerfile <immintrin.h> | ||
| 512 | /// | ||
| 513 | /// This intrinsic corresponds to the \c VPAVGW instruction. | ||
| 514 | /// | ||
| 515 | /// \param __a | ||
| 516 | /// A 256-bit vector of [16 x i16]. | ||
| 517 | /// \param __b | ||
| 518 | /// A 256-bit vector of [16 x i16]. | ||
| 519 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 138 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 520 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 139 | _mm256_avg_epu16(__m256i __a, __m256i __b) | 521 | _mm256_avg_epu16(__m256i __a, __m256i __b) |
| 140 | { | 522 | { |
| 141 | return (__m256i)__builtin_ia32_pavgw256((__v16hi)__a, (__v16hi)__b); | 523 | return (__m256i)__builtin_ia32_pavgw256((__v16hi)__a, (__v16hi)__b); |
| 142 | } | 524 | } |
| 143 | 525 | ||
| 526 | /// Merges 8-bit integer values from either of the two 256-bit vectors | ||
| 527 | /// \a __V1 or \a __V2, as specified by the 256-bit mask \a __M and returns | ||
| 528 | /// the resulting 256-bit integer vector. | ||
| 529 | /// | ||
| 530 | /// \code{.operation} | ||
| 531 | /// FOR i := 0 TO 31 | ||
| 532 | /// j := i*8 | ||
| 533 | /// IF __M[7+i] == 0 | ||
| 534 | /// result[7+j:j] := __V1[7+j:j] | ||
| 535 | /// ELSE | ||
| 536 | /// result[7+j:j] := __V2[7+j:j] | ||
| 537 | /// FI | ||
| 538 | /// ENDFOR | ||
| 539 | /// \endcode | ||
| 540 | /// | ||
| 541 | /// \headerfile <immintrin.h> | ||
| 542 | /// | ||
| 543 | /// This intrinsic corresponds to the \c VPBLENDVB instruction. | ||
| 544 | /// | ||
| 545 | /// \param __V1 | ||
| 546 | /// A 256-bit integer vector containing source values. | ||
| 547 | /// \param __V2 | ||
| 548 | /// A 256-bit integer vector containing source values. | ||
| 549 | /// \param __M | ||
| 550 | /// A 256-bit integer vector, with bit [7] of each byte specifying the | ||
| 551 | /// source for each corresponding byte of the result. When the mask bit | ||
| 552 | /// is 0, the byte is copied from \a __V1; otherwise, it is copied from | ||
| 553 | /// \a __V2. | ||
| 554 | /// \returns A 256-bit integer vector containing the result. | ||
| 144 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 555 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 145 | _mm256_blendv_epi8(__m256i __V1, __m256i __V2, __m256i __M) | 556 | _mm256_blendv_epi8(__m256i __V1, __m256i __V2, __m256i __M) |
| 146 | { | 557 | { |
| ... | @@ -148,34 +559,171 @@ _mm256_blendv_epi8(__m256i __V1, __m256i __V2, __m256i __M) | ... | @@ -148,34 +559,171 @@ _mm256_blendv_epi8(__m256i __V1, __m256i __V2, __m256i __M) |
| 148 | (__v32qi)__M); | 559 | (__v32qi)__M); |
| 149 | } | 560 | } |
| 150 | 561 | ||
| 562 | /// Merges 16-bit integer values from either of the two 256-bit vectors | ||
| 563 | /// \a V1 or \a V2, as specified by the immediate integer operand \a M, | ||
| 564 | /// and returns the resulting 256-bit vector of [16 x i16]. | ||
| 565 | /// | ||
| 566 | /// \code{.operation} | ||
| 567 | /// FOR i := 0 TO 7 | ||
| 568 | /// j := i*16 | ||
| 569 | /// IF M[i] == 0 | ||
| 570 | /// result[7+j:j] := V1[7+j:j] | ||
| 571 | /// result[135+j:128+j] := V1[135+j:128+j] | ||
| 572 | /// ELSE | ||
| 573 | /// result[7+j:j] := V2[7+j:j] | ||
| 574 | /// result[135+j:128+j] := V2[135+j:128+j] | ||
| 575 | /// FI | ||
| 576 | /// ENDFOR | ||
| 577 | /// \endcode | ||
| 578 | /// | ||
| 579 | /// \headerfile <immintrin.h> | ||
| 580 | /// | ||
| 581 | /// \code | ||
| 582 | /// __m256i _mm256_blend_epi16(__m256i V1, __m256i V2, const int M); | ||
| 583 | /// \endcode | ||
| 584 | /// | ||
| 585 | /// This intrinsic corresponds to the \c VPBLENDW instruction. | ||
| 586 | /// | ||
| 587 | /// \param V1 | ||
| 588 | /// A 256-bit vector of [16 x i16] containing source values. | ||
| 589 | /// \param V2 | ||
| 590 | /// A 256-bit vector of [16 x i16] containing source values. | ||
| 591 | /// \param M | ||
| 592 | /// An immediate 8-bit integer operand, with bits [7:0] specifying the | ||
| 593 | /// source for each element of the result. The position of the mask bit | ||
| 594 | /// corresponds to the index of a copied value. When a mask bit is 0, the | ||
| 595 | /// element is copied from \a V1; otherwise, it is copied from \a V2. | ||
| 596 | /// \a M[0] determines the source for elements 0 and 8, \a M[1] for | ||
| 597 | /// elements 1 and 9, and so forth. | ||
| 598 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 151 | #define _mm256_blend_epi16(V1, V2, M) \ | 599 | #define _mm256_blend_epi16(V1, V2, M) \ |
| 152 | ((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(V1), \ | 600 | ((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(V1), \ |
| 153 | (__v16hi)(__m256i)(V2), (int)(M))) | 601 | (__v16hi)(__m256i)(V2), (int)(M))) |
| 154 | 602 | ||
| 603 | /// Compares corresponding bytes in the 256-bit integer vectors in \a __a and | ||
| 604 | /// \a __b for equality and returns the outcomes in the corresponding | ||
| 605 | /// bytes of the 256-bit result. | ||
| 606 | /// | ||
| 607 | /// \code{.operation} | ||
| 608 | /// FOR i := 0 TO 31 | ||
| 609 | /// j := i*8 | ||
| 610 | /// result[j+7:j] := (__a[j+7:j] == __b[j+7:j]) ? 0xFF : 0 | ||
| 611 | /// ENDFOR | ||
| 612 | /// \endcode | ||
| 613 | /// | ||
| 614 | /// \headerfile <immintrin.h> | ||
| 615 | /// | ||
| 616 | /// This intrinsic corresponds to the \c VPCMPEQB instruction. | ||
| 617 | /// | ||
| 618 | /// \param __a | ||
| 619 | /// A 256-bit integer vector containing one of the inputs. | ||
| 620 | /// \param __b | ||
| 621 | /// A 256-bit integer vector containing one of the inputs. | ||
| 622 | /// \returns A 256-bit integer vector containing the result. | ||
| 155 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 623 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 156 | _mm256_cmpeq_epi8(__m256i __a, __m256i __b) | 624 | _mm256_cmpeq_epi8(__m256i __a, __m256i __b) |
| 157 | { | 625 | { |
| 158 | return (__m256i)((__v32qi)__a == (__v32qi)__b); | 626 | return (__m256i)((__v32qi)__a == (__v32qi)__b); |
| 159 | } | 627 | } |
| 160 | 628 | ||
| 629 | /// Compares corresponding elements in the 256-bit vectors of [16 x i16] in | ||
| 630 | /// \a __a and \a __b for equality and returns the outcomes in the | ||
| 631 | /// corresponding elements of the 256-bit result. | ||
| 632 | /// | ||
| 633 | /// \code{.operation} | ||
| 634 | /// FOR i := 0 TO 15 | ||
| 635 | /// j := i*16 | ||
| 636 | /// result[j+15:j] := (__a[j+15:j] == __b[j+15:j]) ? 0xFFFF : 0 | ||
| 637 | /// ENDFOR | ||
| 638 | /// \endcode | ||
| 639 | /// | ||
| 640 | /// \headerfile <immintrin.h> | ||
| 641 | /// | ||
| 642 | /// This intrinsic corresponds to the \c VPCMPEQW instruction. | ||
| 643 | /// | ||
| 644 | /// \param __a | ||
| 645 | /// A 256-bit vector of [16 x i16] containing one of the inputs. | ||
| 646 | /// \param __b | ||
| 647 | /// A 256-bit vector of [16 x i16] containing one of the inputs. | ||
| 648 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 161 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 649 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 162 | _mm256_cmpeq_epi16(__m256i __a, __m256i __b) | 650 | _mm256_cmpeq_epi16(__m256i __a, __m256i __b) |
| 163 | { | 651 | { |
| 164 | return (__m256i)((__v16hi)__a == (__v16hi)__b); | 652 | return (__m256i)((__v16hi)__a == (__v16hi)__b); |
| 165 | } | 653 | } |
| 166 | 654 | ||
| 655 | /// Compares corresponding elements in the 256-bit vectors of [8 x i32] in | ||
| 656 | /// \a __a and \a __b for equality and returns the outcomes in the | ||
| 657 | /// corresponding elements of the 256-bit result. | ||
| 658 | /// | ||
| 659 | /// \code{.operation} | ||
| 660 | /// FOR i := 0 TO 7 | ||
| 661 | /// j := i*32 | ||
| 662 | /// result[j+31:j] := (__a[j+31:j] == __b[j+31:j]) ? 0xFFFFFFFF : 0 | ||
| 663 | /// ENDFOR | ||
| 664 | /// \endcode | ||
| 665 | /// | ||
| 666 | /// \headerfile <immintrin.h> | ||
| 667 | /// | ||
| 668 | /// This intrinsic corresponds to the \c VPCMPEQD instruction. | ||
| 669 | /// | ||
| 670 | /// \param __a | ||
| 671 | /// A 256-bit vector of [8 x i32] containing one of the inputs. | ||
| 672 | /// \param __b | ||
| 673 | /// A 256-bit vector of [8 x i32] containing one of the inputs. | ||
| 674 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 167 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 675 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 168 | _mm256_cmpeq_epi32(__m256i __a, __m256i __b) | 676 | _mm256_cmpeq_epi32(__m256i __a, __m256i __b) |
| 169 | { | 677 | { |
| 170 | return (__m256i)((__v8si)__a == (__v8si)__b); | 678 | return (__m256i)((__v8si)__a == (__v8si)__b); |
| 171 | } | 679 | } |
| 172 | 680 | ||
| 681 | /// Compares corresponding elements in the 256-bit vectors of [4 x i64] in | ||
| 682 | /// \a __a and \a __b for equality and returns the outcomes in the | ||
| 683 | /// corresponding elements of the 256-bit result. | ||
| 684 | /// | ||
| 685 | /// \code{.operation} | ||
| 686 | /// FOR i := 0 TO 3 | ||
| 687 | /// j := i*64 | ||
| 688 | /// result[j+63:j] := (__a[j+63:j] == __b[j+63:j]) ? 0xFFFFFFFFFFFFFFFF : 0 | ||
| 689 | /// ENDFOR | ||
| 690 | /// \endcode | ||
| 691 | /// | ||
| 692 | /// \headerfile <immintrin.h> | ||
| 693 | /// | ||
| 694 | /// This intrinsic corresponds to the \c VPCMPEQQ instruction. | ||
| 695 | /// | ||
| 696 | /// \param __a | ||
| 697 | /// A 256-bit vector of [4 x i64] containing one of the inputs. | ||
| 698 | /// \param __b | ||
| 699 | /// A 256-bit vector of [4 x i64] containing one of the inputs. | ||
| 700 | /// \returns A 256-bit vector of [4 x i64] containing the result. | ||
| 173 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 701 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 174 | _mm256_cmpeq_epi64(__m256i __a, __m256i __b) | 702 | _mm256_cmpeq_epi64(__m256i __a, __m256i __b) |
| 175 | { | 703 | { |
| 176 | return (__m256i)((__v4di)__a == (__v4di)__b); | 704 | return (__m256i)((__v4di)__a == (__v4di)__b); |
| 177 | } | 705 | } |
| 178 | 706 | ||
| 707 | /// Compares corresponding signed bytes in the 256-bit integer vectors in | ||
| 708 | /// \a __a and \a __b for greater-than and returns the outcomes in the | ||
| 709 | /// corresponding bytes of the 256-bit result. | ||
| 710 | /// | ||
| 711 | /// \code{.operation} | ||
| 712 | /// FOR i := 0 TO 31 | ||
| 713 | /// j := i*8 | ||
| 714 | /// result[j+7:j] := (__a[j+7:j] > __b[j+7:j]) ? 0xFF : 0 | ||
| 715 | /// ENDFOR | ||
| 716 | /// \endcode | ||
| 717 | /// | ||
| 718 | /// \headerfile <immintrin.h> | ||
| 719 | /// | ||
| 720 | /// This intrinsic corresponds to the \c VPCMPGTB instruction. | ||
| 721 | /// | ||
| 722 | /// \param __a | ||
| 723 | /// A 256-bit integer vector containing one of the inputs. | ||
| 724 | /// \param __b | ||
| 725 | /// A 256-bit integer vector containing one of the inputs. | ||
| 726 | /// \returns A 256-bit integer vector containing the result. | ||
| 179 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 727 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 180 | _mm256_cmpgt_epi8(__m256i __a, __m256i __b) | 728 | _mm256_cmpgt_epi8(__m256i __a, __m256i __b) |
| 181 | { | 729 | { |
| ... | @@ -184,138 +732,575 @@ _mm256_cmpgt_epi8(__m256i __a, __m256i __b) | ... | @@ -184,138 +732,575 @@ _mm256_cmpgt_epi8(__m256i __a, __m256i __b) |
| 184 | return (__m256i)((__v32qs)__a > (__v32qs)__b); | 732 | return (__m256i)((__v32qs)__a > (__v32qs)__b); |
| 185 | } | 733 | } |
| 186 | 734 | ||
| 735 | /// Compares corresponding signed elements in the 256-bit vectors of | ||
| 736 | /// [16 x i16] in \a __a and \a __b for greater-than and returns the | ||
| 737 | /// outcomes in the corresponding elements of the 256-bit result. | ||
| 738 | /// | ||
| 739 | /// \code{.operation} | ||
| 740 | /// FOR i := 0 TO 15 | ||
| 741 | /// j := i*16 | ||
| 742 | /// result[j+15:j] := (__a[j+15:j] > __b[j+15:j]) ? 0xFFFF : 0 | ||
| 743 | /// ENDFOR | ||
| 744 | /// \endcode | ||
| 745 | /// | ||
| 746 | /// \headerfile <immintrin.h> | ||
| 747 | /// | ||
| 748 | /// This intrinsic corresponds to the \c VPCMPGTW instruction. | ||
| 749 | /// | ||
| 750 | /// \param __a | ||
| 751 | /// A 256-bit vector of [16 x i16] containing one of the inputs. | ||
| 752 | /// \param __b | ||
| 753 | /// A 256-bit vector of [16 x i16] containing one of the inputs. | ||
| 754 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 187 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 755 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 188 | _mm256_cmpgt_epi16(__m256i __a, __m256i __b) | 756 | _mm256_cmpgt_epi16(__m256i __a, __m256i __b) |
| 189 | { | 757 | { |
| 190 | return (__m256i)((__v16hi)__a > (__v16hi)__b); | 758 | return (__m256i)((__v16hi)__a > (__v16hi)__b); |
| 191 | } | 759 | } |
| 192 | 760 | ||
| 761 | /// Compares corresponding signed elements in the 256-bit vectors of | ||
| 762 | /// [8 x i32] in \a __a and \a __b for greater-than and returns the | ||
| 763 | /// outcomes in the corresponding elements of the 256-bit result. | ||
| 764 | /// | ||
| 765 | /// \code{.operation} | ||
| 766 | /// FOR i := 0 TO 7 | ||
| 767 | /// j := i*32 | ||
| 768 | /// result[j+31:j] := (__a[j+31:j] > __b[j+31:j]) ? 0xFFFFFFFF : 0 | ||
| 769 | /// ENDFOR | ||
| 770 | /// \endcode | ||
| 771 | /// | ||
| 772 | /// \headerfile <immintrin.h> | ||
| 773 | /// | ||
| 774 | /// This intrinsic corresponds to the \c VPCMPGTD instruction. | ||
| 775 | /// | ||
| 776 | /// \param __a | ||
| 777 | /// A 256-bit vector of [8 x i32] containing one of the inputs. | ||
| 778 | /// \param __b | ||
| 779 | /// A 256-bit vector of [8 x i32] containing one of the inputs. | ||
| 780 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 193 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 781 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 194 | _mm256_cmpgt_epi32(__m256i __a, __m256i __b) | 782 | _mm256_cmpgt_epi32(__m256i __a, __m256i __b) |
| 195 | { | 783 | { |
| 196 | return (__m256i)((__v8si)__a > (__v8si)__b); | 784 | return (__m256i)((__v8si)__a > (__v8si)__b); |
| 197 | } | 785 | } |
| 198 | 786 | ||
| 787 | /// Compares corresponding signed elements in the 256-bit vectors of | ||
| 788 | /// [4 x i64] in \a __a and \a __b for greater-than and returns the | ||
| 789 | /// outcomes in the corresponding elements of the 256-bit result. | ||
| 790 | /// | ||
| 791 | /// \code{.operation} | ||
| 792 | /// FOR i := 0 TO 3 | ||
| 793 | /// j := i*64 | ||
| 794 | /// result[j+63:j] := (__a[j+63:j] > __b[j+63:j]) ? 0xFFFFFFFFFFFFFFFF : 0 | ||
| 795 | /// ENDFOR | ||
| 796 | /// \endcode | ||
| 797 | /// | ||
| 798 | /// \headerfile <immintrin.h> | ||
| 799 | /// | ||
| 800 | /// This intrinsic corresponds to the \c VPCMPGTQ instruction. | ||
| 801 | /// | ||
| 802 | /// \param __a | ||
| 803 | /// A 256-bit vector of [4 x i64] containing one of the inputs. | ||
| 804 | /// \param __b | ||
| 805 | /// A 256-bit vector of [4 x i64] containing one of the inputs. | ||
| 806 | /// \returns A 256-bit vector of [4 x i64] containing the result. | ||
| 199 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 807 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 200 | _mm256_cmpgt_epi64(__m256i __a, __m256i __b) | 808 | _mm256_cmpgt_epi64(__m256i __a, __m256i __b) |
| 201 | { | 809 | { |
| 202 | return (__m256i)((__v4di)__a > (__v4di)__b); | 810 | return (__m256i)((__v4di)__a > (__v4di)__b); |
| 203 | } | 811 | } |
| 204 | 812 | ||
| 813 | /// Horizontally adds the adjacent pairs of 16-bit integers from two 256-bit | ||
| 814 | /// vectors of [16 x i16] and returns the lower 16 bits of each sum in an | ||
| 815 | /// element of the [16 x i16] result (overflow is ignored). Sums from | ||
| 816 | /// \a __a are returned in the lower 64 bits of each 128-bit half of the | ||
| 817 | /// result; sums from \a __b are returned in the upper 64 bits of each | ||
| 818 | /// 128-bit half of the result. | ||
| 819 | /// | ||
| 820 | /// \code{.operation} | ||
| 821 | /// FOR i := 0 TO 1 | ||
| 822 | /// j := i*128 | ||
| 823 | /// result[j+15:j] := __a[j+15:j] + __a[j+31:j+16] | ||
| 824 | /// result[j+31:j+16] := __a[j+47:j+32] + __a[j+63:j+48] | ||
| 825 | /// result[j+47:j+32] := __a[j+79:j+64] + __a[j+95:j+80] | ||
| 826 | /// result[j+63:j+48] := __a[j+111:j+96] + __a[j+127:j+112] | ||
| 827 | /// result[j+79:j+64] := __b[j+15:j] + __b[j+31:j+16] | ||
| 828 | /// result[j+95:j+80] := __b[j+47:j+32] + __b[j+63:j+48] | ||
| 829 | /// result[j+111:j+96] := __b[j+79:j+64] + __b[j+95:j+80] | ||
| 830 | /// result[j+127:j+112] := __b[j+111:j+96] + __b[j+127:j+112] | ||
| 831 | /// ENDFOR | ||
| 832 | /// \endcode | ||
| 833 | /// | ||
| 834 | /// \headerfile <immintrin.h> | ||
| 835 | /// | ||
| 836 | /// This intrinsic corresponds to the \c VPHADDW instruction. | ||
| 837 | /// | ||
| 838 | /// \param __a | ||
| 839 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 840 | /// \param __b | ||
| 841 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 842 | /// \returns A 256-bit vector of [16 x i16] containing the sums. | ||
| 205 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 843 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 206 | _mm256_hadd_epi16(__m256i __a, __m256i __b) | 844 | _mm256_hadd_epi16(__m256i __a, __m256i __b) |
| 207 | { | 845 | { |
| 208 | return (__m256i)__builtin_ia32_phaddw256((__v16hi)__a, (__v16hi)__b); | 846 | return (__m256i)__builtin_ia32_phaddw256((__v16hi)__a, (__v16hi)__b); |
| 209 | } | 847 | } |
| 210 | 848 | ||
| 849 | /// Horizontally adds the adjacent pairs of 32-bit integers from two 256-bit | ||
| 850 | /// vectors of [8 x i32] and returns the lower 32 bits of each sum in an | ||
| 851 | /// element of the [8 x i32] result (overflow is ignored). Sums from \a __a | ||
| 852 | /// are returned in the lower 64 bits of each 128-bit half of the result; | ||
| 853 | /// sums from \a __b are returned in the upper 64 bits of each 128-bit half | ||
| 854 | /// of the result. | ||
| 855 | /// | ||
| 856 | /// \code{.operation} | ||
| 857 | /// FOR i := 0 TO 1 | ||
| 858 | /// j := i*128 | ||
| 859 | /// result[j+31:j] := __a[j+31:j] + __a[j+63:j+32] | ||
| 860 | /// result[j+63:j+32] := __a[j+95:j+64] + __a[j+127:j+96] | ||
| 861 | /// result[j+95:j+64] := __b[j+31:j] + __b[j+63:j+32] | ||
| 862 | /// result[j+127:j+96] := __b[j+95:j+64] + __b[j+127:j+96] | ||
| 863 | /// ENDFOR | ||
| 864 | /// \endcode | ||
| 865 | /// | ||
| 866 | /// \headerfile <immintrin.h> | ||
| 867 | /// | ||
| 868 | /// This intrinsic corresponds to the \c VPHADDD instruction. | ||
| 869 | /// | ||
| 870 | /// \param __a | ||
| 871 | /// A 256-bit vector of [8 x i32] containing one of the source operands. | ||
| 872 | /// \param __b | ||
| 873 | /// A 256-bit vector of [8 x i32] containing one of the source operands. | ||
| 874 | /// \returns A 256-bit vector of [8 x i32] containing the sums. | ||
| 211 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 875 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 212 | _mm256_hadd_epi32(__m256i __a, __m256i __b) | 876 | _mm256_hadd_epi32(__m256i __a, __m256i __b) |
| 213 | { | 877 | { |
| 214 | return (__m256i)__builtin_ia32_phaddd256((__v8si)__a, (__v8si)__b); | 878 | return (__m256i)__builtin_ia32_phaddd256((__v8si)__a, (__v8si)__b); |
| 215 | } | 879 | } |
| 216 | 880 | ||
| 881 | /// Horizontally adds the adjacent pairs of 16-bit integers from two 256-bit | ||
| 882 | /// vectors of [16 x i16] using signed saturation and returns each sum in | ||
| 883 | /// an element of the [16 x i16] result. Sums from \a __a are returned in | ||
| 884 | /// the lower 64 bits of each 128-bit half of the result; sums from \a __b | ||
| 885 | /// are returned in the upper 64 bits of each 128-bit half of the result. | ||
| 886 | /// | ||
| 887 | /// \code{.operation} | ||
| 888 | /// FOR i := 0 TO 1 | ||
| 889 | /// j := i*128 | ||
| 890 | /// result[j+15:j] := SATURATE16(__a[j+15:j] + __a[j+31:j+16]) | ||
| 891 | /// result[j+31:j+16] := SATURATE16(__a[j+47:j+32] + __a[j+63:j+48]) | ||
| 892 | /// result[j+47:j+32] := SATURATE16(__a[j+79:j+64] + __a[j+95:j+80]) | ||
| 893 | /// result[j+63:j+48] := SATURATE16(__a[j+111:j+96] + __a[j+127:j+112]) | ||
| 894 | /// result[j+79:j+64] := SATURATE16(__b[j+15:j] + __b[j+31:j+16]) | ||
| 895 | /// result[j+95:j+80] := SATURATE16(__b[j+47:j+32] + __b[j+63:j+48]) | ||
| 896 | /// result[j+111:j+96] := SATURATE16(__b[j+79:j+64] + __b[j+95:j+80]) | ||
| 897 | /// result[j+127:j+112] := SATURATE16(__b[j+111:j+96] + __b[j+127:j+112]) | ||
| 898 | /// ENDFOR | ||
| 899 | /// \endcode | ||
| 900 | /// | ||
| 901 | /// \headerfile <immintrin.h> | ||
| 902 | /// | ||
| 903 | /// This intrinsic corresponds to the \c VPHADDSW instruction. | ||
| 904 | /// | ||
| 905 | /// \param __a | ||
| 906 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 907 | /// \param __b | ||
| 908 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 909 | /// \returns A 256-bit vector of [16 x i16] containing the sums. | ||
| 217 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 910 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 218 | _mm256_hadds_epi16(__m256i __a, __m256i __b) | 911 | _mm256_hadds_epi16(__m256i __a, __m256i __b) |
| 219 | { | 912 | { |
| 220 | return (__m256i)__builtin_ia32_phaddsw256((__v16hi)__a, (__v16hi)__b); | 913 | return (__m256i)__builtin_ia32_phaddsw256((__v16hi)__a, (__v16hi)__b); |
| 221 | } | 914 | } |
| 222 | 915 | ||
| 916 | /// Horizontally subtracts adjacent pairs of 16-bit integers from two 256-bit | ||
| 917 | /// vectors of [16 x i16] and returns the lower 16 bits of each difference | ||
| 918 | /// in an element of the [16 x i16] result (overflow is ignored). | ||
| 919 | /// Differences from \a __a are returned in the lower 64 bits of each | ||
| 920 | /// 128-bit half of the result; differences from \a __b are returned in the | ||
| 921 | /// upper 64 bits of each 128-bit half of the result. | ||
| 922 | /// | ||
| 923 | /// \code{.operation} | ||
| 924 | /// FOR i := 0 TO 1 | ||
| 925 | /// j := i*128 | ||
| 926 | /// result[j+15:j] := __a[j+15:j] - __a[j+31:j+16] | ||
| 927 | /// result[j+31:j+16] := __a[j+47:j+32] - __a[j+63:j+48] | ||
| 928 | /// result[j+47:j+32] := __a[j+79:j+64] - __a[j+95:j+80] | ||
| 929 | /// result[j+63:j+48] := __a[j+111:j+96] - __a[j+127:j+112] | ||
| 930 | /// result[j+79:j+64] := __b[j+15:j] - __b[j+31:j+16] | ||
| 931 | /// result[j+95:j+80] := __b[j+47:j+32] - __b[j+63:j+48] | ||
| 932 | /// result[j+111:j+96] := __b[j+79:j+64] - __b[j+95:j+80] | ||
| 933 | /// result[j+127:j+112] := __b[j+111:j+96] - __b[j+127:j+112] | ||
| 934 | /// ENDFOR | ||
| 935 | /// \endcode | ||
| 936 | /// | ||
| 937 | /// \headerfile <immintrin.h> | ||
| 938 | /// | ||
| 939 | /// This intrinsic corresponds to the \c VPHSUBW instruction. | ||
| 940 | /// | ||
| 941 | /// \param __a | ||
| 942 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 943 | /// \param __b | ||
| 944 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 945 | /// \returns A 256-bit vector of [16 x i16] containing the differences. | ||
| 223 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 946 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 224 | _mm256_hsub_epi16(__m256i __a, __m256i __b) | 947 | _mm256_hsub_epi16(__m256i __a, __m256i __b) |
| 225 | { | 948 | { |
| 226 | return (__m256i)__builtin_ia32_phsubw256((__v16hi)__a, (__v16hi)__b); | 949 | return (__m256i)__builtin_ia32_phsubw256((__v16hi)__a, (__v16hi)__b); |
| 227 | } | 950 | } |
| 228 | 951 | ||
| 952 | /// Horizontally subtracts adjacent pairs of 32-bit integers from two 256-bit | ||
| 953 | /// vectors of [8 x i32] and returns the lower 32 bits of each difference in | ||
| 954 | /// an element of the [8 x i32] result (overflow is ignored). Differences | ||
| 955 | /// from \a __a are returned in the lower 64 bits of each 128-bit half of | ||
| 956 | /// the result; differences from \a __b are returned in the upper 64 bits | ||
| 957 | /// of each 128-bit half of the result. | ||
| 958 | /// | ||
| 959 | /// \code{.operation} | ||
| 960 | /// FOR i := 0 TO 1 | ||
| 961 | /// j := i*128 | ||
| 962 | /// result[j+31:j] := __a[j+31:j] - __a[j+63:j+32] | ||
| 963 | /// result[j+63:j+32] := __a[j+95:j+64] - __a[j+127:j+96] | ||
| 964 | /// result[j+95:j+64] := __b[j+31:j] - __b[j+63:j+32] | ||
| 965 | /// result[j+127:j+96] := __b[j+95:j+64] - __b[j+127:j+96] | ||
| 966 | /// ENDFOR | ||
| 967 | /// \endcode | ||
| 968 | /// | ||
| 969 | /// \headerfile <immintrin.h> | ||
| 970 | /// | ||
| 971 | /// This intrinsic corresponds to the \c VPHSUBD instruction. | ||
| 972 | /// | ||
| 973 | /// \param __a | ||
| 974 | /// A 256-bit vector of [8 x i32] containing one of the source operands. | ||
| 975 | /// \param __b | ||
| 976 | /// A 256-bit vector of [8 x i32] containing one of the source operands. | ||
| 977 | /// \returns A 256-bit vector of [8 x i32] containing the differences. | ||
| 229 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 978 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 230 | _mm256_hsub_epi32(__m256i __a, __m256i __b) | 979 | _mm256_hsub_epi32(__m256i __a, __m256i __b) |
| 231 | { | 980 | { |
| 232 | return (__m256i)__builtin_ia32_phsubd256((__v8si)__a, (__v8si)__b); | 981 | return (__m256i)__builtin_ia32_phsubd256((__v8si)__a, (__v8si)__b); |
| 233 | } | 982 | } |
| 234 | 983 | ||
| 984 | /// Horizontally subtracts adjacent pairs of 16-bit integers from two 256-bit | ||
| 985 | /// vectors of [16 x i16] using signed saturation and returns each sum in | ||
| 986 | /// an element of the [16 x i16] result. Differences from \a __a are | ||
| 987 | /// returned in the lower 64 bits of each 128-bit half of the result; | ||
| 988 | /// differences from \a __b are returned in the upper 64 bits of each | ||
| 989 | /// 128-bit half of the result. | ||
| 990 | /// | ||
| 991 | /// \code{.operation} | ||
| 992 | /// FOR i := 0 TO 1 | ||
| 993 | /// j := i*128 | ||
| 994 | /// result[j+15:j] := SATURATE16(__a[j+15:j] - __a[j+31:j+16]) | ||
| 995 | /// result[j+31:j+16] := SATURATE16(__a[j+47:j+32] - __a[j+63:j+48]) | ||
| 996 | /// result[j+47:j+32] := SATURATE16(__a[j+79:j+64] - __a[j+95:j+80]) | ||
| 997 | /// result[j+63:j+48] := SATURATE16(__a[j+111:j+96] - __a[j+127:j+112]) | ||
| 998 | /// result[j+79:j+64] := SATURATE16(__b[j+15:j] - __b[j+31:j+16]) | ||
| 999 | /// result[j+95:j+80] := SATURATE16(__b[j+47:j+32] - __b[j+63:j+48]) | ||
| 1000 | /// result[j+111:j+96] := SATURATE16(__b[j+79:j+64] - __b[j+95:j+80]) | ||
| 1001 | /// result[j+127:j+112] := SATURATE16(__b[j+111:j+96] - __b[j+127:j+112]) | ||
| 1002 | /// ENDFOR | ||
| 1003 | /// \endcode | ||
| 1004 | /// | ||
| 1005 | /// \headerfile <immintrin.h> | ||
| 1006 | /// | ||
| 1007 | /// This intrinsic corresponds to the \c VPHSUBSW instruction. | ||
| 1008 | /// | ||
| 1009 | /// \param __a | ||
| 1010 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 1011 | /// \param __b | ||
| 1012 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 1013 | /// \returns A 256-bit vector of [16 x i16] containing the differences. | ||
| 235 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1014 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 236 | _mm256_hsubs_epi16(__m256i __a, __m256i __b) | 1015 | _mm256_hsubs_epi16(__m256i __a, __m256i __b) |
| 237 | { | 1016 | { |
| 238 | return (__m256i)__builtin_ia32_phsubsw256((__v16hi)__a, (__v16hi)__b); | 1017 | return (__m256i)__builtin_ia32_phsubsw256((__v16hi)__a, (__v16hi)__b); |
| 239 | } | 1018 | } |
| 240 | 1019 | ||
| 1020 | /// Multiplies each unsigned byte from the 256-bit integer vector in \a __a | ||
| 1021 | /// with the corresponding signed byte from the 256-bit integer vector in | ||
| 1022 | /// \a __b, forming signed 16-bit intermediate products. Adds adjacent | ||
| 1023 | /// pairs of those products using signed saturation to form 16-bit sums | ||
| 1024 | /// returned as elements of the [16 x i16] result. | ||
| 1025 | /// | ||
| 1026 | /// \code{.operation} | ||
| 1027 | /// FOR i := 0 TO 15 | ||
| 1028 | /// j := i*16 | ||
| 1029 | /// temp1 := __a[j+7:j] * __b[j+7:j] | ||
| 1030 | /// temp2 := __a[j+15:j+8] * __b[j+15:j+8] | ||
| 1031 | /// result[j+15:j] := SATURATE16(temp1 + temp2) | ||
| 1032 | /// ENDFOR | ||
| 1033 | /// \endcode | ||
| 1034 | /// | ||
| 1035 | /// \headerfile <immintrin.h> | ||
| 1036 | /// | ||
| 1037 | /// This intrinsic corresponds to the \c VPMADDUBSW instruction. | ||
| 1038 | /// | ||
| 1039 | /// \param __a | ||
| 1040 | /// A 256-bit vector containing one of the source operands. | ||
| 1041 | /// \param __b | ||
| 1042 | /// A 256-bit vector containing one of the source operands. | ||
| 1043 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 241 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1044 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 242 | _mm256_maddubs_epi16(__m256i __a, __m256i __b) | 1045 | _mm256_maddubs_epi16(__m256i __a, __m256i __b) |
| 243 | { | 1046 | { |
| 244 | return (__m256i)__builtin_ia32_pmaddubsw256((__v32qi)__a, (__v32qi)__b); | 1047 | return (__m256i)__builtin_ia32_pmaddubsw256((__v32qi)__a, (__v32qi)__b); |
| 245 | } | 1048 | } |
| 246 | 1049 | ||
| 1050 | /// Multiplies corresponding 16-bit elements of two 256-bit vectors of | ||
| 1051 | /// [16 x i16], forming 32-bit intermediate products, and adds pairs of | ||
| 1052 | /// those products to form 32-bit sums returned as elements of the | ||
| 1053 | /// [8 x i32] result. | ||
| 1054 | /// | ||
| 1055 | /// There is only one wraparound case: when all four of the 16-bit sources | ||
| 1056 | /// are \c 0x8000, the result will be \c 0x80000000. | ||
| 1057 | /// | ||
| 1058 | /// \code{.operation} | ||
| 1059 | /// FOR i := 0 TO 7 | ||
| 1060 | /// j := i*32 | ||
| 1061 | /// temp1 := __a[j+15:j] * __b[j+15:j] | ||
| 1062 | /// temp2 := __a[j+31:j+16] * __b[j+31:j+16] | ||
| 1063 | /// result[j+31:j] := temp1 + temp2 | ||
| 1064 | /// ENDFOR | ||
| 1065 | /// \endcode | ||
| 1066 | /// | ||
| 1067 | /// \headerfile <immintrin.h> | ||
| 1068 | /// | ||
| 1069 | /// This intrinsic corresponds to the \c VPMADDWD instruction. | ||
| 1070 | /// | ||
| 1071 | /// \param __a | ||
| 1072 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 1073 | /// \param __b | ||
| 1074 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 1075 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 247 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1076 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 248 | _mm256_madd_epi16(__m256i __a, __m256i __b) | 1077 | _mm256_madd_epi16(__m256i __a, __m256i __b) |
| 249 | { | 1078 | { |
| 250 | return (__m256i)__builtin_ia32_pmaddwd256((__v16hi)__a, (__v16hi)__b); | 1079 | return (__m256i)__builtin_ia32_pmaddwd256((__v16hi)__a, (__v16hi)__b); |
| 251 | } | 1080 | } |
| 252 | 1081 | ||
| 1082 | /// Compares the corresponding signed bytes in the two 256-bit integer vectors | ||
| 1083 | /// in \a __a and \a __b and returns the larger of each pair in the | ||
| 1084 | /// corresponding byte of the 256-bit result. | ||
| 1085 | /// | ||
| 1086 | /// \headerfile <immintrin.h> | ||
| 1087 | /// | ||
| 1088 | /// This intrinsic corresponds to the \c VPMAXSB instruction. | ||
| 1089 | /// | ||
| 1090 | /// \param __a | ||
| 1091 | /// A 256-bit integer vector. | ||
| 1092 | /// \param __b | ||
| 1093 | /// A 256-bit integer vector. | ||
| 1094 | /// \returns A 256-bit integer vector containing the result. | ||
| 253 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1095 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 254 | _mm256_max_epi8(__m256i __a, __m256i __b) | 1096 | _mm256_max_epi8(__m256i __a, __m256i __b) |
| 255 | { | 1097 | { |
| 256 | return (__m256i)__builtin_elementwise_max((__v32qs)__a, (__v32qs)__b); | 1098 | return (__m256i)__builtin_elementwise_max((__v32qs)__a, (__v32qs)__b); |
| 257 | } | 1099 | } |
| 258 | 1100 | ||
| 1101 | /// Compares the corresponding signed 16-bit integers in the two 256-bit | ||
| 1102 | /// vectors of [16 x i16] in \a __a and \a __b and returns the larger of | ||
| 1103 | /// each pair in the corresponding element of the 256-bit result. | ||
| 1104 | /// | ||
| 1105 | /// \headerfile <immintrin.h> | ||
| 1106 | /// | ||
| 1107 | /// This intrinsic corresponds to the \c VPMAXSW instruction. | ||
| 1108 | /// | ||
| 1109 | /// \param __a | ||
| 1110 | /// A 256-bit vector of [16 x i16]. | ||
| 1111 | /// \param __b | ||
| 1112 | /// A 256-bit vector of [16 x i16]. | ||
| 1113 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 259 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1114 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 260 | _mm256_max_epi16(__m256i __a, __m256i __b) | 1115 | _mm256_max_epi16(__m256i __a, __m256i __b) |
| 261 | { | 1116 | { |
| 262 | return (__m256i)__builtin_elementwise_max((__v16hi)__a, (__v16hi)__b); | 1117 | return (__m256i)__builtin_elementwise_max((__v16hi)__a, (__v16hi)__b); |
| 263 | } | 1118 | } |
| 264 | 1119 | ||
| 1120 | /// Compares the corresponding signed 32-bit integers in the two 256-bit | ||
| 1121 | /// vectors of [8 x i32] in \a __a and \a __b and returns the larger of | ||
| 1122 | /// each pair in the corresponding element of the 256-bit result. | ||
| 1123 | /// | ||
| 1124 | /// \headerfile <immintrin.h> | ||
| 1125 | /// | ||
| 1126 | /// This intrinsic corresponds to the \c VPMAXSD instruction. | ||
| 1127 | /// | ||
| 1128 | /// \param __a | ||
| 1129 | /// A 256-bit vector of [8 x i32]. | ||
| 1130 | /// \param __b | ||
| 1131 | /// A 256-bit vector of [8 x i32]. | ||
| 1132 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 265 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1133 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 266 | _mm256_max_epi32(__m256i __a, __m256i __b) | 1134 | _mm256_max_epi32(__m256i __a, __m256i __b) |
| 267 | { | 1135 | { |
| 268 | return (__m256i)__builtin_elementwise_max((__v8si)__a, (__v8si)__b); | 1136 | return (__m256i)__builtin_elementwise_max((__v8si)__a, (__v8si)__b); |
| 269 | } | 1137 | } |
| 270 | 1138 | ||
| 1139 | /// Compares the corresponding unsigned bytes in the two 256-bit integer | ||
| 1140 | /// vectors in \a __a and \a __b and returns the larger of each pair in | ||
| 1141 | /// the corresponding byte of the 256-bit result. | ||
| 1142 | /// | ||
| 1143 | /// \headerfile <immintrin.h> | ||
| 1144 | /// | ||
| 1145 | /// This intrinsic corresponds to the \c VPMAXUB instruction. | ||
| 1146 | /// | ||
| 1147 | /// \param __a | ||
| 1148 | /// A 256-bit integer vector. | ||
| 1149 | /// \param __b | ||
| 1150 | /// A 256-bit integer vector. | ||
| 1151 | /// \returns A 256-bit integer vector containing the result. | ||
| 271 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1152 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 272 | _mm256_max_epu8(__m256i __a, __m256i __b) | 1153 | _mm256_max_epu8(__m256i __a, __m256i __b) |
| 273 | { | 1154 | { |
| 274 | return (__m256i)__builtin_elementwise_max((__v32qu)__a, (__v32qu)__b); | 1155 | return (__m256i)__builtin_elementwise_max((__v32qu)__a, (__v32qu)__b); |
| 275 | } | 1156 | } |
| 276 | 1157 | ||
| 1158 | /// Compares the corresponding unsigned 16-bit integers in the two 256-bit | ||
| 1159 | /// vectors of [16 x i16] in \a __a and \a __b and returns the larger of | ||
| 1160 | /// each pair in the corresponding element of the 256-bit result. | ||
| 1161 | /// | ||
| 1162 | /// \headerfile <immintrin.h> | ||
| 1163 | /// | ||
| 1164 | /// This intrinsic corresponds to the \c VPMAXUW instruction. | ||
| 1165 | /// | ||
| 1166 | /// \param __a | ||
| 1167 | /// A 256-bit vector of [16 x i16]. | ||
| 1168 | /// \param __b | ||
| 1169 | /// A 256-bit vector of [16 x i16]. | ||
| 1170 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 277 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1171 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 278 | _mm256_max_epu16(__m256i __a, __m256i __b) | 1172 | _mm256_max_epu16(__m256i __a, __m256i __b) |
| 279 | { | 1173 | { |
| 280 | return (__m256i)__builtin_elementwise_max((__v16hu)__a, (__v16hu)__b); | 1174 | return (__m256i)__builtin_elementwise_max((__v16hu)__a, (__v16hu)__b); |
| 281 | } | 1175 | } |
| 282 | 1176 | ||
| 1177 | /// Compares the corresponding unsigned 32-bit integers in the two 256-bit | ||
| 1178 | /// vectors of [8 x i32] in \a __a and \a __b and returns the larger of | ||
| 1179 | /// each pair in the corresponding element of the 256-bit result. | ||
| 1180 | /// | ||
| 1181 | /// \headerfile <immintrin.h> | ||
| 1182 | /// | ||
| 1183 | /// This intrinsic corresponds to the \c VPMAXUD instruction. | ||
| 1184 | /// | ||
| 1185 | /// \param __a | ||
| 1186 | /// A 256-bit vector of [8 x i32]. | ||
| 1187 | /// \param __b | ||
| 1188 | /// A 256-bit vector of [8 x i32]. | ||
| 1189 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 283 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1190 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 284 | _mm256_max_epu32(__m256i __a, __m256i __b) | 1191 | _mm256_max_epu32(__m256i __a, __m256i __b) |
| 285 | { | 1192 | { |
| 286 | return (__m256i)__builtin_elementwise_max((__v8su)__a, (__v8su)__b); | 1193 | return (__m256i)__builtin_elementwise_max((__v8su)__a, (__v8su)__b); |
| 287 | } | 1194 | } |
| 288 | 1195 | ||
| 1196 | /// Compares the corresponding signed bytes in the two 256-bit integer vectors | ||
| 1197 | /// in \a __a and \a __b and returns the smaller of each pair in the | ||
| 1198 | /// corresponding byte of the 256-bit result. | ||
| 1199 | /// | ||
| 1200 | /// \headerfile <immintrin.h> | ||
| 1201 | /// | ||
| 1202 | /// This intrinsic corresponds to the \c VPMINSB instruction. | ||
| 1203 | /// | ||
| 1204 | /// \param __a | ||
| 1205 | /// A 256-bit integer vector. | ||
| 1206 | /// \param __b | ||
| 1207 | /// A 256-bit integer vector. | ||
| 1208 | /// \returns A 256-bit integer vector containing the result. | ||
| 289 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1209 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 290 | _mm256_min_epi8(__m256i __a, __m256i __b) | 1210 | _mm256_min_epi8(__m256i __a, __m256i __b) |
| 291 | { | 1211 | { |
| 292 | return (__m256i)__builtin_elementwise_min((__v32qs)__a, (__v32qs)__b); | 1212 | return (__m256i)__builtin_elementwise_min((__v32qs)__a, (__v32qs)__b); |
| 293 | } | 1213 | } |
| 294 | 1214 | ||
| 1215 | /// Compares the corresponding signed 16-bit integers in the two 256-bit | ||
| 1216 | /// vectors of [16 x i16] in \a __a and \a __b and returns the smaller of | ||
| 1217 | /// each pair in the corresponding element of the 256-bit result. | ||
| 1218 | /// | ||
| 1219 | /// \headerfile <immintrin.h> | ||
| 1220 | /// | ||
| 1221 | /// This intrinsic corresponds to the \c VPMINSW instruction. | ||
| 1222 | /// | ||
| 1223 | /// \param __a | ||
| 1224 | /// A 256-bit vector of [16 x i16]. | ||
| 1225 | /// \param __b | ||
| 1226 | /// A 256-bit vector of [16 x i16]. | ||
| 1227 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 295 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1228 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 296 | _mm256_min_epi16(__m256i __a, __m256i __b) | 1229 | _mm256_min_epi16(__m256i __a, __m256i __b) |
| 297 | { | 1230 | { |
| 298 | return (__m256i)__builtin_elementwise_min((__v16hi)__a, (__v16hi)__b); | 1231 | return (__m256i)__builtin_elementwise_min((__v16hi)__a, (__v16hi)__b); |
| 299 | } | 1232 | } |
| 300 | 1233 | ||
| 1234 | /// Compares the corresponding signed 32-bit integers in the two 256-bit | ||
| 1235 | /// vectors of [8 x i32] in \a __a and \a __b and returns the smaller of | ||
| 1236 | /// each pair in the corresponding element of the 256-bit result. | ||
| 1237 | /// | ||
| 1238 | /// \headerfile <immintrin.h> | ||
| 1239 | /// | ||
| 1240 | /// This intrinsic corresponds to the \c VPMINSD instruction. | ||
| 1241 | /// | ||
| 1242 | /// \param __a | ||
| 1243 | /// A 256-bit vector of [8 x i32]. | ||
| 1244 | /// \param __b | ||
| 1245 | /// A 256-bit vector of [8 x i32]. | ||
| 1246 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 301 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1247 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 302 | _mm256_min_epi32(__m256i __a, __m256i __b) | 1248 | _mm256_min_epi32(__m256i __a, __m256i __b) |
| 303 | { | 1249 | { |
| 304 | return (__m256i)__builtin_elementwise_min((__v8si)__a, (__v8si)__b); | 1250 | return (__m256i)__builtin_elementwise_min((__v8si)__a, (__v8si)__b); |
| 305 | } | 1251 | } |
| 306 | 1252 | ||
| 1253 | /// Compares the corresponding unsigned bytes in the two 256-bit integer | ||
| 1254 | /// vectors in \a __a and \a __b and returns the smaller of each pair in | ||
| 1255 | /// the corresponding byte of the 256-bit result. | ||
| 1256 | /// | ||
| 1257 | /// \headerfile <immintrin.h> | ||
| 1258 | /// | ||
| 1259 | /// This intrinsic corresponds to the \c VPMINUB instruction. | ||
| 1260 | /// | ||
| 1261 | /// \param __a | ||
| 1262 | /// A 256-bit integer vector. | ||
| 1263 | /// \param __b | ||
| 1264 | /// A 256-bit integer vector. | ||
| 1265 | /// \returns A 256-bit integer vector containing the result. | ||
| 307 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1266 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 308 | _mm256_min_epu8(__m256i __a, __m256i __b) | 1267 | _mm256_min_epu8(__m256i __a, __m256i __b) |
| 309 | { | 1268 | { |
| 310 | return (__m256i)__builtin_elementwise_min((__v32qu)__a, (__v32qu)__b); | 1269 | return (__m256i)__builtin_elementwise_min((__v32qu)__a, (__v32qu)__b); |
| 311 | } | 1270 | } |
| 312 | 1271 | ||
| 1272 | /// Compares the corresponding unsigned 16-bit integers in the two 256-bit | ||
| 1273 | /// vectors of [16 x i16] in \a __a and \a __b and returns the smaller of | ||
| 1274 | /// each pair in the corresponding element of the 256-bit result. | ||
| 1275 | /// | ||
| 1276 | /// \headerfile <immintrin.h> | ||
| 1277 | /// | ||
| 1278 | /// This intrinsic corresponds to the \c VPMINUW instruction. | ||
| 1279 | /// | ||
| 1280 | /// \param __a | ||
| 1281 | /// A 256-bit vector of [16 x i16]. | ||
| 1282 | /// \param __b | ||
| 1283 | /// A 256-bit vector of [16 x i16]. | ||
| 1284 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 313 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1285 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 314 | _mm256_min_epu16(__m256i __a, __m256i __b) | 1286 | _mm256_min_epu16(__m256i __a, __m256i __b) |
| 315 | { | 1287 | { |
| 316 | return (__m256i)__builtin_elementwise_min((__v16hu)__a, (__v16hu)__b); | 1288 | return (__m256i)__builtin_elementwise_min((__v16hu)__a, (__v16hu)__b); |
| 317 | } | 1289 | } |
| 318 | 1290 | ||
| 1291 | /// Compares the corresponding unsigned 32-bit integers in the two 256-bit | ||
| 1292 | /// vectors of [8 x i32] in \a __a and \a __b and returns the smaller of | ||
| 1293 | /// each pair in the corresponding element of the 256-bit result. | ||
| 1294 | /// | ||
| 1295 | /// \headerfile <immintrin.h> | ||
| 1296 | /// | ||
| 1297 | /// This intrinsic corresponds to the \c VPMINUD instruction. | ||
| 1298 | /// | ||
| 1299 | /// \param __a | ||
| 1300 | /// A 256-bit vector of [8 x i32]. | ||
| 1301 | /// \param __b | ||
| 1302 | /// A 256-bit vector of [8 x i32]. | ||
| 1303 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 319 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1304 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 320 | _mm256_min_epu32(__m256i __a, __m256i __b) | 1305 | _mm256_min_epu32(__m256i __a, __m256i __b) |
| 321 | { | 1306 | { |
| ... | @@ -328,6 +1313,26 @@ _mm256_movemask_epi8(__m256i __a) | ... | @@ -328,6 +1313,26 @@ _mm256_movemask_epi8(__m256i __a) |
| 328 | return __builtin_ia32_pmovmskb256((__v32qi)__a); | 1313 | return __builtin_ia32_pmovmskb256((__v32qi)__a); |
| 329 | } | 1314 | } |
| 330 | 1315 | ||
| 1316 | /// Sign-extends bytes from the 128-bit integer vector in \a __V and returns | ||
| 1317 | /// the 16-bit values in the corresponding elements of a 256-bit vector | ||
| 1318 | /// of [16 x i16]. | ||
| 1319 | /// | ||
| 1320 | /// \code{.operation} | ||
| 1321 | /// FOR i := 0 TO 15 | ||
| 1322 | /// j := i*8 | ||
| 1323 | /// k := i*16 | ||
| 1324 | /// result[k+15:k] := SignExtend(__V[j+7:j]) | ||
| 1325 | /// ENDFOR | ||
| 1326 | /// \endcode | ||
| 1327 | /// | ||
| 1328 | /// \headerfile <immintrin.h> | ||
| 1329 | /// | ||
| 1330 | /// This intrinsic corresponds to the \c VPMOVSXBW instruction. | ||
| 1331 | /// | ||
| 1332 | /// \param __V | ||
| 1333 | /// A 128-bit integer vector containing the source bytes. | ||
| 1334 | /// \returns A 256-bit vector of [16 x i16] containing the sign-extended | ||
| 1335 | /// values. | ||
| 331 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1336 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 332 | _mm256_cvtepi8_epi16(__m128i __V) | 1337 | _mm256_cvtepi8_epi16(__m128i __V) |
| 333 | { | 1338 | { |
| ... | @@ -336,6 +1341,26 @@ _mm256_cvtepi8_epi16(__m128i __V) | ... | @@ -336,6 +1341,26 @@ _mm256_cvtepi8_epi16(__m128i __V) |
| 336 | return (__m256i)__builtin_convertvector((__v16qs)__V, __v16hi); | 1341 | return (__m256i)__builtin_convertvector((__v16qs)__V, __v16hi); |
| 337 | } | 1342 | } |
| 338 | 1343 | ||
| 1344 | /// Sign-extends bytes from the lower half of the 128-bit integer vector in | ||
| 1345 | /// \a __V and returns the 32-bit values in the corresponding elements of a | ||
| 1346 | /// 256-bit vector of [8 x i32]. | ||
| 1347 | /// | ||
| 1348 | /// \code{.operation} | ||
| 1349 | /// FOR i := 0 TO 7 | ||
| 1350 | /// j := i*8 | ||
| 1351 | /// k := i*32 | ||
| 1352 | /// result[k+31:k] := SignExtend(__V[j+7:j]) | ||
| 1353 | /// ENDFOR | ||
| 1354 | /// \endcode | ||
| 1355 | /// | ||
| 1356 | /// \headerfile <immintrin.h> | ||
| 1357 | /// | ||
| 1358 | /// This intrinsic corresponds to the \c VPMOVSXBD instruction. | ||
| 1359 | /// | ||
| 1360 | /// \param __V | ||
| 1361 | /// A 128-bit integer vector containing the source bytes. | ||
| 1362 | /// \returns A 256-bit vector of [8 x i32] containing the sign-extended | ||
| 1363 | /// values. | ||
| 339 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1364 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 340 | _mm256_cvtepi8_epi32(__m128i __V) | 1365 | _mm256_cvtepi8_epi32(__m128i __V) |
| 341 | { | 1366 | { |
| ... | @@ -344,6 +1369,25 @@ _mm256_cvtepi8_epi32(__m128i __V) | ... | @@ -344,6 +1369,25 @@ _mm256_cvtepi8_epi32(__m128i __V) |
| 344 | return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8si); | 1369 | return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8si); |
| 345 | } | 1370 | } |
| 346 | 1371 | ||
| 1372 | /// Sign-extends the first four bytes from the 128-bit integer vector in | ||
| 1373 | /// \a __V and returns the 64-bit values in the corresponding elements of a | ||
| 1374 | /// 256-bit vector of [4 x i64]. | ||
| 1375 | /// | ||
| 1376 | /// \code{.operation} | ||
| 1377 | /// result[63:0] := SignExtend(__V[7:0]) | ||
| 1378 | /// result[127:64] := SignExtend(__V[15:8]) | ||
| 1379 | /// result[191:128] := SignExtend(__V[23:16]) | ||
| 1380 | /// result[255:192] := SignExtend(__V[31:24]) | ||
| 1381 | /// \endcode | ||
| 1382 | /// | ||
| 1383 | /// \headerfile <immintrin.h> | ||
| 1384 | /// | ||
| 1385 | /// This intrinsic corresponds to the \c VPMOVSXBQ instruction. | ||
| 1386 | /// | ||
| 1387 | /// \param __V | ||
| 1388 | /// A 128-bit integer vector containing the source bytes. | ||
| 1389 | /// \returns A 256-bit vector of [4 x i64] containing the sign-extended | ||
| 1390 | /// values. | ||
| 347 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1391 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 348 | _mm256_cvtepi8_epi64(__m128i __V) | 1392 | _mm256_cvtepi8_epi64(__m128i __V) |
| 349 | { | 1393 | { |
| ... | @@ -352,357 +1396,1571 @@ _mm256_cvtepi8_epi64(__m128i __V) | ... | @@ -352,357 +1396,1571 @@ _mm256_cvtepi8_epi64(__m128i __V) |
| 352 | return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3), __v4di); | 1396 | return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3), __v4di); |
| 353 | } | 1397 | } |
| 354 | 1398 | ||
| 1399 | /// Sign-extends 16-bit elements from the 128-bit vector of [8 x i16] in | ||
| 1400 | /// \a __V and returns the 32-bit values in the corresponding elements of a | ||
| 1401 | /// 256-bit vector of [8 x i32]. | ||
| 1402 | /// | ||
| 1403 | /// \code{.operation} | ||
| 1404 | /// FOR i := 0 TO 7 | ||
| 1405 | /// j := i*16 | ||
| 1406 | /// k := i*32 | ||
| 1407 | /// result[k+31:k] := SignExtend(__V[j+15:j]) | ||
| 1408 | /// ENDFOR | ||
| 1409 | /// \endcode | ||
| 1410 | /// | ||
| 1411 | /// \headerfile <immintrin.h> | ||
| 1412 | /// | ||
| 1413 | /// This intrinsic corresponds to the \c VPMOVSXWD instruction. | ||
| 1414 | /// | ||
| 1415 | /// \param __V | ||
| 1416 | /// A 128-bit vector of [8 x i16] containing the source values. | ||
| 1417 | /// \returns A 256-bit vector of [8 x i32] containing the sign-extended | ||
| 1418 | /// values. | ||
| 355 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1419 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 356 | _mm256_cvtepi16_epi32(__m128i __V) | 1420 | _mm256_cvtepi16_epi32(__m128i __V) |
| 357 | { | 1421 | { |
| 358 | return (__m256i)__builtin_convertvector((__v8hi)__V, __v8si); | 1422 | return (__m256i)__builtin_convertvector((__v8hi)__V, __v8si); |
| 359 | } | 1423 | } |
| 360 | 1424 | ||
| 1425 | /// Sign-extends 16-bit elements from the lower half of the 128-bit vector of | ||
| 1426 | /// [8 x i16] in \a __V and returns the 64-bit values in the corresponding | ||
| 1427 | /// elements of a 256-bit vector of [4 x i64]. | ||
| 1428 | /// | ||
| 1429 | /// \code{.operation} | ||
| 1430 | /// result[63:0] := SignExtend(__V[15:0]) | ||
| 1431 | /// result[127:64] := SignExtend(__V[31:16]) | ||
| 1432 | /// result[191:128] := SignExtend(__V[47:32]) | ||
| 1433 | /// result[255:192] := SignExtend(__V[64:48]) | ||
| 1434 | /// \endcode | ||
| 1435 | /// | ||
| 1436 | /// \headerfile <immintrin.h> | ||
| 1437 | /// | ||
| 1438 | /// This intrinsic corresponds to the \c VPMOVSXWQ instruction. | ||
| 1439 | /// | ||
| 1440 | /// \param __V | ||
| 1441 | /// A 128-bit vector of [8 x i16] containing the source values. | ||
| 1442 | /// \returns A 256-bit vector of [4 x i64] containing the sign-extended | ||
| 1443 | /// values. | ||
| 361 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1444 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 362 | _mm256_cvtepi16_epi64(__m128i __V) | 1445 | _mm256_cvtepi16_epi64(__m128i __V) |
| 363 | { | 1446 | { |
| 364 | return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1, 2, 3), __v4di); | 1447 | return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1, 2, 3), __v4di); |
| 365 | } | 1448 | } |
| 366 | 1449 | ||
| 1450 | /// Sign-extends 32-bit elements from the 128-bit vector of [4 x i32] in | ||
| 1451 | /// \a __V and returns the 64-bit values in the corresponding elements of a | ||
| 1452 | /// 256-bit vector of [4 x i64]. | ||
| 1453 | /// | ||
| 1454 | /// \code{.operation} | ||
| 1455 | /// result[63:0] := SignExtend(__V[31:0]) | ||
| 1456 | /// result[127:64] := SignExtend(__V[63:32]) | ||
| 1457 | /// result[191:128] := SignExtend(__V[95:64]) | ||
| 1458 | /// result[255:192] := SignExtend(__V[127:96]) | ||
| 1459 | /// \endcode | ||
| 1460 | /// | ||
| 1461 | /// \headerfile <immintrin.h> | ||
| 1462 | /// | ||
| 1463 | /// This intrinsic corresponds to the \c VPMOVSXDQ instruction. | ||
| 1464 | /// | ||
| 1465 | /// \param __V | ||
| 1466 | /// A 128-bit vector of [4 x i32] containing the source values. | ||
| 1467 | /// \returns A 256-bit vector of [4 x i64] containing the sign-extended | ||
| 1468 | /// values. | ||
| 367 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1469 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 368 | _mm256_cvtepi32_epi64(__m128i __V) | 1470 | _mm256_cvtepi32_epi64(__m128i __V) |
| 369 | { | 1471 | { |
| 370 | return (__m256i)__builtin_convertvector((__v4si)__V, __v4di); | 1472 | return (__m256i)__builtin_convertvector((__v4si)__V, __v4di); |
| 371 | } | 1473 | } |
| 372 | 1474 | ||
| 1475 | /// Zero-extends bytes from the 128-bit integer vector in \a __V and returns | ||
| 1476 | /// the 16-bit values in the corresponding elements of a 256-bit vector | ||
| 1477 | /// of [16 x i16]. | ||
| 1478 | /// | ||
| 1479 | /// \code{.operation} | ||
| 1480 | /// FOR i := 0 TO 15 | ||
| 1481 | /// j := i*8 | ||
| 1482 | /// k := i*16 | ||
| 1483 | /// result[k+15:k] := ZeroExtend(__V[j+7:j]) | ||
| 1484 | /// ENDFOR | ||
| 1485 | /// \endcode | ||
| 1486 | /// | ||
| 1487 | /// \headerfile <immintrin.h> | ||
| 1488 | /// | ||
| 1489 | /// This intrinsic corresponds to the \c VPMOVZXBW instruction. | ||
| 1490 | /// | ||
| 1491 | /// \param __V | ||
| 1492 | /// A 128-bit integer vector containing the source bytes. | ||
| 1493 | /// \returns A 256-bit vector of [16 x i16] containing the zero-extended | ||
| 1494 | /// values. | ||
| 373 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1495 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 374 | _mm256_cvtepu8_epi16(__m128i __V) | 1496 | _mm256_cvtepu8_epi16(__m128i __V) |
| 375 | { | 1497 | { |
| 376 | return (__m256i)__builtin_convertvector((__v16qu)__V, __v16hi); | 1498 | return (__m256i)__builtin_convertvector((__v16qu)__V, __v16hi); |
| 377 | } | 1499 | } |
| 378 | 1500 | ||
| 1501 | /// Zero-extends bytes from the lower half of the 128-bit integer vector in | ||
| 1502 | /// \a __V and returns the 32-bit values in the corresponding elements of a | ||
| 1503 | /// 256-bit vector of [8 x i32]. | ||
| 1504 | /// | ||
| 1505 | /// \code{.operation} | ||
| 1506 | /// FOR i := 0 TO 7 | ||
| 1507 | /// j := i*8 | ||
| 1508 | /// k := i*32 | ||
| 1509 | /// result[k+31:k] := ZeroExtend(__V[j+7:j]) | ||
| 1510 | /// ENDFOR | ||
| 1511 | /// \endcode | ||
| 1512 | /// | ||
| 1513 | /// \headerfile <immintrin.h> | ||
| 1514 | /// | ||
| 1515 | /// This intrinsic corresponds to the \c VPMOVZXBD instruction. | ||
| 1516 | /// | ||
| 1517 | /// \param __V | ||
| 1518 | /// A 128-bit integer vector containing the source bytes. | ||
| 1519 | /// \returns A 256-bit vector of [8 x i32] containing the zero-extended | ||
| 1520 | /// values. | ||
| 379 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1521 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 380 | _mm256_cvtepu8_epi32(__m128i __V) | 1522 | _mm256_cvtepu8_epi32(__m128i __V) |
| 381 | { | 1523 | { |
| 382 | return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8si); | 1524 | return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8si); |
| 383 | } | 1525 | } |
| 384 | 1526 | ||
| 1527 | /// Zero-extends the first four bytes from the 128-bit integer vector in | ||
| 1528 | /// \a __V and returns the 64-bit values in the corresponding elements of a | ||
| 1529 | /// 256-bit vector of [4 x i64]. | ||
| 1530 | /// | ||
| 1531 | /// \code{.operation} | ||
| 1532 | /// result[63:0] := ZeroExtend(__V[7:0]) | ||
| 1533 | /// result[127:64] := ZeroExtend(__V[15:8]) | ||
| 1534 | /// result[191:128] := ZeroExtend(__V[23:16]) | ||
| 1535 | /// result[255:192] := ZeroExtend(__V[31:24]) | ||
| 1536 | /// \endcode | ||
| 1537 | /// | ||
| 1538 | /// \headerfile <immintrin.h> | ||
| 1539 | /// | ||
| 1540 | /// This intrinsic corresponds to the \c VPMOVZXBQ instruction. | ||
| 1541 | /// | ||
| 1542 | /// \param __V | ||
| 1543 | /// A 128-bit integer vector containing the source bytes. | ||
| 1544 | /// \returns A 256-bit vector of [4 x i64] containing the zero-extended | ||
| 1545 | /// values. | ||
| 385 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1546 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 386 | _mm256_cvtepu8_epi64(__m128i __V) | 1547 | _mm256_cvtepu8_epi64(__m128i __V) |
| 387 | { | 1548 | { |
| 388 | return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3), __v4di); | 1549 | return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3), __v4di); |
| 389 | } | 1550 | } |
| 390 | 1551 | ||
| 1552 | /// Zero-extends 16-bit elements from the 128-bit vector of [8 x i16] in | ||
| 1553 | /// \a __V and returns the 32-bit values in the corresponding elements of a | ||
| 1554 | /// 256-bit vector of [8 x i32]. | ||
| 1555 | /// | ||
| 1556 | /// \code{.operation} | ||
| 1557 | /// FOR i := 0 TO 7 | ||
| 1558 | /// j := i*16 | ||
| 1559 | /// k := i*32 | ||
| 1560 | /// result[k+31:k] := ZeroExtend(__V[j+15:j]) | ||
| 1561 | /// ENDFOR | ||
| 1562 | /// \endcode | ||
| 1563 | /// | ||
| 1564 | /// \headerfile <immintrin.h> | ||
| 1565 | /// | ||
| 1566 | /// This intrinsic corresponds to the \c VPMOVZXWD instruction. | ||
| 1567 | /// | ||
| 1568 | /// \param __V | ||
| 1569 | /// A 128-bit vector of [8 x i16] containing the source values. | ||
| 1570 | /// \returns A 256-bit vector of [8 x i32] containing the zero-extended | ||
| 1571 | /// values. | ||
| 391 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1572 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 392 | _mm256_cvtepu16_epi32(__m128i __V) | 1573 | _mm256_cvtepu16_epi32(__m128i __V) |
| 393 | { | 1574 | { |
| 394 | return (__m256i)__builtin_convertvector((__v8hu)__V, __v8si); | 1575 | return (__m256i)__builtin_convertvector((__v8hu)__V, __v8si); |
| 395 | } | 1576 | } |
| 396 | 1577 | ||
| 1578 | /// Zero-extends 16-bit elements from the lower half of the 128-bit vector of | ||
| 1579 | /// [8 x i16] in \a __V and returns the 64-bit values in the corresponding | ||
| 1580 | /// elements of a 256-bit vector of [4 x i64]. | ||
| 1581 | /// | ||
| 1582 | /// \code{.operation} | ||
| 1583 | /// result[63:0] := ZeroExtend(__V[15:0]) | ||
| 1584 | /// result[127:64] := ZeroExtend(__V[31:16]) | ||
| 1585 | /// result[191:128] := ZeroExtend(__V[47:32]) | ||
| 1586 | /// result[255:192] := ZeroExtend(__V[64:48]) | ||
| 1587 | /// \endcode | ||
| 1588 | /// | ||
| 1589 | /// \headerfile <immintrin.h> | ||
| 1590 | /// | ||
| 1591 | /// This intrinsic corresponds to the \c VPMOVSXWQ instruction. | ||
| 1592 | /// | ||
| 1593 | /// \param __V | ||
| 1594 | /// A 128-bit vector of [8 x i16] containing the source values. | ||
| 1595 | /// \returns A 256-bit vector of [4 x i64] containing the zero-extended | ||
| 1596 | /// values. | ||
| 397 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1597 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 398 | _mm256_cvtepu16_epi64(__m128i __V) | 1598 | _mm256_cvtepu16_epi64(__m128i __V) |
| 399 | { | 1599 | { |
| 400 | return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v8hu)__V, (__v8hu)__V, 0, 1, 2, 3), __v4di); | 1600 | return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v8hu)__V, (__v8hu)__V, 0, 1, 2, 3), __v4di); |
| 401 | } | 1601 | } |
| 402 | 1602 | ||
| 1603 | /// Zero-extends 32-bit elements from the 128-bit vector of [4 x i32] in | ||
| 1604 | /// \a __V and returns the 64-bit values in the corresponding elements of a | ||
| 1605 | /// 256-bit vector of [4 x i64]. | ||
| 1606 | /// | ||
| 1607 | /// \code{.operation} | ||
| 1608 | /// result[63:0] := ZeroExtend(__V[31:0]) | ||
| 1609 | /// result[127:64] := ZeroExtend(__V[63:32]) | ||
| 1610 | /// result[191:128] := ZeroExtend(__V[95:64]) | ||
| 1611 | /// result[255:192] := ZeroExtend(__V[127:96]) | ||
| 1612 | /// \endcode | ||
| 1613 | /// | ||
| 1614 | /// \headerfile <immintrin.h> | ||
| 1615 | /// | ||
| 1616 | /// This intrinsic corresponds to the \c VPMOVZXDQ instruction. | ||
| 1617 | /// | ||
| 1618 | /// \param __V | ||
| 1619 | /// A 128-bit vector of [4 x i32] containing the source values. | ||
| 1620 | /// \returns A 256-bit vector of [4 x i64] containing the zero-extended | ||
| 1621 | /// values. | ||
| 403 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1622 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 404 | _mm256_cvtepu32_epi64(__m128i __V) | 1623 | _mm256_cvtepu32_epi64(__m128i __V) |
| 405 | { | 1624 | { |
| 406 | return (__m256i)__builtin_convertvector((__v4su)__V, __v4di); | 1625 | return (__m256i)__builtin_convertvector((__v4su)__V, __v4di); |
| 407 | } | 1626 | } |
| 408 | 1627 | ||
| 1628 | /// Multiplies signed 32-bit integers from even-numbered elements of two | ||
| 1629 | /// 256-bit vectors of [8 x i32] and returns the 64-bit products in the | ||
| 1630 | /// [4 x i64] result. | ||
| 1631 | /// | ||
| 1632 | /// \code{.operation} | ||
| 1633 | /// result[63:0] := __a[31:0] * __b[31:0] | ||
| 1634 | /// result[127:64] := __a[95:64] * __b[95:64] | ||
| 1635 | /// result[191:128] := __a[159:128] * __b[159:128] | ||
| 1636 | /// result[255:192] := __a[223:192] * __b[223:192] | ||
| 1637 | /// \endcode | ||
| 1638 | /// | ||
| 1639 | /// \headerfile <immintrin.h> | ||
| 1640 | /// | ||
| 1641 | /// This intrinsic corresponds to the \c VPMULDQ instruction. | ||
| 1642 | /// | ||
| 1643 | /// \param __a | ||
| 1644 | /// A 256-bit vector of [8 x i32] containing one of the source operands. | ||
| 1645 | /// \param __b | ||
| 1646 | /// A 256-bit vector of [8 x i32] containing one of the source operands. | ||
| 1647 | /// \returns A 256-bit vector of [4 x i64] containing the products. | ||
| 409 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1648 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 410 | _mm256_mul_epi32(__m256i __a, __m256i __b) | 1649 | _mm256_mul_epi32(__m256i __a, __m256i __b) |
| 411 | { | 1650 | { |
| 412 | return (__m256i)__builtin_ia32_pmuldq256((__v8si)__a, (__v8si)__b); | 1651 | return (__m256i)__builtin_ia32_pmuldq256((__v8si)__a, (__v8si)__b); |
| 413 | } | 1652 | } |
| 414 | 1653 | ||
| 1654 | /// Multiplies signed 16-bit integer elements of two 256-bit vectors of | ||
| 1655 | /// [16 x i16], truncates the 32-bit results to the most significant 18 | ||
| 1656 | /// bits, rounds by adding 1, and returns bits [16:1] of each rounded | ||
| 1657 | /// product in the [16 x i16] result. | ||
| 1658 | /// | ||
| 1659 | /// \code{.operation} | ||
| 1660 | /// FOR i := 0 TO 15 | ||
| 1661 | /// j := i*16 | ||
| 1662 | /// temp := ((__a[j+15:j] * __b[j+15:j]) >> 14) + 1 | ||
| 1663 | /// result[j+15:j] := temp[16:1] | ||
| 1664 | /// \endcode | ||
| 1665 | /// | ||
| 1666 | /// \headerfile <immintrin.h> | ||
| 1667 | /// | ||
| 1668 | /// This intrinsic corresponds to the \c VPMULHRSW instruction. | ||
| 1669 | /// | ||
| 1670 | /// \param __a | ||
| 1671 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 1672 | /// \param __b | ||
| 1673 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 1674 | /// \returns A 256-bit vector of [16 x i16] containing the rounded products. | ||
| 415 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1675 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 416 | _mm256_mulhrs_epi16(__m256i __a, __m256i __b) | 1676 | _mm256_mulhrs_epi16(__m256i __a, __m256i __b) |
| 417 | { | 1677 | { |
| 418 | return (__m256i)__builtin_ia32_pmulhrsw256((__v16hi)__a, (__v16hi)__b); | 1678 | return (__m256i)__builtin_ia32_pmulhrsw256((__v16hi)__a, (__v16hi)__b); |
| 419 | } | 1679 | } |
| 420 | 1680 | ||
| 1681 | /// Multiplies unsigned 16-bit integer elements of two 256-bit vectors of | ||
| 1682 | /// [16 x i16], and returns the upper 16 bits of each 32-bit product in the | ||
| 1683 | /// [16 x i16] result. | ||
| 1684 | /// | ||
| 1685 | /// \headerfile <immintrin.h> | ||
| 1686 | /// | ||
| 1687 | /// This intrinsic corresponds to the \c VPMULHUW instruction. | ||
| 1688 | /// | ||
| 1689 | /// \param __a | ||
| 1690 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 1691 | /// \param __b | ||
| 1692 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 1693 | /// \returns A 256-bit vector of [16 x i16] containing the products. | ||
| 421 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1694 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 422 | _mm256_mulhi_epu16(__m256i __a, __m256i __b) | 1695 | _mm256_mulhi_epu16(__m256i __a, __m256i __b) |
| 423 | { | 1696 | { |
| 424 | return (__m256i)__builtin_ia32_pmulhuw256((__v16hi)__a, (__v16hi)__b); | 1697 | return (__m256i)__builtin_ia32_pmulhuw256((__v16hi)__a, (__v16hi)__b); |
| 425 | } | 1698 | } |
| 426 | 1699 | ||
| 1700 | /// Multiplies signed 16-bit integer elements of two 256-bit vectors of | ||
| 1701 | /// [16 x i16], and returns the upper 16 bits of each 32-bit product in the | ||
| 1702 | /// [16 x i16] result. | ||
| 1703 | /// | ||
| 1704 | /// \headerfile <immintrin.h> | ||
| 1705 | /// | ||
| 1706 | /// This intrinsic corresponds to the \c VPMULHW instruction. | ||
| 1707 | /// | ||
| 1708 | /// \param __a | ||
| 1709 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 1710 | /// \param __b | ||
| 1711 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 1712 | /// \returns A 256-bit vector of [16 x i16] containing the products. | ||
| 427 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1713 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 428 | _mm256_mulhi_epi16(__m256i __a, __m256i __b) | 1714 | _mm256_mulhi_epi16(__m256i __a, __m256i __b) |
| 429 | { | 1715 | { |
| 430 | return (__m256i)__builtin_ia32_pmulhw256((__v16hi)__a, (__v16hi)__b); | 1716 | return (__m256i)__builtin_ia32_pmulhw256((__v16hi)__a, (__v16hi)__b); |
| 431 | } | 1717 | } |
| 432 | 1718 | ||
| 1719 | /// Multiplies signed 16-bit integer elements of two 256-bit vectors of | ||
| 1720 | /// [16 x i16], and returns the lower 16 bits of each 32-bit product in the | ||
| 1721 | /// [16 x i16] result. | ||
| 1722 | /// | ||
| 1723 | /// \headerfile <immintrin.h> | ||
| 1724 | /// | ||
| 1725 | /// This intrinsic corresponds to the \c VPMULLW instruction. | ||
| 1726 | /// | ||
| 1727 | /// \param __a | ||
| 1728 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 1729 | /// \param __b | ||
| 1730 | /// A 256-bit vector of [16 x i16] containing one of the source operands. | ||
| 1731 | /// \returns A 256-bit vector of [16 x i16] containing the products. | ||
| 433 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1732 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 434 | _mm256_mullo_epi16(__m256i __a, __m256i __b) | 1733 | _mm256_mullo_epi16(__m256i __a, __m256i __b) |
| 435 | { | 1734 | { |
| 436 | return (__m256i)((__v16hu)__a * (__v16hu)__b); | 1735 | return (__m256i)((__v16hu)__a * (__v16hu)__b); |
| 437 | } | 1736 | } |
| 438 | 1737 | ||
| 1738 | /// Multiplies signed 32-bit integer elements of two 256-bit vectors of | ||
| 1739 | /// [8 x i32], and returns the lower 32 bits of each 64-bit product in the | ||
| 1740 | /// [8 x i32] result. | ||
| 1741 | /// | ||
| 1742 | /// \headerfile <immintrin.h> | ||
| 1743 | /// | ||
| 1744 | /// This intrinsic corresponds to the \c VPMULLD instruction. | ||
| 1745 | /// | ||
| 1746 | /// \param __a | ||
| 1747 | /// A 256-bit vector of [8 x i32] containing one of the source operands. | ||
| 1748 | /// \param __b | ||
| 1749 | /// A 256-bit vector of [8 x i32] containing one of the source operands. | ||
| 1750 | /// \returns A 256-bit vector of [8 x i32] containing the products. | ||
| 439 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1751 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 440 | _mm256_mullo_epi32 (__m256i __a, __m256i __b) | 1752 | _mm256_mullo_epi32 (__m256i __a, __m256i __b) |
| 441 | { | 1753 | { |
| 442 | return (__m256i)((__v8su)__a * (__v8su)__b); | 1754 | return (__m256i)((__v8su)__a * (__v8su)__b); |
| 443 | } | 1755 | } |
| 444 | 1756 | ||
| 1757 | /// Multiplies unsigned 32-bit integers from even-numered elements of two | ||
| 1758 | /// 256-bit vectors of [8 x i32] and returns the 64-bit products in the | ||
| 1759 | /// [4 x i64] result. | ||
| 1760 | /// | ||
| 1761 | /// \code{.operation} | ||
| 1762 | /// result[63:0] := __a[31:0] * __b[31:0] | ||
| 1763 | /// result[127:64] := __a[95:64] * __b[95:64] | ||
| 1764 | /// result[191:128] := __a[159:128] * __b[159:128] | ||
| 1765 | /// result[255:192] := __a[223:192] * __b[223:192] | ||
| 1766 | /// \endcode | ||
| 1767 | /// | ||
| 1768 | /// \headerfile <immintrin.h> | ||
| 1769 | /// | ||
| 1770 | /// This intrinsic corresponds to the \c VPMULUDQ instruction. | ||
| 1771 | /// | ||
| 1772 | /// \param __a | ||
| 1773 | /// A 256-bit vector of [8 x i32] containing one of the source operands. | ||
| 1774 | /// \param __b | ||
| 1775 | /// A 256-bit vector of [8 x i32] containing one of the source operands. | ||
| 1776 | /// \returns A 256-bit vector of [4 x i64] containing the products. | ||
| 445 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1777 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 446 | _mm256_mul_epu32(__m256i __a, __m256i __b) | 1778 | _mm256_mul_epu32(__m256i __a, __m256i __b) |
| 447 | { | 1779 | { |
| 448 | return __builtin_ia32_pmuludq256((__v8si)__a, (__v8si)__b); | 1780 | return __builtin_ia32_pmuludq256((__v8si)__a, (__v8si)__b); |
| 449 | } | 1781 | } |
| 450 | 1782 | ||
| 1783 | /// Computes the bitwise OR of the 256-bit integer vectors in \a __a and | ||
| 1784 | /// \a __b. | ||
| 1785 | /// | ||
| 1786 | /// \headerfile <immintrin.h> | ||
| 1787 | /// | ||
| 1788 | /// This intrinsic corresponds to the \c VPOR instruction. | ||
| 1789 | /// | ||
| 1790 | /// \param __a | ||
| 1791 | /// A 256-bit integer vector. | ||
| 1792 | /// \param __b | ||
| 1793 | /// A 256-bit integer vector. | ||
| 1794 | /// \returns A 256-bit integer vector containing the result. | ||
| 451 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1795 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 452 | _mm256_or_si256(__m256i __a, __m256i __b) | 1796 | _mm256_or_si256(__m256i __a, __m256i __b) |
| 453 | { | 1797 | { |
| 454 | return (__m256i)((__v4du)__a | (__v4du)__b); | 1798 | return (__m256i)((__v4du)__a | (__v4du)__b); |
| 455 | } | 1799 | } |
| 456 | 1800 | ||
| 1801 | /// Computes four sum of absolute difference (SAD) operations on sets of eight | ||
| 1802 | /// unsigned 8-bit integers from the 256-bit integer vectors \a __a and | ||
| 1803 | /// \a __b. | ||
| 1804 | /// | ||
| 1805 | /// One SAD result is computed for each set of eight bytes from \a __a and | ||
| 1806 | /// eight bytes from \a __b. The zero-extended SAD value is returned in the | ||
| 1807 | /// corresponding 64-bit element of the result. | ||
| 1808 | /// | ||
| 1809 | /// A single SAD operation takes the differences between the corresponding | ||
| 1810 | /// bytes of \a __a and \a __b, takes the absolute value of each difference, | ||
| 1811 | /// and sums these eight values to form one 16-bit result. This operation | ||
| 1812 | /// is repeated four times with successive sets of eight bytes. | ||
| 1813 | /// | ||
| 1814 | /// \code{.operation} | ||
| 1815 | /// FOR i := 0 TO 3 | ||
| 1816 | /// j := i*64 | ||
| 1817 | /// temp0 := ABS(__a[j+7:j] - __b[j+7:j]) | ||
| 1818 | /// temp1 := ABS(__a[j+15:j+8] - __b[j+15:j+8]) | ||
| 1819 | /// temp2 := ABS(__a[j+23:j+16] - __b[j+23:j+16]) | ||
| 1820 | /// temp3 := ABS(__a[j+31:j+24] - __b[j+31:j+24]) | ||
| 1821 | /// temp4 := ABS(__a[j+39:j+32] - __b[j+39:j+32]) | ||
| 1822 | /// temp5 := ABS(__a[j+47:j+40] - __b[j+47:j+40]) | ||
| 1823 | /// temp6 := ABS(__a[j+55:j+48] - __b[j+55:j+48]) | ||
| 1824 | /// temp7 := ABS(__a[j+63:j+56] - __b[j+63:j+56]) | ||
| 1825 | /// result[j+15:j] := temp0 + temp1 + temp2 + temp3 + | ||
| 1826 | /// temp4 + temp5 + temp6 + temp7 | ||
| 1827 | /// result[j+63:j+16] := 0 | ||
| 1828 | /// ENDFOR | ||
| 1829 | /// \endcode | ||
| 1830 | /// | ||
| 1831 | /// \headerfile <immintrin.h> | ||
| 1832 | /// | ||
| 1833 | /// This intrinsic corresponds to the \c VPSADBW instruction. | ||
| 1834 | /// | ||
| 1835 | /// \param __a | ||
| 1836 | /// A 256-bit integer vector. | ||
| 1837 | /// \param __b | ||
| 1838 | /// A 256-bit integer vector. | ||
| 1839 | /// \returns A 256-bit integer vector containing the result. | ||
| 457 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1840 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 458 | _mm256_sad_epu8(__m256i __a, __m256i __b) | 1841 | _mm256_sad_epu8(__m256i __a, __m256i __b) |
| 459 | { | 1842 | { |
| 460 | return __builtin_ia32_psadbw256((__v32qi)__a, (__v32qi)__b); | 1843 | return __builtin_ia32_psadbw256((__v32qi)__a, (__v32qi)__b); |
| 461 | } | 1844 | } |
| 462 | 1845 | ||
| 1846 | /// Shuffles 8-bit integers in the 256-bit integer vector \a __a according | ||
| 1847 | /// to control information in the 256-bit integer vector \a __b, and | ||
| 1848 | /// returns the 256-bit result. In effect there are two separate 128-bit | ||
| 1849 | /// shuffles in the lower and upper halves. | ||
| 1850 | /// | ||
| 1851 | /// \code{.operation} | ||
| 1852 | /// FOR i := 0 TO 31 | ||
| 1853 | /// j := i*8 | ||
| 1854 | /// IF __b[j+7] == 1 | ||
| 1855 | /// result[j+7:j] := 0 | ||
| 1856 | /// ELSE | ||
| 1857 | /// k := __b[j+3:j] * 8 | ||
| 1858 | /// IF i > 15 | ||
| 1859 | /// k := k + 128 | ||
| 1860 | /// FI | ||
| 1861 | /// result[j+7:j] := __a[k+7:k] | ||
| 1862 | /// FI | ||
| 1863 | /// ENDFOR | ||
| 1864 | /// \endcode | ||
| 1865 | /// | ||
| 1866 | /// \headerfile <immintrin.h> | ||
| 1867 | /// | ||
| 1868 | /// This intrinsic corresponds to the \c VPSHUFB instruction. | ||
| 1869 | /// | ||
| 1870 | /// \param __a | ||
| 1871 | /// A 256-bit integer vector containing source values. | ||
| 1872 | /// \param __b | ||
| 1873 | /// A 256-bit integer vector containing control information to determine | ||
| 1874 | /// what goes into the corresponding byte of the result. If bit 7 of the | ||
| 1875 | /// control byte is 1, the result byte is 0; otherwise, bits 3:0 of the | ||
| 1876 | /// control byte specify the index (within the same 128-bit half) of \a __a | ||
| 1877 | /// to copy to the result byte. | ||
| 1878 | /// \returns A 256-bit integer vector containing the result. | ||
| 463 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 1879 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 464 | _mm256_shuffle_epi8(__m256i __a, __m256i __b) | 1880 | _mm256_shuffle_epi8(__m256i __a, __m256i __b) |
| 465 | { | 1881 | { |
| 466 | return (__m256i)__builtin_ia32_pshufb256((__v32qi)__a, (__v32qi)__b); | 1882 | return (__m256i)__builtin_ia32_pshufb256((__v32qi)__a, (__v32qi)__b); |
| 467 | } | 1883 | } |
| 468 | 1884 | ||
| 1885 | /// Shuffles 32-bit integers from the 256-bit vector of [8 x i32] in \a a | ||
| 1886 | /// according to control information in the integer literal \a imm, and | ||
| 1887 | /// returns the 256-bit result. In effect there are two parallel 128-bit | ||
| 1888 | /// shuffles in the lower and upper halves. | ||
| 1889 | /// | ||
| 1890 | /// \code{.operation} | ||
| 1891 | /// FOR i := 0 to 3 | ||
| 1892 | /// j := i*32 | ||
| 1893 | /// k := (imm >> i*2)[1:0] * 32 | ||
| 1894 | /// result[j+31:j] := a[k+31:k] | ||
| 1895 | /// result[128+j+31:128+j] := a[128+k+31:128+k] | ||
| 1896 | /// ENDFOR | ||
| 1897 | /// \endcode | ||
| 1898 | /// | ||
| 1899 | /// \headerfile <immintrin.h> | ||
| 1900 | /// | ||
| 1901 | /// \code | ||
| 1902 | /// __m256i _mm256_shuffle_epi32(__m256i a, const int imm); | ||
| 1903 | /// \endcode | ||
| 1904 | /// | ||
| 1905 | /// This intrinsic corresponds to the \c VPSHUFB instruction. | ||
| 1906 | /// | ||
| 1907 | /// \param a | ||
| 1908 | /// A 256-bit vector of [8 x i32] containing source values. | ||
| 1909 | /// \param imm | ||
| 1910 | /// An immediate 8-bit value specifying which elements to copy from \a a. | ||
| 1911 | /// \a imm[1:0] specifies the index in \a a for elements 0 and 4 of the | ||
| 1912 | /// result, \a imm[3:2] specifies the index for elements 1 and 5, and so | ||
| 1913 | /// forth. | ||
| 1914 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 469 | #define _mm256_shuffle_epi32(a, imm) \ | 1915 | #define _mm256_shuffle_epi32(a, imm) \ |
| 470 | ((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(a), (int)(imm))) | 1916 | ((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(a), (int)(imm))) |
| 471 | 1917 | ||
| 1918 | /// Shuffles 16-bit integers from the 256-bit vector of [16 x i16] in \a a | ||
| 1919 | /// according to control information in the integer literal \a imm, and | ||
| 1920 | /// returns the 256-bit result. The upper 64 bits of each 128-bit half | ||
| 1921 | /// are shuffled in parallel; the lower 64 bits of each 128-bit half are | ||
| 1922 | /// copied from \a a unchanged. | ||
| 1923 | /// | ||
| 1924 | /// \code{.operation} | ||
| 1925 | /// result[63:0] := a[63:0] | ||
| 1926 | /// result[191:128] := a[191:128] | ||
| 1927 | /// FOR i := 0 TO 3 | ||
| 1928 | /// j := i * 16 + 64 | ||
| 1929 | /// k := (imm >> i*2)[1:0] * 16 + 64 | ||
| 1930 | /// result[j+15:j] := a[k+15:k] | ||
| 1931 | /// result[128+j+15:128+j] := a[128+k+15:128+k] | ||
| 1932 | /// ENDFOR | ||
| 1933 | /// \endcode | ||
| 1934 | /// | ||
| 1935 | /// \headerfile <immintrin.h> | ||
| 1936 | /// | ||
| 1937 | /// \code | ||
| 1938 | /// __m256i _mm256_shufflehi_epi16(__m256i a, const int imm); | ||
| 1939 | /// \endcode | ||
| 1940 | /// | ||
| 1941 | /// This intrinsic corresponds to the \c VPSHUFHW instruction. | ||
| 1942 | /// | ||
| 1943 | /// \param a | ||
| 1944 | /// A 256-bit vector of [16 x i16] containing source values. | ||
| 1945 | /// \param imm | ||
| 1946 | /// An immediate 8-bit value specifying which elements to copy from \a a. | ||
| 1947 | /// \a imm[1:0] specifies the index in \a a for elements 4 and 8 of the | ||
| 1948 | /// result, \a imm[3:2] specifies the index for elements 5 and 9, and so | ||
| 1949 | /// forth. Indexes are offset by 4 (so 0 means index 4, and so forth). | ||
| 1950 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 472 | #define _mm256_shufflehi_epi16(a, imm) \ | 1951 | #define _mm256_shufflehi_epi16(a, imm) \ |
| 473 | ((__m256i)__builtin_ia32_pshufhw256((__v16hi)(__m256i)(a), (int)(imm))) | 1952 | ((__m256i)__builtin_ia32_pshufhw256((__v16hi)(__m256i)(a), (int)(imm))) |
| 474 | 1953 | ||
| 1954 | /// Shuffles 16-bit integers from the 256-bit vector of [16 x i16] \a a | ||
| 1955 | /// according to control information in the integer literal \a imm, and | ||
| 1956 | /// returns the 256-bit [16 x i16] result. The lower 64 bits of each | ||
| 1957 | /// 128-bit half are shuffled; the upper 64 bits of each 128-bit half are | ||
| 1958 | /// copied from \a a unchanged. | ||
| 1959 | /// | ||
| 1960 | /// \code{.operation} | ||
| 1961 | /// result[127:64] := a[127:64] | ||
| 1962 | /// result[255:192] := a[255:192] | ||
| 1963 | /// FOR i := 0 TO 3 | ||
| 1964 | /// j := i * 16 | ||
| 1965 | /// k := (imm >> i*2)[1:0] * 16 | ||
| 1966 | /// result[j+15:j] := a[k+15:k] | ||
| 1967 | /// result[128+j+15:128+j] := a[128+k+15:128+k] | ||
| 1968 | /// ENDFOR | ||
| 1969 | /// \endcode | ||
| 1970 | /// | ||
| 1971 | /// \headerfile <immintrin.h> | ||
| 1972 | /// | ||
| 1973 | /// \code | ||
| 1974 | /// __m256i _mm256_shufflelo_epi16(__m256i a, const int imm); | ||
| 1975 | /// \endcode | ||
| 1976 | /// | ||
| 1977 | /// This intrinsic corresponds to the \c VPSHUFLW instruction. | ||
| 1978 | /// | ||
| 1979 | /// \param a | ||
| 1980 | /// A 256-bit vector of [16 x i16] to use as a source of data for the | ||
| 1981 | /// result. | ||
| 1982 | /// \param imm | ||
| 1983 | /// An immediate 8-bit value specifying which elements to copy from \a a. | ||
| 1984 | /// \a imm[1:0] specifies the index in \a a for elements 0 and 8 of the | ||
| 1985 | /// result, \a imm[3:2] specifies the index for elements 1 and 9, and so | ||
| 1986 | /// forth. | ||
| 1987 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 475 | #define _mm256_shufflelo_epi16(a, imm) \ | 1988 | #define _mm256_shufflelo_epi16(a, imm) \ |
| 476 | ((__m256i)__builtin_ia32_pshuflw256((__v16hi)(__m256i)(a), (int)(imm))) | 1989 | ((__m256i)__builtin_ia32_pshuflw256((__v16hi)(__m256i)(a), (int)(imm))) |
| 477 | 1990 | ||
| 1991 | /// Sets each byte of the result to the corresponding byte of the 256-bit | ||
| 1992 | /// integer vector in \a __a, the negative of that byte, or zero, depending | ||
| 1993 | /// on whether the corresponding byte of the 256-bit integer vector in | ||
| 1994 | /// \a __b is greater than zero, less than zero, or equal to zero, | ||
| 1995 | /// respectively. | ||
| 1996 | /// | ||
| 1997 | /// \headerfile <immintrin.h> | ||
| 1998 | /// | ||
| 1999 | /// This intrinsic corresponds to the \c VPSIGNB instruction. | ||
| 2000 | /// | ||
| 2001 | /// \param __a | ||
| 2002 | /// A 256-bit integer vector. | ||
| 2003 | /// \param __b | ||
| 2004 | /// A 256-bit integer vector]. | ||
| 2005 | /// \returns A 256-bit integer vector containing the result. | ||
| 478 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2006 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 479 | _mm256_sign_epi8(__m256i __a, __m256i __b) | 2007 | _mm256_sign_epi8(__m256i __a, __m256i __b) |
| 480 | { | 2008 | { |
| 481 | return (__m256i)__builtin_ia32_psignb256((__v32qi)__a, (__v32qi)__b); | 2009 | return (__m256i)__builtin_ia32_psignb256((__v32qi)__a, (__v32qi)__b); |
| 482 | } | 2010 | } |
| 483 | 2011 | ||
| 2012 | /// Sets each element of the result to the corresponding element of the | ||
| 2013 | /// 256-bit vector of [16 x i16] in \a __a, the negative of that element, | ||
| 2014 | /// or zero, depending on whether the corresponding element of the 256-bit | ||
| 2015 | /// vector of [16 x i16] in \a __b is greater than zero, less than zero, or | ||
| 2016 | /// equal to zero, respectively. | ||
| 2017 | /// | ||
| 2018 | /// \headerfile <immintrin.h> | ||
| 2019 | /// | ||
| 2020 | /// This intrinsic corresponds to the \c VPSIGNW instruction. | ||
| 2021 | /// | ||
| 2022 | /// \param __a | ||
| 2023 | /// A 256-bit vector of [16 x i16]. | ||
| 2024 | /// \param __b | ||
| 2025 | /// A 256-bit vector of [16 x i16]. | ||
| 2026 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 484 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2027 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 485 | _mm256_sign_epi16(__m256i __a, __m256i __b) | 2028 | _mm256_sign_epi16(__m256i __a, __m256i __b) |
| 486 | { | 2029 | { |
| 487 | return (__m256i)__builtin_ia32_psignw256((__v16hi)__a, (__v16hi)__b); | 2030 | return (__m256i)__builtin_ia32_psignw256((__v16hi)__a, (__v16hi)__b); |
| 488 | } | 2031 | } |
| 489 | 2032 | ||
| 2033 | /// Sets each element of the result to the corresponding element of the | ||
| 2034 | /// 256-bit vector of [8 x i32] in \a __a, the negative of that element, or | ||
| 2035 | /// zero, depending on whether the corresponding element of the 256-bit | ||
| 2036 | /// vector of [8 x i32] in \a __b is greater than zero, less than zero, or | ||
| 2037 | /// equal to zero, respectively. | ||
| 2038 | /// | ||
| 2039 | /// \headerfile <immintrin.h> | ||
| 2040 | /// | ||
| 2041 | /// This intrinsic corresponds to the \c VPSIGND instruction. | ||
| 2042 | /// | ||
| 2043 | /// \param __a | ||
| 2044 | /// A 256-bit vector of [8 x i32]. | ||
| 2045 | /// \param __b | ||
| 2046 | /// A 256-bit vector of [8 x i32]. | ||
| 2047 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 490 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2048 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 491 | _mm256_sign_epi32(__m256i __a, __m256i __b) | 2049 | _mm256_sign_epi32(__m256i __a, __m256i __b) |
| 492 | { | 2050 | { |
| 493 | return (__m256i)__builtin_ia32_psignd256((__v8si)__a, (__v8si)__b); | 2051 | return (__m256i)__builtin_ia32_psignd256((__v8si)__a, (__v8si)__b); |
| 494 | } | 2052 | } |
| 495 | 2053 | ||
| 2054 | /// Shifts each 128-bit half of the 256-bit integer vector \a a left by | ||
| 2055 | /// \a imm bytes, shifting in zero bytes, and returns the result. If \a imm | ||
| 2056 | /// is greater than 15, the returned result is all zeroes. | ||
| 2057 | /// | ||
| 2058 | /// \headerfile <immintrin.h> | ||
| 2059 | /// | ||
| 2060 | /// \code | ||
| 2061 | /// __m256i _mm256_slli_si256(__m256i a, const int imm); | ||
| 2062 | /// \endcode | ||
| 2063 | /// | ||
| 2064 | /// This intrinsic corresponds to the \c VPSLLDQ instruction. | ||
| 2065 | /// | ||
| 2066 | /// \param a | ||
| 2067 | /// A 256-bit integer vector to be shifted. | ||
| 2068 | /// \param imm | ||
| 2069 | /// An unsigned immediate value specifying the shift count (in bytes). | ||
| 2070 | /// \returns A 256-bit integer vector containing the result. | ||
| 496 | #define _mm256_slli_si256(a, imm) \ | 2071 | #define _mm256_slli_si256(a, imm) \ |
| 497 | ((__m256i)__builtin_ia32_pslldqi256_byteshift((__v4di)(__m256i)(a), (int)(imm))) | 2072 | ((__m256i)__builtin_ia32_pslldqi256_byteshift((__v4di)(__m256i)(a), (int)(imm))) |
| 498 | 2073 | ||
| 2074 | /// Shifts each 128-bit half of the 256-bit integer vector \a a left by | ||
| 2075 | /// \a imm bytes, shifting in zero bytes, and returns the result. If \a imm | ||
| 2076 | /// is greater than 15, the returned result is all zeroes. | ||
| 2077 | /// | ||
| 2078 | /// \headerfile <immintrin.h> | ||
| 2079 | /// | ||
| 2080 | /// \code | ||
| 2081 | /// __m256i _mm256_bslli_epi128(__m256i a, const int imm); | ||
| 2082 | /// \endcode | ||
| 2083 | /// | ||
| 2084 | /// This intrinsic corresponds to the \c VPSLLDQ instruction. | ||
| 2085 | /// | ||
| 2086 | /// \param a | ||
| 2087 | /// A 256-bit integer vector to be shifted. | ||
| 2088 | /// \param imm | ||
| 2089 | /// An unsigned immediate value specifying the shift count (in bytes). | ||
| 2090 | /// \returns A 256-bit integer vector containing the result. | ||
| 499 | #define _mm256_bslli_epi128(a, imm) \ | 2091 | #define _mm256_bslli_epi128(a, imm) \ |
| 500 | ((__m256i)__builtin_ia32_pslldqi256_byteshift((__v4di)(__m256i)(a), (int)(imm))) | 2092 | ((__m256i)__builtin_ia32_pslldqi256_byteshift((__v4di)(__m256i)(a), (int)(imm))) |
| 501 | 2093 | ||
| 2094 | /// Shifts each 16-bit element of the 256-bit vector of [16 x i16] in \a __a | ||
| 2095 | /// left by \a __count bits, shifting in zero bits, and returns the result. | ||
| 2096 | /// If \a __count is greater than 15, the returned result is all zeroes. | ||
| 2097 | /// | ||
| 2098 | /// \headerfile <immintrin.h> | ||
| 2099 | /// | ||
| 2100 | /// This intrinsic corresponds to the \c VPSLLW instruction. | ||
| 2101 | /// | ||
| 2102 | /// \param __a | ||
| 2103 | /// A 256-bit vector of [16 x i16] to be shifted. | ||
| 2104 | /// \param __count | ||
| 2105 | /// An unsigned integer value specifying the shift count (in bits). | ||
| 2106 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 502 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2107 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 503 | _mm256_slli_epi16(__m256i __a, int __count) | 2108 | _mm256_slli_epi16(__m256i __a, int __count) |
| 504 | { | 2109 | { |
| 505 | return (__m256i)__builtin_ia32_psllwi256((__v16hi)__a, __count); | 2110 | return (__m256i)__builtin_ia32_psllwi256((__v16hi)__a, __count); |
| 506 | } | 2111 | } |
| 507 | 2112 | ||
| 2113 | /// Shifts each 16-bit element of the 256-bit vector of [16 x i16] in \a __a | ||
| 2114 | /// left by the number of bits specified by the lower 64 bits of \a __count, | ||
| 2115 | /// shifting in zero bits, and returns the result. If \a __count is greater | ||
| 2116 | /// than 15, the returned result is all zeroes. | ||
| 2117 | /// | ||
| 2118 | /// \headerfile <immintrin.h> | ||
| 2119 | /// | ||
| 2120 | /// This intrinsic corresponds to the \c VPSLLW instruction. | ||
| 2121 | /// | ||
| 2122 | /// \param __a | ||
| 2123 | /// A 256-bit vector of [16 x i16] to be shifted. | ||
| 2124 | /// \param __count | ||
| 2125 | /// A 128-bit vector of [2 x i64] whose lower element gives the unsigned | ||
| 2126 | /// shift count (in bits). The upper element is ignored. | ||
| 2127 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 508 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2128 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 509 | _mm256_sll_epi16(__m256i __a, __m128i __count) | 2129 | _mm256_sll_epi16(__m256i __a, __m128i __count) |
| 510 | { | 2130 | { |
| 511 | return (__m256i)__builtin_ia32_psllw256((__v16hi)__a, (__v8hi)__count); | 2131 | return (__m256i)__builtin_ia32_psllw256((__v16hi)__a, (__v8hi)__count); |
| 512 | } | 2132 | } |
| 513 | 2133 | ||
| 2134 | /// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __a | ||
| 2135 | /// left by \a __count bits, shifting in zero bits, and returns the result. | ||
| 2136 | /// If \a __count is greater than 31, the returned result is all zeroes. | ||
| 2137 | /// | ||
| 2138 | /// \headerfile <immintrin.h> | ||
| 2139 | /// | ||
| 2140 | /// This intrinsic corresponds to the \c VPSLLD instruction. | ||
| 2141 | /// | ||
| 2142 | /// \param __a | ||
| 2143 | /// A 256-bit vector of [8 x i32] to be shifted. | ||
| 2144 | /// \param __count | ||
| 2145 | /// An unsigned integer value specifying the shift count (in bits). | ||
| 2146 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 514 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2147 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 515 | _mm256_slli_epi32(__m256i __a, int __count) | 2148 | _mm256_slli_epi32(__m256i __a, int __count) |
| 516 | { | 2149 | { |
| 517 | return (__m256i)__builtin_ia32_pslldi256((__v8si)__a, __count); | 2150 | return (__m256i)__builtin_ia32_pslldi256((__v8si)__a, __count); |
| 518 | } | 2151 | } |
| 519 | 2152 | ||
| 2153 | /// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __a | ||
| 2154 | /// left by the number of bits given in the lower 64 bits of \a __count, | ||
| 2155 | /// shifting in zero bits, and returns the result. If \a __count is greater | ||
| 2156 | /// than 31, the returned result is all zeroes. | ||
| 2157 | /// | ||
| 2158 | /// \headerfile <immintrin.h> | ||
| 2159 | /// | ||
| 2160 | /// This intrinsic corresponds to the \c VPSLLD instruction. | ||
| 2161 | /// | ||
| 2162 | /// \param __a | ||
| 2163 | /// A 256-bit vector of [8 x i32] to be shifted. | ||
| 2164 | /// \param __count | ||
| 2165 | /// A 128-bit vector of [2 x i64] whose lower element gives the unsigned | ||
| 2166 | /// shift count (in bits). The upper element is ignored. | ||
| 2167 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 520 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2168 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 521 | _mm256_sll_epi32(__m256i __a, __m128i __count) | 2169 | _mm256_sll_epi32(__m256i __a, __m128i __count) |
| 522 | { | 2170 | { |
| 523 | return (__m256i)__builtin_ia32_pslld256((__v8si)__a, (__v4si)__count); | 2171 | return (__m256i)__builtin_ia32_pslld256((__v8si)__a, (__v4si)__count); |
| 524 | } | 2172 | } |
| 525 | 2173 | ||
| 2174 | /// Shifts each 64-bit element of the 256-bit vector of [4 x i64] in \a __a | ||
| 2175 | /// left by \a __count bits, shifting in zero bits, and returns the result. | ||
| 2176 | /// If \a __count is greater than 63, the returned result is all zeroes. | ||
| 2177 | /// | ||
| 2178 | /// \headerfile <immintrin.h> | ||
| 2179 | /// | ||
| 2180 | /// This intrinsic corresponds to the \c VPSLLQ instruction. | ||
| 2181 | /// | ||
| 2182 | /// \param __a | ||
| 2183 | /// A 256-bit vector of [4 x i64] to be shifted. | ||
| 2184 | /// \param __count | ||
| 2185 | /// An unsigned integer value specifying the shift count (in bits). | ||
| 2186 | /// \returns A 256-bit vector of [4 x i64] containing the result. | ||
| 526 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2187 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 527 | _mm256_slli_epi64(__m256i __a, int __count) | 2188 | _mm256_slli_epi64(__m256i __a, int __count) |
| 528 | { | 2189 | { |
| 529 | return __builtin_ia32_psllqi256((__v4di)__a, __count); | 2190 | return __builtin_ia32_psllqi256((__v4di)__a, __count); |
| 530 | } | 2191 | } |
| 531 | 2192 | ||
| 2193 | /// Shifts each 64-bit element of the 256-bit vector of [4 x i64] in \a __a | ||
| 2194 | /// left by the number of bits given in the lower 64 bits of \a __count, | ||
| 2195 | /// shifting in zero bits, and returns the result. If \a __count is greater | ||
| 2196 | /// than 63, the returned result is all zeroes. | ||
| 2197 | /// | ||
| 2198 | /// \headerfile <immintrin.h> | ||
| 2199 | /// | ||
| 2200 | /// This intrinsic corresponds to the \c VPSLLQ instruction. | ||
| 2201 | /// | ||
| 2202 | /// \param __a | ||
| 2203 | /// A 256-bit vector of [4 x i64] to be shifted. | ||
| 2204 | /// \param __count | ||
| 2205 | /// A 128-bit vector of [2 x i64] whose lower element gives the unsigned | ||
| 2206 | /// shift count (in bits). The upper element is ignored. | ||
| 2207 | /// \returns A 256-bit vector of [4 x i64] containing the result. | ||
| 532 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2208 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 533 | _mm256_sll_epi64(__m256i __a, __m128i __count) | 2209 | _mm256_sll_epi64(__m256i __a, __m128i __count) |
| 534 | { | 2210 | { |
| 535 | return __builtin_ia32_psllq256((__v4di)__a, __count); | 2211 | return __builtin_ia32_psllq256((__v4di)__a, __count); |
| 536 | } | 2212 | } |
| 537 | 2213 | ||
| 2214 | /// Shifts each 16-bit element of the 256-bit vector of [16 x i16] in \a __a | ||
| 2215 | /// right by \a __count bits, shifting in sign bits, and returns the result. | ||
| 2216 | /// If \a __count is greater than 15, each element of the result is either | ||
| 2217 | /// 0 or -1 according to the corresponding input sign bit. | ||
| 2218 | /// | ||
| 2219 | /// \headerfile <immintrin.h> | ||
| 2220 | /// | ||
| 2221 | /// This intrinsic corresponds to the \c VPSRAW instruction. | ||
| 2222 | /// | ||
| 2223 | /// \param __a | ||
| 2224 | /// A 256-bit vector of [16 x i16] to be shifted. | ||
| 2225 | /// \param __count | ||
| 2226 | /// An unsigned integer value specifying the shift count (in bits). | ||
| 2227 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 538 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2228 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 539 | _mm256_srai_epi16(__m256i __a, int __count) | 2229 | _mm256_srai_epi16(__m256i __a, int __count) |
| 540 | { | 2230 | { |
| 541 | return (__m256i)__builtin_ia32_psrawi256((__v16hi)__a, __count); | 2231 | return (__m256i)__builtin_ia32_psrawi256((__v16hi)__a, __count); |
| 542 | } | 2232 | } |
| 543 | 2233 | ||
| 2234 | /// Shifts each 16-bit element of the 256-bit vector of [16 x i16] in \a __a | ||
| 2235 | /// right by the number of bits given in the lower 64 bits of \a __count, | ||
| 2236 | /// shifting in sign bits, and returns the result. If \a __count is greater | ||
| 2237 | /// than 15, each element of the result is either 0 or -1 according to the | ||
| 2238 | /// corresponding input sign bit. | ||
| 2239 | /// | ||
| 2240 | /// \headerfile <immintrin.h> | ||
| 2241 | /// | ||
| 2242 | /// This intrinsic corresponds to the \c VPSRAW instruction. | ||
| 2243 | /// | ||
| 2244 | /// \param __a | ||
| 2245 | /// A 256-bit vector of [16 x i16] to be shifted. | ||
| 2246 | /// \param __count | ||
| 2247 | /// A 128-bit vector of [2 x i64] whose lower element gives the unsigned | ||
| 2248 | /// shift count (in bits). The upper element is ignored. | ||
| 2249 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 544 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2250 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 545 | _mm256_sra_epi16(__m256i __a, __m128i __count) | 2251 | _mm256_sra_epi16(__m256i __a, __m128i __count) |
| 546 | { | 2252 | { |
| 547 | return (__m256i)__builtin_ia32_psraw256((__v16hi)__a, (__v8hi)__count); | 2253 | return (__m256i)__builtin_ia32_psraw256((__v16hi)__a, (__v8hi)__count); |
| 548 | } | 2254 | } |
| 549 | 2255 | ||
| 2256 | /// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __a | ||
| 2257 | /// right by \a __count bits, shifting in sign bits, and returns the result. | ||
| 2258 | /// If \a __count is greater than 31, each element of the result is either | ||
| 2259 | /// 0 or -1 according to the corresponding input sign bit. | ||
| 2260 | /// | ||
| 2261 | /// \headerfile <immintrin.h> | ||
| 2262 | /// | ||
| 2263 | /// This intrinsic corresponds to the \c VPSRAD instruction. | ||
| 2264 | /// | ||
| 2265 | /// \param __a | ||
| 2266 | /// A 256-bit vector of [8 x i32] to be shifted. | ||
| 2267 | /// \param __count | ||
| 2268 | /// An unsigned integer value specifying the shift count (in bits). | ||
| 2269 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 550 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2270 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 551 | _mm256_srai_epi32(__m256i __a, int __count) | 2271 | _mm256_srai_epi32(__m256i __a, int __count) |
| 552 | { | 2272 | { |
| 553 | return (__m256i)__builtin_ia32_psradi256((__v8si)__a, __count); | 2273 | return (__m256i)__builtin_ia32_psradi256((__v8si)__a, __count); |
| 554 | } | 2274 | } |
| 555 | 2275 | ||
| 2276 | /// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __a | ||
| 2277 | /// right by the number of bits given in the lower 64 bits of \a __count, | ||
| 2278 | /// shifting in sign bits, and returns the result. If \a __count is greater | ||
| 2279 | /// than 31, each element of the result is either 0 or -1 according to the | ||
| 2280 | /// corresponding input sign bit. | ||
| 2281 | /// | ||
| 2282 | /// \headerfile <immintrin.h> | ||
| 2283 | /// | ||
| 2284 | /// This intrinsic corresponds to the \c VPSRAD instruction. | ||
| 2285 | /// | ||
| 2286 | /// \param __a | ||
| 2287 | /// A 256-bit vector of [8 x i32] to be shifted. | ||
| 2288 | /// \param __count | ||
| 2289 | /// A 128-bit vector of [2 x i64] whose lower element gives the unsigned | ||
| 2290 | /// shift count (in bits). The upper element is ignored. | ||
| 2291 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 556 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2292 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 557 | _mm256_sra_epi32(__m256i __a, __m128i __count) | 2293 | _mm256_sra_epi32(__m256i __a, __m128i __count) |
| 558 | { | 2294 | { |
| 559 | return (__m256i)__builtin_ia32_psrad256((__v8si)__a, (__v4si)__count); | 2295 | return (__m256i)__builtin_ia32_psrad256((__v8si)__a, (__v4si)__count); |
| 560 | } | 2296 | } |
| 561 | 2297 | ||
| 2298 | /// Shifts each 128-bit half of the 256-bit integer vector in \a a right by | ||
| 2299 | /// \a imm bytes, shifting in zero bytes, and returns the result. If | ||
| 2300 | /// \a imm is greater than 15, the returned result is all zeroes. | ||
| 2301 | /// | ||
| 2302 | /// \headerfile <immintrin.h> | ||
| 2303 | /// | ||
| 2304 | /// \code | ||
| 2305 | /// __m256i _mm256_srli_si256(__m256i a, const int imm); | ||
| 2306 | /// \endcode | ||
| 2307 | /// | ||
| 2308 | /// This intrinsic corresponds to the \c VPSRLDQ instruction. | ||
| 2309 | /// | ||
| 2310 | /// \param a | ||
| 2311 | /// A 256-bit integer vector to be shifted. | ||
| 2312 | /// \param imm | ||
| 2313 | /// An unsigned immediate value specifying the shift count (in bytes). | ||
| 2314 | /// \returns A 256-bit integer vector containing the result. | ||
| 562 | #define _mm256_srli_si256(a, imm) \ | 2315 | #define _mm256_srli_si256(a, imm) \ |
| 563 | ((__m256i)__builtin_ia32_psrldqi256_byteshift((__m256i)(a), (int)(imm))) | 2316 | ((__m256i)__builtin_ia32_psrldqi256_byteshift((__m256i)(a), (int)(imm))) |
| 564 | 2317 | ||
| 2318 | /// Shifts each 128-bit half of the 256-bit integer vector in \a a right by | ||
| 2319 | /// \a imm bytes, shifting in zero bytes, and returns the result. If | ||
| 2320 | /// \a imm is greater than 15, the returned result is all zeroes. | ||
| 2321 | /// | ||
| 2322 | /// \headerfile <immintrin.h> | ||
| 2323 | /// | ||
| 2324 | /// \code | ||
| 2325 | /// __m256i _mm256_bsrli_epi128(__m256i a, const int imm); | ||
| 2326 | /// \endcode | ||
| 2327 | /// | ||
| 2328 | /// This intrinsic corresponds to the \c VPSRLDQ instruction. | ||
| 2329 | /// | ||
| 2330 | /// \param a | ||
| 2331 | /// A 256-bit integer vector to be shifted. | ||
| 2332 | /// \param imm | ||
| 2333 | /// An unsigned immediate value specifying the shift count (in bytes). | ||
| 2334 | /// \returns A 256-bit integer vector containing the result. | ||
| 565 | #define _mm256_bsrli_epi128(a, imm) \ | 2335 | #define _mm256_bsrli_epi128(a, imm) \ |
| 566 | ((__m256i)__builtin_ia32_psrldqi256_byteshift((__m256i)(a), (int)(imm))) | 2336 | ((__m256i)__builtin_ia32_psrldqi256_byteshift((__m256i)(a), (int)(imm))) |
| 567 | 2337 | ||
| 2338 | /// Shifts each 16-bit element of the 256-bit vector of [16 x i16] in \a __a | ||
| 2339 | /// right by \a __count bits, shifting in zero bits, and returns the result. | ||
| 2340 | /// If \a __count is greater than 15, the returned result is all zeroes. | ||
| 2341 | /// | ||
| 2342 | /// \headerfile <immintrin.h> | ||
| 2343 | /// | ||
| 2344 | /// This intrinsic corresponds to the \c VPSRLW instruction. | ||
| 2345 | /// | ||
| 2346 | /// \param __a | ||
| 2347 | /// A 256-bit vector of [16 x i16] to be shifted. | ||
| 2348 | /// \param __count | ||
| 2349 | /// An unsigned integer value specifying the shift count (in bits). | ||
| 2350 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 568 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2351 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 569 | _mm256_srli_epi16(__m256i __a, int __count) | 2352 | _mm256_srli_epi16(__m256i __a, int __count) |
| 570 | { | 2353 | { |
| 571 | return (__m256i)__builtin_ia32_psrlwi256((__v16hi)__a, __count); | 2354 | return (__m256i)__builtin_ia32_psrlwi256((__v16hi)__a, __count); |
| 572 | } | 2355 | } |
| 573 | 2356 | ||
| 2357 | /// Shifts each 16-bit element of the 256-bit vector of [16 x i16] in \a __a | ||
| 2358 | /// right by the number of bits given in the lower 64 bits of \a __count, | ||
| 2359 | /// shifting in zero bits, and returns the result. If \a __count is greater | ||
| 2360 | /// than 15, the returned result is all zeroes. | ||
| 2361 | /// | ||
| 2362 | /// \headerfile <immintrin.h> | ||
| 2363 | /// | ||
| 2364 | /// This intrinsic corresponds to the \c VPSRLW instruction. | ||
| 2365 | /// | ||
| 2366 | /// \param __a | ||
| 2367 | /// A 256-bit vector of [16 x i16] to be shifted. | ||
| 2368 | /// \param __count | ||
| 2369 | /// A 128-bit vector of [2 x i64] whose lower element gives the unsigned | ||
| 2370 | /// shift count (in bits). The upper element is ignored. | ||
| 2371 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 574 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2372 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 575 | _mm256_srl_epi16(__m256i __a, __m128i __count) | 2373 | _mm256_srl_epi16(__m256i __a, __m128i __count) |
| 576 | { | 2374 | { |
| 577 | return (__m256i)__builtin_ia32_psrlw256((__v16hi)__a, (__v8hi)__count); | 2375 | return (__m256i)__builtin_ia32_psrlw256((__v16hi)__a, (__v8hi)__count); |
| 578 | } | 2376 | } |
| 579 | 2377 | ||
| 2378 | /// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __a | ||
| 2379 | /// right by \a __count bits, shifting in zero bits, and returns the result. | ||
| 2380 | /// If \a __count is greater than 31, the returned result is all zeroes. | ||
| 2381 | /// | ||
| 2382 | /// \headerfile <immintrin.h> | ||
| 2383 | /// | ||
| 2384 | /// This intrinsic corresponds to the \c VPSRLD instruction. | ||
| 2385 | /// | ||
| 2386 | /// \param __a | ||
| 2387 | /// A 256-bit vector of [8 x i32] to be shifted. | ||
| 2388 | /// \param __count | ||
| 2389 | /// An unsigned integer value specifying the shift count (in bits). | ||
| 2390 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 580 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2391 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 581 | _mm256_srli_epi32(__m256i __a, int __count) | 2392 | _mm256_srli_epi32(__m256i __a, int __count) |
| 582 | { | 2393 | { |
| 583 | return (__m256i)__builtin_ia32_psrldi256((__v8si)__a, __count); | 2394 | return (__m256i)__builtin_ia32_psrldi256((__v8si)__a, __count); |
| 584 | } | 2395 | } |
| 585 | 2396 | ||
| 2397 | /// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __a | ||
| 2398 | /// right by the number of bits given in the lower 64 bits of \a __count, | ||
| 2399 | /// shifting in zero bits, and returns the result. If \a __count is greater | ||
| 2400 | /// than 31, the returned result is all zeroes. | ||
| 2401 | /// | ||
| 2402 | /// \headerfile <immintrin.h> | ||
| 2403 | /// | ||
| 2404 | /// This intrinsic corresponds to the \c VPSRLD instruction. | ||
| 2405 | /// | ||
| 2406 | /// \param __a | ||
| 2407 | /// A 256-bit vector of [8 x i32] to be shifted. | ||
| 2408 | /// \param __count | ||
| 2409 | /// A 128-bit vector of [2 x i64] whose lower element gives the unsigned | ||
| 2410 | /// shift count (in bits). The upper element is ignored. | ||
| 2411 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 586 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2412 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 587 | _mm256_srl_epi32(__m256i __a, __m128i __count) | 2413 | _mm256_srl_epi32(__m256i __a, __m128i __count) |
| 588 | { | 2414 | { |
| 589 | return (__m256i)__builtin_ia32_psrld256((__v8si)__a, (__v4si)__count); | 2415 | return (__m256i)__builtin_ia32_psrld256((__v8si)__a, (__v4si)__count); |
| 590 | } | 2416 | } |
| 591 | 2417 | ||
| 2418 | /// Shifts each 64-bit element of the 256-bit vector of [4 x i64] in \a __a | ||
| 2419 | /// right by \a __count bits, shifting in zero bits, and returns the result. | ||
| 2420 | /// If \a __count is greater than 63, the returned result is all zeroes. | ||
| 2421 | /// | ||
| 2422 | /// \headerfile <immintrin.h> | ||
| 2423 | /// | ||
| 2424 | /// This intrinsic corresponds to the \c VPSRLQ instruction. | ||
| 2425 | /// | ||
| 2426 | /// \param __a | ||
| 2427 | /// A 256-bit vector of [4 x i64] to be shifted. | ||
| 2428 | /// \param __count | ||
| 2429 | /// An unsigned integer value specifying the shift count (in bits). | ||
| 2430 | /// \returns A 256-bit vector of [4 x i64] containing the result. | ||
| 592 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2431 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 593 | _mm256_srli_epi64(__m256i __a, int __count) | 2432 | _mm256_srli_epi64(__m256i __a, int __count) |
| 594 | { | 2433 | { |
| 595 | return __builtin_ia32_psrlqi256((__v4di)__a, __count); | 2434 | return __builtin_ia32_psrlqi256((__v4di)__a, __count); |
| 596 | } | 2435 | } |
| 597 | 2436 | ||
| 2437 | /// Shifts each 64-bit element of the 256-bit vector of [4 x i64] in \a __a | ||
| 2438 | /// right by the number of bits given in the lower 64 bits of \a __count, | ||
| 2439 | /// shifting in zero bits, and returns the result. If \a __count is greater | ||
| 2440 | /// than 63, the returned result is all zeroes. | ||
| 2441 | /// | ||
| 2442 | /// \headerfile <immintrin.h> | ||
| 2443 | /// | ||
| 2444 | /// This intrinsic corresponds to the \c VPSRLQ instruction. | ||
| 2445 | /// | ||
| 2446 | /// \param __a | ||
| 2447 | /// A 256-bit vector of [4 x i64] to be shifted. | ||
| 2448 | /// \param __count | ||
| 2449 | /// A 128-bit vector of [2 x i64] whose lower element gives the unsigned | ||
| 2450 | /// shift count (in bits). The upper element is ignored. | ||
| 2451 | /// \returns A 256-bit vector of [4 x i64] containing the result. | ||
| 598 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2452 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 599 | _mm256_srl_epi64(__m256i __a, __m128i __count) | 2453 | _mm256_srl_epi64(__m256i __a, __m128i __count) |
| 600 | { | 2454 | { |
| 601 | return __builtin_ia32_psrlq256((__v4di)__a, __count); | 2455 | return __builtin_ia32_psrlq256((__v4di)__a, __count); |
| 602 | } | 2456 | } |
| 603 | 2457 | ||
| 2458 | /// Subtracts 8-bit integers from corresponding bytes of two 256-bit integer | ||
| 2459 | /// vectors. Returns the lower 8 bits of each difference in the | ||
| 2460 | /// corresponding byte of the 256-bit integer vector result (overflow is | ||
| 2461 | /// ignored). | ||
| 2462 | /// | ||
| 2463 | /// \code{.operation} | ||
| 2464 | /// FOR i := 0 TO 31 | ||
| 2465 | /// j := i*8 | ||
| 2466 | /// result[j+7:j] := __a[j+7:j] - __b[j+7:j] | ||
| 2467 | /// ENDFOR | ||
| 2468 | /// \endcode | ||
| 2469 | /// | ||
| 2470 | /// \headerfile <immintrin.h> | ||
| 2471 | /// | ||
| 2472 | /// This intrinsic corresponds to the \c VPSUBB instruction. | ||
| 2473 | /// | ||
| 2474 | /// \param __a | ||
| 2475 | /// A 256-bit integer vector containing the minuends. | ||
| 2476 | /// \param __b | ||
| 2477 | /// A 256-bit integer vector containing the subtrahends. | ||
| 2478 | /// \returns A 256-bit integer vector containing the differences. | ||
| 604 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2479 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 605 | _mm256_sub_epi8(__m256i __a, __m256i __b) | 2480 | _mm256_sub_epi8(__m256i __a, __m256i __b) |
| 606 | { | 2481 | { |
| 607 | return (__m256i)((__v32qu)__a - (__v32qu)__b); | 2482 | return (__m256i)((__v32qu)__a - (__v32qu)__b); |
| 608 | } | 2483 | } |
| 609 | 2484 | ||
| 2485 | /// Subtracts 16-bit integers from corresponding elements of two 256-bit | ||
| 2486 | /// vectors of [16 x i16]. Returns the lower 16 bits of each difference in | ||
| 2487 | /// the corresponding element of the [16 x i16] result (overflow is | ||
| 2488 | /// ignored). | ||
| 2489 | /// | ||
| 2490 | /// \code{.operation} | ||
| 2491 | /// FOR i := 0 TO 15 | ||
| 2492 | /// j := i*16 | ||
| 2493 | /// result[j+15:j] := __a[j+15:j] - __b[j+15:j] | ||
| 2494 | /// ENDFOR | ||
| 2495 | /// \endcode | ||
| 2496 | /// | ||
| 2497 | /// \headerfile <immintrin.h> | ||
| 2498 | /// | ||
| 2499 | /// This intrinsic corresponds to the \c VPSUBW instruction. | ||
| 2500 | /// | ||
| 2501 | /// \param __a | ||
| 2502 | /// A 256-bit vector of [16 x i16] containing the minuends. | ||
| 2503 | /// \param __b | ||
| 2504 | /// A 256-bit vector of [16 x i16] containing the subtrahends. | ||
| 2505 | /// \returns A 256-bit vector of [16 x i16] containing the differences. | ||
| 610 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2506 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 611 | _mm256_sub_epi16(__m256i __a, __m256i __b) | 2507 | _mm256_sub_epi16(__m256i __a, __m256i __b) |
| 612 | { | 2508 | { |
| 613 | return (__m256i)((__v16hu)__a - (__v16hu)__b); | 2509 | return (__m256i)((__v16hu)__a - (__v16hu)__b); |
| 614 | } | 2510 | } |
| 615 | 2511 | ||
| 2512 | /// Subtracts 32-bit integers from corresponding elements of two 256-bit | ||
| 2513 | /// vectors of [8 x i32]. Returns the lower 32 bits of each difference in | ||
| 2514 | /// the corresponding element of the [8 x i32] result (overflow is ignored). | ||
| 2515 | /// | ||
| 2516 | /// \code{.operation} | ||
| 2517 | /// FOR i := 0 TO 7 | ||
| 2518 | /// j := i*32 | ||
| 2519 | /// result[j+31:j] := __a[j+31:j] - __b[j+31:j] | ||
| 2520 | /// ENDFOR | ||
| 2521 | /// \endcode | ||
| 2522 | /// | ||
| 2523 | /// \headerfile <immintrin.h> | ||
| 2524 | /// | ||
| 2525 | /// This intrinsic corresponds to the \c VPSUBD instruction. | ||
| 2526 | /// | ||
| 2527 | /// \param __a | ||
| 2528 | /// A 256-bit vector of [8 x i32] containing the minuends. | ||
| 2529 | /// \param __b | ||
| 2530 | /// A 256-bit vector of [8 x i32] containing the subtrahends. | ||
| 2531 | /// \returns A 256-bit vector of [8 x i32] containing the differences. | ||
| 616 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2532 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 617 | _mm256_sub_epi32(__m256i __a, __m256i __b) | 2533 | _mm256_sub_epi32(__m256i __a, __m256i __b) |
| 618 | { | 2534 | { |
| 619 | return (__m256i)((__v8su)__a - (__v8su)__b); | 2535 | return (__m256i)((__v8su)__a - (__v8su)__b); |
| 620 | } | 2536 | } |
| 621 | 2537 | ||
| 2538 | /// Subtracts 64-bit integers from corresponding elements of two 256-bit | ||
| 2539 | /// vectors of [4 x i64]. Returns the lower 64 bits of each difference in | ||
| 2540 | /// the corresponding element of the [4 x i64] result (overflow is ignored). | ||
| 2541 | /// | ||
| 2542 | /// \code{.operation} | ||
| 2543 | /// FOR i := 0 TO 3 | ||
| 2544 | /// j := i*64 | ||
| 2545 | /// result[j+63:j] := __a[j+63:j] - __b[j+63:j] | ||
| 2546 | /// ENDFOR | ||
| 2547 | /// \endcode | ||
| 2548 | /// | ||
| 2549 | /// \headerfile <immintrin.h> | ||
| 2550 | /// | ||
| 2551 | /// This intrinsic corresponds to the \c VPSUBQ instruction. | ||
| 2552 | /// | ||
| 2553 | /// \param __a | ||
| 2554 | /// A 256-bit vector of [4 x i64] containing the minuends. | ||
| 2555 | /// \param __b | ||
| 2556 | /// A 256-bit vector of [4 x i64] containing the subtrahends. | ||
| 2557 | /// \returns A 256-bit vector of [4 x i64] containing the differences. | ||
| 622 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2558 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 623 | _mm256_sub_epi64(__m256i __a, __m256i __b) | 2559 | _mm256_sub_epi64(__m256i __a, __m256i __b) |
| 624 | { | 2560 | { |
| 625 | return (__m256i)((__v4du)__a - (__v4du)__b); | 2561 | return (__m256i)((__v4du)__a - (__v4du)__b); |
| 626 | } | 2562 | } |
| 627 | 2563 | ||
| 2564 | /// Subtracts 8-bit integers from corresponding bytes of two 256-bit integer | ||
| 2565 | /// vectors using signed saturation, and returns each differences in the | ||
| 2566 | /// corresponding byte of the 256-bit integer vector result. | ||
| 2567 | /// | ||
| 2568 | /// \code{.operation} | ||
| 2569 | /// FOR i := 0 TO 31 | ||
| 2570 | /// j := i*8 | ||
| 2571 | /// result[j+7:j] := SATURATE8(__a[j+7:j] - __b[j+7:j]) | ||
| 2572 | /// ENDFOR | ||
| 2573 | /// \endcode | ||
| 2574 | /// | ||
| 2575 | /// \headerfile <immintrin.h> | ||
| 2576 | /// | ||
| 2577 | /// This intrinsic corresponds to the \c VPSUBSB instruction. | ||
| 2578 | /// | ||
| 2579 | /// \param __a | ||
| 2580 | /// A 256-bit integer vector containing the minuends. | ||
| 2581 | /// \param __b | ||
| 2582 | /// A 256-bit integer vector containing the subtrahends. | ||
| 2583 | /// \returns A 256-bit integer vector containing the differences. | ||
| 628 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2584 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 629 | _mm256_subs_epi8(__m256i __a, __m256i __b) | 2585 | _mm256_subs_epi8(__m256i __a, __m256i __b) |
| 630 | { | 2586 | { |
| 631 | return (__m256i)__builtin_elementwise_sub_sat((__v32qs)__a, (__v32qs)__b); | 2587 | return (__m256i)__builtin_elementwise_sub_sat((__v32qs)__a, (__v32qs)__b); |
| 632 | } | 2588 | } |
| 633 | 2589 | ||
| 2590 | /// Subtracts 16-bit integers from corresponding elements of two 256-bit | ||
| 2591 | /// vectors of [16 x i16] using signed saturation, and returns each | ||
| 2592 | /// difference in the corresponding element of the [16 x i16] result. | ||
| 2593 | /// | ||
| 2594 | /// \code{.operation} | ||
| 2595 | /// FOR i := 0 TO 15 | ||
| 2596 | /// j := i*16 | ||
| 2597 | /// result[j+7:j] := SATURATE16(__a[j+7:j] - __b[j+7:j]) | ||
| 2598 | /// ENDFOR | ||
| 2599 | /// \endcode | ||
| 2600 | /// | ||
| 2601 | /// \headerfile <immintrin.h> | ||
| 2602 | /// | ||
| 2603 | /// This intrinsic corresponds to the \c VPSUBSW instruction. | ||
| 2604 | /// | ||
| 2605 | /// \param __a | ||
| 2606 | /// A 256-bit vector of [16 x i16] containing the minuends. | ||
| 2607 | /// \param __b | ||
| 2608 | /// A 256-bit vector of [16 x i16] containing the subtrahends. | ||
| 2609 | /// \returns A 256-bit vector of [16 x i16] containing the differences. | ||
| 634 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2610 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 635 | _mm256_subs_epi16(__m256i __a, __m256i __b) | 2611 | _mm256_subs_epi16(__m256i __a, __m256i __b) |
| 636 | { | 2612 | { |
| 637 | return (__m256i)__builtin_elementwise_sub_sat((__v16hi)__a, (__v16hi)__b); | 2613 | return (__m256i)__builtin_elementwise_sub_sat((__v16hi)__a, (__v16hi)__b); |
| 638 | } | 2614 | } |
| 639 | 2615 | ||
| 2616 | /// Subtracts 8-bit integers from corresponding bytes of two 256-bit integer | ||
| 2617 | /// vectors using unsigned saturation, and returns each difference in the | ||
| 2618 | /// corresponding byte of the 256-bit integer vector result. For each byte, | ||
| 2619 | /// computes <c> result = __a - __b </c>. | ||
| 2620 | /// | ||
| 2621 | /// \code{.operation} | ||
| 2622 | /// FOR i := 0 TO 31 | ||
| 2623 | /// j := i*8 | ||
| 2624 | /// result[j+7:j] := SATURATE8U(__a[j+7:j] - __b[j+7:j]) | ||
| 2625 | /// ENDFOR | ||
| 2626 | /// \endcode | ||
| 2627 | /// | ||
| 2628 | /// \headerfile <immintrin.h> | ||
| 2629 | /// | ||
| 2630 | /// This intrinsic corresponds to the \c VPSUBUSB instruction. | ||
| 2631 | /// | ||
| 2632 | /// \param __a | ||
| 2633 | /// A 256-bit integer vector containing the minuends. | ||
| 2634 | /// \param __b | ||
| 2635 | /// A 256-bit integer vector containing the subtrahends. | ||
| 2636 | /// \returns A 256-bit integer vector containing the differences. | ||
| 640 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2637 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 641 | _mm256_subs_epu8(__m256i __a, __m256i __b) | 2638 | _mm256_subs_epu8(__m256i __a, __m256i __b) |
| 642 | { | 2639 | { |
| 643 | return (__m256i)__builtin_elementwise_sub_sat((__v32qu)__a, (__v32qu)__b); | 2640 | return (__m256i)__builtin_elementwise_sub_sat((__v32qu)__a, (__v32qu)__b); |
| 644 | } | 2641 | } |
| 645 | 2642 | ||
| 2643 | /// Subtracts 16-bit integers from corresponding elements of two 256-bit | ||
| 2644 | /// vectors of [16 x i16] using unsigned saturation, and returns each | ||
| 2645 | /// difference in the corresponding element of the [16 x i16] result. | ||
| 2646 | /// | ||
| 2647 | /// \code{.operation} | ||
| 2648 | /// FOR i := 0 TO 15 | ||
| 2649 | /// j := i*16 | ||
| 2650 | /// result[j+15:j] := SATURATE16U(__a[j+15:j] - __b[j+15:j]) | ||
| 2651 | /// ENDFOR | ||
| 2652 | /// \endcode | ||
| 2653 | /// | ||
| 2654 | /// \headerfile <immintrin.h> | ||
| 2655 | /// | ||
| 2656 | /// This intrinsic corresponds to the \c VPSUBUSW instruction. | ||
| 2657 | /// | ||
| 2658 | /// \param __a | ||
| 2659 | /// A 256-bit vector of [16 x i16] containing the minuends. | ||
| 2660 | /// \param __b | ||
| 2661 | /// A 256-bit vector of [16 x i16] containing the subtrahends. | ||
| 2662 | /// \returns A 256-bit vector of [16 x i16] containing the differences. | ||
| 646 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2663 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 647 | _mm256_subs_epu16(__m256i __a, __m256i __b) | 2664 | _mm256_subs_epu16(__m256i __a, __m256i __b) |
| 648 | { | 2665 | { |
| 649 | return (__m256i)__builtin_elementwise_sub_sat((__v16hu)__a, (__v16hu)__b); | 2666 | return (__m256i)__builtin_elementwise_sub_sat((__v16hu)__a, (__v16hu)__b); |
| 650 | } | 2667 | } |
| 651 | 2668 | ||
| 2669 | /// Unpacks and interleaves 8-bit integers from parts of the 256-bit integer | ||
| 2670 | /// vectors in \a __a and \a __b to form the 256-bit result. Specifically, | ||
| 2671 | /// uses the upper 64 bits of each 128-bit half of \a __a and \a __b as | ||
| 2672 | /// input; other bits in these parameters are ignored. | ||
| 2673 | /// | ||
| 2674 | /// \code{.operation} | ||
| 2675 | /// result[7:0] := __a[71:64] | ||
| 2676 | /// result[15:8] := __b[71:64] | ||
| 2677 | /// result[23:16] := __a[79:72] | ||
| 2678 | /// result[31:24] := __b[79:72] | ||
| 2679 | /// . . . | ||
| 2680 | /// result[127:120] := __b[127:120] | ||
| 2681 | /// result[135:128] := __a[199:192] | ||
| 2682 | /// . . . | ||
| 2683 | /// result[255:248] := __b[255:248] | ||
| 2684 | /// \endcode | ||
| 2685 | /// | ||
| 2686 | /// \headerfile <immintrin.h> | ||
| 2687 | /// | ||
| 2688 | /// This intrinsic corresponds to the \c VPUNPCKHBW instruction. | ||
| 2689 | /// | ||
| 2690 | /// \param __a | ||
| 2691 | /// A 256-bit integer vector used as the source for the even-numbered bytes | ||
| 2692 | /// of the result. | ||
| 2693 | /// \param __b | ||
| 2694 | /// A 256-bit integer vector used as the source for the odd-numbered bytes | ||
| 2695 | /// of the result. | ||
| 2696 | /// \returns A 256-bit integer vector containing the result. | ||
| 652 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2697 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 653 | _mm256_unpackhi_epi8(__m256i __a, __m256i __b) | 2698 | _mm256_unpackhi_epi8(__m256i __a, __m256i __b) |
| 654 | { | 2699 | { |
| 655 | return (__m256i)__builtin_shufflevector((__v32qi)__a, (__v32qi)__b, 8, 32+8, 9, 32+9, 10, 32+10, 11, 32+11, 12, 32+12, 13, 32+13, 14, 32+14, 15, 32+15, 24, 32+24, 25, 32+25, 26, 32+26, 27, 32+27, 28, 32+28, 29, 32+29, 30, 32+30, 31, 32+31); | 2700 | return (__m256i)__builtin_shufflevector((__v32qi)__a, (__v32qi)__b, 8, 32+8, 9, 32+9, 10, 32+10, 11, 32+11, 12, 32+12, 13, 32+13, 14, 32+14, 15, 32+15, 24, 32+24, 25, 32+25, 26, 32+26, 27, 32+27, 28, 32+28, 29, 32+29, 30, 32+30, 31, 32+31); |
| 656 | } | 2701 | } |
| 657 | 2702 | ||
| 2703 | /// Unpacks and interleaves 16-bit integers from parts of the 256-bit vectors | ||
| 2704 | /// of [16 x i16] in \a __a and \a __b to return the resulting 256-bit | ||
| 2705 | /// vector of [16 x i16]. Specifically, uses the upper 64 bits of each | ||
| 2706 | /// 128-bit half of \a __a and \a __b as input; other bits in these | ||
| 2707 | /// parameters are ignored. | ||
| 2708 | /// | ||
| 2709 | /// \code{.operation} | ||
| 2710 | /// result[15:0] := __a[79:64] | ||
| 2711 | /// result[31:16] := __b[79:64] | ||
| 2712 | /// result[47:32] := __a[95:80] | ||
| 2713 | /// result[63:48] := __b[95:80] | ||
| 2714 | /// . . . | ||
| 2715 | /// result[127:112] := __b[127:112] | ||
| 2716 | /// result[143:128] := __a[211:196] | ||
| 2717 | /// . . . | ||
| 2718 | /// result[255:240] := __b[255:240] | ||
| 2719 | /// \endcode | ||
| 2720 | /// | ||
| 2721 | /// \headerfile <immintrin.h> | ||
| 2722 | /// | ||
| 2723 | /// This intrinsic corresponds to the \c VPUNPCKHWD instruction. | ||
| 2724 | /// | ||
| 2725 | /// \param __a | ||
| 2726 | /// A 256-bit vector of [16 x i16] used as the source for the even-numbered | ||
| 2727 | /// elements of the result. | ||
| 2728 | /// \param __b | ||
| 2729 | /// A 256-bit vector of [16 x i16] used as the source for the odd-numbered | ||
| 2730 | /// elements of the result. | ||
| 2731 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 658 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2732 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 659 | _mm256_unpackhi_epi16(__m256i __a, __m256i __b) | 2733 | _mm256_unpackhi_epi16(__m256i __a, __m256i __b) |
| 660 | { | 2734 | { |
| 661 | return (__m256i)__builtin_shufflevector((__v16hi)__a, (__v16hi)__b, 4, 16+4, 5, 16+5, 6, 16+6, 7, 16+7, 12, 16+12, 13, 16+13, 14, 16+14, 15, 16+15); | 2735 | return (__m256i)__builtin_shufflevector((__v16hi)__a, (__v16hi)__b, 4, 16+4, 5, 16+5, 6, 16+6, 7, 16+7, 12, 16+12, 13, 16+13, 14, 16+14, 15, 16+15); |
| 662 | } | 2736 | } |
| 663 | 2737 | ||
| 2738 | /// Unpacks and interleaves 32-bit integers from parts of the 256-bit vectors | ||
| 2739 | /// of [8 x i32] in \a __a and \a __b to return the resulting 256-bit vector | ||
| 2740 | /// of [8 x i32]. Specifically, uses the upper 64 bits of each 128-bit half | ||
| 2741 | /// of \a __a and \a __b as input; other bits in these parameters are | ||
| 2742 | /// ignored. | ||
| 2743 | /// | ||
| 2744 | /// \code{.operation} | ||
| 2745 | /// result[31:0] := __a[95:64] | ||
| 2746 | /// result[63:32] := __b[95:64] | ||
| 2747 | /// result[95:64] := __a[127:96] | ||
| 2748 | /// result[127:96] := __b[127:96] | ||
| 2749 | /// result[159:128] := __a[223:192] | ||
| 2750 | /// result[191:160] := __b[223:192] | ||
| 2751 | /// result[223:192] := __a[255:224] | ||
| 2752 | /// result[255:224] := __b[255:224] | ||
| 2753 | /// \endcode | ||
| 2754 | /// | ||
| 2755 | /// \headerfile <immintrin.h> | ||
| 2756 | /// | ||
| 2757 | /// This intrinsic corresponds to the \c VPUNPCKHDQ instruction. | ||
| 2758 | /// | ||
| 2759 | /// \param __a | ||
| 2760 | /// A 256-bit vector of [8 x i32] used as the source for the even-numbered | ||
| 2761 | /// elements of the result. | ||
| 2762 | /// \param __b | ||
| 2763 | /// A 256-bit vector of [8 x i32] used as the source for the odd-numbered | ||
| 2764 | /// elements of the result. | ||
| 2765 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 664 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2766 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 665 | _mm256_unpackhi_epi32(__m256i __a, __m256i __b) | 2767 | _mm256_unpackhi_epi32(__m256i __a, __m256i __b) |
| 666 | { | 2768 | { |
| 667 | return (__m256i)__builtin_shufflevector((__v8si)__a, (__v8si)__b, 2, 8+2, 3, 8+3, 6, 8+6, 7, 8+7); | 2769 | return (__m256i)__builtin_shufflevector((__v8si)__a, (__v8si)__b, 2, 8+2, 3, 8+3, 6, 8+6, 7, 8+7); |
| 668 | } | 2770 | } |
| 669 | 2771 | ||
| 2772 | /// Unpacks and interleaves 64-bit integers from parts of the 256-bit vectors | ||
| 2773 | /// of [4 x i64] in \a __a and \a __b to return the resulting 256-bit vector | ||
| 2774 | /// of [4 x i64]. Specifically, uses the upper 64 bits of each 128-bit half | ||
| 2775 | /// of \a __a and \a __b as input; other bits in these parameters are | ||
| 2776 | /// ignored. | ||
| 2777 | /// | ||
| 2778 | /// \code{.operation} | ||
| 2779 | /// result[63:0] := __a[127:64] | ||
| 2780 | /// result[127:64] := __b[127:64] | ||
| 2781 | /// result[191:128] := __a[255:192] | ||
| 2782 | /// result[255:192] := __b[255:192] | ||
| 2783 | /// \endcode | ||
| 2784 | /// | ||
| 2785 | /// \headerfile <immintrin.h> | ||
| 2786 | /// | ||
| 2787 | /// This intrinsic corresponds to the \c VPUNPCKHQDQ instruction. | ||
| 2788 | /// | ||
| 2789 | /// \param __a | ||
| 2790 | /// A 256-bit vector of [4 x i64] used as the source for the even-numbered | ||
| 2791 | /// elements of the result. | ||
| 2792 | /// \param __b | ||
| 2793 | /// A 256-bit vector of [4 x i64] used as the source for the odd-numbered | ||
| 2794 | /// elements of the result. | ||
| 2795 | /// \returns A 256-bit vector of [4 x i64] containing the result. | ||
| 670 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2796 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 671 | _mm256_unpackhi_epi64(__m256i __a, __m256i __b) | 2797 | _mm256_unpackhi_epi64(__m256i __a, __m256i __b) |
| 672 | { | 2798 | { |
| 673 | return (__m256i)__builtin_shufflevector((__v4di)__a, (__v4di)__b, 1, 4+1, 3, 4+3); | 2799 | return (__m256i)__builtin_shufflevector((__v4di)__a, (__v4di)__b, 1, 4+1, 3, 4+3); |
| 674 | } | 2800 | } |
| 675 | 2801 | ||
| 2802 | /// Unpacks and interleaves 8-bit integers from parts of the 256-bit integer | ||
| 2803 | /// vectors in \a __a and \a __b to form the 256-bit result. Specifically, | ||
| 2804 | /// uses the lower 64 bits of each 128-bit half of \a __a and \a __b as | ||
| 2805 | /// input; other bits in these parameters are ignored. | ||
| 2806 | /// | ||
| 2807 | /// \code{.operation} | ||
| 2808 | /// result[7:0] := __a[7:0] | ||
| 2809 | /// result[15:8] := __b[7:0] | ||
| 2810 | /// result[23:16] := __a[15:8] | ||
| 2811 | /// result[31:24] := __b[15:8] | ||
| 2812 | /// . . . | ||
| 2813 | /// result[127:120] := __b[63:56] | ||
| 2814 | /// result[135:128] := __a[135:128] | ||
| 2815 | /// . . . | ||
| 2816 | /// result[255:248] := __b[191:184] | ||
| 2817 | /// \endcode | ||
| 2818 | /// | ||
| 2819 | /// \headerfile <immintrin.h> | ||
| 2820 | /// | ||
| 2821 | /// This intrinsic corresponds to the \c VPUNPCKLBW instruction. | ||
| 2822 | /// | ||
| 2823 | /// \param __a | ||
| 2824 | /// A 256-bit integer vector used as the source for the even-numbered bytes | ||
| 2825 | /// of the result. | ||
| 2826 | /// \param __b | ||
| 2827 | /// A 256-bit integer vector used as the source for the odd-numbered bytes | ||
| 2828 | /// of the result. | ||
| 2829 | /// \returns A 256-bit integer vector containing the result. | ||
| 676 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2830 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 677 | _mm256_unpacklo_epi8(__m256i __a, __m256i __b) | 2831 | _mm256_unpacklo_epi8(__m256i __a, __m256i __b) |
| 678 | { | 2832 | { |
| 679 | return (__m256i)__builtin_shufflevector((__v32qi)__a, (__v32qi)__b, 0, 32+0, 1, 32+1, 2, 32+2, 3, 32+3, 4, 32+4, 5, 32+5, 6, 32+6, 7, 32+7, 16, 32+16, 17, 32+17, 18, 32+18, 19, 32+19, 20, 32+20, 21, 32+21, 22, 32+22, 23, 32+23); | 2833 | return (__m256i)__builtin_shufflevector((__v32qi)__a, (__v32qi)__b, 0, 32+0, 1, 32+1, 2, 32+2, 3, 32+3, 4, 32+4, 5, 32+5, 6, 32+6, 7, 32+7, 16, 32+16, 17, 32+17, 18, 32+18, 19, 32+19, 20, 32+20, 21, 32+21, 22, 32+22, 23, 32+23); |
| 680 | } | 2834 | } |
| 681 | 2835 | ||
| 2836 | /// Unpacks and interleaves 16-bit integers from parts of the 256-bit vectors | ||
| 2837 | /// of [16 x i16] in \a __a and \a __b to return the resulting 256-bit | ||
| 2838 | /// vector of [16 x i16]. Specifically, uses the lower 64 bits of each | ||
| 2839 | /// 128-bit half of \a __a and \a __b as input; other bits in these | ||
| 2840 | /// parameters are ignored. | ||
| 2841 | /// | ||
| 2842 | /// \code{.operation} | ||
| 2843 | /// result[15:0] := __a[15:0] | ||
| 2844 | /// result[31:16] := __b[15:0] | ||
| 2845 | /// result[47:32] := __a[31:16] | ||
| 2846 | /// result[63:48] := __b[31:16] | ||
| 2847 | /// . . . | ||
| 2848 | /// result[127:112] := __b[63:48] | ||
| 2849 | /// result[143:128] := __a[143:128] | ||
| 2850 | /// . . . | ||
| 2851 | /// result[255:239] := __b[191:176] | ||
| 2852 | /// \endcode | ||
| 2853 | /// | ||
| 2854 | /// \headerfile <immintrin.h> | ||
| 2855 | /// | ||
| 2856 | /// This intrinsic corresponds to the \c VPUNPCKLWD instruction. | ||
| 2857 | /// | ||
| 2858 | /// \param __a | ||
| 2859 | /// A 256-bit vector of [16 x i16] used as the source for the even-numbered | ||
| 2860 | /// elements of the result. | ||
| 2861 | /// \param __b | ||
| 2862 | /// A 256-bit vector of [16 x i16] used as the source for the odd-numbered | ||
| 2863 | /// elements of the result. | ||
| 2864 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 682 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2865 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 683 | _mm256_unpacklo_epi16(__m256i __a, __m256i __b) | 2866 | _mm256_unpacklo_epi16(__m256i __a, __m256i __b) |
| 684 | { | 2867 | { |
| 685 | return (__m256i)__builtin_shufflevector((__v16hi)__a, (__v16hi)__b, 0, 16+0, 1, 16+1, 2, 16+2, 3, 16+3, 8, 16+8, 9, 16+9, 10, 16+10, 11, 16+11); | 2868 | return (__m256i)__builtin_shufflevector((__v16hi)__a, (__v16hi)__b, 0, 16+0, 1, 16+1, 2, 16+2, 3, 16+3, 8, 16+8, 9, 16+9, 10, 16+10, 11, 16+11); |
| 686 | } | 2869 | } |
| 687 | 2870 | ||
| 2871 | /// Unpacks and interleaves 32-bit integers from parts of the 256-bit vectors | ||
| 2872 | /// of [8 x i32] in \a __a and \a __b to return the resulting 256-bit vector | ||
| 2873 | /// of [8 x i32]. Specifically, uses the lower 64 bits of each 128-bit half | ||
| 2874 | /// of \a __a and \a __b as input; other bits in these parameters are | ||
| 2875 | /// ignored. | ||
| 2876 | /// | ||
| 2877 | /// \code{.operation} | ||
| 2878 | /// result[31:0] := __a[31:0] | ||
| 2879 | /// result[63:32] := __b[31:0] | ||
| 2880 | /// result[95:64] := __a[63:32] | ||
| 2881 | /// result[127:96] := __b[63:32] | ||
| 2882 | /// result[159:128] := __a[159:128] | ||
| 2883 | /// result[191:160] := __b[159:128] | ||
| 2884 | /// result[223:192] := __a[191:160] | ||
| 2885 | /// result[255:224] := __b[191:190] | ||
| 2886 | /// \endcode | ||
| 2887 | /// | ||
| 2888 | /// \headerfile <immintrin.h> | ||
| 2889 | /// | ||
| 2890 | /// This intrinsic corresponds to the \c VPUNPCKLDQ instruction. | ||
| 2891 | /// | ||
| 2892 | /// \param __a | ||
| 2893 | /// A 256-bit vector of [8 x i32] used as the source for the even-numbered | ||
| 2894 | /// elements of the result. | ||
| 2895 | /// \param __b | ||
| 2896 | /// A 256-bit vector of [8 x i32] used as the source for the odd-numbered | ||
| 2897 | /// elements of the result. | ||
| 2898 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 688 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2899 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 689 | _mm256_unpacklo_epi32(__m256i __a, __m256i __b) | 2900 | _mm256_unpacklo_epi32(__m256i __a, __m256i __b) |
| 690 | { | 2901 | { |
| 691 | return (__m256i)__builtin_shufflevector((__v8si)__a, (__v8si)__b, 0, 8+0, 1, 8+1, 4, 8+4, 5, 8+5); | 2902 | return (__m256i)__builtin_shufflevector((__v8si)__a, (__v8si)__b, 0, 8+0, 1, 8+1, 4, 8+4, 5, 8+5); |
| 692 | } | 2903 | } |
| 693 | 2904 | ||
| 2905 | /// Unpacks and interleaves 64-bit integers from parts of the 256-bit vectors | ||
| 2906 | /// of [4 x i64] in \a __a and \a __b to return the resulting 256-bit vector | ||
| 2907 | /// of [4 x i64]. Specifically, uses the lower 64 bits of each 128-bit half | ||
| 2908 | /// of \a __a and \a __b as input; other bits in these parameters are | ||
| 2909 | /// ignored. | ||
| 2910 | /// | ||
| 2911 | /// \code{.operation} | ||
| 2912 | /// result[63:0] := __a[63:0] | ||
| 2913 | /// result[127:64] := __b[63:0] | ||
| 2914 | /// result[191:128] := __a[191:128] | ||
| 2915 | /// result[255:192] := __b[191:128] | ||
| 2916 | /// \endcode | ||
| 2917 | /// | ||
| 2918 | /// \headerfile <immintrin.h> | ||
| 2919 | /// | ||
| 2920 | /// This intrinsic corresponds to the \c VPUNPCKLQDQ instruction. | ||
| 2921 | /// | ||
| 2922 | /// \param __a | ||
| 2923 | /// A 256-bit vector of [4 x i64] used as the source for the even-numbered | ||
| 2924 | /// elements of the result. | ||
| 2925 | /// \param __b | ||
| 2926 | /// A 256-bit vector of [4 x i64] used as the source for the odd-numbered | ||
| 2927 | /// elements of the result. | ||
| 2928 | /// \returns A 256-bit vector of [4 x i64] containing the result. | ||
| 694 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2929 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 695 | _mm256_unpacklo_epi64(__m256i __a, __m256i __b) | 2930 | _mm256_unpacklo_epi64(__m256i __a, __m256i __b) |
| 696 | { | 2931 | { |
| 697 | return (__m256i)__builtin_shufflevector((__v4di)__a, (__v4di)__b, 0, 4+0, 2, 4+2); | 2932 | return (__m256i)__builtin_shufflevector((__v4di)__a, (__v4di)__b, 0, 4+0, 2, 4+2); |
| 698 | } | 2933 | } |
| 699 | 2934 | ||
| 2935 | /// Computes the bitwise XOR of the 256-bit integer vectors in \a __a and | ||
| 2936 | /// \a __b. | ||
| 2937 | /// | ||
| 2938 | /// \headerfile <immintrin.h> | ||
| 2939 | /// | ||
| 2940 | /// This intrinsic corresponds to the \c VPXOR instruction. | ||
| 2941 | /// | ||
| 2942 | /// \param __a | ||
| 2943 | /// A 256-bit integer vector. | ||
| 2944 | /// \param __b | ||
| 2945 | /// A 256-bit integer vector. | ||
| 2946 | /// \returns A 256-bit integer vector containing the result. | ||
| 700 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2947 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 701 | _mm256_xor_si256(__m256i __a, __m256i __b) | 2948 | _mm256_xor_si256(__m256i __a, __m256i __b) |
| 702 | { | 2949 | { |
| 703 | return (__m256i)((__v4du)__a ^ (__v4du)__b); | 2950 | return (__m256i)((__v4du)__a ^ (__v4du)__b); |
| 704 | } | 2951 | } |
| 705 | 2952 | ||
| 2953 | /// Loads the 256-bit integer vector from memory \a __V using a non-temporal | ||
| 2954 | /// memory hint and returns the vector. \a __V must be aligned on a 32-byte | ||
| 2955 | /// boundary. | ||
| 2956 | /// | ||
| 2957 | /// \headerfile <immintrin.h> | ||
| 2958 | /// | ||
| 2959 | /// This intrinsic corresponds to the \c VMOVNTDQA instruction. | ||
| 2960 | /// | ||
| 2961 | /// \param __V | ||
| 2962 | /// A pointer to the 32-byte aligned memory containing the vector to load. | ||
| 2963 | /// \returns A 256-bit integer vector loaded from memory. | ||
| 706 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 2964 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 707 | _mm256_stream_load_si256(__m256i const *__V) | 2965 | _mm256_stream_load_si256(__m256i const *__V) |
| 708 | { | 2966 | { |
| ... | @@ -710,30 +2968,84 @@ _mm256_stream_load_si256(__m256i const *__V) | ... | @@ -710,30 +2968,84 @@ _mm256_stream_load_si256(__m256i const *__V) |
| 710 | return (__m256i)__builtin_nontemporal_load((const __v4di_aligned *)__V); | 2968 | return (__m256i)__builtin_nontemporal_load((const __v4di_aligned *)__V); |
| 711 | } | 2969 | } |
| 712 | 2970 | ||
| 2971 | /// Broadcasts the 32-bit floating-point value from the low element of the | ||
| 2972 | /// 128-bit vector of [4 x float] in \a __X to all elements of the result's | ||
| 2973 | /// 128-bit vector of [4 x float]. | ||
| 2974 | /// | ||
| 2975 | /// \headerfile <immintrin.h> | ||
| 2976 | /// | ||
| 2977 | /// This intrinsic corresponds to the \c VBROADCASTSS instruction. | ||
| 2978 | /// | ||
| 2979 | /// \param __X | ||
| 2980 | /// A 128-bit vector of [4 x float] whose low element will be broadcast. | ||
| 2981 | /// \returns A 128-bit vector of [4 x float] containing the result. | ||
| 713 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 | 2982 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 |
| 714 | _mm_broadcastss_ps(__m128 __X) | 2983 | _mm_broadcastss_ps(__m128 __X) |
| 715 | { | 2984 | { |
| 716 | return (__m128)__builtin_shufflevector((__v4sf)__X, (__v4sf)__X, 0, 0, 0, 0); | 2985 | return (__m128)__builtin_shufflevector((__v4sf)__X, (__v4sf)__X, 0, 0, 0, 0); |
| 717 | } | 2986 | } |
| 718 | 2987 | ||
| 2988 | /// Broadcasts the 64-bit floating-point value from the low element of the | ||
| 2989 | /// 128-bit vector of [2 x double] in \a __a to both elements of the | ||
| 2990 | /// result's 128-bit vector of [2 x double]. | ||
| 2991 | /// | ||
| 2992 | /// \headerfile <immintrin.h> | ||
| 2993 | /// | ||
| 2994 | /// This intrinsic corresponds to the \c MOVDDUP instruction. | ||
| 2995 | /// | ||
| 2996 | /// \param __a | ||
| 2997 | /// A 128-bit vector of [2 x double] whose low element will be broadcast. | ||
| 2998 | /// \returns A 128-bit vector of [2 x double] containing the result. | ||
| 719 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 | 2999 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 |
| 720 | _mm_broadcastsd_pd(__m128d __a) | 3000 | _mm_broadcastsd_pd(__m128d __a) |
| 721 | { | 3001 | { |
| 722 | return __builtin_shufflevector((__v2df)__a, (__v2df)__a, 0, 0); | 3002 | return __builtin_shufflevector((__v2df)__a, (__v2df)__a, 0, 0); |
| 723 | } | 3003 | } |
| 724 | 3004 | ||
| 3005 | /// Broadcasts the 32-bit floating-point value from the low element of the | ||
| 3006 | /// 128-bit vector of [4 x float] in \a __X to all elements of the | ||
| 3007 | /// result's 256-bit vector of [8 x float]. | ||
| 3008 | /// | ||
| 3009 | /// \headerfile <immintrin.h> | ||
| 3010 | /// | ||
| 3011 | /// This intrinsic corresponds to the \c VBROADCASTSS instruction. | ||
| 3012 | /// | ||
| 3013 | /// \param __X | ||
| 3014 | /// A 128-bit vector of [4 x float] whose low element will be broadcast. | ||
| 3015 | /// \returns A 256-bit vector of [8 x float] containing the result. | ||
| 725 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 | 3016 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 |
| 726 | _mm256_broadcastss_ps(__m128 __X) | 3017 | _mm256_broadcastss_ps(__m128 __X) |
| 727 | { | 3018 | { |
| 728 | return (__m256)__builtin_shufflevector((__v4sf)__X, (__v4sf)__X, 0, 0, 0, 0, 0, 0, 0, 0); | 3019 | return (__m256)__builtin_shufflevector((__v4sf)__X, (__v4sf)__X, 0, 0, 0, 0, 0, 0, 0, 0); |
| 729 | } | 3020 | } |
| 730 | 3021 | ||
| 3022 | /// Broadcasts the 64-bit floating-point value from the low element of the | ||
| 3023 | /// 128-bit vector of [2 x double] in \a __X to all elements of the | ||
| 3024 | /// result's 256-bit vector of [4 x double]. | ||
| 3025 | /// | ||
| 3026 | /// \headerfile <immintrin.h> | ||
| 3027 | /// | ||
| 3028 | /// This intrinsic corresponds to the \c VBROADCASTSD instruction. | ||
| 3029 | /// | ||
| 3030 | /// \param __X | ||
| 3031 | /// A 128-bit vector of [2 x double] whose low element will be broadcast. | ||
| 3032 | /// \returns A 256-bit vector of [4 x double] containing the result. | ||
| 731 | static __inline__ __m256d __DEFAULT_FN_ATTRS256 | 3033 | static __inline__ __m256d __DEFAULT_FN_ATTRS256 |
| 732 | _mm256_broadcastsd_pd(__m128d __X) | 3034 | _mm256_broadcastsd_pd(__m128d __X) |
| 733 | { | 3035 | { |
| 734 | return (__m256d)__builtin_shufflevector((__v2df)__X, (__v2df)__X, 0, 0, 0, 0); | 3036 | return (__m256d)__builtin_shufflevector((__v2df)__X, (__v2df)__X, 0, 0, 0, 0); |
| 735 | } | 3037 | } |
| 736 | 3038 | ||
| 3039 | /// Broadcasts the 128-bit integer data from \a __X to both the lower and | ||
| 3040 | /// upper halves of the 256-bit result. | ||
| 3041 | /// | ||
| 3042 | /// \headerfile <immintrin.h> | ||
| 3043 | /// | ||
| 3044 | /// This intrinsic corresponds to the \c VBROADCASTI128 instruction. | ||
| 3045 | /// | ||
| 3046 | /// \param __X | ||
| 3047 | /// A 128-bit integer vector to be broadcast. | ||
| 3048 | /// \returns A 256-bit integer vector containing the result. | ||
| 737 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 3049 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 738 | _mm256_broadcastsi128_si256(__m128i __X) | 3050 | _mm256_broadcastsi128_si256(__m128i __X) |
| 739 | { | 3051 | { |
| ... | @@ -742,295 +3054,1688 @@ _mm256_broadcastsi128_si256(__m128i __X) | ... | @@ -742,295 +3054,1688 @@ _mm256_broadcastsi128_si256(__m128i __X) |
| 742 | 3054 | ||
| 743 | #define _mm_broadcastsi128_si256(X) _mm256_broadcastsi128_si256(X) | 3055 | #define _mm_broadcastsi128_si256(X) _mm256_broadcastsi128_si256(X) |
| 744 | 3056 | ||
| 3057 | /// Merges 32-bit integer elements from either of the two 128-bit vectors of | ||
| 3058 | /// [4 x i32] in \a V1 or \a V2 to the result's 128-bit vector of [4 x i32], | ||
| 3059 | /// as specified by the immediate integer operand \a M. | ||
| 3060 | /// | ||
| 3061 | /// \code{.operation} | ||
| 3062 | /// FOR i := 0 TO 3 | ||
| 3063 | /// j := i*32 | ||
| 3064 | /// IF M[i] == 0 | ||
| 3065 | /// result[31+j:j] := V1[31+j:j] | ||
| 3066 | /// ELSE | ||
| 3067 | /// result[31+j:j] := V2[32+j:j] | ||
| 3068 | /// FI | ||
| 3069 | /// ENDFOR | ||
| 3070 | /// \endcode | ||
| 3071 | /// | ||
| 3072 | /// \headerfile <immintrin.h> | ||
| 3073 | /// | ||
| 3074 | /// \code | ||
| 3075 | /// __m128i _mm_blend_epi32(__m128i V1, __m128i V2, const int M); | ||
| 3076 | /// \endcode | ||
| 3077 | /// | ||
| 3078 | /// This intrinsic corresponds to the \c VPBLENDDD instruction. | ||
| 3079 | /// | ||
| 3080 | /// \param V1 | ||
| 3081 | /// A 128-bit vector of [4 x i32] containing source values. | ||
| 3082 | /// \param V2 | ||
| 3083 | /// A 128-bit vector of [4 x i32] containing source values. | ||
| 3084 | /// \param M | ||
| 3085 | /// An immediate 8-bit integer operand, with bits [3:0] specifying the | ||
| 3086 | /// source for each element of the result. The position of the mask bit | ||
| 3087 | /// corresponds to the index of a copied value. When a mask bit is 0, the | ||
| 3088 | /// element is copied from \a V1; otherwise, it is copied from \a V2. | ||
| 3089 | /// \returns A 128-bit vector of [4 x i32] containing the result. | ||
| 745 | #define _mm_blend_epi32(V1, V2, M) \ | 3090 | #define _mm_blend_epi32(V1, V2, M) \ |
| 746 | ((__m128i)__builtin_ia32_pblendd128((__v4si)(__m128i)(V1), \ | 3091 | ((__m128i)__builtin_ia32_pblendd128((__v4si)(__m128i)(V1), \ |
| 747 | (__v4si)(__m128i)(V2), (int)(M))) | 3092 | (__v4si)(__m128i)(V2), (int)(M))) |
| 748 | 3093 | ||
| 3094 | /// Merges 32-bit integer elements from either of the two 256-bit vectors of | ||
| 3095 | /// [8 x i32] in \a V1 or \a V2 to return a 256-bit vector of [8 x i32], | ||
| 3096 | /// as specified by the immediate integer operand \a M. | ||
| 3097 | /// | ||
| 3098 | /// \code{.operation} | ||
| 3099 | /// FOR i := 0 TO 7 | ||
| 3100 | /// j := i*32 | ||
| 3101 | /// IF M[i] == 0 | ||
| 3102 | /// result[31+j:j] := V1[31+j:j] | ||
| 3103 | /// ELSE | ||
| 3104 | /// result[31+j:j] := V2[32+j:j] | ||
| 3105 | /// FI | ||
| 3106 | /// ENDFOR | ||
| 3107 | /// \endcode | ||
| 3108 | /// | ||
| 3109 | /// \headerfile <immintrin.h> | ||
| 3110 | /// | ||
| 3111 | /// \code | ||
| 3112 | /// __m256i _mm256_blend_epi32(__m256i V1, __m256i V2, const int M); | ||
| 3113 | /// \endcode | ||
| 3114 | /// | ||
| 3115 | /// This intrinsic corresponds to the \c VPBLENDDD instruction. | ||
| 3116 | /// | ||
| 3117 | /// \param V1 | ||
| 3118 | /// A 256-bit vector of [8 x i32] containing source values. | ||
| 3119 | /// \param V2 | ||
| 3120 | /// A 256-bit vector of [8 x i32] containing source values. | ||
| 3121 | /// \param M | ||
| 3122 | /// An immediate 8-bit integer operand, with bits [7:0] specifying the | ||
| 3123 | /// source for each element of the result. The position of the mask bit | ||
| 3124 | /// corresponds to the index of a copied value. When a mask bit is 0, the | ||
| 3125 | /// element is copied from \a V1; otherwise, it is is copied from \a V2. | ||
| 3126 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 749 | #define _mm256_blend_epi32(V1, V2, M) \ | 3127 | #define _mm256_blend_epi32(V1, V2, M) \ |
| 750 | ((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(V1), \ | 3128 | ((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(V1), \ |
| 751 | (__v8si)(__m256i)(V2), (int)(M))) | 3129 | (__v8si)(__m256i)(V2), (int)(M))) |
| 752 | 3130 | ||
| 3131 | /// Broadcasts the low byte from the 128-bit integer vector in \a __X to all | ||
| 3132 | /// bytes of the 256-bit result. | ||
| 3133 | /// | ||
| 3134 | /// \headerfile <immintrin.h> | ||
| 3135 | /// | ||
| 3136 | /// This intrinsic corresponds to the \c VPBROADCASTB instruction. | ||
| 3137 | /// | ||
| 3138 | /// \param __X | ||
| 3139 | /// A 128-bit integer vector whose low byte will be broadcast. | ||
| 3140 | /// \returns A 256-bit integer vector containing the result. | ||
| 753 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 3141 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 754 | _mm256_broadcastb_epi8(__m128i __X) | 3142 | _mm256_broadcastb_epi8(__m128i __X) |
| 755 | { | 3143 | { |
| 756 | return (__m256i)__builtin_shufflevector((__v16qi)__X, (__v16qi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); | 3144 | return (__m256i)__builtin_shufflevector((__v16qi)__X, (__v16qi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); |
| 757 | } | 3145 | } |
| 758 | 3146 | ||
| 3147 | /// Broadcasts the low element from the 128-bit vector of [8 x i16] in \a __X | ||
| 3148 | /// to all elements of the result's 256-bit vector of [16 x i16]. | ||
| 3149 | /// | ||
| 3150 | /// \headerfile <immintrin.h> | ||
| 3151 | /// | ||
| 3152 | /// This intrinsic corresponds to the \c VPBROADCASTW instruction. | ||
| 3153 | /// | ||
| 3154 | /// \param __X | ||
| 3155 | /// A 128-bit vector of [8 x i16] whose low element will be broadcast. | ||
| 3156 | /// \returns A 256-bit vector of [16 x i16] containing the result. | ||
| 759 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 3157 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 760 | _mm256_broadcastw_epi16(__m128i __X) | 3158 | _mm256_broadcastw_epi16(__m128i __X) |
| 761 | { | 3159 | { |
| 762 | return (__m256i)__builtin_shufflevector((__v8hi)__X, (__v8hi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); | 3160 | return (__m256i)__builtin_shufflevector((__v8hi)__X, (__v8hi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); |
| 763 | } | 3161 | } |
| 764 | 3162 | ||
| 3163 | /// Broadcasts the low element from the 128-bit vector of [4 x i32] in \a __X | ||
| 3164 | /// to all elements of the result's 256-bit vector of [8 x i32]. | ||
| 3165 | /// | ||
| 3166 | /// \headerfile <immintrin.h> | ||
| 3167 | /// | ||
| 3168 | /// This intrinsic corresponds to the \c VPBROADCASTD instruction. | ||
| 3169 | /// | ||
| 3170 | /// \param __X | ||
| 3171 | /// A 128-bit vector of [4 x i32] whose low element will be broadcast. | ||
| 3172 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 765 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 3173 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 766 | _mm256_broadcastd_epi32(__m128i __X) | 3174 | _mm256_broadcastd_epi32(__m128i __X) |
| 767 | { | 3175 | { |
| 768 | return (__m256i)__builtin_shufflevector((__v4si)__X, (__v4si)__X, 0, 0, 0, 0, 0, 0, 0, 0); | 3176 | return (__m256i)__builtin_shufflevector((__v4si)__X, (__v4si)__X, 0, 0, 0, 0, 0, 0, 0, 0); |
| 769 | } | 3177 | } |
| 770 | 3178 | ||
| 3179 | /// Broadcasts the low element from the 128-bit vector of [2 x i64] in \a __X | ||
| 3180 | /// to all elements of the result's 256-bit vector of [4 x i64]. | ||
| 3181 | /// | ||
| 3182 | /// \headerfile <immintrin.h> | ||
| 3183 | /// | ||
| 3184 | /// This intrinsic corresponds to the \c VPBROADCASTQ instruction. | ||
| 3185 | /// | ||
| 3186 | /// \param __X | ||
| 3187 | /// A 128-bit vector of [2 x i64] whose low element will be broadcast. | ||
| 3188 | /// \returns A 256-bit vector of [4 x i64] containing the result. | ||
| 771 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 3189 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 772 | _mm256_broadcastq_epi64(__m128i __X) | 3190 | _mm256_broadcastq_epi64(__m128i __X) |
| 773 | { | 3191 | { |
| 774 | return (__m256i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 0, 0, 0); | 3192 | return (__m256i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 0, 0, 0); |
| 775 | } | 3193 | } |
| 776 | 3194 | ||
| 3195 | /// Broadcasts the low byte from the 128-bit integer vector in \a __X to all | ||
| 3196 | /// bytes of the 128-bit result. | ||
| 3197 | /// | ||
| 3198 | /// \headerfile <immintrin.h> | ||
| 3199 | /// | ||
| 3200 | /// This intrinsic corresponds to the \c VPBROADCASTB instruction. | ||
| 3201 | /// | ||
| 3202 | /// \param __X | ||
| 3203 | /// A 128-bit integer vector whose low byte will be broadcast. | ||
| 3204 | /// \returns A 128-bit integer vector containing the result. | ||
| 777 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 3205 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 778 | _mm_broadcastb_epi8(__m128i __X) | 3206 | _mm_broadcastb_epi8(__m128i __X) |
| 779 | { | 3207 | { |
| 780 | return (__m128i)__builtin_shufflevector((__v16qi)__X, (__v16qi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); | 3208 | return (__m128i)__builtin_shufflevector((__v16qi)__X, (__v16qi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); |
| 781 | } | 3209 | } |
| 782 | 3210 | ||
| 3211 | /// Broadcasts the low element from the 128-bit vector of [8 x i16] in | ||
| 3212 | /// \a __X to all elements of the result's 128-bit vector of [8 x i16]. | ||
| 3213 | /// | ||
| 3214 | /// \headerfile <immintrin.h> | ||
| 3215 | /// | ||
| 3216 | /// This intrinsic corresponds to the \c VPBROADCASTW instruction. | ||
| 3217 | /// | ||
| 3218 | /// \param __X | ||
| 3219 | /// A 128-bit vector of [8 x i16] whose low element will be broadcast. | ||
| 3220 | /// \returns A 128-bit vector of [8 x i16] containing the result. | ||
| 783 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 3221 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 784 | _mm_broadcastw_epi16(__m128i __X) | 3222 | _mm_broadcastw_epi16(__m128i __X) |
| 785 | { | 3223 | { |
| 786 | return (__m128i)__builtin_shufflevector((__v8hi)__X, (__v8hi)__X, 0, 0, 0, 0, 0, 0, 0, 0); | 3224 | return (__m128i)__builtin_shufflevector((__v8hi)__X, (__v8hi)__X, 0, 0, 0, 0, 0, 0, 0, 0); |
| 787 | } | 3225 | } |
| 788 | 3226 | ||
| 789 | 3227 | /// Broadcasts the low element from the 128-bit vector of [4 x i32] in \a __X | |
| 3228 | /// to all elements of the result's vector of [4 x i32]. | ||
| 3229 | /// | ||
| 3230 | /// \headerfile <immintrin.h> | ||
| 3231 | /// | ||
| 3232 | /// This intrinsic corresponds to the \c VPBROADCASTD instruction. | ||
| 3233 | /// | ||
| 3234 | /// \param __X | ||
| 3235 | /// A 128-bit vector of [4 x i32] whose low element will be broadcast. | ||
| 3236 | /// \returns A 128-bit vector of [4 x i32] containing the result. | ||
| 790 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 3237 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 791 | _mm_broadcastd_epi32(__m128i __X) | 3238 | _mm_broadcastd_epi32(__m128i __X) |
| 792 | { | 3239 | { |
| 793 | return (__m128i)__builtin_shufflevector((__v4si)__X, (__v4si)__X, 0, 0, 0, 0); | 3240 | return (__m128i)__builtin_shufflevector((__v4si)__X, (__v4si)__X, 0, 0, 0, 0); |
| 794 | } | 3241 | } |
| 795 | 3242 | ||
| 3243 | /// Broadcasts the low element from the 128-bit vector of [2 x i64] in \a __X | ||
| 3244 | /// to both elements of the result's 128-bit vector of [2 x i64]. | ||
| 3245 | /// | ||
| 3246 | /// \headerfile <immintrin.h> | ||
| 3247 | /// | ||
| 3248 | /// This intrinsic corresponds to the \c VPBROADCASTQ instruction. | ||
| 3249 | /// | ||
| 3250 | /// \param __X | ||
| 3251 | /// A 128-bit vector of [2 x i64] whose low element will be broadcast. | ||
| 3252 | /// \returns A 128-bit vector of [2 x i64] containing the result. | ||
| 796 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 3253 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 797 | _mm_broadcastq_epi64(__m128i __X) | 3254 | _mm_broadcastq_epi64(__m128i __X) |
| 798 | { | 3255 | { |
| 799 | return (__m128i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 0); | 3256 | return (__m128i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 0); |
| 800 | } | 3257 | } |
| 801 | 3258 | ||
| 3259 | /// Sets the result's 256-bit vector of [8 x i32] to copies of elements of the | ||
| 3260 | /// 256-bit vector of [8 x i32] in \a __a as specified by indexes in the | ||
| 3261 | /// elements of the 256-bit vector of [8 x i32] in \a __b. | ||
| 3262 | /// | ||
| 3263 | /// \code{.operation} | ||
| 3264 | /// FOR i := 0 TO 7 | ||
| 3265 | /// j := i*32 | ||
| 3266 | /// k := __b[j+2:j] * 32 | ||
| 3267 | /// result[j+31:j] := __a[k+31:k] | ||
| 3268 | /// ENDFOR | ||
| 3269 | /// \endcode | ||
| 3270 | /// | ||
| 3271 | /// \headerfile <immintrin.h> | ||
| 3272 | /// | ||
| 3273 | /// This intrinsic corresponds to the \c VPERMD instruction. | ||
| 3274 | /// | ||
| 3275 | /// \param __a | ||
| 3276 | /// A 256-bit vector of [8 x i32] containing the source values. | ||
| 3277 | /// \param __b | ||
| 3278 | /// A 256-bit vector of [8 x i32] containing indexes of values to use from | ||
| 3279 | /// \a __a. | ||
| 3280 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 802 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 3281 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 803 | _mm256_permutevar8x32_epi32(__m256i __a, __m256i __b) | 3282 | _mm256_permutevar8x32_epi32(__m256i __a, __m256i __b) |
| 804 | { | 3283 | { |
| 805 | return (__m256i)__builtin_ia32_permvarsi256((__v8si)__a, (__v8si)__b); | 3284 | return (__m256i)__builtin_ia32_permvarsi256((__v8si)__a, (__v8si)__b); |
| 806 | } | 3285 | } |
| 807 | 3286 | ||
| 3287 | /// Sets the result's 256-bit vector of [4 x double] to copies of elements of | ||
| 3288 | /// the 256-bit vector of [4 x double] in \a V as specified by the | ||
| 3289 | /// immediate value \a M. | ||
| 3290 | /// | ||
| 3291 | /// \code{.operation} | ||
| 3292 | /// FOR i := 0 TO 3 | ||
| 3293 | /// j := i*64 | ||
| 3294 | /// k := (M >> i*2)[1:0] * 64 | ||
| 3295 | /// result[j+63:j] := V[k+63:k] | ||
| 3296 | /// ENDFOR | ||
| 3297 | /// \endcode | ||
| 3298 | /// | ||
| 3299 | /// \headerfile <immintrin.h> | ||
| 3300 | /// | ||
| 3301 | /// \code | ||
| 3302 | /// __m256d _mm256_permute4x64_pd(__m256d V, const int M); | ||
| 3303 | /// \endcode | ||
| 3304 | /// | ||
| 3305 | /// This intrinsic corresponds to the \c VPERMPD instruction. | ||
| 3306 | /// | ||
| 3307 | /// \param V | ||
| 3308 | /// A 256-bit vector of [4 x double] containing the source values. | ||
| 3309 | /// \param M | ||
| 3310 | /// An immediate 8-bit value specifying which elements to copy from \a V. | ||
| 3311 | /// \a M[1:0] specifies the index in \a a for element 0 of the result, | ||
| 3312 | /// \a M[3:2] specifies the index for element 1, and so forth. | ||
| 3313 | /// \returns A 256-bit vector of [4 x double] containing the result. | ||
| 808 | #define _mm256_permute4x64_pd(V, M) \ | 3314 | #define _mm256_permute4x64_pd(V, M) \ |
| 809 | ((__m256d)__builtin_ia32_permdf256((__v4df)(__m256d)(V), (int)(M))) | 3315 | ((__m256d)__builtin_ia32_permdf256((__v4df)(__m256d)(V), (int)(M))) |
| 810 | 3316 | ||
| 3317 | /// Sets the result's 256-bit vector of [8 x float] to copies of elements of | ||
| 3318 | /// the 256-bit vector of [8 x float] in \a __a as specified by indexes in | ||
| 3319 | /// the elements of the 256-bit vector of [8 x i32] in \a __b. | ||
| 3320 | /// | ||
| 3321 | /// \code{.operation} | ||
| 3322 | /// FOR i := 0 TO 7 | ||
| 3323 | /// j := i*32 | ||
| 3324 | /// k := __b[j+2:j] * 32 | ||
| 3325 | /// result[j+31:j] := __a[k+31:k] | ||
| 3326 | /// ENDFOR | ||
| 3327 | /// \endcode | ||
| 3328 | /// | ||
| 3329 | /// \headerfile <immintrin.h> | ||
| 3330 | /// | ||
| 3331 | /// This intrinsic corresponds to the \c VPERMPS instruction. | ||
| 3332 | /// | ||
| 3333 | /// \param __a | ||
| 3334 | /// A 256-bit vector of [8 x float] containing the source values. | ||
| 3335 | /// \param __b | ||
| 3336 | /// A 256-bit vector of [8 x i32] containing indexes of values to use from | ||
| 3337 | /// \a __a. | ||
| 3338 | /// \returns A 256-bit vector of [8 x float] containing the result. | ||
| 811 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 | 3339 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 |
| 812 | _mm256_permutevar8x32_ps(__m256 __a, __m256i __b) | 3340 | _mm256_permutevar8x32_ps(__m256 __a, __m256i __b) |
| 813 | { | 3341 | { |
| 814 | return (__m256)__builtin_ia32_permvarsf256((__v8sf)__a, (__v8si)__b); | 3342 | return (__m256)__builtin_ia32_permvarsf256((__v8sf)__a, (__v8si)__b); |
| 815 | } | 3343 | } |
| 816 | 3344 | ||
| 3345 | /// Sets the result's 256-bit vector of [4 x i64] result to copies of elements | ||
| 3346 | /// of the 256-bit vector of [4 x i64] in \a V as specified by the | ||
| 3347 | /// immediate value \a M. | ||
| 3348 | /// | ||
| 3349 | /// \code{.operation} | ||
| 3350 | /// FOR i := 0 TO 3 | ||
| 3351 | /// j := i*64 | ||
| 3352 | /// k := (M >> i*2)[1:0] * 64 | ||
| 3353 | /// result[j+63:j] := V[k+63:k] | ||
| 3354 | /// ENDFOR | ||
| 3355 | /// \endcode | ||
| 3356 | /// | ||
| 3357 | /// \headerfile <immintrin.h> | ||
| 3358 | /// | ||
| 3359 | /// \code | ||
| 3360 | /// __m256i _mm256_permute4x64_epi64(__m256i V, const int M); | ||
| 3361 | /// \endcode | ||
| 3362 | /// | ||
| 3363 | /// This intrinsic corresponds to the \c VPERMQ instruction. | ||
| 3364 | /// | ||
| 3365 | /// \param V | ||
| 3366 | /// A 256-bit vector of [4 x i64] containing the source values. | ||
| 3367 | /// \param M | ||
| 3368 | /// An immediate 8-bit value specifying which elements to copy from \a V. | ||
| 3369 | /// \a M[1:0] specifies the index in \a a for element 0 of the result, | ||
| 3370 | /// \a M[3:2] specifies the index for element 1, and so forth. | ||
| 3371 | /// \returns A 256-bit vector of [4 x i64] containing the result. | ||
| 817 | #define _mm256_permute4x64_epi64(V, M) \ | 3372 | #define _mm256_permute4x64_epi64(V, M) \ |
| 818 | ((__m256i)__builtin_ia32_permdi256((__v4di)(__m256i)(V), (int)(M))) | 3373 | ((__m256i)__builtin_ia32_permdi256((__v4di)(__m256i)(V), (int)(M))) |
| 819 | 3374 | ||
| 3375 | /// Sets each half of the 256-bit result either to zero or to one of the | ||
| 3376 | /// four possible 128-bit halves of the 256-bit vectors \a V1 and \a V2, | ||
| 3377 | /// as specified by the immediate value \a M. | ||
| 3378 | /// | ||
| 3379 | /// \code{.operation} | ||
| 3380 | /// FOR i := 0 TO 1 | ||
| 3381 | /// j := i*128 | ||
| 3382 | /// k := M >> (i*4) | ||
| 3383 | /// IF k[3] == 0 | ||
| 3384 | /// CASE (k[1:0]) OF | ||
| 3385 | /// 0: result[127+j:j] := V1[127:0] | ||
| 3386 | /// 1: result[127+j:j] := V1[255:128] | ||
| 3387 | /// 2: result[127+j:j] := V2[127:0] | ||
| 3388 | /// 3: result[127+j:j] := V2[255:128] | ||
| 3389 | /// ESAC | ||
| 3390 | /// ELSE | ||
| 3391 | /// result[127+j:j] := 0 | ||
| 3392 | /// FI | ||
| 3393 | /// ENDFOR | ||
| 3394 | /// \endcode | ||
| 3395 | /// | ||
| 3396 | /// \headerfile <immintrin.h> | ||
| 3397 | /// | ||
| 3398 | /// \code | ||
| 3399 | /// __m256i _mm256_permute2x128_si256(__m256i V1, __m256i V2, const int M); | ||
| 3400 | /// \endcode | ||
| 3401 | /// | ||
| 3402 | /// This intrinsic corresponds to the \c VPERM2I128 instruction. | ||
| 3403 | /// | ||
| 3404 | /// \param V1 | ||
| 3405 | /// A 256-bit integer vector containing source values. | ||
| 3406 | /// \param V2 | ||
| 3407 | /// A 256-bit integer vector containing source values. | ||
| 3408 | /// \param M | ||
| 3409 | /// An immediate value specifying how to form the result. Bits [3:0] | ||
| 3410 | /// control the lower half of the result, bits [7:4] control the upper half. | ||
| 3411 | /// Within each 4-bit control value, if bit 3 is 1, the result is zero, | ||
| 3412 | /// otherwise bits [1:0] determine the source as follows. \n | ||
| 3413 | /// 0: the lower half of \a V1 \n | ||
| 3414 | /// 1: the upper half of \a V1 \n | ||
| 3415 | /// 2: the lower half of \a V2 \n | ||
| 3416 | /// 3: the upper half of \a V2 | ||
| 3417 | /// \returns A 256-bit integer vector containing the result. | ||
| 820 | #define _mm256_permute2x128_si256(V1, V2, M) \ | 3418 | #define _mm256_permute2x128_si256(V1, V2, M) \ |
| 821 | ((__m256i)__builtin_ia32_permti256((__m256i)(V1), (__m256i)(V2), (int)(M))) | 3419 | ((__m256i)__builtin_ia32_permti256((__m256i)(V1), (__m256i)(V2), (int)(M))) |
| 822 | 3420 | ||
| 3421 | /// Extracts half of the 256-bit vector \a V to the 128-bit result. If bit 0 | ||
| 3422 | /// of the immediate \a M is zero, extracts the lower half of the result; | ||
| 3423 | /// otherwise, extracts the upper half. | ||
| 3424 | /// | ||
| 3425 | /// \headerfile <immintrin.h> | ||
| 3426 | /// | ||
| 3427 | /// \code | ||
| 3428 | /// __m128i _mm256_extracti128_si256(__m256i V, const int M); | ||
| 3429 | /// \endcode | ||
| 3430 | /// | ||
| 3431 | /// This intrinsic corresponds to the \c VEXTRACTI128 instruction. | ||
| 3432 | /// | ||
| 3433 | /// \param V | ||
| 3434 | /// A 256-bit integer vector containing the source values. | ||
| 3435 | /// \param M | ||
| 3436 | /// An immediate value specifying which half of \a V to extract. | ||
| 3437 | /// \returns A 128-bit integer vector containing the result. | ||
| 823 | #define _mm256_extracti128_si256(V, M) \ | 3438 | #define _mm256_extracti128_si256(V, M) \ |
| 824 | ((__m128i)__builtin_ia32_extract128i256((__v4di)(__m256i)(V), (int)(M))) | 3439 | ((__m128i)__builtin_ia32_extract128i256((__v4di)(__m256i)(V), (int)(M))) |
| 825 | 3440 | ||
| 3441 | /// Copies the 256-bit vector \a V1 to the result, then overwrites half of the | ||
| 3442 | /// result with the 128-bit vector \a V2. If bit 0 of the immediate \a M | ||
| 3443 | /// is zero, overwrites the lower half of the result; otherwise, | ||
| 3444 | /// overwrites the upper half. | ||
| 3445 | /// | ||
| 3446 | /// \headerfile <immintrin.h> | ||
| 3447 | /// | ||
| 3448 | /// \code | ||
| 3449 | /// __m256i _mm256_inserti128_si256(__m256i V1, __m128i V2, const int M); | ||
| 3450 | /// \endcode | ||
| 3451 | /// | ||
| 3452 | /// This intrinsic corresponds to the \c VINSERTI128 instruction. | ||
| 3453 | /// | ||
| 3454 | /// \param V1 | ||
| 3455 | /// A 256-bit integer vector containing a source value. | ||
| 3456 | /// \param V2 | ||
| 3457 | /// A 128-bit integer vector containing a source value. | ||
| 3458 | /// \param M | ||
| 3459 | /// An immediate value specifying where to put \a V2 in the result. | ||
| 3460 | /// \returns A 256-bit integer vector containing the result. | ||
| 826 | #define _mm256_inserti128_si256(V1, V2, M) \ | 3461 | #define _mm256_inserti128_si256(V1, V2, M) \ |
| 827 | ((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(V1), \ | 3462 | ((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(V1), \ |
| 828 | (__v2di)(__m128i)(V2), (int)(M))) | 3463 | (__v2di)(__m128i)(V2), (int)(M))) |
| 829 | 3464 | ||
| 3465 | /// Conditionally loads eight 32-bit integer elements from memory \a __X, if | ||
| 3466 | /// the most significant bit of the corresponding element in the mask | ||
| 3467 | /// \a __M is set; otherwise, sets that element of the result to zero. | ||
| 3468 | /// Returns the 256-bit [8 x i32] result. | ||
| 3469 | /// | ||
| 3470 | /// \code{.operation} | ||
| 3471 | /// FOR i := 0 TO 7 | ||
| 3472 | /// j := i*32 | ||
| 3473 | /// IF __M[j+31] == 1 | ||
| 3474 | /// result[j+31:j] := Load32(__X+(i*4)) | ||
| 3475 | /// ELSE | ||
| 3476 | /// result[j+31:j] := 0 | ||
| 3477 | /// FI | ||
| 3478 | /// ENDFOR | ||
| 3479 | /// \endcode | ||
| 3480 | /// | ||
| 3481 | /// \headerfile <immintrin.h> | ||
| 3482 | /// | ||
| 3483 | /// This intrinsic corresponds to the \c VPMASKMOVD instruction. | ||
| 3484 | /// | ||
| 3485 | /// \param __X | ||
| 3486 | /// A pointer to the memory used for loading values. | ||
| 3487 | /// \param __M | ||
| 3488 | /// A 256-bit vector of [8 x i32] containing the mask bits. | ||
| 3489 | /// \returns A 256-bit vector of [8 x i32] containing the loaded or zeroed | ||
| 3490 | /// elements. | ||
| 830 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 3491 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 831 | _mm256_maskload_epi32(int const *__X, __m256i __M) | 3492 | _mm256_maskload_epi32(int const *__X, __m256i __M) |
| 832 | { | 3493 | { |
| 833 | return (__m256i)__builtin_ia32_maskloadd256((const __v8si *)__X, (__v8si)__M); | 3494 | return (__m256i)__builtin_ia32_maskloadd256((const __v8si *)__X, (__v8si)__M); |
| 834 | } | 3495 | } |
| 835 | 3496 | ||
| 3497 | /// Conditionally loads four 64-bit integer elements from memory \a __X, if | ||
| 3498 | /// the most significant bit of the corresponding element in the mask | ||
| 3499 | /// \a __M is set; otherwise, sets that element of the result to zero. | ||
| 3500 | /// Returns the 256-bit [4 x i64] result. | ||
| 3501 | /// | ||
| 3502 | /// \code{.operation} | ||
| 3503 | /// FOR i := 0 TO 3 | ||
| 3504 | /// j := i*64 | ||
| 3505 | /// IF __M[j+63] == 1 | ||
| 3506 | /// result[j+63:j] := Load64(__X+(i*8)) | ||
| 3507 | /// ELSE | ||
| 3508 | /// result[j+63:j] := 0 | ||
| 3509 | /// FI | ||
| 3510 | /// ENDFOR | ||
| 3511 | /// \endcode | ||
| 3512 | /// | ||
| 3513 | /// \headerfile <immintrin.h> | ||
| 3514 | /// | ||
| 3515 | /// This intrinsic corresponds to the \c VPMASKMOVQ instruction. | ||
| 3516 | /// | ||
| 3517 | /// \param __X | ||
| 3518 | /// A pointer to the memory used for loading values. | ||
| 3519 | /// \param __M | ||
| 3520 | /// A 256-bit vector of [4 x i64] containing the mask bits. | ||
| 3521 | /// \returns A 256-bit vector of [4 x i64] containing the loaded or zeroed | ||
| 3522 | /// elements. | ||
| 836 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 3523 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 837 | _mm256_maskload_epi64(long long const *__X, __m256i __M) | 3524 | _mm256_maskload_epi64(long long const *__X, __m256i __M) |
| 838 | { | 3525 | { |
| 839 | return (__m256i)__builtin_ia32_maskloadq256((const __v4di *)__X, (__v4di)__M); | 3526 | return (__m256i)__builtin_ia32_maskloadq256((const __v4di *)__X, (__v4di)__M); |
| 840 | } | 3527 | } |
| 841 | 3528 | ||
| 3529 | /// Conditionally loads four 32-bit integer elements from memory \a __X, if | ||
| 3530 | /// the most significant bit of the corresponding element in the mask | ||
| 3531 | /// \a __M is set; otherwise, sets that element of the result to zero. | ||
| 3532 | /// Returns the 128-bit [4 x i32] result. | ||
| 3533 | /// | ||
| 3534 | /// \code{.operation} | ||
| 3535 | /// FOR i := 0 TO 3 | ||
| 3536 | /// j := i*32 | ||
| 3537 | /// IF __M[j+31] == 1 | ||
| 3538 | /// result[j+31:j] := Load32(__X+(i*4)) | ||
| 3539 | /// ELSE | ||
| 3540 | /// result[j+31:j] := 0 | ||
| 3541 | /// FI | ||
| 3542 | /// ENDFOR | ||
| 3543 | /// \endcode | ||
| 3544 | /// | ||
| 3545 | /// \headerfile <immintrin.h> | ||
| 3546 | /// | ||
| 3547 | /// This intrinsic corresponds to the \c VPMASKMOVD instruction. | ||
| 3548 | /// | ||
| 3549 | /// \param __X | ||
| 3550 | /// A pointer to the memory used for loading values. | ||
| 3551 | /// \param __M | ||
| 3552 | /// A 128-bit vector of [4 x i32] containing the mask bits. | ||
| 3553 | /// \returns A 128-bit vector of [4 x i32] containing the loaded or zeroed | ||
| 3554 | /// elements. | ||
| 842 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 3555 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 843 | _mm_maskload_epi32(int const *__X, __m128i __M) | 3556 | _mm_maskload_epi32(int const *__X, __m128i __M) |
| 844 | { | 3557 | { |
| 845 | return (__m128i)__builtin_ia32_maskloadd((const __v4si *)__X, (__v4si)__M); | 3558 | return (__m128i)__builtin_ia32_maskloadd((const __v4si *)__X, (__v4si)__M); |
| 846 | } | 3559 | } |
| 847 | 3560 | ||
| 3561 | /// Conditionally loads two 64-bit integer elements from memory \a __X, if | ||
| 3562 | /// the most significant bit of the corresponding element in the mask | ||
| 3563 | /// \a __M is set; otherwise, sets that element of the result to zero. | ||
| 3564 | /// Returns the 128-bit [2 x i64] result. | ||
| 3565 | /// | ||
| 3566 | /// \code{.operation} | ||
| 3567 | /// FOR i := 0 TO 1 | ||
| 3568 | /// j := i*64 | ||
| 3569 | /// IF __M[j+63] == 1 | ||
| 3570 | /// result[j+63:j] := Load64(__X+(i*8)) | ||
| 3571 | /// ELSE | ||
| 3572 | /// result[j+63:j] := 0 | ||
| 3573 | /// FI | ||
| 3574 | /// ENDFOR | ||
| 3575 | /// \endcode | ||
| 3576 | /// | ||
| 3577 | /// \headerfile <immintrin.h> | ||
| 3578 | /// | ||
| 3579 | /// This intrinsic corresponds to the \c VPMASKMOVQ instruction. | ||
| 3580 | /// | ||
| 3581 | /// \param __X | ||
| 3582 | /// A pointer to the memory used for loading values. | ||
| 3583 | /// \param __M | ||
| 3584 | /// A 128-bit vector of [2 x i64] containing the mask bits. | ||
| 3585 | /// \returns A 128-bit vector of [2 x i64] containing the loaded or zeroed | ||
| 3586 | /// elements. | ||
| 848 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 3587 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 849 | _mm_maskload_epi64(long long const *__X, __m128i __M) | 3588 | _mm_maskload_epi64(long long const *__X, __m128i __M) |
| 850 | { | 3589 | { |
| 851 | return (__m128i)__builtin_ia32_maskloadq((const __v2di *)__X, (__v2di)__M); | 3590 | return (__m128i)__builtin_ia32_maskloadq((const __v2di *)__X, (__v2di)__M); |
| 852 | } | 3591 | } |
| 853 | 3592 | ||
| 3593 | /// Conditionally stores eight 32-bit integer elements from the 256-bit vector | ||
| 3594 | /// of [8 x i32] in \a __Y to memory \a __X, if the most significant bit of | ||
| 3595 | /// the corresponding element in the mask \a __M is set; otherwise, the | ||
| 3596 | /// memory element is unchanged. | ||
| 3597 | /// | ||
| 3598 | /// \code{.operation} | ||
| 3599 | /// FOR i := 0 TO 7 | ||
| 3600 | /// j := i*32 | ||
| 3601 | /// IF __M[j+31] == 1 | ||
| 3602 | /// Store32(__X+(i*4), __Y[j+31:j]) | ||
| 3603 | /// FI | ||
| 3604 | /// ENDFOR | ||
| 3605 | /// \endcode | ||
| 3606 | /// | ||
| 3607 | /// \headerfile <immintrin.h> | ||
| 3608 | /// | ||
| 3609 | /// This intrinsic corresponds to the \c VPMASKMOVD instruction. | ||
| 3610 | /// | ||
| 3611 | /// \param __X | ||
| 3612 | /// A pointer to the memory used for storing values. | ||
| 3613 | /// \param __M | ||
| 3614 | /// A 256-bit vector of [8 x i32] containing the mask bits. | ||
| 3615 | /// \param __Y | ||
| 3616 | /// A 256-bit vector of [8 x i32] containing the values to store. | ||
| 854 | static __inline__ void __DEFAULT_FN_ATTRS256 | 3617 | static __inline__ void __DEFAULT_FN_ATTRS256 |
| 855 | _mm256_maskstore_epi32(int *__X, __m256i __M, __m256i __Y) | 3618 | _mm256_maskstore_epi32(int *__X, __m256i __M, __m256i __Y) |
| 856 | { | 3619 | { |
| 857 | __builtin_ia32_maskstored256((__v8si *)__X, (__v8si)__M, (__v8si)__Y); | 3620 | __builtin_ia32_maskstored256((__v8si *)__X, (__v8si)__M, (__v8si)__Y); |
| 858 | } | 3621 | } |
| 859 | 3622 | ||
| 3623 | /// Conditionally stores four 64-bit integer elements from the 256-bit vector | ||
| 3624 | /// of [4 x i64] in \a __Y to memory \a __X, if the most significant bit of | ||
| 3625 | /// the corresponding element in the mask \a __M is set; otherwise, the | ||
| 3626 | /// memory element is unchanged. | ||
| 3627 | /// | ||
| 3628 | /// \code{.operation} | ||
| 3629 | /// FOR i := 0 TO 3 | ||
| 3630 | /// j := i*64 | ||
| 3631 | /// IF __M[j+63] == 1 | ||
| 3632 | /// Store64(__X+(i*8), __Y[j+63:j]) | ||
| 3633 | /// FI | ||
| 3634 | /// ENDFOR | ||
| 3635 | /// \endcode | ||
| 3636 | /// | ||
| 3637 | /// \headerfile <immintrin.h> | ||
| 3638 | /// | ||
| 3639 | /// This intrinsic corresponds to the \c VPMASKMOVQ instruction. | ||
| 3640 | /// | ||
| 3641 | /// \param __X | ||
| 3642 | /// A pointer to the memory used for storing values. | ||
| 3643 | /// \param __M | ||
| 3644 | /// A 256-bit vector of [4 x i64] containing the mask bits. | ||
| 3645 | /// \param __Y | ||
| 3646 | /// A 256-bit vector of [4 x i64] containing the values to store. | ||
| 860 | static __inline__ void __DEFAULT_FN_ATTRS256 | 3647 | static __inline__ void __DEFAULT_FN_ATTRS256 |
| 861 | _mm256_maskstore_epi64(long long *__X, __m256i __M, __m256i __Y) | 3648 | _mm256_maskstore_epi64(long long *__X, __m256i __M, __m256i __Y) |
| 862 | { | 3649 | { |
| 863 | __builtin_ia32_maskstoreq256((__v4di *)__X, (__v4di)__M, (__v4di)__Y); | 3650 | __builtin_ia32_maskstoreq256((__v4di *)__X, (__v4di)__M, (__v4di)__Y); |
| 864 | } | 3651 | } |
| 865 | 3652 | ||
| 3653 | /// Conditionally stores four 32-bit integer elements from the 128-bit vector | ||
| 3654 | /// of [4 x i32] in \a __Y to memory \a __X, if the most significant bit of | ||
| 3655 | /// the corresponding element in the mask \a __M is set; otherwise, the | ||
| 3656 | /// memory element is unchanged. | ||
| 3657 | /// | ||
| 3658 | /// \code{.operation} | ||
| 3659 | /// FOR i := 0 TO 3 | ||
| 3660 | /// j := i*32 | ||
| 3661 | /// IF __M[j+31] == 1 | ||
| 3662 | /// Store32(__X+(i*4), __Y[j+31:j]) | ||
| 3663 | /// FI | ||
| 3664 | /// ENDFOR | ||
| 3665 | /// \endcode | ||
| 3666 | /// | ||
| 3667 | /// \headerfile <immintrin.h> | ||
| 3668 | /// | ||
| 3669 | /// This intrinsic corresponds to the \c VPMASKMOVD instruction. | ||
| 3670 | /// | ||
| 3671 | /// \param __X | ||
| 3672 | /// A pointer to the memory used for storing values. | ||
| 3673 | /// \param __M | ||
| 3674 | /// A 128-bit vector of [4 x i32] containing the mask bits. | ||
| 3675 | /// \param __Y | ||
| 3676 | /// A 128-bit vector of [4 x i32] containing the values to store. | ||
| 866 | static __inline__ void __DEFAULT_FN_ATTRS128 | 3677 | static __inline__ void __DEFAULT_FN_ATTRS128 |
| 867 | _mm_maskstore_epi32(int *__X, __m128i __M, __m128i __Y) | 3678 | _mm_maskstore_epi32(int *__X, __m128i __M, __m128i __Y) |
| 868 | { | 3679 | { |
| 869 | __builtin_ia32_maskstored((__v4si *)__X, (__v4si)__M, (__v4si)__Y); | 3680 | __builtin_ia32_maskstored((__v4si *)__X, (__v4si)__M, (__v4si)__Y); |
| 870 | } | 3681 | } |
| 871 | 3682 | ||
| 3683 | /// Conditionally stores two 64-bit integer elements from the 128-bit vector | ||
| 3684 | /// of [2 x i64] in \a __Y to memory \a __X, if the most significant bit of | ||
| 3685 | /// the corresponding element in the mask \a __M is set; otherwise, the | ||
| 3686 | /// memory element is unchanged. | ||
| 3687 | /// | ||
| 3688 | /// \code{.operation} | ||
| 3689 | /// FOR i := 0 TO 1 | ||
| 3690 | /// j := i*64 | ||
| 3691 | /// IF __M[j+63] == 1 | ||
| 3692 | /// Store64(__X+(i*8), __Y[j+63:j]) | ||
| 3693 | /// FI | ||
| 3694 | /// ENDFOR | ||
| 3695 | /// \endcode | ||
| 3696 | /// | ||
| 3697 | /// \headerfile <immintrin.h> | ||
| 3698 | /// | ||
| 3699 | /// This intrinsic corresponds to the \c VPMASKMOVQ instruction. | ||
| 3700 | /// | ||
| 3701 | /// \param __X | ||
| 3702 | /// A pointer to the memory used for storing values. | ||
| 3703 | /// \param __M | ||
| 3704 | /// A 128-bit vector of [2 x i64] containing the mask bits. | ||
| 3705 | /// \param __Y | ||
| 3706 | /// A 128-bit vector of [2 x i64] containing the values to store. | ||
| 872 | static __inline__ void __DEFAULT_FN_ATTRS128 | 3707 | static __inline__ void __DEFAULT_FN_ATTRS128 |
| 873 | _mm_maskstore_epi64(long long *__X, __m128i __M, __m128i __Y) | 3708 | _mm_maskstore_epi64(long long *__X, __m128i __M, __m128i __Y) |
| 874 | { | 3709 | { |
| 875 | __builtin_ia32_maskstoreq(( __v2di *)__X, (__v2di)__M, (__v2di)__Y); | 3710 | __builtin_ia32_maskstoreq(( __v2di *)__X, (__v2di)__M, (__v2di)__Y); |
| 876 | } | 3711 | } |
| 877 | 3712 | ||
| 3713 | /// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __X | ||
| 3714 | /// left by the number of bits given in the corresponding element of the | ||
| 3715 | /// 256-bit vector of [8 x i32] in \a __Y, shifting in zero bits, and | ||
| 3716 | /// returns the result. If the shift count for any element is greater than | ||
| 3717 | /// 31, the result for that element is zero. | ||
| 3718 | /// | ||
| 3719 | /// \headerfile <immintrin.h> | ||
| 3720 | /// | ||
| 3721 | /// This intrinsic corresponds to the \c VPSLLVD instruction. | ||
| 3722 | /// | ||
| 3723 | /// \param __X | ||
| 3724 | /// A 256-bit vector of [8 x i32] to be shifted. | ||
| 3725 | /// \param __Y | ||
| 3726 | /// A 256-bit vector of [8 x i32] containing the unsigned shift counts (in | ||
| 3727 | /// bits). | ||
| 3728 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 878 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 3729 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 879 | _mm256_sllv_epi32(__m256i __X, __m256i __Y) | 3730 | _mm256_sllv_epi32(__m256i __X, __m256i __Y) |
| 880 | { | 3731 | { |
| 881 | return (__m256i)__builtin_ia32_psllv8si((__v8si)__X, (__v8si)__Y); | 3732 | return (__m256i)__builtin_ia32_psllv8si((__v8si)__X, (__v8si)__Y); |
| 882 | } | 3733 | } |
| 883 | 3734 | ||
| 3735 | /// Shifts each 32-bit element of the 128-bit vector of [4 x i32] in \a __X | ||
| 3736 | /// left by the number of bits given in the corresponding element of the | ||
| 3737 | /// 128-bit vector of [4 x i32] in \a __Y, shifting in zero bits, and | ||
| 3738 | /// returns the result. If the shift count for any element is greater than | ||
| 3739 | /// 31, the result for that element is zero. | ||
| 3740 | /// | ||
| 3741 | /// \headerfile <immintrin.h> | ||
| 3742 | /// | ||
| 3743 | /// This intrinsic corresponds to the \c VPSLLVD instruction. | ||
| 3744 | /// | ||
| 3745 | /// \param __X | ||
| 3746 | /// A 128-bit vector of [4 x i32] to be shifted. | ||
| 3747 | /// \param __Y | ||
| 3748 | /// A 128-bit vector of [4 x i32] containing the unsigned shift counts (in | ||
| 3749 | /// bits). | ||
| 3750 | /// \returns A 128-bit vector of [4 x i32] containing the result. | ||
| 884 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 3751 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 885 | _mm_sllv_epi32(__m128i __X, __m128i __Y) | 3752 | _mm_sllv_epi32(__m128i __X, __m128i __Y) |
| 886 | { | 3753 | { |
| 887 | return (__m128i)__builtin_ia32_psllv4si((__v4si)__X, (__v4si)__Y); | 3754 | return (__m128i)__builtin_ia32_psllv4si((__v4si)__X, (__v4si)__Y); |
| 888 | } | 3755 | } |
| 889 | 3756 | ||
| 3757 | /// Shifts each 64-bit element of the 256-bit vector of [4 x i64] in \a __X | ||
| 3758 | /// left by the number of bits given in the corresponding element of the | ||
| 3759 | /// 128-bit vector of [4 x i64] in \a __Y, shifting in zero bits, and | ||
| 3760 | /// returns the result. If the shift count for any element is greater than | ||
| 3761 | /// 63, the result for that element is zero. | ||
| 3762 | /// | ||
| 3763 | /// \headerfile <immintrin.h> | ||
| 3764 | /// | ||
| 3765 | /// This intrinsic corresponds to the \c VPSLLVQ instruction. | ||
| 3766 | /// | ||
| 3767 | /// \param __X | ||
| 3768 | /// A 256-bit vector of [4 x i64] to be shifted. | ||
| 3769 | /// \param __Y | ||
| 3770 | /// A 256-bit vector of [4 x i64] containing the unsigned shift counts (in | ||
| 3771 | /// bits). | ||
| 3772 | /// \returns A 256-bit vector of [4 x i64] containing the result. | ||
| 890 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 3773 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 891 | _mm256_sllv_epi64(__m256i __X, __m256i __Y) | 3774 | _mm256_sllv_epi64(__m256i __X, __m256i __Y) |
| 892 | { | 3775 | { |
| 893 | return (__m256i)__builtin_ia32_psllv4di((__v4di)__X, (__v4di)__Y); | 3776 | return (__m256i)__builtin_ia32_psllv4di((__v4di)__X, (__v4di)__Y); |
| 894 | } | 3777 | } |
| 895 | 3778 | ||
| 3779 | /// Shifts each 64-bit element of the 128-bit vector of [2 x i64] in \a __X | ||
| 3780 | /// left by the number of bits given in the corresponding element of the | ||
| 3781 | /// 128-bit vector of [2 x i64] in \a __Y, shifting in zero bits, and | ||
| 3782 | /// returns the result. If the shift count for any element is greater than | ||
| 3783 | /// 63, the result for that element is zero. | ||
| 3784 | /// | ||
| 3785 | /// \headerfile <immintrin.h> | ||
| 3786 | /// | ||
| 3787 | /// This intrinsic corresponds to the \c VPSLLVQ instruction. | ||
| 3788 | /// | ||
| 3789 | /// \param __X | ||
| 3790 | /// A 128-bit vector of [2 x i64] to be shifted. | ||
| 3791 | /// \param __Y | ||
| 3792 | /// A 128-bit vector of [2 x i64] containing the unsigned shift counts (in | ||
| 3793 | /// bits). | ||
| 3794 | /// \returns A 128-bit vector of [2 x i64] containing the result. | ||
| 896 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 3795 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 897 | _mm_sllv_epi64(__m128i __X, __m128i __Y) | 3796 | _mm_sllv_epi64(__m128i __X, __m128i __Y) |
| 898 | { | 3797 | { |
| 899 | return (__m128i)__builtin_ia32_psllv2di((__v2di)__X, (__v2di)__Y); | 3798 | return (__m128i)__builtin_ia32_psllv2di((__v2di)__X, (__v2di)__Y); |
| 900 | } | 3799 | } |
| 901 | 3800 | ||
| 3801 | /// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __X | ||
| 3802 | /// right by the number of bits given in the corresponding element of the | ||
| 3803 | /// 256-bit vector of [8 x i32] in \a __Y, shifting in sign bits, and | ||
| 3804 | /// returns the result. If the shift count for any element is greater than | ||
| 3805 | /// 31, the result for that element is 0 or -1 according to the sign bit | ||
| 3806 | /// for that element. | ||
| 3807 | /// | ||
| 3808 | /// \headerfile <immintrin.h> | ||
| 3809 | /// | ||
| 3810 | /// This intrinsic corresponds to the \c VPSRAVD instruction. | ||
| 3811 | /// | ||
| 3812 | /// \param __X | ||
| 3813 | /// A 256-bit vector of [8 x i32] to be shifted. | ||
| 3814 | /// \param __Y | ||
| 3815 | /// A 256-bit vector of [8 x i32] containing the unsigned shift counts (in | ||
| 3816 | /// bits). | ||
| 3817 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 902 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 3818 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 903 | _mm256_srav_epi32(__m256i __X, __m256i __Y) | 3819 | _mm256_srav_epi32(__m256i __X, __m256i __Y) |
| 904 | { | 3820 | { |
| 905 | return (__m256i)__builtin_ia32_psrav8si((__v8si)__X, (__v8si)__Y); | 3821 | return (__m256i)__builtin_ia32_psrav8si((__v8si)__X, (__v8si)__Y); |
| 906 | } | 3822 | } |
| 907 | 3823 | ||
| 3824 | /// Shifts each 32-bit element of the 128-bit vector of [4 x i32] in \a __X | ||
| 3825 | /// right by the number of bits given in the corresponding element of the | ||
| 3826 | /// 128-bit vector of [4 x i32] in \a __Y, shifting in sign bits, and | ||
| 3827 | /// returns the result. If the shift count for any element is greater than | ||
| 3828 | /// 31, the result for that element is 0 or -1 according to the sign bit | ||
| 3829 | /// for that element. | ||
| 3830 | /// | ||
| 3831 | /// \headerfile <immintrin.h> | ||
| 3832 | /// | ||
| 3833 | /// This intrinsic corresponds to the \c VPSRAVD instruction. | ||
| 3834 | /// | ||
| 3835 | /// \param __X | ||
| 3836 | /// A 128-bit vector of [4 x i32] to be shifted. | ||
| 3837 | /// \param __Y | ||
| 3838 | /// A 128-bit vector of [4 x i32] containing the unsigned shift counts (in | ||
| 3839 | /// bits). | ||
| 3840 | /// \returns A 128-bit vector of [4 x i32] containing the result. | ||
| 908 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 3841 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 909 | _mm_srav_epi32(__m128i __X, __m128i __Y) | 3842 | _mm_srav_epi32(__m128i __X, __m128i __Y) |
| 910 | { | 3843 | { |
| 911 | return (__m128i)__builtin_ia32_psrav4si((__v4si)__X, (__v4si)__Y); | 3844 | return (__m128i)__builtin_ia32_psrav4si((__v4si)__X, (__v4si)__Y); |
| 912 | } | 3845 | } |
| 913 | 3846 | ||
| 3847 | /// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __X | ||
| 3848 | /// right by the number of bits given in the corresponding element of the | ||
| 3849 | /// 256-bit vector of [8 x i32] in \a __Y, shifting in zero bits, and | ||
| 3850 | /// returns the result. If the shift count for any element is greater than | ||
| 3851 | /// 31, the result for that element is zero. | ||
| 3852 | /// | ||
| 3853 | /// \headerfile <immintrin.h> | ||
| 3854 | /// | ||
| 3855 | /// This intrinsic corresponds to the \c VPSRLVD instruction. | ||
| 3856 | /// | ||
| 3857 | /// \param __X | ||
| 3858 | /// A 256-bit vector of [8 x i32] to be shifted. | ||
| 3859 | /// \param __Y | ||
| 3860 | /// A 256-bit vector of [8 x i32] containing the unsigned shift counts (in | ||
| 3861 | /// bits). | ||
| 3862 | /// \returns A 256-bit vector of [8 x i32] containing the result. | ||
| 914 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 3863 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 915 | _mm256_srlv_epi32(__m256i __X, __m256i __Y) | 3864 | _mm256_srlv_epi32(__m256i __X, __m256i __Y) |
| 916 | { | 3865 | { |
| 917 | return (__m256i)__builtin_ia32_psrlv8si((__v8si)__X, (__v8si)__Y); | 3866 | return (__m256i)__builtin_ia32_psrlv8si((__v8si)__X, (__v8si)__Y); |
| 918 | } | 3867 | } |
| 919 | 3868 | ||
| 3869 | /// Shifts each 32-bit element of the 128-bit vector of [4 x i32] in \a __X | ||
| 3870 | /// right by the number of bits given in the corresponding element of the | ||
| 3871 | /// 128-bit vector of [4 x i32] in \a __Y, shifting in zero bits, and | ||
| 3872 | /// returns the result. If the shift count for any element is greater than | ||
| 3873 | /// 31, the result for that element is zero. | ||
| 3874 | /// | ||
| 3875 | /// \headerfile <immintrin.h> | ||
| 3876 | /// | ||
| 3877 | /// This intrinsic corresponds to the \c VPSRLVD instruction. | ||
| 3878 | /// | ||
| 3879 | /// \param __X | ||
| 3880 | /// A 128-bit vector of [4 x i32] to be shifted. | ||
| 3881 | /// \param __Y | ||
| 3882 | /// A 128-bit vector of [4 x i32] containing the unsigned shift counts (in | ||
| 3883 | /// bits). | ||
| 3884 | /// \returns A 128-bit vector of [4 x i32] containing the result. | ||
| 920 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 3885 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 921 | _mm_srlv_epi32(__m128i __X, __m128i __Y) | 3886 | _mm_srlv_epi32(__m128i __X, __m128i __Y) |
| 922 | { | 3887 | { |
| 923 | return (__m128i)__builtin_ia32_psrlv4si((__v4si)__X, (__v4si)__Y); | 3888 | return (__m128i)__builtin_ia32_psrlv4si((__v4si)__X, (__v4si)__Y); |
| 924 | } | 3889 | } |
| 925 | 3890 | ||
| 3891 | /// Shifts each 64-bit element of the 256-bit vector of [4 x i64] in \a __X | ||
| 3892 | /// right by the number of bits given in the corresponding element of the | ||
| 3893 | /// 128-bit vector of [4 x i64] in \a __Y, shifting in zero bits, and | ||
| 3894 | /// returns the result. If the shift count for any element is greater than | ||
| 3895 | /// 63, the result for that element is zero. | ||
| 3896 | /// | ||
| 3897 | /// \headerfile <immintrin.h> | ||
| 3898 | /// | ||
| 3899 | /// This intrinsic corresponds to the \c VPSRLVQ instruction. | ||
| 3900 | /// | ||
| 3901 | /// \param __X | ||
| 3902 | /// A 256-bit vector of [4 x i64] to be shifted. | ||
| 3903 | /// \param __Y | ||
| 3904 | /// A 256-bit vector of [4 x i64] containing the unsigned shift counts (in | ||
| 3905 | /// bits). | ||
| 3906 | /// \returns A 256-bit vector of [4 x i64] containing the result. | ||
| 926 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | 3907 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 |
| 927 | _mm256_srlv_epi64(__m256i __X, __m256i __Y) | 3908 | _mm256_srlv_epi64(__m256i __X, __m256i __Y) |
| 928 | { | 3909 | { |
| 929 | return (__m256i)__builtin_ia32_psrlv4di((__v4di)__X, (__v4di)__Y); | 3910 | return (__m256i)__builtin_ia32_psrlv4di((__v4di)__X, (__v4di)__Y); |
| 930 | } | 3911 | } |
| 931 | 3912 | ||
| 3913 | /// Shifts each 64-bit element of the 128-bit vector of [2 x i64] in \a __X | ||
| 3914 | /// right by the number of bits given in the corresponding element of the | ||
| 3915 | /// 128-bit vector of [2 x i64] in \a __Y, shifting in zero bits, and | ||
| 3916 | /// returns the result. If the shift count for any element is greater than | ||
| 3917 | /// 63, the result for that element is zero. | ||
| 3918 | /// | ||
| 3919 | /// \headerfile <immintrin.h> | ||
| 3920 | /// | ||
| 3921 | /// This intrinsic corresponds to the \c VPSRLVQ instruction. | ||
| 3922 | /// | ||
| 3923 | /// \param __X | ||
| 3924 | /// A 128-bit vector of [2 x i64] to be shifted. | ||
| 3925 | /// \param __Y | ||
| 3926 | /// A 128-bit vector of [2 x i64] containing the unsigned shift counts (in | ||
| 3927 | /// bits). | ||
| 3928 | /// \returns A 128-bit vector of [2 x i64] containing the result. | ||
| 932 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 | 3929 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 |
| 933 | _mm_srlv_epi64(__m128i __X, __m128i __Y) | 3930 | _mm_srlv_epi64(__m128i __X, __m128i __Y) |
| 934 | { | 3931 | { |
| 935 | return (__m128i)__builtin_ia32_psrlv2di((__v2di)__X, (__v2di)__Y); | 3932 | return (__m128i)__builtin_ia32_psrlv2di((__v2di)__X, (__v2di)__Y); |
| 936 | } | 3933 | } |
| 937 | 3934 | ||
| 3935 | /// Conditionally gathers two 64-bit floating-point values, either from the | ||
| 3936 | /// 128-bit vector of [2 x double] in \a a, or from memory \a m using scaled | ||
| 3937 | /// indexes from the 128-bit vector of [4 x i32] in \a i. The 128-bit vector | ||
| 3938 | /// of [2 x double] in \a mask determines the source for each element. | ||
| 3939 | /// | ||
| 3940 | /// \code{.operation} | ||
| 3941 | /// FOR element := 0 to 1 | ||
| 3942 | /// j := element*64 | ||
| 3943 | /// k := element*32 | ||
| 3944 | /// IF mask[j+63] == 0 | ||
| 3945 | /// result[j+63:j] := a[j+63:j] | ||
| 3946 | /// ELSE | ||
| 3947 | /// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s) | ||
| 3948 | /// FI | ||
| 3949 | /// ENDFOR | ||
| 3950 | /// \endcode | ||
| 3951 | /// | ||
| 3952 | /// \headerfile <immintrin.h> | ||
| 3953 | /// | ||
| 3954 | /// \code | ||
| 3955 | /// __m128d _mm_mask_i32gather_pd(__m128d a, const double *m, __m128i i, | ||
| 3956 | /// __m128d mask, const int s); | ||
| 3957 | /// \endcode | ||
| 3958 | /// | ||
| 3959 | /// This intrinsic corresponds to the \c VGATHERDPD instruction. | ||
| 3960 | /// | ||
| 3961 | /// \param a | ||
| 3962 | /// A 128-bit vector of [2 x double] used as the source when a mask bit is | ||
| 3963 | /// zero. | ||
| 3964 | /// \param m | ||
| 3965 | /// A pointer to the memory used for loading values. | ||
| 3966 | /// \param i | ||
| 3967 | /// A 128-bit vector of [4 x i32] containing signed indexes into \a m. Only | ||
| 3968 | /// the first two elements are used. | ||
| 3969 | /// \param mask | ||
| 3970 | /// A 128-bit vector of [2 x double] containing the mask. The most | ||
| 3971 | /// significant bit of each element in the mask vector represents the mask | ||
| 3972 | /// bits. If a mask bit is zero, the corresponding value from vector \a a | ||
| 3973 | /// is gathered; otherwise the value is loaded from memory. | ||
| 3974 | /// \param s | ||
| 3975 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 3976 | /// 1, 2, 4, or 8. | ||
| 3977 | /// \returns A 128-bit vector of [2 x double] containing the gathered values. | ||
| 938 | #define _mm_mask_i32gather_pd(a, m, i, mask, s) \ | 3978 | #define _mm_mask_i32gather_pd(a, m, i, mask, s) \ |
| 939 | ((__m128d)__builtin_ia32_gatherd_pd((__v2df)(__m128i)(a), \ | 3979 | ((__m128d)__builtin_ia32_gatherd_pd((__v2df)(__m128i)(a), \ |
| 940 | (double const *)(m), \ | 3980 | (double const *)(m), \ |
| 941 | (__v4si)(__m128i)(i), \ | 3981 | (__v4si)(__m128i)(i), \ |
| 942 | (__v2df)(__m128d)(mask), (s))) | 3982 | (__v2df)(__m128d)(mask), (s))) |
| 943 | 3983 | ||
| 3984 | /// Conditionally gathers four 64-bit floating-point values, either from the | ||
| 3985 | /// 256-bit vector of [4 x double] in \a a, or from memory \a m using scaled | ||
| 3986 | /// indexes from the 128-bit vector of [4 x i32] in \a i. The 256-bit vector | ||
| 3987 | /// of [4 x double] in \a mask determines the source for each element. | ||
| 3988 | /// | ||
| 3989 | /// \code{.operation} | ||
| 3990 | /// FOR element := 0 to 3 | ||
| 3991 | /// j := element*64 | ||
| 3992 | /// k := element*32 | ||
| 3993 | /// IF mask[j+63] == 0 | ||
| 3994 | /// result[j+63:j] := a[j+63:j] | ||
| 3995 | /// ELSE | ||
| 3996 | /// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s) | ||
| 3997 | /// FI | ||
| 3998 | /// ENDFOR | ||
| 3999 | /// \endcode | ||
| 4000 | /// | ||
| 4001 | /// \headerfile <immintrin.h> | ||
| 4002 | /// | ||
| 4003 | /// \code | ||
| 4004 | /// __m256d _mm256_mask_i32gather_pd(__m256d a, const double *m, __m128i i, | ||
| 4005 | /// __m256d mask, const int s); | ||
| 4006 | /// \endcode | ||
| 4007 | /// | ||
| 4008 | /// This intrinsic corresponds to the \c VGATHERDPD instruction. | ||
| 4009 | /// | ||
| 4010 | /// \param a | ||
| 4011 | /// A 256-bit vector of [4 x double] used as the source when a mask bit is | ||
| 4012 | /// zero. | ||
| 4013 | /// \param m | ||
| 4014 | /// A pointer to the memory used for loading values. | ||
| 4015 | /// \param i | ||
| 4016 | /// A 128-bit vector of [4 x i32] containing signed indexes into \a m. | ||
| 4017 | /// \param mask | ||
| 4018 | /// A 256-bit vector of [4 x double] containing the mask. The most | ||
| 4019 | /// significant bit of each element in the mask vector represents the mask | ||
| 4020 | /// bits. If a mask bit is zero, the corresponding value from vector \a a | ||
| 4021 | /// is gathered; otherwise the value is loaded from memory. | ||
| 4022 | /// \param s | ||
| 4023 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4024 | /// 1, 2, 4, or 8. | ||
| 4025 | /// \returns A 256-bit vector of [4 x double] containing the gathered values. | ||
| 944 | #define _mm256_mask_i32gather_pd(a, m, i, mask, s) \ | 4026 | #define _mm256_mask_i32gather_pd(a, m, i, mask, s) \ |
| 945 | ((__m256d)__builtin_ia32_gatherd_pd256((__v4df)(__m256d)(a), \ | 4027 | ((__m256d)__builtin_ia32_gatherd_pd256((__v4df)(__m256d)(a), \ |
| 946 | (double const *)(m), \ | 4028 | (double const *)(m), \ |
| 947 | (__v4si)(__m128i)(i), \ | 4029 | (__v4si)(__m128i)(i), \ |
| 948 | (__v4df)(__m256d)(mask), (s))) | 4030 | (__v4df)(__m256d)(mask), (s))) |
| 949 | 4031 | ||
| 4032 | /// Conditionally gathers two 64-bit floating-point values, either from the | ||
| 4033 | /// 128-bit vector of [2 x double] in \a a, or from memory \a m using scaled | ||
| 4034 | /// indexes from the 128-bit vector of [2 x i64] in \a i. The 128-bit vector | ||
| 4035 | /// of [2 x double] in \a mask determines the source for each element. | ||
| 4036 | /// | ||
| 4037 | /// \code{.operation} | ||
| 4038 | /// FOR element := 0 to 1 | ||
| 4039 | /// j := element*64 | ||
| 4040 | /// k := element*64 | ||
| 4041 | /// IF mask[j+63] == 0 | ||
| 4042 | /// result[j+63:j] := a[j+63:j] | ||
| 4043 | /// ELSE | ||
| 4044 | /// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s) | ||
| 4045 | /// FI | ||
| 4046 | /// ENDFOR | ||
| 4047 | /// \endcode | ||
| 4048 | /// | ||
| 4049 | /// \headerfile <immintrin.h> | ||
| 4050 | /// | ||
| 4051 | /// \code | ||
| 4052 | /// __m128d _mm_mask_i64gather_pd(__m128d a, const double *m, __m128i i, | ||
| 4053 | /// __m128d mask, const int s); | ||
| 4054 | /// \endcode | ||
| 4055 | /// | ||
| 4056 | /// This intrinsic corresponds to the \c VGATHERQPD instruction. | ||
| 4057 | /// | ||
| 4058 | /// \param a | ||
| 4059 | /// A 128-bit vector of [2 x double] used as the source when a mask bit is | ||
| 4060 | /// zero. | ||
| 4061 | /// \param m | ||
| 4062 | /// A pointer to the memory used for loading values. | ||
| 4063 | /// \param i | ||
| 4064 | /// A 128-bit vector of [2 x i64] containing signed indexes into \a m. | ||
| 4065 | /// \param mask | ||
| 4066 | /// A 128-bit vector of [2 x double] containing the mask. The most | ||
| 4067 | /// significant bit of each element in the mask vector represents the mask | ||
| 4068 | /// bits. If a mask bit is zero, the corresponding value from vector \a a | ||
| 4069 | /// is gathered; otherwise the value is loaded from memory. | ||
| 4070 | /// \param s | ||
| 4071 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4072 | /// 1, 2, 4, or 8. | ||
| 4073 | /// \returns A 128-bit vector of [2 x double] containing the gathered values. | ||
| 950 | #define _mm_mask_i64gather_pd(a, m, i, mask, s) \ | 4074 | #define _mm_mask_i64gather_pd(a, m, i, mask, s) \ |
| 951 | ((__m128d)__builtin_ia32_gatherq_pd((__v2df)(__m128d)(a), \ | 4075 | ((__m128d)__builtin_ia32_gatherq_pd((__v2df)(__m128d)(a), \ |
| 952 | (double const *)(m), \ | 4076 | (double const *)(m), \ |
| 953 | (__v2di)(__m128i)(i), \ | 4077 | (__v2di)(__m128i)(i), \ |
| 954 | (__v2df)(__m128d)(mask), (s))) | 4078 | (__v2df)(__m128d)(mask), (s))) |
| 955 | 4079 | ||
| 4080 | /// Conditionally gathers four 64-bit floating-point values, either from the | ||
| 4081 | /// 256-bit vector of [4 x double] in \a a, or from memory \a m using scaled | ||
| 4082 | /// indexes from the 256-bit vector of [4 x i64] in \a i. The 256-bit vector | ||
| 4083 | /// of [4 x double] in \a mask determines the source for each element. | ||
| 4084 | /// | ||
| 4085 | /// \code{.operation} | ||
| 4086 | /// FOR element := 0 to 3 | ||
| 4087 | /// j := element*64 | ||
| 4088 | /// k := element*64 | ||
| 4089 | /// IF mask[j+63] == 0 | ||
| 4090 | /// result[j+63:j] := a[j+63:j] | ||
| 4091 | /// ELSE | ||
| 4092 | /// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s) | ||
| 4093 | /// FI | ||
| 4094 | /// ENDFOR | ||
| 4095 | /// \endcode | ||
| 4096 | /// | ||
| 4097 | /// \headerfile <immintrin.h> | ||
| 4098 | /// | ||
| 4099 | /// \code | ||
| 4100 | /// __m256d _mm256_mask_i64gather_pd(__m256d a, const double *m, __m256i i, | ||
| 4101 | /// __m256d mask, const int s); | ||
| 4102 | /// \endcode | ||
| 4103 | /// | ||
| 4104 | /// This intrinsic corresponds to the \c VGATHERQPD instruction. | ||
| 4105 | /// | ||
| 4106 | /// \param a | ||
| 4107 | /// A 256-bit vector of [4 x double] used as the source when a mask bit is | ||
| 4108 | /// zero. | ||
| 4109 | /// \param m | ||
| 4110 | /// A pointer to the memory used for loading values. | ||
| 4111 | /// \param i | ||
| 4112 | /// A 256-bit vector of [4 x i64] containing signed indexes into \a m. | ||
| 4113 | /// \param mask | ||
| 4114 | /// A 256-bit vector of [4 x double] containing the mask. The most | ||
| 4115 | /// significant bit of each element in the mask vector represents the mask | ||
| 4116 | /// bits. If a mask bit is zero, the corresponding value from vector \a a | ||
| 4117 | /// is gathered; otherwise the value is loaded from memory. | ||
| 4118 | /// \param s | ||
| 4119 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4120 | /// 1, 2, 4, or 8. | ||
| 4121 | /// \returns A 256-bit vector of [4 x double] containing the gathered values. | ||
| 956 | #define _mm256_mask_i64gather_pd(a, m, i, mask, s) \ | 4122 | #define _mm256_mask_i64gather_pd(a, m, i, mask, s) \ |
| 957 | ((__m256d)__builtin_ia32_gatherq_pd256((__v4df)(__m256d)(a), \ | 4123 | ((__m256d)__builtin_ia32_gatherq_pd256((__v4df)(__m256d)(a), \ |
| 958 | (double const *)(m), \ | 4124 | (double const *)(m), \ |
| 959 | (__v4di)(__m256i)(i), \ | 4125 | (__v4di)(__m256i)(i), \ |
| 960 | (__v4df)(__m256d)(mask), (s))) | 4126 | (__v4df)(__m256d)(mask), (s))) |
| 961 | 4127 | ||
| 4128 | /// Conditionally gathers four 32-bit floating-point values, either from the | ||
| 4129 | /// 128-bit vector of [4 x float] in \a a, or from memory \a m using scaled | ||
| 4130 | /// indexes from the 128-bit vector of [4 x i32] in \a i. The 128-bit vector | ||
| 4131 | /// of [4 x float] in \a mask determines the source for each element. | ||
| 4132 | /// | ||
| 4133 | /// \code{.operation} | ||
| 4134 | /// FOR element := 0 to 3 | ||
| 4135 | /// j := element*32 | ||
| 4136 | /// k := element*32 | ||
| 4137 | /// IF mask[j+31] == 0 | ||
| 4138 | /// result[j+31:j] := a[j+31:j] | ||
| 4139 | /// ELSE | ||
| 4140 | /// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s) | ||
| 4141 | /// FI | ||
| 4142 | /// ENDFOR | ||
| 4143 | /// \endcode | ||
| 4144 | /// | ||
| 4145 | /// \headerfile <immintrin.h> | ||
| 4146 | /// | ||
| 4147 | /// \code | ||
| 4148 | /// __m128 _mm_mask_i32gather_ps(__m128 a, const float *m, __m128i i, | ||
| 4149 | /// __m128 mask, const int s); | ||
| 4150 | /// \endcode | ||
| 4151 | /// | ||
| 4152 | /// This intrinsic corresponds to the \c VGATHERDPS instruction. | ||
| 4153 | /// | ||
| 4154 | /// \param a | ||
| 4155 | /// A 128-bit vector of [4 x float] used as the source when a mask bit is | ||
| 4156 | /// zero. | ||
| 4157 | /// \param m | ||
| 4158 | /// A pointer to the memory used for loading values. | ||
| 4159 | /// \param i | ||
| 4160 | /// A 128-bit vector of [4 x i32] containing signed indexes into \a m. | ||
| 4161 | /// \param mask | ||
| 4162 | /// A 128-bit vector of [4 x float] containing the mask. The most | ||
| 4163 | /// significant bit of each element in the mask vector represents the mask | ||
| 4164 | /// bits. If a mask bit is zero, the corresponding value from vector \a a | ||
| 4165 | /// is gathered; otherwise the value is loaded from memory. | ||
| 4166 | /// \param s | ||
| 4167 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4168 | /// 1, 2, 4, or 8. | ||
| 4169 | /// \returns A 128-bit vector of [4 x float] containing the gathered values. | ||
| 962 | #define _mm_mask_i32gather_ps(a, m, i, mask, s) \ | 4170 | #define _mm_mask_i32gather_ps(a, m, i, mask, s) \ |
| 963 | ((__m128)__builtin_ia32_gatherd_ps((__v4sf)(__m128)(a), \ | 4171 | ((__m128)__builtin_ia32_gatherd_ps((__v4sf)(__m128)(a), \ |
| 964 | (float const *)(m), \ | 4172 | (float const *)(m), \ |
| 965 | (__v4si)(__m128i)(i), \ | 4173 | (__v4si)(__m128i)(i), \ |
| 966 | (__v4sf)(__m128)(mask), (s))) | 4174 | (__v4sf)(__m128)(mask), (s))) |
| 967 | 4175 | ||
| 4176 | /// Conditionally gathers eight 32-bit floating-point values, either from the | ||
| 4177 | /// 256-bit vector of [8 x float] in \a a, or from memory \a m using scaled | ||
| 4178 | /// indexes from the 256-bit vector of [8 x i32] in \a i. The 256-bit vector | ||
| 4179 | /// of [8 x float] in \a mask determines the source for each element. | ||
| 4180 | /// | ||
| 4181 | /// \code{.operation} | ||
| 4182 | /// FOR element := 0 to 7 | ||
| 4183 | /// j := element*32 | ||
| 4184 | /// k := element*32 | ||
| 4185 | /// IF mask[j+31] == 0 | ||
| 4186 | /// result[j+31:j] := a[j+31:j] | ||
| 4187 | /// ELSE | ||
| 4188 | /// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s) | ||
| 4189 | /// FI | ||
| 4190 | /// ENDFOR | ||
| 4191 | /// \endcode | ||
| 4192 | /// | ||
| 4193 | /// \headerfile <immintrin.h> | ||
| 4194 | /// | ||
| 4195 | /// \code | ||
| 4196 | /// __m256 _mm256_mask_i32gather_ps(__m256 a, const float *m, __m256i i, | ||
| 4197 | /// __m256 mask, const int s); | ||
| 4198 | /// \endcode | ||
| 4199 | /// | ||
| 4200 | /// This intrinsic corresponds to the \c VGATHERDPS instruction. | ||
| 4201 | /// | ||
| 4202 | /// \param a | ||
| 4203 | /// A 256-bit vector of [8 x float] used as the source when a mask bit is | ||
| 4204 | /// zero. | ||
| 4205 | /// \param m | ||
| 4206 | /// A pointer to the memory used for loading values. | ||
| 4207 | /// \param i | ||
| 4208 | /// A 256-bit vector of [8 x i32] containing signed indexes into \a m. | ||
| 4209 | /// \param mask | ||
| 4210 | /// A 256-bit vector of [8 x float] containing the mask. The most | ||
| 4211 | /// significant bit of each element in the mask vector represents the mask | ||
| 4212 | /// bits. If a mask bit is zero, the corresponding value from vector \a a | ||
| 4213 | /// is gathered; otherwise the value is loaded from memory. | ||
| 4214 | /// \param s | ||
| 4215 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4216 | /// 1, 2, 4, or 8. | ||
| 4217 | /// \returns A 256-bit vector of [8 x float] containing the gathered values. | ||
| 968 | #define _mm256_mask_i32gather_ps(a, m, i, mask, s) \ | 4218 | #define _mm256_mask_i32gather_ps(a, m, i, mask, s) \ |
| 969 | ((__m256)__builtin_ia32_gatherd_ps256((__v8sf)(__m256)(a), \ | 4219 | ((__m256)__builtin_ia32_gatherd_ps256((__v8sf)(__m256)(a), \ |
| 970 | (float const *)(m), \ | 4220 | (float const *)(m), \ |
| 971 | (__v8si)(__m256i)(i), \ | 4221 | (__v8si)(__m256i)(i), \ |
| 972 | (__v8sf)(__m256)(mask), (s))) | 4222 | (__v8sf)(__m256)(mask), (s))) |
| 973 | 4223 | ||
| 4224 | /// Conditionally gathers two 32-bit floating-point values, either from the | ||
| 4225 | /// 128-bit vector of [4 x float] in \a a, or from memory \a m using scaled | ||
| 4226 | /// indexes from the 128-bit vector of [2 x i64] in \a i. The 128-bit vector | ||
| 4227 | /// of [4 x float] in \a mask determines the source for the lower two | ||
| 4228 | /// elements. The upper two elements of the result are zeroed. | ||
| 4229 | /// | ||
| 4230 | /// \code{.operation} | ||
| 4231 | /// FOR element := 0 to 1 | ||
| 4232 | /// j := element*32 | ||
| 4233 | /// k := element*64 | ||
| 4234 | /// IF mask[j+31] == 0 | ||
| 4235 | /// result[j+31:j] := a[j+31:j] | ||
| 4236 | /// ELSE | ||
| 4237 | /// result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s) | ||
| 4238 | /// FI | ||
| 4239 | /// ENDFOR | ||
| 4240 | /// result[127:64] := 0 | ||
| 4241 | /// \endcode | ||
| 4242 | /// | ||
| 4243 | /// \headerfile <immintrin.h> | ||
| 4244 | /// | ||
| 4245 | /// \code | ||
| 4246 | /// __m128 _mm_mask_i64gather_ps(__m128 a, const float *m, __m128i i, | ||
| 4247 | /// __m128 mask, const int s); | ||
| 4248 | /// \endcode | ||
| 4249 | /// | ||
| 4250 | /// This intrinsic corresponds to the \c VGATHERQPS instruction. | ||
| 4251 | /// | ||
| 4252 | /// \param a | ||
| 4253 | /// A 128-bit vector of [4 x float] used as the source when a mask bit is | ||
| 4254 | /// zero. Only the first two elements are used. | ||
| 4255 | /// \param m | ||
| 4256 | /// A pointer to the memory used for loading values. | ||
| 4257 | /// \param i | ||
| 4258 | /// A 128-bit vector of [2 x i64] containing signed indexes into \a m. | ||
| 4259 | /// \param mask | ||
| 4260 | /// A 128-bit vector of [4 x float] containing the mask. The most | ||
| 4261 | /// significant bit of each element in the mask vector represents the mask | ||
| 4262 | /// bits. If a mask bit is zero, the corresponding value from vector \a a | ||
| 4263 | /// is gathered; otherwise the value is loaded from memory. Only the first | ||
| 4264 | /// two elements are used. | ||
| 4265 | /// \param s | ||
| 4266 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4267 | /// 1, 2, 4, or 8. | ||
| 4268 | /// \returns A 128-bit vector of [4 x float] containing the gathered values. | ||
| 974 | #define _mm_mask_i64gather_ps(a, m, i, mask, s) \ | 4269 | #define _mm_mask_i64gather_ps(a, m, i, mask, s) \ |
| 975 | ((__m128)__builtin_ia32_gatherq_ps((__v4sf)(__m128)(a), \ | 4270 | ((__m128)__builtin_ia32_gatherq_ps((__v4sf)(__m128)(a), \ |
| 976 | (float const *)(m), \ | 4271 | (float const *)(m), \ |
| 977 | (__v2di)(__m128i)(i), \ | 4272 | (__v2di)(__m128i)(i), \ |
| 978 | (__v4sf)(__m128)(mask), (s))) | 4273 | (__v4sf)(__m128)(mask), (s))) |
| 979 | 4274 | ||
| 4275 | /// Conditionally gathers four 32-bit floating-point values, either from the | ||
| 4276 | /// 128-bit vector of [4 x float] in \a a, or from memory \a m using scaled | ||
| 4277 | /// indexes from the 256-bit vector of [4 x i64] in \a i. The 128-bit vector | ||
| 4278 | /// of [4 x float] in \a mask determines the source for each element. | ||
| 4279 | /// | ||
| 4280 | /// \code{.operation} | ||
| 4281 | /// FOR element := 0 to 3 | ||
| 4282 | /// j := element*32 | ||
| 4283 | /// k := element*64 | ||
| 4284 | /// IF mask[j+31] == 0 | ||
| 4285 | /// result[j+31:j] := a[j+31:j] | ||
| 4286 | /// ELSE | ||
| 4287 | /// result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s) | ||
| 4288 | /// FI | ||
| 4289 | /// ENDFOR | ||
| 4290 | /// \endcode | ||
| 4291 | /// | ||
| 4292 | /// \headerfile <immintrin.h> | ||
| 4293 | /// | ||
| 4294 | /// \code | ||
| 4295 | /// __m128 _mm256_mask_i64gather_ps(__m128 a, const float *m, __m256i i, | ||
| 4296 | /// __m128 mask, const int s); | ||
| 4297 | /// \endcode | ||
| 4298 | /// | ||
| 4299 | /// This intrinsic corresponds to the \c VGATHERQPS instruction. | ||
| 4300 | /// | ||
| 4301 | /// \param a | ||
| 4302 | /// A 128-bit vector of [4 x float] used as the source when a mask bit is | ||
| 4303 | /// zero. | ||
| 4304 | /// \param m | ||
| 4305 | /// A pointer to the memory used for loading values. | ||
| 4306 | /// \param i | ||
| 4307 | /// A 256-bit vector of [4 x i64] containing signed indexes into \a m. | ||
| 4308 | /// \param mask | ||
| 4309 | /// A 128-bit vector of [4 x float] containing the mask. The most | ||
| 4310 | /// significant bit of each element in the mask vector represents the mask | ||
| 4311 | /// bits. If a mask bit is zero, the corresponding value from vector \a a | ||
| 4312 | /// is gathered; otherwise the value is loaded from memory. | ||
| 4313 | /// \param s | ||
| 4314 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4315 | /// 1, 2, 4, or 8. | ||
| 4316 | /// \returns A 128-bit vector of [4 x float] containing the gathered values. | ||
| 980 | #define _mm256_mask_i64gather_ps(a, m, i, mask, s) \ | 4317 | #define _mm256_mask_i64gather_ps(a, m, i, mask, s) \ |
| 981 | ((__m128)__builtin_ia32_gatherq_ps256((__v4sf)(__m128)(a), \ | 4318 | ((__m128)__builtin_ia32_gatherq_ps256((__v4sf)(__m128)(a), \ |
| 982 | (float const *)(m), \ | 4319 | (float const *)(m), \ |
| 983 | (__v4di)(__m256i)(i), \ | 4320 | (__v4di)(__m256i)(i), \ |
| 984 | (__v4sf)(__m128)(mask), (s))) | 4321 | (__v4sf)(__m128)(mask), (s))) |
| 985 | 4322 | ||
| 4323 | /// Conditionally gathers four 32-bit integer values, either from the | ||
| 4324 | /// 128-bit vector of [4 x i32] in \a a, or from memory \a m using scaled | ||
| 4325 | /// indexes from the 128-bit vector of [4 x i32] in \a i. The 128-bit vector | ||
| 4326 | /// of [4 x i32] in \a mask determines the source for each element. | ||
| 4327 | /// | ||
| 4328 | /// \code{.operation} | ||
| 4329 | /// FOR element := 0 to 3 | ||
| 4330 | /// j := element*32 | ||
| 4331 | /// k := element*32 | ||
| 4332 | /// IF mask[j+31] == 0 | ||
| 4333 | /// result[j+31:j] := a[j+31:j] | ||
| 4334 | /// ELSE | ||
| 4335 | /// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s) | ||
| 4336 | /// FI | ||
| 4337 | /// ENDFOR | ||
| 4338 | /// \endcode | ||
| 4339 | /// | ||
| 4340 | /// \headerfile <immintrin.h> | ||
| 4341 | /// | ||
| 4342 | /// \code | ||
| 4343 | /// __m128i _mm_mask_i32gather_epi32(__m128i a, const int *m, __m128i i, | ||
| 4344 | /// __m128i mask, const int s); | ||
| 4345 | /// \endcode | ||
| 4346 | /// | ||
| 4347 | /// This intrinsic corresponds to the \c VPGATHERDD instruction. | ||
| 4348 | /// | ||
| 4349 | /// \param a | ||
| 4350 | /// A 128-bit vector of [4 x i32] used as the source when a mask bit is | ||
| 4351 | /// zero. | ||
| 4352 | /// \param m | ||
| 4353 | /// A pointer to the memory used for loading values. | ||
| 4354 | /// \param i | ||
| 4355 | /// A 128-bit vector of [4 x i32] containing signed indexes into \a m. | ||
| 4356 | /// \param mask | ||
| 4357 | /// A 128-bit vector of [4 x i32] containing the mask. The most significant | ||
| 4358 | /// bit of each element in the mask vector represents the mask bits. If a | ||
| 4359 | /// mask bit is zero, the corresponding value from vector \a a is gathered; | ||
| 4360 | /// otherwise the value is loaded from memory. | ||
| 4361 | /// \param s | ||
| 4362 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4363 | /// 1, 2, 4, or 8. | ||
| 4364 | /// \returns A 128-bit vector of [4 x i32] containing the gathered values. | ||
| 986 | #define _mm_mask_i32gather_epi32(a, m, i, mask, s) \ | 4365 | #define _mm_mask_i32gather_epi32(a, m, i, mask, s) \ |
| 987 | ((__m128i)__builtin_ia32_gatherd_d((__v4si)(__m128i)(a), \ | 4366 | ((__m128i)__builtin_ia32_gatherd_d((__v4si)(__m128i)(a), \ |
| 988 | (int const *)(m), \ | 4367 | (int const *)(m), \ |
| 989 | (__v4si)(__m128i)(i), \ | 4368 | (__v4si)(__m128i)(i), \ |
| 990 | (__v4si)(__m128i)(mask), (s))) | 4369 | (__v4si)(__m128i)(mask), (s))) |
| 991 | 4370 | ||
| 4371 | /// Conditionally gathers eight 32-bit integer values, either from the | ||
| 4372 | /// 256-bit vector of [8 x i32] in \a a, or from memory \a m using scaled | ||
| 4373 | /// indexes from the 256-bit vector of [8 x i32] in \a i. The 256-bit vector | ||
| 4374 | /// of [8 x i32] in \a mask determines the source for each element. | ||
| 4375 | /// | ||
| 4376 | /// \code{.operation} | ||
| 4377 | /// FOR element := 0 to 7 | ||
| 4378 | /// j := element*32 | ||
| 4379 | /// k := element*32 | ||
| 4380 | /// IF mask[j+31] == 0 | ||
| 4381 | /// result[j+31:j] := a[j+31:j] | ||
| 4382 | /// ELSE | ||
| 4383 | /// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s) | ||
| 4384 | /// FI | ||
| 4385 | /// ENDFOR | ||
| 4386 | /// \endcode | ||
| 4387 | /// | ||
| 4388 | /// \headerfile <immintrin.h> | ||
| 4389 | /// | ||
| 4390 | /// \code | ||
| 4391 | /// __m256i _mm256_mask_i32gather_epi32(__m256i a, const int *m, __m256i i, | ||
| 4392 | /// __m256i mask, const int s); | ||
| 4393 | /// \endcode | ||
| 4394 | /// | ||
| 4395 | /// This intrinsic corresponds to the \c VPGATHERDD instruction. | ||
| 4396 | /// | ||
| 4397 | /// \param a | ||
| 4398 | /// A 256-bit vector of [8 x i32] used as the source when a mask bit is | ||
| 4399 | /// zero. | ||
| 4400 | /// \param m | ||
| 4401 | /// A pointer to the memory used for loading values. | ||
| 4402 | /// \param i | ||
| 4403 | /// A 256-bit vector of [8 x i32] containing signed indexes into \a m. | ||
| 4404 | /// \param mask | ||
| 4405 | /// A 256-bit vector of [8 x i32] containing the mask. The most significant | ||
| 4406 | /// bit of each element in the mask vector represents the mask bits. If a | ||
| 4407 | /// mask bit is zero, the corresponding value from vector \a a is gathered; | ||
| 4408 | /// otherwise the value is loaded from memory. | ||
| 4409 | /// \param s | ||
| 4410 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4411 | /// 1, 2, 4, or 8. | ||
| 4412 | /// \returns A 256-bit vector of [8 x i32] containing the gathered values. | ||
| 992 | #define _mm256_mask_i32gather_epi32(a, m, i, mask, s) \ | 4413 | #define _mm256_mask_i32gather_epi32(a, m, i, mask, s) \ |
| 993 | ((__m256i)__builtin_ia32_gatherd_d256((__v8si)(__m256i)(a), \ | 4414 | ((__m256i)__builtin_ia32_gatherd_d256((__v8si)(__m256i)(a), \ |
| 994 | (int const *)(m), \ | 4415 | (int const *)(m), \ |
| 995 | (__v8si)(__m256i)(i), \ | 4416 | (__v8si)(__m256i)(i), \ |
| 996 | (__v8si)(__m256i)(mask), (s))) | 4417 | (__v8si)(__m256i)(mask), (s))) |
| 997 | 4418 | ||
| 4419 | /// Conditionally gathers two 32-bit integer values, either from the | ||
| 4420 | /// 128-bit vector of [4 x i32] in \a a, or from memory \a m using scaled | ||
| 4421 | /// indexes from the 128-bit vector of [2 x i64] in \a i. The 128-bit vector | ||
| 4422 | /// of [4 x i32] in \a mask determines the source for the lower two | ||
| 4423 | /// elements. The upper two elements of the result are zeroed. | ||
| 4424 | /// | ||
| 4425 | /// \code{.operation} | ||
| 4426 | /// FOR element := 0 to 1 | ||
| 4427 | /// j := element*32 | ||
| 4428 | /// k := element*64 | ||
| 4429 | /// IF mask[j+31] == 0 | ||
| 4430 | /// result[j+31:j] := a[j+31:j] | ||
| 4431 | /// ELSE | ||
| 4432 | /// result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s) | ||
| 4433 | /// FI | ||
| 4434 | /// ENDFOR | ||
| 4435 | /// result[127:64] := 0 | ||
| 4436 | /// \endcode | ||
| 4437 | /// | ||
| 4438 | /// \headerfile <immintrin.h> | ||
| 4439 | /// | ||
| 4440 | /// \code | ||
| 4441 | /// __m128i _mm_mask_i64gather_epi32(__m128i a, const int *m, __m128i i, | ||
| 4442 | /// __m128i mask, const int s); | ||
| 4443 | /// \endcode | ||
| 4444 | /// | ||
| 4445 | /// This intrinsic corresponds to the \c VPGATHERQD instruction. | ||
| 4446 | /// | ||
| 4447 | /// \param a | ||
| 4448 | /// A 128-bit vector of [4 x i32] used as the source when a mask bit is | ||
| 4449 | /// zero. Only the first two elements are used. | ||
| 4450 | /// \param m | ||
| 4451 | /// A pointer to the memory used for loading values. | ||
| 4452 | /// \param i | ||
| 4453 | /// A 128-bit vector of [2 x i64] containing indexes into \a m. | ||
| 4454 | /// \param mask | ||
| 4455 | /// A 128-bit vector of [4 x i32] containing the mask. The most significant | ||
| 4456 | /// bit of each element in the mask vector represents the mask bits. If a | ||
| 4457 | /// mask bit is zero, the corresponding value from vector \a a is gathered; | ||
| 4458 | /// otherwise the value is loaded from memory. Only the first two elements | ||
| 4459 | /// are used. | ||
| 4460 | /// \param s | ||
| 4461 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4462 | /// 1, 2, 4, or 8. | ||
| 4463 | /// \returns A 128-bit vector of [4 x i32] containing the gathered values. | ||
| 998 | #define _mm_mask_i64gather_epi32(a, m, i, mask, s) \ | 4464 | #define _mm_mask_i64gather_epi32(a, m, i, mask, s) \ |
| 999 | ((__m128i)__builtin_ia32_gatherq_d((__v4si)(__m128i)(a), \ | 4465 | ((__m128i)__builtin_ia32_gatherq_d((__v4si)(__m128i)(a), \ |
| 1000 | (int const *)(m), \ | 4466 | (int const *)(m), \ |
| 1001 | (__v2di)(__m128i)(i), \ | 4467 | (__v2di)(__m128i)(i), \ |
| 1002 | (__v4si)(__m128i)(mask), (s))) | 4468 | (__v4si)(__m128i)(mask), (s))) |
| 1003 | 4469 | ||
| 4470 | /// Conditionally gathers four 32-bit integer values, either from the | ||
| 4471 | /// 128-bit vector of [4 x i32] in \a a, or from memory \a m using scaled | ||
| 4472 | /// indexes from the 256-bit vector of [4 x i64] in \a i. The 128-bit vector | ||
| 4473 | /// of [4 x i32] in \a mask determines the source for each element. | ||
| 4474 | /// | ||
| 4475 | /// \code{.operation} | ||
| 4476 | /// FOR element := 0 to 3 | ||
| 4477 | /// j := element*32 | ||
| 4478 | /// k := element*64 | ||
| 4479 | /// IF mask[j+31] == 0 | ||
| 4480 | /// result[j+31:j] := a[j+31:j] | ||
| 4481 | /// ELSE | ||
| 4482 | /// result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s) | ||
| 4483 | /// FI | ||
| 4484 | /// ENDFOR | ||
| 4485 | /// \endcode | ||
| 4486 | /// | ||
| 4487 | /// \headerfile <immintrin.h> | ||
| 4488 | /// | ||
| 4489 | /// \code | ||
| 4490 | /// __m128i _mm256_mask_i64gather_epi32(__m128i a, const int *m, __m256i i, | ||
| 4491 | /// __m128i mask, const int s); | ||
| 4492 | /// \endcode | ||
| 4493 | /// | ||
| 4494 | /// This intrinsic corresponds to the \c VPGATHERQD instruction. | ||
| 4495 | /// | ||
| 4496 | /// \param a | ||
| 4497 | /// A 128-bit vector of [4 x i32] used as the source when a mask bit is | ||
| 4498 | /// zero. | ||
| 4499 | /// \param m | ||
| 4500 | /// A pointer to the memory used for loading values. | ||
| 4501 | /// \param i | ||
| 4502 | /// A 256-bit vector of [4 x i64] containing signed indexes into \a m. | ||
| 4503 | /// \param mask | ||
| 4504 | /// A 128-bit vector of [4 x i32] containing the mask. The most significant | ||
| 4505 | /// bit of each element in the mask vector represents the mask bits. If a | ||
| 4506 | /// mask bit is zero, the corresponding value from vector \a a is gathered; | ||
| 4507 | /// otherwise the value is loaded from memory. | ||
| 4508 | /// \param s | ||
| 4509 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4510 | /// 1, 2, 4, or 8. | ||
| 4511 | /// \returns A 128-bit vector of [4 x i32] containing the gathered values. | ||
| 1004 | #define _mm256_mask_i64gather_epi32(a, m, i, mask, s) \ | 4512 | #define _mm256_mask_i64gather_epi32(a, m, i, mask, s) \ |
| 1005 | ((__m128i)__builtin_ia32_gatherq_d256((__v4si)(__m128i)(a), \ | 4513 | ((__m128i)__builtin_ia32_gatherq_d256((__v4si)(__m128i)(a), \ |
| 1006 | (int const *)(m), \ | 4514 | (int const *)(m), \ |
| 1007 | (__v4di)(__m256i)(i), \ | 4515 | (__v4di)(__m256i)(i), \ |
| 1008 | (__v4si)(__m128i)(mask), (s))) | 4516 | (__v4si)(__m128i)(mask), (s))) |
| 1009 | 4517 | ||
| 4518 | /// Conditionally gathers two 64-bit integer values, either from the | ||
| 4519 | /// 128-bit vector of [2 x i64] in \a a, or from memory \a m using scaled | ||
| 4520 | /// indexes from the 128-bit vector of [4 x i32] in \a i. The 128-bit vector | ||
| 4521 | /// of [2 x i64] in \a mask determines the source for each element. | ||
| 4522 | /// | ||
| 4523 | /// \code{.operation} | ||
| 4524 | /// FOR element := 0 to 1 | ||
| 4525 | /// j := element*64 | ||
| 4526 | /// k := element*32 | ||
| 4527 | /// IF mask[j+63] == 0 | ||
| 4528 | /// result[j+63:j] := a[j+63:j] | ||
| 4529 | /// ELSE | ||
| 4530 | /// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s) | ||
| 4531 | /// FI | ||
| 4532 | /// ENDFOR | ||
| 4533 | /// \endcode | ||
| 4534 | /// | ||
| 4535 | /// \headerfile <immintrin.h> | ||
| 4536 | /// | ||
| 4537 | /// \code | ||
| 4538 | /// __m128i _mm_mask_i32gather_epi64(__m128i a, const long long *m, __m128i i, | ||
| 4539 | /// __m128i mask, const int s); | ||
| 4540 | /// \endcode | ||
| 4541 | /// | ||
| 4542 | /// This intrinsic corresponds to the \c VPGATHERDQ instruction. | ||
| 4543 | /// | ||
| 4544 | /// \param a | ||
| 4545 | /// A 128-bit vector of [2 x i64] used as the source when a mask bit is | ||
| 4546 | /// zero. | ||
| 4547 | /// \param m | ||
| 4548 | /// A pointer to the memory used for loading values. | ||
| 4549 | /// \param i | ||
| 4550 | /// A 128-bit vector of [4 x i32] containing signed indexes into \a m. Only | ||
| 4551 | /// the first two elements are used. | ||
| 4552 | /// \param mask | ||
| 4553 | /// A 128-bit vector of [2 x i64] containing the mask. The most significant | ||
| 4554 | /// bit of each element in the mask vector represents the mask bits. If a | ||
| 4555 | /// mask bit is zero, the corresponding value from vector \a a is gathered; | ||
| 4556 | /// otherwise the value is loaded from memory. | ||
| 4557 | /// \param s | ||
| 4558 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4559 | /// 1, 2, 4, or 8. | ||
| 4560 | /// \returns A 128-bit vector of [2 x i64] containing the gathered values. | ||
| 1010 | #define _mm_mask_i32gather_epi64(a, m, i, mask, s) \ | 4561 | #define _mm_mask_i32gather_epi64(a, m, i, mask, s) \ |
| 1011 | ((__m128i)__builtin_ia32_gatherd_q((__v2di)(__m128i)(a), \ | 4562 | ((__m128i)__builtin_ia32_gatherd_q((__v2di)(__m128i)(a), \ |
| 1012 | (long long const *)(m), \ | 4563 | (long long const *)(m), \ |
| 1013 | (__v4si)(__m128i)(i), \ | 4564 | (__v4si)(__m128i)(i), \ |
| 1014 | (__v2di)(__m128i)(mask), (s))) | 4565 | (__v2di)(__m128i)(mask), (s))) |
| 1015 | 4566 | ||
| 4567 | /// Conditionally gathers four 64-bit integer values, either from the | ||
| 4568 | /// 256-bit vector of [4 x i64] in \a a, or from memory \a m using scaled | ||
| 4569 | /// indexes from the 128-bit vector of [4 x i32] in \a i. The 256-bit vector | ||
| 4570 | /// of [4 x i64] in \a mask determines the source for each element. | ||
| 4571 | /// | ||
| 4572 | /// \code{.operation} | ||
| 4573 | /// FOR element := 0 to 3 | ||
| 4574 | /// j := element*64 | ||
| 4575 | /// k := element*32 | ||
| 4576 | /// IF mask[j+63] == 0 | ||
| 4577 | /// result[j+63:j] := a[j+63:j] | ||
| 4578 | /// ELSE | ||
| 4579 | /// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s) | ||
| 4580 | /// FI | ||
| 4581 | /// ENDFOR | ||
| 4582 | /// \endcode | ||
| 4583 | /// | ||
| 4584 | /// \headerfile <immintrin.h> | ||
| 4585 | /// | ||
| 4586 | /// \code | ||
| 4587 | /// __m256i _mm256_mask_i32gather_epi64(__m256i a, const long long *m, | ||
| 4588 | /// __m128i i, __m256i mask, const int s); | ||
| 4589 | /// \endcode | ||
| 4590 | /// | ||
| 4591 | /// This intrinsic corresponds to the \c VPGATHERDQ instruction. | ||
| 4592 | /// | ||
| 4593 | /// \param a | ||
| 4594 | /// A 256-bit vector of [4 x i64] used as the source when a mask bit is | ||
| 4595 | /// zero. | ||
| 4596 | /// \param m | ||
| 4597 | /// A pointer to the memory used for loading values. | ||
| 4598 | /// \param i | ||
| 4599 | /// A 128-bit vector of [4 x i32] containing signed indexes into \a m. | ||
| 4600 | /// \param mask | ||
| 4601 | /// A 256-bit vector of [4 x i64] containing the mask. The most significant | ||
| 4602 | /// bit of each element in the mask vector represents the mask bits. If a | ||
| 4603 | /// mask bit is zero, the corresponding value from vector \a a is gathered; | ||
| 4604 | /// otherwise the value is loaded from memory. | ||
| 4605 | /// \param s | ||
| 4606 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4607 | /// 1, 2, 4, or 8. | ||
| 4608 | /// \returns A 256-bit vector of [4 x i64] containing the gathered values. | ||
| 1016 | #define _mm256_mask_i32gather_epi64(a, m, i, mask, s) \ | 4609 | #define _mm256_mask_i32gather_epi64(a, m, i, mask, s) \ |
| 1017 | ((__m256i)__builtin_ia32_gatherd_q256((__v4di)(__m256i)(a), \ | 4610 | ((__m256i)__builtin_ia32_gatherd_q256((__v4di)(__m256i)(a), \ |
| 1018 | (long long const *)(m), \ | 4611 | (long long const *)(m), \ |
| 1019 | (__v4si)(__m128i)(i), \ | 4612 | (__v4si)(__m128i)(i), \ |
| 1020 | (__v4di)(__m256i)(mask), (s))) | 4613 | (__v4di)(__m256i)(mask), (s))) |
| 1021 | 4614 | ||
| 4615 | /// Conditionally gathers two 64-bit integer values, either from the | ||
| 4616 | /// 128-bit vector of [2 x i64] in \a a, or from memory \a m using scaled | ||
| 4617 | /// indexes from the 128-bit vector of [2 x i64] in \a i. The 128-bit vector | ||
| 4618 | /// of [2 x i64] in \a mask determines the source for each element. | ||
| 4619 | /// | ||
| 4620 | /// \code{.operation} | ||
| 4621 | /// FOR element := 0 to 1 | ||
| 4622 | /// j := element*64 | ||
| 4623 | /// k := element*64 | ||
| 4624 | /// IF mask[j+63] == 0 | ||
| 4625 | /// result[j+63:j] := a[j+63:j] | ||
| 4626 | /// ELSE | ||
| 4627 | /// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s) | ||
| 4628 | /// FI | ||
| 4629 | /// ENDFOR | ||
| 4630 | /// \endcode | ||
| 4631 | /// | ||
| 4632 | /// \headerfile <immintrin.h> | ||
| 4633 | /// | ||
| 4634 | /// \code | ||
| 4635 | /// __m128i _mm_mask_i64gather_epi64(__m128i a, const long long *m, __m128i i, | ||
| 4636 | /// __m128i mask, const int s); | ||
| 4637 | /// \endcode | ||
| 4638 | /// | ||
| 4639 | /// This intrinsic corresponds to the \c VPGATHERQQ instruction. | ||
| 4640 | /// | ||
| 4641 | /// \param a | ||
| 4642 | /// A 128-bit vector of [2 x i64] used as the source when a mask bit is | ||
| 4643 | /// zero. | ||
| 4644 | /// \param m | ||
| 4645 | /// A pointer to the memory used for loading values. | ||
| 4646 | /// \param i | ||
| 4647 | /// A 128-bit vector of [2 x i64] containing signed indexes into \a m. | ||
| 4648 | /// \param mask | ||
| 4649 | /// A 128-bit vector of [2 x i64] containing the mask. The most significant | ||
| 4650 | /// bit of each element in the mask vector represents the mask bits. If a | ||
| 4651 | /// mask bit is zero, the corresponding value from vector \a a is gathered; | ||
| 4652 | /// otherwise the value is loaded from memory. | ||
| 4653 | /// \param s | ||
| 4654 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4655 | /// 1, 2, 4, or 8. | ||
| 4656 | /// \returns A 128-bit vector of [2 x i64] containing the gathered values. | ||
| 1022 | #define _mm_mask_i64gather_epi64(a, m, i, mask, s) \ | 4657 | #define _mm_mask_i64gather_epi64(a, m, i, mask, s) \ |
| 1023 | ((__m128i)__builtin_ia32_gatherq_q((__v2di)(__m128i)(a), \ | 4658 | ((__m128i)__builtin_ia32_gatherq_q((__v2di)(__m128i)(a), \ |
| 1024 | (long long const *)(m), \ | 4659 | (long long const *)(m), \ |
| 1025 | (__v2di)(__m128i)(i), \ | 4660 | (__v2di)(__m128i)(i), \ |
| 1026 | (__v2di)(__m128i)(mask), (s))) | 4661 | (__v2di)(__m128i)(mask), (s))) |
| 1027 | 4662 | ||
| 4663 | /// Conditionally gathers four 64-bit integer values, either from the | ||
| 4664 | /// 256-bit vector of [4 x i64] in \a a, or from memory \a m using scaled | ||
| 4665 | /// indexes from the 256-bit vector of [4 x i64] in \a i. The 256-bit vector | ||
| 4666 | /// of [4 x i64] in \a mask determines the source for each element. | ||
| 4667 | /// | ||
| 4668 | /// \code{.operation} | ||
| 4669 | /// FOR element := 0 to 3 | ||
| 4670 | /// j := element*64 | ||
| 4671 | /// k := element*64 | ||
| 4672 | /// IF mask[j+63] == 0 | ||
| 4673 | /// result[j+63:j] := a[j+63:j] | ||
| 4674 | /// ELSE | ||
| 4675 | /// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s) | ||
| 4676 | /// FI | ||
| 4677 | /// ENDFOR | ||
| 4678 | /// \endcode | ||
| 4679 | /// | ||
| 4680 | /// \headerfile <immintrin.h> | ||
| 4681 | /// | ||
| 4682 | /// \code | ||
| 4683 | /// __m256i _mm256_mask_i64gather_epi64(__m256i a, const long long *m, | ||
| 4684 | /// __m256i i, __m256i mask, const int s); | ||
| 4685 | /// \endcode | ||
| 4686 | /// | ||
| 4687 | /// This intrinsic corresponds to the \c VPGATHERQQ instruction. | ||
| 4688 | /// | ||
| 4689 | /// \param a | ||
| 4690 | /// A 256-bit vector of [4 x i64] used as the source when a mask bit is | ||
| 4691 | /// zero. | ||
| 4692 | /// \param m | ||
| 4693 | /// A pointer to the memory used for loading values. | ||
| 4694 | /// \param i | ||
| 4695 | /// A 256-bit vector of [4 x i64] containing signed indexes into \a m. | ||
| 4696 | /// \param mask | ||
| 4697 | /// A 256-bit vector of [4 x i64] containing the mask. The most significant | ||
| 4698 | /// bit of each element in the mask vector represents the mask bits. If a | ||
| 4699 | /// mask bit is zero, the corresponding value from vector \a a is gathered; | ||
| 4700 | /// otherwise the value is loaded from memory. | ||
| 4701 | /// \param s | ||
| 4702 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4703 | /// 1, 2, 4, or 8. | ||
| 4704 | /// \returns A 256-bit vector of [4 x i64] containing the gathered values. | ||
| 1028 | #define _mm256_mask_i64gather_epi64(a, m, i, mask, s) \ | 4705 | #define _mm256_mask_i64gather_epi64(a, m, i, mask, s) \ |
| 1029 | ((__m256i)__builtin_ia32_gatherq_q256((__v4di)(__m256i)(a), \ | 4706 | ((__m256i)__builtin_ia32_gatherq_q256((__v4di)(__m256i)(a), \ |
| 1030 | (long long const *)(m), \ | 4707 | (long long const *)(m), \ |
| 1031 | (__v4di)(__m256i)(i), \ | 4708 | (__v4di)(__m256i)(i), \ |
| 1032 | (__v4di)(__m256i)(mask), (s))) | 4709 | (__v4di)(__m256i)(mask), (s))) |
| 1033 | 4710 | ||
| 4711 | /// Gathers two 64-bit floating-point values from memory \a m using scaled | ||
| 4712 | /// indexes from the 128-bit vector of [4 x i32] in \a i. | ||
| 4713 | /// | ||
| 4714 | /// \code{.operation} | ||
| 4715 | /// FOR element := 0 to 1 | ||
| 4716 | /// j := element*64 | ||
| 4717 | /// k := element*32 | ||
| 4718 | /// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s) | ||
| 4719 | /// ENDFOR | ||
| 4720 | /// \endcode | ||
| 4721 | /// | ||
| 4722 | /// \headerfile <immintrin.h> | ||
| 4723 | /// | ||
| 4724 | /// \code | ||
| 4725 | /// __m128d _mm_i32gather_pd(const double *m, __m128i i, const int s); | ||
| 4726 | /// \endcode | ||
| 4727 | /// | ||
| 4728 | /// This intrinsic corresponds to the \c VGATHERDPD instruction. | ||
| 4729 | /// | ||
| 4730 | /// \param m | ||
| 4731 | /// A pointer to the memory used for loading values. | ||
| 4732 | /// \param i | ||
| 4733 | /// A 128-bit vector of [4 x i32] containing signed indexes into \a m. Only | ||
| 4734 | /// the first two elements are used. | ||
| 4735 | /// \param s | ||
| 4736 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4737 | /// 1, 2, 4, or 8. | ||
| 4738 | /// \returns A 128-bit vector of [2 x double] containing the gathered values. | ||
| 1034 | #define _mm_i32gather_pd(m, i, s) \ | 4739 | #define _mm_i32gather_pd(m, i, s) \ |
| 1035 | ((__m128d)__builtin_ia32_gatherd_pd((__v2df)_mm_undefined_pd(), \ | 4740 | ((__m128d)__builtin_ia32_gatherd_pd((__v2df)_mm_undefined_pd(), \ |
| 1036 | (double const *)(m), \ | 4741 | (double const *)(m), \ |
| ... | @@ -1039,6 +4744,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) | ... | @@ -1039,6 +4744,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) |
| 1039 | _mm_setzero_pd()), \ | 4744 | _mm_setzero_pd()), \ |
| 1040 | (s))) | 4745 | (s))) |
| 1041 | 4746 | ||
| 4747 | /// Gathers four 64-bit floating-point values from memory \a m using scaled | ||
| 4748 | /// indexes from the 128-bit vector of [4 x i32] in \a i. | ||
| 4749 | /// | ||
| 4750 | /// \code{.operation} | ||
| 4751 | /// FOR element := 0 to 3 | ||
| 4752 | /// j := element*64 | ||
| 4753 | /// k := element*32 | ||
| 4754 | /// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s) | ||
| 4755 | /// ENDFOR | ||
| 4756 | /// \endcode | ||
| 4757 | /// | ||
| 4758 | /// \headerfile <immintrin.h> | ||
| 4759 | /// | ||
| 4760 | /// \code | ||
| 4761 | /// __m256d _mm256_i32gather_pd(const double *m, __m128i i, const int s); | ||
| 4762 | /// \endcode | ||
| 4763 | /// | ||
| 4764 | /// This intrinsic corresponds to the \c VGATHERDPD instruction. | ||
| 4765 | /// | ||
| 4766 | /// \param m | ||
| 4767 | /// A pointer to the memory used for loading values. | ||
| 4768 | /// \param i | ||
| 4769 | /// A 128-bit vector of [4 x i32] containing signed indexes into \a m. | ||
| 4770 | /// \param s | ||
| 4771 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4772 | /// 1, 2, 4, or 8. | ||
| 4773 | /// \returns A 256-bit vector of [4 x double] containing the gathered values. | ||
| 1042 | #define _mm256_i32gather_pd(m, i, s) \ | 4774 | #define _mm256_i32gather_pd(m, i, s) \ |
| 1043 | ((__m256d)__builtin_ia32_gatherd_pd256((__v4df)_mm256_undefined_pd(), \ | 4775 | ((__m256d)__builtin_ia32_gatherd_pd256((__v4df)_mm256_undefined_pd(), \ |
| 1044 | (double const *)(m), \ | 4776 | (double const *)(m), \ |
| ... | @@ -1048,6 +4780,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) | ... | @@ -1048,6 +4780,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) |
| 1048 | _CMP_EQ_OQ), \ | 4780 | _CMP_EQ_OQ), \ |
| 1049 | (s))) | 4781 | (s))) |
| 1050 | 4782 | ||
| 4783 | /// Gathers two 64-bit floating-point values from memory \a m using scaled | ||
| 4784 | /// indexes from the 128-bit vector of [2 x i64] in \a i. | ||
| 4785 | /// | ||
| 4786 | /// \code{.operation} | ||
| 4787 | /// FOR element := 0 to 1 | ||
| 4788 | /// j := element*64 | ||
| 4789 | /// k := element*64 | ||
| 4790 | /// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s) | ||
| 4791 | /// ENDFOR | ||
| 4792 | /// \endcode | ||
| 4793 | /// | ||
| 4794 | /// \headerfile <immintrin.h> | ||
| 4795 | /// | ||
| 4796 | /// \code | ||
| 4797 | /// __m128d _mm_i64gather_pd(const double *m, __m128i i, const int s); | ||
| 4798 | /// \endcode | ||
| 4799 | /// | ||
| 4800 | /// This intrinsic corresponds to the \c VGATHERQPD instruction. | ||
| 4801 | /// | ||
| 4802 | /// \param m | ||
| 4803 | /// A pointer to the memory used for loading values. | ||
| 4804 | /// \param i | ||
| 4805 | /// A 128-bit vector of [2 x i64] containing signed indexes into \a m. | ||
| 4806 | /// \param s | ||
| 4807 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4808 | /// 1, 2, 4, or 8. | ||
| 4809 | /// \returns A 128-bit vector of [2 x double] containing the gathered values. | ||
| 1051 | #define _mm_i64gather_pd(m, i, s) \ | 4810 | #define _mm_i64gather_pd(m, i, s) \ |
| 1052 | ((__m128d)__builtin_ia32_gatherq_pd((__v2df)_mm_undefined_pd(), \ | 4811 | ((__m128d)__builtin_ia32_gatherq_pd((__v2df)_mm_undefined_pd(), \ |
| 1053 | (double const *)(m), \ | 4812 | (double const *)(m), \ |
| ... | @@ -1056,6 +4815,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) | ... | @@ -1056,6 +4815,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) |
| 1056 | _mm_setzero_pd()), \ | 4815 | _mm_setzero_pd()), \ |
| 1057 | (s))) | 4816 | (s))) |
| 1058 | 4817 | ||
| 4818 | /// Gathers four 64-bit floating-point values from memory \a m using scaled | ||
| 4819 | /// indexes from the 256-bit vector of [4 x i64] in \a i. | ||
| 4820 | /// | ||
| 4821 | /// \code{.operation} | ||
| 4822 | /// FOR element := 0 to 3 | ||
| 4823 | /// j := element*64 | ||
| 4824 | /// k := element*64 | ||
| 4825 | /// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s) | ||
| 4826 | /// ENDFOR | ||
| 4827 | /// \endcode | ||
| 4828 | /// | ||
| 4829 | /// \headerfile <immintrin.h> | ||
| 4830 | /// | ||
| 4831 | /// \code | ||
| 4832 | /// __m256d _mm256_i64gather_pd(const double *m, __m256i i, const int s); | ||
| 4833 | /// \endcode | ||
| 4834 | /// | ||
| 4835 | /// This intrinsic corresponds to the \c VGATHERQPD instruction. | ||
| 4836 | /// | ||
| 4837 | /// \param m | ||
| 4838 | /// A pointer to the memory used for loading values. | ||
| 4839 | /// \param i | ||
| 4840 | /// A 256-bit vector of [4 x i64] containing signed indexes into \a m. | ||
| 4841 | /// \param s | ||
| 4842 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4843 | /// 1, 2, 4, or 8. | ||
| 4844 | /// \returns A 256-bit vector of [4 x double] containing the gathered values. | ||
| 1059 | #define _mm256_i64gather_pd(m, i, s) \ | 4845 | #define _mm256_i64gather_pd(m, i, s) \ |
| 1060 | ((__m256d)__builtin_ia32_gatherq_pd256((__v4df)_mm256_undefined_pd(), \ | 4846 | ((__m256d)__builtin_ia32_gatherq_pd256((__v4df)_mm256_undefined_pd(), \ |
| 1061 | (double const *)(m), \ | 4847 | (double const *)(m), \ |
| ... | @@ -1065,6 +4851,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) | ... | @@ -1065,6 +4851,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) |
| 1065 | _CMP_EQ_OQ), \ | 4851 | _CMP_EQ_OQ), \ |
| 1066 | (s))) | 4852 | (s))) |
| 1067 | 4853 | ||
| 4854 | /// Gathers four 32-bit floating-point values from memory \a m using scaled | ||
| 4855 | /// indexes from the 128-bit vector of [4 x i32] in \a i. | ||
| 4856 | /// | ||
| 4857 | /// \code{.operation} | ||
| 4858 | /// FOR element := 0 to 3 | ||
| 4859 | /// j := element*32 | ||
| 4860 | /// k := element*32 | ||
| 4861 | /// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s) | ||
| 4862 | /// ENDFOR | ||
| 4863 | /// \endcode | ||
| 4864 | /// | ||
| 4865 | /// \headerfile <immintrin.h> | ||
| 4866 | /// | ||
| 4867 | /// \code | ||
| 4868 | /// __m128 _mm_i32gather_ps(const float *m, __m128i i, const int s); | ||
| 4869 | /// \endcode | ||
| 4870 | /// | ||
| 4871 | /// This intrinsic corresponds to the \c VGATHERDPS instruction. | ||
| 4872 | /// | ||
| 4873 | /// \param m | ||
| 4874 | /// A pointer to the memory used for loading values. | ||
| 4875 | /// \param i | ||
| 4876 | /// A 128-bit vector of [4 x i32] containing signed indexes into \a m. | ||
| 4877 | /// \param s | ||
| 4878 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4879 | /// 1, 2, 4, or 8. | ||
| 4880 | /// \returns A 128-bit vector of [4 x float] containing the gathered values. | ||
| 1068 | #define _mm_i32gather_ps(m, i, s) \ | 4881 | #define _mm_i32gather_ps(m, i, s) \ |
| 1069 | ((__m128)__builtin_ia32_gatherd_ps((__v4sf)_mm_undefined_ps(), \ | 4882 | ((__m128)__builtin_ia32_gatherd_ps((__v4sf)_mm_undefined_ps(), \ |
| 1070 | (float const *)(m), \ | 4883 | (float const *)(m), \ |
| ... | @@ -1073,6 +4886,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) | ... | @@ -1073,6 +4886,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) |
| 1073 | _mm_setzero_ps()), \ | 4886 | _mm_setzero_ps()), \ |
| 1074 | (s))) | 4887 | (s))) |
| 1075 | 4888 | ||
| 4889 | /// Gathers eight 32-bit floating-point values from memory \a m using scaled | ||
| 4890 | /// indexes from the 256-bit vector of [8 x i32] in \a i. | ||
| 4891 | /// | ||
| 4892 | /// \code{.operation} | ||
| 4893 | /// FOR element := 0 to 7 | ||
| 4894 | /// j := element*32 | ||
| 4895 | /// k := element*32 | ||
| 4896 | /// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s) | ||
| 4897 | /// ENDFOR | ||
| 4898 | /// \endcode | ||
| 4899 | /// | ||
| 4900 | /// \headerfile <immintrin.h> | ||
| 4901 | /// | ||
| 4902 | /// \code | ||
| 4903 | /// __m256 _mm256_i32gather_ps(const float *m, __m256i i, const int s); | ||
| 4904 | /// \endcode | ||
| 4905 | /// | ||
| 4906 | /// This intrinsic corresponds to the \c VGATHERDPS instruction. | ||
| 4907 | /// | ||
| 4908 | /// \param m | ||
| 4909 | /// A pointer to the memory used for loading values. | ||
| 4910 | /// \param i | ||
| 4911 | /// A 256-bit vector of [8 x i32] containing signed indexes into \a m. | ||
| 4912 | /// \param s | ||
| 4913 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4914 | /// 1, 2, 4, or 8. | ||
| 4915 | /// \returns A 256-bit vector of [8 x float] containing the gathered values. | ||
| 1076 | #define _mm256_i32gather_ps(m, i, s) \ | 4916 | #define _mm256_i32gather_ps(m, i, s) \ |
| 1077 | ((__m256)__builtin_ia32_gatherd_ps256((__v8sf)_mm256_undefined_ps(), \ | 4917 | ((__m256)__builtin_ia32_gatherd_ps256((__v8sf)_mm256_undefined_ps(), \ |
| 1078 | (float const *)(m), \ | 4918 | (float const *)(m), \ |
| ... | @@ -1082,6 +4922,35 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) | ... | @@ -1082,6 +4922,35 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) |
| 1082 | _CMP_EQ_OQ), \ | 4922 | _CMP_EQ_OQ), \ |
| 1083 | (s))) | 4923 | (s))) |
| 1084 | 4924 | ||
| 4925 | /// Gathers two 32-bit floating-point values from memory \a m using scaled | ||
| 4926 | /// indexes from the 128-bit vector of [2 x i64] in \a i. The upper two | ||
| 4927 | /// elements of the result are zeroed. | ||
| 4928 | /// | ||
| 4929 | /// \code{.operation} | ||
| 4930 | /// FOR element := 0 to 1 | ||
| 4931 | /// j := element*32 | ||
| 4932 | /// k := element*64 | ||
| 4933 | /// result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s) | ||
| 4934 | /// ENDFOR | ||
| 4935 | /// result[127:64] := 0 | ||
| 4936 | /// \endcode | ||
| 4937 | /// | ||
| 4938 | /// \headerfile <immintrin.h> | ||
| 4939 | /// | ||
| 4940 | /// \code | ||
| 4941 | /// __m128 _mm_i64gather_ps(const float *m, __m128i i, const int s); | ||
| 4942 | /// \endcode | ||
| 4943 | /// | ||
| 4944 | /// This intrinsic corresponds to the \c VGATHERQPS instruction. | ||
| 4945 | /// | ||
| 4946 | /// \param m | ||
| 4947 | /// A pointer to the memory used for loading values. | ||
| 4948 | /// \param i | ||
| 4949 | /// A 128-bit vector of [2 x i64] containing signed indexes into \a m. | ||
| 4950 | /// \param s | ||
| 4951 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4952 | /// 1, 2, 4, or 8. | ||
| 4953 | /// \returns A 128-bit vector of [4 x float] containing the gathered values. | ||
| 1085 | #define _mm_i64gather_ps(m, i, s) \ | 4954 | #define _mm_i64gather_ps(m, i, s) \ |
| 1086 | ((__m128)__builtin_ia32_gatherq_ps((__v4sf)_mm_undefined_ps(), \ | 4955 | ((__m128)__builtin_ia32_gatherq_ps((__v4sf)_mm_undefined_ps(), \ |
| 1087 | (float const *)(m), \ | 4956 | (float const *)(m), \ |
| ... | @@ -1090,6 +4959,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) | ... | @@ -1090,6 +4959,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) |
| 1090 | _mm_setzero_ps()), \ | 4959 | _mm_setzero_ps()), \ |
| 1091 | (s))) | 4960 | (s))) |
| 1092 | 4961 | ||
| 4962 | /// Gathers four 32-bit floating-point values from memory \a m using scaled | ||
| 4963 | /// indexes from the 256-bit vector of [4 x i64] in \a i. | ||
| 4964 | /// | ||
| 4965 | /// \code{.operation} | ||
| 4966 | /// FOR element := 0 to 3 | ||
| 4967 | /// j := element*32 | ||
| 4968 | /// k := element*64 | ||
| 4969 | /// result[j+31:j] := Load32(m + SignExtend(i[k+64:k])*s) | ||
| 4970 | /// ENDFOR | ||
| 4971 | /// \endcode | ||
| 4972 | /// | ||
| 4973 | /// \headerfile <immintrin.h> | ||
| 4974 | /// | ||
| 4975 | /// \code | ||
| 4976 | /// __m128 _mm256_i64gather_ps(const float *m, __m256i i, const int s); | ||
| 4977 | /// \endcode | ||
| 4978 | /// | ||
| 4979 | /// This intrinsic corresponds to the \c VGATHERQPS instruction. | ||
| 4980 | /// | ||
| 4981 | /// \param m | ||
| 4982 | /// A pointer to the memory used for loading values. | ||
| 4983 | /// \param i | ||
| 4984 | /// A 256-bit vector of [4 x i64] containing signed indexes into \a m. | ||
| 4985 | /// \param s | ||
| 4986 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 4987 | /// 1, 2, 4, or 8. | ||
| 4988 | /// \returns A 128-bit vector of [4 x float] containing the gathered values. | ||
| 1093 | #define _mm256_i64gather_ps(m, i, s) \ | 4989 | #define _mm256_i64gather_ps(m, i, s) \ |
| 1094 | ((__m128)__builtin_ia32_gatherq_ps256((__v4sf)_mm_undefined_ps(), \ | 4990 | ((__m128)__builtin_ia32_gatherq_ps256((__v4sf)_mm_undefined_ps(), \ |
| 1095 | (float const *)(m), \ | 4991 | (float const *)(m), \ |
| ... | @@ -1098,44 +4994,263 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) | ... | @@ -1098,44 +4994,263 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) |
| 1098 | _mm_setzero_ps()), \ | 4994 | _mm_setzero_ps()), \ |
| 1099 | (s))) | 4995 | (s))) |
| 1100 | 4996 | ||
| 4997 | /// Gathers four 32-bit floating-point values from memory \a m using scaled | ||
| 4998 | /// indexes from the 128-bit vector of [4 x i32] in \a i. | ||
| 4999 | /// | ||
| 5000 | /// \code{.operation} | ||
| 5001 | /// FOR element := 0 to 3 | ||
| 5002 | /// j := element*32 | ||
| 5003 | /// k := element*32 | ||
| 5004 | /// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s) | ||
| 5005 | /// ENDFOR | ||
| 5006 | /// \endcode | ||
| 5007 | /// | ||
| 5008 | /// \headerfile <immintrin.h> | ||
| 5009 | /// | ||
| 5010 | /// \code | ||
| 5011 | /// __m128i _mm_i32gather_epi32(const int *m, __m128i i, const int s); | ||
| 5012 | /// \endcode | ||
| 5013 | /// | ||
| 5014 | /// This intrinsic corresponds to the \c VPGATHERDD instruction. | ||
| 5015 | /// | ||
| 5016 | /// \param m | ||
| 5017 | /// A pointer to the memory used for loading values. | ||
| 5018 | /// \param i | ||
| 5019 | /// A 128-bit vector of [4 x i32] containing signed indexes into \a m. | ||
| 5020 | /// \param s | ||
| 5021 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 5022 | /// 1, 2, 4, or 8. | ||
| 5023 | /// \returns A 128-bit vector of [4 x i32] containing the gathered values. | ||
| 1101 | #define _mm_i32gather_epi32(m, i, s) \ | 5024 | #define _mm_i32gather_epi32(m, i, s) \ |
| 1102 | ((__m128i)__builtin_ia32_gatherd_d((__v4si)_mm_undefined_si128(), \ | 5025 | ((__m128i)__builtin_ia32_gatherd_d((__v4si)_mm_undefined_si128(), \ |
| 1103 | (int const *)(m), (__v4si)(__m128i)(i), \ | 5026 | (int const *)(m), (__v4si)(__m128i)(i), \ |
| 1104 | (__v4si)_mm_set1_epi32(-1), (s))) | 5027 | (__v4si)_mm_set1_epi32(-1), (s))) |
| 1105 | 5028 | ||
| 5029 | /// Gathers eight 32-bit floating-point values from memory \a m using scaled | ||
| 5030 | /// indexes from the 256-bit vector of [8 x i32] in \a i. | ||
| 5031 | /// | ||
| 5032 | /// \code{.operation} | ||
| 5033 | /// FOR element := 0 to 7 | ||
| 5034 | /// j := element*32 | ||
| 5035 | /// k := element*32 | ||
| 5036 | /// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s) | ||
| 5037 | /// ENDFOR | ||
| 5038 | /// \endcode | ||
| 5039 | /// | ||
| 5040 | /// \headerfile <immintrin.h> | ||
| 5041 | /// | ||
| 5042 | /// \code | ||
| 5043 | /// __m256i _mm256_i32gather_epi32(const int *m, __m256i i, const int s); | ||
| 5044 | /// \endcode | ||
| 5045 | /// | ||
| 5046 | /// This intrinsic corresponds to the \c VPGATHERDD instruction. | ||
| 5047 | /// | ||
| 5048 | /// \param m | ||
| 5049 | /// A pointer to the memory used for loading values. | ||
| 5050 | /// \param i | ||
| 5051 | /// A 256-bit vector of [8 x i32] containing signed indexes into \a m. | ||
| 5052 | /// \param s | ||
| 5053 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 5054 | /// 1, 2, 4, or 8. | ||
| 5055 | /// \returns A 256-bit vector of [8 x i32] containing the gathered values. | ||
| 1106 | #define _mm256_i32gather_epi32(m, i, s) \ | 5056 | #define _mm256_i32gather_epi32(m, i, s) \ |
| 1107 | ((__m256i)__builtin_ia32_gatherd_d256((__v8si)_mm256_undefined_si256(), \ | 5057 | ((__m256i)__builtin_ia32_gatherd_d256((__v8si)_mm256_undefined_si256(), \ |
| 1108 | (int const *)(m), (__v8si)(__m256i)(i), \ | 5058 | (int const *)(m), (__v8si)(__m256i)(i), \ |
| 1109 | (__v8si)_mm256_set1_epi32(-1), (s))) | 5059 | (__v8si)_mm256_set1_epi32(-1), (s))) |
| 1110 | 5060 | ||
| 5061 | /// Gathers two 32-bit integer values from memory \a m using scaled indexes | ||
| 5062 | /// from the 128-bit vector of [2 x i64] in \a i. The upper two elements | ||
| 5063 | /// of the result are zeroed. | ||
| 5064 | /// | ||
| 5065 | /// \code{.operation} | ||
| 5066 | /// FOR element := 0 to 1 | ||
| 5067 | /// j := element*32 | ||
| 5068 | /// k := element*64 | ||
| 5069 | /// result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s) | ||
| 5070 | /// ENDFOR | ||
| 5071 | /// result[127:64] := 0 | ||
| 5072 | /// \endcode | ||
| 5073 | /// | ||
| 5074 | /// \headerfile <immintrin.h> | ||
| 5075 | /// | ||
| 5076 | /// \code | ||
| 5077 | /// __m128i _mm_i64gather_epi32(const int *m, __m128i i, const int s); | ||
| 5078 | /// \endcode | ||
| 5079 | /// | ||
| 5080 | /// This intrinsic corresponds to the \c VPGATHERQD instruction. | ||
| 5081 | /// | ||
| 5082 | /// \param m | ||
| 5083 | /// A pointer to the memory used for loading values. | ||
| 5084 | /// \param i | ||
| 5085 | /// A 128-bit vector of [2 x i64] containing signed indexes into \a m. | ||
| 5086 | /// \param s | ||
| 5087 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 5088 | /// 1, 2, 4, or 8. | ||
| 5089 | /// \returns A 128-bit vector of [4 x i32] containing the gathered values. | ||
| 1111 | #define _mm_i64gather_epi32(m, i, s) \ | 5090 | #define _mm_i64gather_epi32(m, i, s) \ |
| 1112 | ((__m128i)__builtin_ia32_gatherq_d((__v4si)_mm_undefined_si128(), \ | 5091 | ((__m128i)__builtin_ia32_gatherq_d((__v4si)_mm_undefined_si128(), \ |
| 1113 | (int const *)(m), (__v2di)(__m128i)(i), \ | 5092 | (int const *)(m), (__v2di)(__m128i)(i), \ |
| 1114 | (__v4si)_mm_set1_epi32(-1), (s))) | 5093 | (__v4si)_mm_set1_epi32(-1), (s))) |
| 1115 | 5094 | ||
| 5095 | /// Gathers four 32-bit integer values from memory \a m using scaled indexes | ||
| 5096 | /// from the 256-bit vector of [4 x i64] in \a i. | ||
| 5097 | /// | ||
| 5098 | /// \code{.operation} | ||
| 5099 | /// FOR element := 0 to 3 | ||
| 5100 | /// j := element*32 | ||
| 5101 | /// k := element*64 | ||
| 5102 | /// result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s) | ||
| 5103 | /// ENDFOR | ||
| 5104 | /// \endcode | ||
| 5105 | /// | ||
| 5106 | /// \headerfile <immintrin.h> | ||
| 5107 | /// | ||
| 5108 | /// \code | ||
| 5109 | /// __m128i _mm256_i64gather_epi32(const int *m, __m256i i, const int s); | ||
| 5110 | /// \endcode | ||
| 5111 | /// | ||
| 5112 | /// This intrinsic corresponds to the \c VPGATHERQD instruction. | ||
| 5113 | /// | ||
| 5114 | /// \param m | ||
| 5115 | /// A pointer to the memory used for loading values. | ||
| 5116 | /// \param i | ||
| 5117 | /// A 256-bit vector of [4 x i64] containing signed indexes into \a m. | ||
| 5118 | /// \param s | ||
| 5119 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 5120 | /// 1, 2, 4, or 8. | ||
| 5121 | /// \returns A 128-bit vector of [4 x i32] containing the gathered values. | ||
| 1116 | #define _mm256_i64gather_epi32(m, i, s) \ | 5122 | #define _mm256_i64gather_epi32(m, i, s) \ |
| 1117 | ((__m128i)__builtin_ia32_gatherq_d256((__v4si)_mm_undefined_si128(), \ | 5123 | ((__m128i)__builtin_ia32_gatherq_d256((__v4si)_mm_undefined_si128(), \ |
| 1118 | (int const *)(m), (__v4di)(__m256i)(i), \ | 5124 | (int const *)(m), (__v4di)(__m256i)(i), \ |
| 1119 | (__v4si)_mm_set1_epi32(-1), (s))) | 5125 | (__v4si)_mm_set1_epi32(-1), (s))) |
| 1120 | 5126 | ||
| 5127 | /// Gathers two 64-bit integer values from memory \a m using scaled indexes | ||
| 5128 | /// from the 128-bit vector of [4 x i32] in \a i. | ||
| 5129 | /// | ||
| 5130 | /// \code{.operation} | ||
| 5131 | /// FOR element := 0 to 1 | ||
| 5132 | /// j := element*64 | ||
| 5133 | /// k := element*32 | ||
| 5134 | /// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s) | ||
| 5135 | /// ENDFOR | ||
| 5136 | /// \endcode | ||
| 5137 | /// | ||
| 5138 | /// \headerfile <immintrin.h> | ||
| 5139 | /// | ||
| 5140 | /// \code | ||
| 5141 | /// __m128i _mm_i32gather_epi64(const long long *m, __m128i i, const int s); | ||
| 5142 | /// \endcode | ||
| 5143 | /// | ||
| 5144 | /// This intrinsic corresponds to the \c VPGATHERDQ instruction. | ||
| 5145 | /// | ||
| 5146 | /// \param m | ||
| 5147 | /// A pointer to the memory used for loading values. | ||
| 5148 | /// \param i | ||
| 5149 | /// A 128-bit vector of [4 x i32] containing signed indexes into \a m. Only | ||
| 5150 | /// the first two elements are used. | ||
| 5151 | /// \param s | ||
| 5152 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 5153 | /// 1, 2, 4, or 8. | ||
| 5154 | /// \returns A 128-bit vector of [2 x i64] containing the gathered values. | ||
| 1121 | #define _mm_i32gather_epi64(m, i, s) \ | 5155 | #define _mm_i32gather_epi64(m, i, s) \ |
| 1122 | ((__m128i)__builtin_ia32_gatherd_q((__v2di)_mm_undefined_si128(), \ | 5156 | ((__m128i)__builtin_ia32_gatherd_q((__v2di)_mm_undefined_si128(), \ |
| 1123 | (long long const *)(m), \ | 5157 | (long long const *)(m), \ |
| 1124 | (__v4si)(__m128i)(i), \ | 5158 | (__v4si)(__m128i)(i), \ |
| 1125 | (__v2di)_mm_set1_epi64x(-1), (s))) | 5159 | (__v2di)_mm_set1_epi64x(-1), (s))) |
| 1126 | 5160 | ||
| 5161 | /// Gathers four 64-bit integer values from memory \a m using scaled indexes | ||
| 5162 | /// from the 128-bit vector of [4 x i32] in \a i. | ||
| 5163 | /// | ||
| 5164 | /// \code{.operation} | ||
| 5165 | /// FOR element := 0 to 3 | ||
| 5166 | /// j := element*64 | ||
| 5167 | /// k := element*32 | ||
| 5168 | /// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s) | ||
| 5169 | /// ENDFOR | ||
| 5170 | /// \endcode | ||
| 5171 | /// | ||
| 5172 | /// \headerfile <immintrin.h> | ||
| 5173 | /// | ||
| 5174 | /// \code | ||
| 5175 | /// __m256i _mm256_i32gather_epi64(const long long *m, __m128i i, const int s); | ||
| 5176 | /// \endcode | ||
| 5177 | /// | ||
| 5178 | /// This intrinsic corresponds to the \c VPGATHERDQ instruction. | ||
| 5179 | /// | ||
| 5180 | /// \param m | ||
| 5181 | /// A pointer to the memory used for loading values. | ||
| 5182 | /// \param i | ||
| 5183 | /// A 128-bit vector of [4 x i32] containing signed indexes into \a m. | ||
| 5184 | /// \param s | ||
| 5185 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 5186 | /// 1, 2, 4, or 8. | ||
| 5187 | /// \returns A 256-bit vector of [4 x i64] containing the gathered values. | ||
| 1127 | #define _mm256_i32gather_epi64(m, i, s) \ | 5188 | #define _mm256_i32gather_epi64(m, i, s) \ |
| 1128 | ((__m256i)__builtin_ia32_gatherd_q256((__v4di)_mm256_undefined_si256(), \ | 5189 | ((__m256i)__builtin_ia32_gatherd_q256((__v4di)_mm256_undefined_si256(), \ |
| 1129 | (long long const *)(m), \ | 5190 | (long long const *)(m), \ |
| 1130 | (__v4si)(__m128i)(i), \ | 5191 | (__v4si)(__m128i)(i), \ |
| 1131 | (__v4di)_mm256_set1_epi64x(-1), (s))) | 5192 | (__v4di)_mm256_set1_epi64x(-1), (s))) |
| 1132 | 5193 | ||
| 5194 | /// Gathers two 64-bit integer values from memory \a m using scaled indexes | ||
| 5195 | /// from the 128-bit vector of [2 x i64] in \a i. | ||
| 5196 | /// | ||
| 5197 | /// \code{.operation} | ||
| 5198 | /// FOR element := 0 to 1 | ||
| 5199 | /// j := element*64 | ||
| 5200 | /// k := element*64 | ||
| 5201 | /// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s) | ||
| 5202 | /// ENDFOR | ||
| 5203 | /// \endcode | ||
| 5204 | /// | ||
| 5205 | /// \headerfile <immintrin.h> | ||
| 5206 | /// | ||
| 5207 | /// \code | ||
| 5208 | /// __m128i _mm_i64gather_epi64(const long long *m, __m128i i, const int s); | ||
| 5209 | /// \endcode | ||
| 5210 | /// | ||
| 5211 | /// This intrinsic corresponds to the \c VPGATHERQQ instruction. | ||
| 5212 | /// | ||
| 5213 | /// \param m | ||
| 5214 | /// A pointer to the memory used for loading values. | ||
| 5215 | /// \param i | ||
| 5216 | /// A 128-bit vector of [2 x i64] containing signed indexes into \a m. | ||
| 5217 | /// \param s | ||
| 5218 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 5219 | /// 1, 2, 4, or 8. | ||
| 5220 | /// \returns A 128-bit vector of [2 x i64] containing the gathered values. | ||
| 1133 | #define _mm_i64gather_epi64(m, i, s) \ | 5221 | #define _mm_i64gather_epi64(m, i, s) \ |
| 1134 | ((__m128i)__builtin_ia32_gatherq_q((__v2di)_mm_undefined_si128(), \ | 5222 | ((__m128i)__builtin_ia32_gatherq_q((__v2di)_mm_undefined_si128(), \ |
| 1135 | (long long const *)(m), \ | 5223 | (long long const *)(m), \ |
| 1136 | (__v2di)(__m128i)(i), \ | 5224 | (__v2di)(__m128i)(i), \ |
| 1137 | (__v2di)_mm_set1_epi64x(-1), (s))) | 5225 | (__v2di)_mm_set1_epi64x(-1), (s))) |
| 1138 | 5226 | ||
| 5227 | /// Gathers four 64-bit integer values from memory \a m using scaled indexes | ||
| 5228 | /// from the 256-bit vector of [4 x i64] in \a i. | ||
| 5229 | /// | ||
| 5230 | /// \code{.operation} | ||
| 5231 | /// FOR element := 0 to 3 | ||
| 5232 | /// j := element*64 | ||
| 5233 | /// k := element*64 | ||
| 5234 | /// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s) | ||
| 5235 | /// ENDFOR | ||
| 5236 | /// \endcode | ||
| 5237 | /// | ||
| 5238 | /// \headerfile <immintrin.h> | ||
| 5239 | /// | ||
| 5240 | /// \code | ||
| 5241 | /// __m256i _mm256_i64gather_epi64(const long long *m, __m256i i, const int s); | ||
| 5242 | /// \endcode | ||
| 5243 | /// | ||
| 5244 | /// This intrinsic corresponds to the \c VPGATHERQQ instruction. | ||
| 5245 | /// | ||
| 5246 | /// \param m | ||
| 5247 | /// A pointer to the memory used for loading values. | ||
| 5248 | /// \param i | ||
| 5249 | /// A 256-bit vector of [4 x i64] containing signed indexes into \a m. | ||
| 5250 | /// \param s | ||
| 5251 | /// A literal constant scale factor for the indexes in \a i. Must be | ||
| 5252 | /// 1, 2, 4, or 8. | ||
| 5253 | /// \returns A 256-bit vector of [4 x i64] containing the gathered values. | ||
| 1139 | #define _mm256_i64gather_epi64(m, i, s) \ | 5254 | #define _mm256_i64gather_epi64(m, i, s) \ |
| 1140 | ((__m256i)__builtin_ia32_gatherq_q256((__v4di)_mm256_undefined_si256(), \ | 5255 | ((__m256i)__builtin_ia32_gatherq_q256((__v4di)_mm256_undefined_si256(), \ |
| 1141 | (long long const *)(m), \ | 5256 | (long long const *)(m), \ |
lib/include/avx512fintrin.h+17-7
| ... | @@ -397,14 +397,15 @@ _mm512_broadcastsd_pd(__m128d __A) | ... | @@ -397,14 +397,15 @@ _mm512_broadcastsd_pd(__m128d __A) |
| 397 | static __inline __m512d __DEFAULT_FN_ATTRS512 | 397 | static __inline __m512d __DEFAULT_FN_ATTRS512 |
| 398 | _mm512_castpd256_pd512(__m256d __a) | 398 | _mm512_castpd256_pd512(__m256d __a) |
| 399 | { | 399 | { |
| 400 | return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, -1, -1, -1, -1); | 400 | return __builtin_shufflevector(__a, __builtin_nondeterministic_value(__a), 0, |
| 401 | 1, 2, 3, 4, 5, 6, 7); | ||
| 401 | } | 402 | } |
| 402 | 403 | ||
| 403 | static __inline __m512 __DEFAULT_FN_ATTRS512 | 404 | static __inline __m512 __DEFAULT_FN_ATTRS512 |
| 404 | _mm512_castps256_ps512(__m256 __a) | 405 | _mm512_castps256_ps512(__m256 __a) |
| 405 | { | 406 | { |
| 406 | return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, | 407 | return __builtin_shufflevector(__a, __builtin_nondeterministic_value(__a), 0, |
| 407 | -1, -1, -1, -1, -1, -1, -1, -1); | 408 | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); |
| 408 | } | 409 | } |
| 409 | 410 | ||
| 410 | static __inline __m128d __DEFAULT_FN_ATTRS512 | 411 | static __inline __m128d __DEFAULT_FN_ATTRS512 |
| ... | @@ -446,7 +447,10 @@ _mm512_castpd_si512 (__m512d __A) | ... | @@ -446,7 +447,10 @@ _mm512_castpd_si512 (__m512d __A) |
| 446 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 | 447 | static __inline__ __m512d __DEFAULT_FN_ATTRS512 |
| 447 | _mm512_castpd128_pd512 (__m128d __A) | 448 | _mm512_castpd128_pd512 (__m128d __A) |
| 448 | { | 449 | { |
| 449 | return __builtin_shufflevector( __A, __A, 0, 1, -1, -1, -1, -1, -1, -1); | 450 | __m256d __B = __builtin_nondeterministic_value(__B); |
| 451 | return __builtin_shufflevector( | ||
| 452 | __builtin_shufflevector(__A, __builtin_nondeterministic_value(__A), 0, 1, 2, 3), | ||
| 453 | __B, 0, 1, 2, 3, 4, 5, 6, 7); | ||
| 450 | } | 454 | } |
| 451 | 455 | ||
| 452 | static __inline __m512d __DEFAULT_FN_ATTRS512 | 456 | static __inline __m512d __DEFAULT_FN_ATTRS512 |
| ... | @@ -464,19 +468,25 @@ _mm512_castps_si512 (__m512 __A) | ... | @@ -464,19 +468,25 @@ _mm512_castps_si512 (__m512 __A) |
| 464 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 | 468 | static __inline__ __m512 __DEFAULT_FN_ATTRS512 |
| 465 | _mm512_castps128_ps512 (__m128 __A) | 469 | _mm512_castps128_ps512 (__m128 __A) |
| 466 | { | 470 | { |
| 467 | return __builtin_shufflevector( __A, __A, 0, 1, 2, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1); | 471 | __m256 __B = __builtin_nondeterministic_value(__B); |
| 472 | return __builtin_shufflevector( | ||
| 473 | __builtin_shufflevector(__A, __builtin_nondeterministic_value(__A), 0, 1, 2, 3, 4, 5, 6, 7), | ||
| 474 | __B, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); | ||
| 468 | } | 475 | } |
| 469 | 476 | ||
| 470 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 | 477 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 471 | _mm512_castsi128_si512 (__m128i __A) | 478 | _mm512_castsi128_si512 (__m128i __A) |
| 472 | { | 479 | { |
| 473 | return __builtin_shufflevector( __A, __A, 0, 1, -1, -1, -1, -1, -1, -1); | 480 | __m256i __B = __builtin_nondeterministic_value(__B); |
| 481 | return __builtin_shufflevector( | ||
| 482 | __builtin_shufflevector(__A, __builtin_nondeterministic_value(__A), 0, 1, 2, 3), | ||
| 483 | __B, 0, 1, 2, 3, 4, 5, 6, 7); | ||
| 474 | } | 484 | } |
| 475 | 485 | ||
| 476 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 | 486 | static __inline__ __m512i __DEFAULT_FN_ATTRS512 |
| 477 | _mm512_castsi256_si512 (__m256i __A) | 487 | _mm512_castsi256_si512 (__m256i __A) |
| 478 | { | 488 | { |
| 479 | return __builtin_shufflevector( __A, __A, 0, 1, 2, 3, -1, -1, -1, -1); | 489 | return __builtin_shufflevector( __A, __builtin_nondeterministic_value(__A), 0, 1, 2, 3, 4, 5, 6, 7); |
| 480 | } | 490 | } |
| 481 | 491 | ||
| 482 | static __inline __m512 __DEFAULT_FN_ATTRS512 | 492 | static __inline __m512 __DEFAULT_FN_ATTRS512 |
lib/include/avx512fp16intrin.h+12-8
| ... | @@ -192,22 +192,26 @@ _mm512_castph512_ph256(__m512h __a) { | ... | @@ -192,22 +192,26 @@ _mm512_castph512_ph256(__m512h __a) { |
| 192 | 192 | ||
| 193 | static __inline__ __m256h __DEFAULT_FN_ATTRS256 | 193 | static __inline__ __m256h __DEFAULT_FN_ATTRS256 |
| 194 | _mm256_castph128_ph256(__m128h __a) { | 194 | _mm256_castph128_ph256(__m128h __a) { |
| 195 | return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, -1, -1, -1, | 195 | return __builtin_shufflevector(__a, __builtin_nondeterministic_value(__a), |
| 196 | -1, -1, -1, -1, -1); | 196 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); |
| 197 | } | 197 | } |
| 198 | 198 | ||
| 199 | static __inline__ __m512h __DEFAULT_FN_ATTRS512 | 199 | static __inline__ __m512h __DEFAULT_FN_ATTRS512 |
| 200 | _mm512_castph128_ph512(__m128h __a) { | 200 | _mm512_castph128_ph512(__m128h __a) { |
| 201 | return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, -1, -1, -1, | 201 | __m256h __b = __builtin_nondeterministic_value(__b); |
| 202 | -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, | 202 | return __builtin_shufflevector( |
| 203 | -1, -1, -1, -1, -1, -1, -1, -1, -1); | 203 | __builtin_shufflevector(__a, __builtin_nondeterministic_value(__a), |
| 204 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), | ||
| 205 | __b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, | ||
| 206 | 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); | ||
| 204 | } | 207 | } |
| 205 | 208 | ||
| 206 | static __inline__ __m512h __DEFAULT_FN_ATTRS512 | 209 | static __inline__ __m512h __DEFAULT_FN_ATTRS512 |
| 207 | _mm512_castph256_ph512(__m256h __a) { | 210 | _mm512_castph256_ph512(__m256h __a) { |
| 208 | return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, | 211 | return __builtin_shufflevector(__a, __builtin_nondeterministic_value(__a), 0, |
| 209 | 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, | 212 | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, |
| 210 | -1, -1, -1, -1, -1, -1, -1, -1); | 213 | 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, |
| 214 | 27, 28, 29, 30, 31); | ||
| 211 | } | 215 | } |
| 212 | 216 | ||
| 213 | /// Constructs a 256-bit floating-point vector of [16 x half] from a | 217 | /// Constructs a 256-bit floating-point vector of [16 x half] from a |
lib/include/avxintrin.h+20-7
| ... | @@ -3017,8 +3017,11 @@ _mm256_zeroupper(void) | ... | @@ -3017,8 +3017,11 @@ _mm256_zeroupper(void) |
| 3017 | static __inline __m128 __DEFAULT_FN_ATTRS128 | 3017 | static __inline __m128 __DEFAULT_FN_ATTRS128 |
| 3018 | _mm_broadcast_ss(float const *__a) | 3018 | _mm_broadcast_ss(float const *__a) |
| 3019 | { | 3019 | { |
| 3020 | float __f = *__a; | 3020 | struct __mm_broadcast_ss_struct { |
| 3021 | return __extension__ (__m128)(__v4sf){ __f, __f, __f, __f }; | 3021 | float __f; |
| 3022 | } __attribute__((__packed__, __may_alias__)); | ||
| 3023 | float __f = ((const struct __mm_broadcast_ss_struct*)__a)->__f; | ||
| 3024 | return __extension__ (__m128){ __f, __f, __f, __f }; | ||
| 3022 | } | 3025 | } |
| 3023 | 3026 | ||
| 3024 | /// Loads a scalar double-precision floating point value from the | 3027 | /// Loads a scalar double-precision floating point value from the |
| ... | @@ -3036,7 +3039,10 @@ _mm_broadcast_ss(float const *__a) | ... | @@ -3036,7 +3039,10 @@ _mm_broadcast_ss(float const *__a) |
| 3036 | static __inline __m256d __DEFAULT_FN_ATTRS | 3039 | static __inline __m256d __DEFAULT_FN_ATTRS |
| 3037 | _mm256_broadcast_sd(double const *__a) | 3040 | _mm256_broadcast_sd(double const *__a) |
| 3038 | { | 3041 | { |
| 3039 | double __d = *__a; | 3042 | struct __mm256_broadcast_sd_struct { |
| 3043 | double __d; | ||
| 3044 | } __attribute__((__packed__, __may_alias__)); | ||
| 3045 | double __d = ((const struct __mm256_broadcast_sd_struct*)__a)->__d; | ||
| 3040 | return __extension__ (__m256d)(__v4df){ __d, __d, __d, __d }; | 3046 | return __extension__ (__m256d)(__v4df){ __d, __d, __d, __d }; |
| 3041 | } | 3047 | } |
| 3042 | 3048 | ||
| ... | @@ -3055,7 +3061,10 @@ _mm256_broadcast_sd(double const *__a) | ... | @@ -3055,7 +3061,10 @@ _mm256_broadcast_sd(double const *__a) |
| 3055 | static __inline __m256 __DEFAULT_FN_ATTRS | 3061 | static __inline __m256 __DEFAULT_FN_ATTRS |
| 3056 | _mm256_broadcast_ss(float const *__a) | 3062 | _mm256_broadcast_ss(float const *__a) |
| 3057 | { | 3063 | { |
| 3058 | float __f = *__a; | 3064 | struct __mm256_broadcast_ss_struct { |
| 3065 | float __f; | ||
| 3066 | } __attribute__((__packed__, __may_alias__)); | ||
| 3067 | float __f = ((const struct __mm256_broadcast_ss_struct*)__a)->__f; | ||
| 3059 | return __extension__ (__m256)(__v8sf){ __f, __f, __f, __f, __f, __f, __f, __f }; | 3068 | return __extension__ (__m256)(__v8sf){ __f, __f, __f, __f, __f, __f, __f, __f }; |
| 3060 | } | 3069 | } |
| 3061 | 3070 | ||
| ... | @@ -4499,7 +4508,8 @@ _mm256_castsi256_si128(__m256i __a) | ... | @@ -4499,7 +4508,8 @@ _mm256_castsi256_si128(__m256i __a) |
| 4499 | static __inline __m256d __DEFAULT_FN_ATTRS | 4508 | static __inline __m256d __DEFAULT_FN_ATTRS |
| 4500 | _mm256_castpd128_pd256(__m128d __a) | 4509 | _mm256_castpd128_pd256(__m128d __a) |
| 4501 | { | 4510 | { |
| 4502 | return __builtin_shufflevector((__v2df)__a, (__v2df)__a, 0, 1, -1, -1); | 4511 | return __builtin_shufflevector( |
| 4512 | (__v2df)__a, (__v2df)__builtin_nondeterministic_value(__a), 0, 1, 2, 3); | ||
| 4503 | } | 4513 | } |
| 4504 | 4514 | ||
| 4505 | /// Constructs a 256-bit floating-point vector of [8 x float] from a | 4515 | /// Constructs a 256-bit floating-point vector of [8 x float] from a |
| ... | @@ -4520,7 +4530,9 @@ _mm256_castpd128_pd256(__m128d __a) | ... | @@ -4520,7 +4530,9 @@ _mm256_castpd128_pd256(__m128d __a) |
| 4520 | static __inline __m256 __DEFAULT_FN_ATTRS | 4530 | static __inline __m256 __DEFAULT_FN_ATTRS |
| 4521 | _mm256_castps128_ps256(__m128 __a) | 4531 | _mm256_castps128_ps256(__m128 __a) |
| 4522 | { | 4532 | { |
| 4523 | return __builtin_shufflevector((__v4sf)__a, (__v4sf)__a, 0, 1, 2, 3, -1, -1, -1, -1); | 4533 | return __builtin_shufflevector((__v4sf)__a, |
| 4534 | (__v4sf)__builtin_nondeterministic_value(__a), | ||
| 4535 | 0, 1, 2, 3, 4, 5, 6, 7); | ||
| 4524 | } | 4536 | } |
| 4525 | 4537 | ||
| 4526 | /// Constructs a 256-bit integer vector from a 128-bit integer vector. | 4538 | /// Constructs a 256-bit integer vector from a 128-bit integer vector. |
| ... | @@ -4539,7 +4551,8 @@ _mm256_castps128_ps256(__m128 __a) | ... | @@ -4539,7 +4551,8 @@ _mm256_castps128_ps256(__m128 __a) |
| 4539 | static __inline __m256i __DEFAULT_FN_ATTRS | 4551 | static __inline __m256i __DEFAULT_FN_ATTRS |
| 4540 | _mm256_castsi128_si256(__m128i __a) | 4552 | _mm256_castsi128_si256(__m128i __a) |
| 4541 | { | 4553 | { |
| 4542 | return __builtin_shufflevector((__v2di)__a, (__v2di)__a, 0, 1, -1, -1); | 4554 | return __builtin_shufflevector( |
| 4555 | (__v2di)__a, (__v2di)__builtin_nondeterministic_value(__a), 0, 1, 2, 3); | ||
| 4543 | } | 4556 | } |
| 4544 | 4557 | ||
| 4545 | /// Constructs a 256-bit floating-point vector of [4 x double] from a | 4558 | /// Constructs a 256-bit floating-point vector of [4 x double] from a |
lib/include/avxvnniint16intrin.h created+473
| ... | @@ -0,0 +1,473 @@ | ||
| 1 | /*===----------- avxvnniint16intrin.h - AVXVNNIINT16 intrinsics-------------=== | ||
| 2 | * | ||
| 3 | * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| 4 | * See https://llvm.org/LICENSE.txt for license information. | ||
| 5 | * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| 6 | * | ||
| 7 | *===-----------------------------------------------------------------------=== | ||
| 8 | */ | ||
| 9 | |||
| 10 | #ifndef __IMMINTRIN_H | ||
| 11 | #error \ | ||
| 12 | "Never use <avxvnniint16intrin.h> directly; include <immintrin.h> instead." | ||
| 13 | #endif // __IMMINTRIN_H | ||
| 14 | |||
| 15 | #ifndef __AVXVNNIINT16INTRIN_H | ||
| 16 | #define __AVXVNNIINT16INTRIN_H | ||
| 17 | |||
| 18 | /* Define the default attributes for the functions in this file. */ | ||
| 19 | #define __DEFAULT_FN_ATTRS128 \ | ||
| 20 | __attribute__((__always_inline__, __nodebug__, __target__("avxvnniint16"), \ | ||
| 21 | __min_vector_width__(128))) | ||
| 22 | #define __DEFAULT_FN_ATTRS256 \ | ||
| 23 | __attribute__((__always_inline__, __nodebug__, __target__("avxvnniint16"), \ | ||
| 24 | __min_vector_width__(256))) | ||
| 25 | |||
| 26 | /// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with | ||
| 27 | /// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate | ||
| 28 | /// signed 16-bit results. Sum these 2 results with the corresponding | ||
| 29 | /// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst. | ||
| 30 | /// | ||
| 31 | /// \headerfile <immintrin.h> | ||
| 32 | /// | ||
| 33 | /// \code | ||
| 34 | /// __m128i _mm_dpwsud_epi32(__m128i __W, __m128i __A, __m128i __B) | ||
| 35 | /// \endcode | ||
| 36 | /// | ||
| 37 | /// This intrinsic corresponds to the \c VPDPWSUD instruction. | ||
| 38 | /// | ||
| 39 | /// \param __W | ||
| 40 | /// A 128-bit vector of [4 x int]. | ||
| 41 | /// \param __A | ||
| 42 | /// A 128-bit vector of [8 x short]. | ||
| 43 | /// \param __B | ||
| 44 | /// A 128-bit vector of [8 x unsigned short]. | ||
| 45 | /// \returns | ||
| 46 | /// A 128-bit vector of [4 x int]. | ||
| 47 | /// | ||
| 48 | /// \code{.operation} | ||
| 49 | /// FOR j := 0 to 3 | ||
| 50 | /// tmp1.dword := SignExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j]) | ||
| 51 | /// tmp2.dword := SignExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1]) | ||
| 52 | /// dst.dword[j] := __W.dword[j] + tmp1 + tmp2 | ||
| 53 | /// ENDFOR | ||
| 54 | /// dst[MAX:128] := 0 | ||
| 55 | /// \endcode | ||
| 56 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsud_epi32(__m128i __W, | ||
| 57 | __m128i __A, | ||
| 58 | __m128i __B) { | ||
| 59 | return (__m128i)__builtin_ia32_vpdpwsud128((__v4si)__W, (__v4si)__A, | ||
| 60 | (__v4si)__B); | ||
| 61 | } | ||
| 62 | |||
| 63 | /// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with | ||
| 64 | /// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate | ||
| 65 | /// signed 16-bit results. Sum these 2 results with the corresponding | ||
| 66 | /// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst. | ||
| 67 | /// | ||
| 68 | /// \headerfile <immintrin.h> | ||
| 69 | /// | ||
| 70 | /// \code | ||
| 71 | /// __m256i _mm256_dpwsud_epi32(__m256i __W, __m256i __A, __m256i __B) | ||
| 72 | /// \endcode | ||
| 73 | /// | ||
| 74 | /// This intrinsic corresponds to the \c VPDPWSUD instruction. | ||
| 75 | /// | ||
| 76 | /// \param __W | ||
| 77 | /// A 256-bit vector of [8 x int]. | ||
| 78 | /// \param __A | ||
| 79 | /// A 256-bit vector of [16 x short]. | ||
| 80 | /// \param __B | ||
| 81 | /// A 256-bit vector of [16 x unsigned short]. | ||
| 82 | /// \returns | ||
| 83 | /// A 256-bit vector of [8 x int]. | ||
| 84 | /// | ||
| 85 | /// \code{.operation} | ||
| 86 | /// FOR j := 0 to 7 | ||
| 87 | /// tmp1.dword := SignExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j]) | ||
| 88 | /// tmp2.dword := SignExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1]) | ||
| 89 | /// dst.dword[j] := __W.dword[j] + tmp1 + tmp2 | ||
| 90 | /// ENDFOR | ||
| 91 | /// dst[MAX:256] := 0 | ||
| 92 | /// \endcode | ||
| 93 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 94 | _mm256_dpwsud_epi32(__m256i __W, __m256i __A, __m256i __B) { | ||
| 95 | return (__m256i)__builtin_ia32_vpdpwsud256((__v8si)__W, (__v8si)__A, | ||
| 96 | (__v8si)__B); | ||
| 97 | } | ||
| 98 | |||
| 99 | /// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with | ||
| 100 | /// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate | ||
| 101 | /// signed 16-bit results. Sum these 2 results with the corresponding | ||
| 102 | /// 32-bit integer in \a __W with signed saturation, and store the packed | ||
| 103 | /// 32-bit results in \a dst. | ||
| 104 | /// | ||
| 105 | /// \headerfile <immintrin.h> | ||
| 106 | /// | ||
| 107 | /// \code | ||
| 108 | /// __m128i _mm_dpwsuds_epi32(__m128i __W, __m128i __A, __m128i __B) | ||
| 109 | /// \endcode | ||
| 110 | /// | ||
| 111 | /// This intrinsic corresponds to the \c VPDPWSUDS instruction. | ||
| 112 | /// | ||
| 113 | /// \param __W | ||
| 114 | /// A 128-bit vector of [4 x int]. | ||
| 115 | /// \param __A | ||
| 116 | /// A 128-bit vector of [8 x short]. | ||
| 117 | /// \param __B | ||
| 118 | /// A 128-bit vector of [8 x unsigned short]. | ||
| 119 | /// \returns | ||
| 120 | /// A 128-bit vector of [4 x int]. | ||
| 121 | /// | ||
| 122 | /// \code{.operation} | ||
| 123 | /// FOR j := 0 to 3 | ||
| 124 | /// tmp1.dword := SignExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j]) | ||
| 125 | /// tmp2.dword := SignExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1]) | ||
| 126 | /// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2) | ||
| 127 | /// ENDFOR | ||
| 128 | /// dst[MAX:128] := 0 | ||
| 129 | /// \endcode | ||
| 130 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsuds_epi32(__m128i __W, | ||
| 131 | __m128i __A, | ||
| 132 | __m128i __B) { | ||
| 133 | return (__m128i)__builtin_ia32_vpdpwsuds128((__v4si)__W, (__v4si)__A, | ||
| 134 | (__v4si)__B); | ||
| 135 | } | ||
| 136 | |||
| 137 | /// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with | ||
| 138 | /// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate | ||
| 139 | /// signed 16-bit results. Sum these 2 results with the corresponding | ||
| 140 | /// 32-bit integer in \a __W with signed saturation, and store the packed | ||
| 141 | /// 32-bit results in \a dst. | ||
| 142 | /// | ||
| 143 | /// \headerfile <immintrin.h> | ||
| 144 | /// | ||
| 145 | /// \code | ||
| 146 | /// __m256i _mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B) | ||
| 147 | /// \endcode | ||
| 148 | /// | ||
| 149 | /// This intrinsic corresponds to the \c VPDPWSUDS instruction. | ||
| 150 | /// | ||
| 151 | /// \param __W | ||
| 152 | /// A 256-bit vector of [8 x int]. | ||
| 153 | /// \param __A | ||
| 154 | /// A 256-bit vector of [16 x short]. | ||
| 155 | /// \param __B | ||
| 156 | /// A 256-bit vector of [16 x unsigned short]. | ||
| 157 | /// \returns | ||
| 158 | /// A 256-bit vector of [8 x int]. | ||
| 159 | /// | ||
| 160 | /// \code{.operation} | ||
| 161 | /// FOR j := 0 to 7 | ||
| 162 | /// tmp1.dword := SignExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j]) | ||
| 163 | /// tmp2.dword := SignExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1]) | ||
| 164 | /// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2) | ||
| 165 | /// ENDFOR | ||
| 166 | /// dst[MAX:256] := 0 | ||
| 167 | /// \endcode | ||
| 168 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 169 | _mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B) { | ||
| 170 | return (__m256i)__builtin_ia32_vpdpwsuds256((__v8si)__W, (__v8si)__A, | ||
| 171 | (__v8si)__B); | ||
| 172 | } | ||
| 173 | |||
| 174 | /// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with | ||
| 175 | /// corresponding signed 16-bit integers in \a __B, producing 2 intermediate | ||
| 176 | /// signed 16-bit results. Sum these 2 results with the corresponding | ||
| 177 | /// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst. | ||
| 178 | /// | ||
| 179 | /// \headerfile <immintrin.h> | ||
| 180 | /// | ||
| 181 | /// \code | ||
| 182 | /// __m128i _mm_dpbusd_epi32(__m128i __W, __m128i __A, __m128i __B) | ||
| 183 | /// \endcode | ||
| 184 | /// | ||
| 185 | /// This intrinsic corresponds to the \c VPDPWUSD instruction. | ||
| 186 | /// | ||
| 187 | /// \param __W | ||
| 188 | /// A 128-bit vector of [4 x int]. | ||
| 189 | /// \param __A | ||
| 190 | /// A 128-bit vector of [8 x unsigned short]. | ||
| 191 | /// \param __B | ||
| 192 | /// A 128-bit vector of [8 x short]. | ||
| 193 | /// \returns | ||
| 194 | /// A 128-bit vector of [4 x int]. | ||
| 195 | /// | ||
| 196 | /// \code{.operation} | ||
| 197 | /// FOR j := 0 to 3 | ||
| 198 | /// tmp1.dword := ZeroExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j]) | ||
| 199 | /// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1]) | ||
| 200 | /// dst.dword[j] := __W.dword[j] + tmp1 + tmp2 | ||
| 201 | /// ENDFOR | ||
| 202 | /// dst[MAX:128] := 0 | ||
| 203 | /// \endcode | ||
| 204 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusd_epi32(__m128i __W, | ||
| 205 | __m128i __A, | ||
| 206 | __m128i __B) { | ||
| 207 | return (__m128i)__builtin_ia32_vpdpwusd128((__v4si)__W, (__v4si)__A, | ||
| 208 | (__v4si)__B); | ||
| 209 | } | ||
| 210 | |||
| 211 | /// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with | ||
| 212 | /// corresponding signed 16-bit integers in \a __B, producing 2 intermediate | ||
| 213 | /// signed 16-bit results. Sum these 2 results with the corresponding | ||
| 214 | /// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst. | ||
| 215 | /// | ||
| 216 | /// \headerfile <immintrin.h> | ||
| 217 | /// | ||
| 218 | /// \code | ||
| 219 | /// __m256i _mm256_dpwusd_epi32(__m256i __W, __m256i __A, __m256i __B) | ||
| 220 | /// \endcode | ||
| 221 | /// | ||
| 222 | /// This intrinsic corresponds to the \c VPDPWUSD instruction. | ||
| 223 | /// | ||
| 224 | /// \param __W | ||
| 225 | /// A 256-bit vector of [8 x int]. | ||
| 226 | /// \param __A | ||
| 227 | /// A 256-bit vector of [16 x unsigned short]. | ||
| 228 | /// \param __B | ||
| 229 | /// A 256-bit vector of [16 x short]. | ||
| 230 | /// \returns | ||
| 231 | /// A 256-bit vector of [8 x int]. | ||
| 232 | /// | ||
| 233 | /// \code{.operation} | ||
| 234 | /// FOR j := 0 to 7 | ||
| 235 | /// tmp1.dword := ZeroExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j]) | ||
| 236 | /// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1]) | ||
| 237 | /// dst.dword[j] := __W.dword[j] + tmp1 + tmp2 | ||
| 238 | /// ENDFOR | ||
| 239 | /// dst[MAX:256] := 0 | ||
| 240 | /// \endcode | ||
| 241 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 242 | _mm256_dpwusd_epi32(__m256i __W, __m256i __A, __m256i __B) { | ||
| 243 | return (__m256i)__builtin_ia32_vpdpwusd256((__v8si)__W, (__v8si)__A, | ||
| 244 | (__v8si)__B); | ||
| 245 | } | ||
| 246 | |||
| 247 | /// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with | ||
| 248 | /// corresponding signed 16-bit integers in \a __B, producing 2 intermediate | ||
| 249 | /// signed 16-bit results. Sum these 2 results with the corresponding | ||
| 250 | /// 32-bit integer in \a __W with signed saturation, and store the packed | ||
| 251 | /// 32-bit results in \a dst. | ||
| 252 | /// | ||
| 253 | /// \headerfile <immintrin.h> | ||
| 254 | /// | ||
| 255 | /// \code | ||
| 256 | /// __m128i _mm_dpwusds_epi32(__m128i __W, __m128i __A, __m128i __B) | ||
| 257 | /// \endcode | ||
| 258 | /// | ||
| 259 | /// This intrinsic corresponds to the \c VPDPWSUDS instruction. | ||
| 260 | /// | ||
| 261 | /// \param __W | ||
| 262 | /// A 128-bit vector of [4 x int]. | ||
| 263 | /// \param __A | ||
| 264 | /// A 128-bit vector of [8 x unsigned short]. | ||
| 265 | /// \param __B | ||
| 266 | /// A 128-bit vector of [8 x short]. | ||
| 267 | /// \returns | ||
| 268 | /// A 128-bit vector of [4 x int]. | ||
| 269 | /// | ||
| 270 | /// \code{.operation} | ||
| 271 | /// FOR j := 0 to 3 | ||
| 272 | /// tmp1.dword := ZeroExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j]) | ||
| 273 | /// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1]) | ||
| 274 | /// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2) | ||
| 275 | /// ENDFOR | ||
| 276 | /// dst[MAX:128] := 0 | ||
| 277 | /// \endcode | ||
| 278 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusds_epi32(__m128i __W, | ||
| 279 | __m128i __A, | ||
| 280 | __m128i __B) { | ||
| 281 | return (__m128i)__builtin_ia32_vpdpwusds128((__v4si)__W, (__v4si)__A, | ||
| 282 | (__v4si)__B); | ||
| 283 | } | ||
| 284 | |||
| 285 | /// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with | ||
| 286 | /// corresponding signed 16-bit integers in \a __B, producing 2 intermediate | ||
| 287 | /// signed 16-bit results. Sum these 2 results with the corresponding | ||
| 288 | /// 32-bit integer in \a __W with signed saturation, and store the packed | ||
| 289 | /// 32-bit results in \a dst. | ||
| 290 | /// | ||
| 291 | /// \headerfile <immintrin.h> | ||
| 292 | /// | ||
| 293 | /// \code | ||
| 294 | /// __m256i _mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B) | ||
| 295 | /// \endcode | ||
| 296 | /// | ||
| 297 | /// This intrinsic corresponds to the \c VPDPWSUDS instruction. | ||
| 298 | /// | ||
| 299 | /// \param __W | ||
| 300 | /// A 256-bit vector of [8 x int]. | ||
| 301 | /// \param __A | ||
| 302 | /// A 256-bit vector of [16 x unsigned short]. | ||
| 303 | /// \param __B | ||
| 304 | /// A 256-bit vector of [16 x short]. | ||
| 305 | /// \returns | ||
| 306 | /// A 256-bit vector of [8 x int]. | ||
| 307 | /// | ||
| 308 | /// \code{.operation} | ||
| 309 | /// FOR j := 0 to 7 | ||
| 310 | /// tmp1.dword := ZeroExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j]) | ||
| 311 | /// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1]) | ||
| 312 | /// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2) | ||
| 313 | /// ENDFOR | ||
| 314 | /// dst[MAX:256] := 0 | ||
| 315 | /// \endcode | ||
| 316 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 317 | _mm256_dpwusds_epi32(__m256i __W, __m256i __A, __m256i __B) { | ||
| 318 | return (__m256i)__builtin_ia32_vpdpwusds256((__v8si)__W, (__v8si)__A, | ||
| 319 | (__v8si)__B); | ||
| 320 | } | ||
| 321 | |||
| 322 | /// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with | ||
| 323 | /// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate | ||
| 324 | /// signed 16-bit results. Sum these 2 results with the corresponding | ||
| 325 | /// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst. | ||
| 326 | /// | ||
| 327 | /// \headerfile <immintrin.h> | ||
| 328 | /// | ||
| 329 | /// \code | ||
| 330 | /// __m128i _mm_dpwuud_epi32(__m128i __W, __m128i __A, __m128i __B) | ||
| 331 | /// \endcode | ||
| 332 | /// | ||
| 333 | /// This intrinsic corresponds to the \c VPDPWUUD instruction. | ||
| 334 | /// | ||
| 335 | /// \param __W | ||
| 336 | /// A 128-bit vector of [4 x unsigned int]. | ||
| 337 | /// \param __A | ||
| 338 | /// A 128-bit vector of [8 x unsigned short]. | ||
| 339 | /// \param __B | ||
| 340 | /// A 128-bit vector of [8 x unsigned short]. | ||
| 341 | /// \returns | ||
| 342 | /// A 128-bit vector of [4 x unsigned int]. | ||
| 343 | /// | ||
| 344 | /// \code{.operation} | ||
| 345 | /// FOR j := 0 to 3 | ||
| 346 | /// tmp1.dword := ZeroExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j]) | ||
| 347 | /// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1]) | ||
| 348 | /// dst.dword[j] := __W.dword[j] + tmp1 + tmp2 | ||
| 349 | /// ENDFOR | ||
| 350 | /// dst[MAX:128] := 0 | ||
| 351 | /// \endcode | ||
| 352 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuud_epi32(__m128i __W, | ||
| 353 | __m128i __A, | ||
| 354 | __m128i __B) { | ||
| 355 | return (__m128i)__builtin_ia32_vpdpwuud128((__v4si)__W, (__v4si)__A, | ||
| 356 | (__v4si)__B); | ||
| 357 | } | ||
| 358 | |||
| 359 | /// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with | ||
| 360 | /// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate | ||
| 361 | /// signed 16-bit results. Sum these 2 results with the corresponding | ||
| 362 | /// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst. | ||
| 363 | /// | ||
| 364 | /// \headerfile <immintrin.h> | ||
| 365 | /// | ||
| 366 | /// \code | ||
| 367 | /// __m256i _mm256_dpwuud_epi32(__m256i __W, __m256i __A, __m256i __B) | ||
| 368 | /// \endcode | ||
| 369 | /// | ||
| 370 | /// This intrinsic corresponds to the \c VPDPWUUD instruction. | ||
| 371 | /// | ||
| 372 | /// \param __W | ||
| 373 | /// A 256-bit vector of [8 x unsigned int]. | ||
| 374 | /// \param __A | ||
| 375 | /// A 256-bit vector of [16 x unsigned short]. | ||
| 376 | /// \param __B | ||
| 377 | /// A 256-bit vector of [16 x unsigned short]. | ||
| 378 | /// \returns | ||
| 379 | /// A 256-bit vector of [8 x unsigned int]. | ||
| 380 | /// | ||
| 381 | /// \code{.operation} | ||
| 382 | /// FOR j := 0 to 7 | ||
| 383 | /// tmp1.dword := ZeroExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j]) | ||
| 384 | /// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1]) | ||
| 385 | /// dst.dword[j] := __W.dword[j] + tmp1 + tmp2 | ||
| 386 | /// ENDFOR | ||
| 387 | /// dst[MAX:256] := 0 | ||
| 388 | /// \endcode | ||
| 389 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 390 | _mm256_dpwuud_epi32(__m256i __W, __m256i __A, __m256i __B) { | ||
| 391 | return (__m256i)__builtin_ia32_vpdpwuud256((__v8si)__W, (__v8si)__A, | ||
| 392 | (__v8si)__B); | ||
| 393 | } | ||
| 394 | |||
| 395 | /// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with | ||
| 396 | /// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate | ||
| 397 | /// signed 16-bit results. Sum these 2 results with the corresponding | ||
| 398 | /// 32-bit integer in \a __W with signed saturation, and store the packed | ||
| 399 | /// 32-bit results in \a dst. | ||
| 400 | /// | ||
| 401 | /// \headerfile <immintrin.h> | ||
| 402 | /// | ||
| 403 | /// \code | ||
| 404 | /// __m128i _mm_dpwsuds_epi32(__m128i __W, __m128i __A, __m128i __B) | ||
| 405 | /// \endcode | ||
| 406 | /// | ||
| 407 | /// This intrinsic corresponds to the \c VPDPWSUDS instruction. | ||
| 408 | /// | ||
| 409 | /// \param __W | ||
| 410 | /// A 128-bit vector of [4 x unsigned int]. | ||
| 411 | /// \param __A | ||
| 412 | /// A 128-bit vector of [8 x unsigned short]. | ||
| 413 | /// \param __B | ||
| 414 | /// A 128-bit vector of [8 x unsigned short]. | ||
| 415 | /// \returns | ||
| 416 | /// A 128-bit vector of [4 x unsigned int]. | ||
| 417 | /// | ||
| 418 | /// \code{.operation} | ||
| 419 | /// FOR j := 0 to 3 | ||
| 420 | /// tmp1.dword := ZeroExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j]) | ||
| 421 | /// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1]) | ||
| 422 | /// dst.dword[j] := UNSIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2) | ||
| 423 | /// ENDFOR | ||
| 424 | /// dst[MAX:128] := 0 | ||
| 425 | /// \endcode | ||
| 426 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuuds_epi32(__m128i __W, | ||
| 427 | __m128i __A, | ||
| 428 | __m128i __B) { | ||
| 429 | return (__m128i)__builtin_ia32_vpdpwuuds128((__v4si)__W, (__v4si)__A, | ||
| 430 | (__v4si)__B); | ||
| 431 | } | ||
| 432 | |||
| 433 | /// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with | ||
| 434 | /// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate | ||
| 435 | /// signed 16-bit results. Sum these 2 results with the corresponding | ||
| 436 | /// 32-bit integer in \a __W with signed saturation, and store the packed | ||
| 437 | /// 32-bit results in \a dst. | ||
| 438 | /// | ||
| 439 | /// \headerfile <immintrin.h> | ||
| 440 | /// | ||
| 441 | /// \code | ||
| 442 | /// __m256i _mm256_dpwuuds_epi32(__m256i __W, __m256i __A, __m256i __B) | ||
| 443 | /// \endcode | ||
| 444 | /// | ||
| 445 | /// This intrinsic corresponds to the \c VPDPWSUDS instruction. | ||
| 446 | /// | ||
| 447 | /// \param __W | ||
| 448 | /// A 256-bit vector of [8 x unsigned int]. | ||
| 449 | /// \param __A | ||
| 450 | /// A 256-bit vector of [16 x unsigned short]. | ||
| 451 | /// \param __B | ||
| 452 | /// A 256-bit vector of [16 x unsigned short]. | ||
| 453 | /// \returns | ||
| 454 | /// A 256-bit vector of [8 x unsigned int]. | ||
| 455 | /// | ||
| 456 | /// \code{.operation} | ||
| 457 | /// FOR j := 0 to 7 | ||
| 458 | /// tmp1.dword := ZeroExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j]) | ||
| 459 | /// tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1]) | ||
| 460 | /// dst.dword[j] := UNSIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2) | ||
| 461 | /// ENDFOR | ||
| 462 | /// dst[MAX:256] := 0 | ||
| 463 | /// \endcode | ||
| 464 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 465 | _mm256_dpwuuds_epi32(__m256i __W, __m256i __A, __m256i __B) { | ||
| 466 | return (__m256i)__builtin_ia32_vpdpwuuds256((__v8si)__W, (__v8si)__A, | ||
| 467 | (__v8si)__B); | ||
| 468 | } | ||
| 469 | |||
| 470 | #undef __DEFAULT_FN_ATTRS128 | ||
| 471 | #undef __DEFAULT_FN_ATTRS256 | ||
| 472 | |||
| 473 | #endif // __AVXVNNIINT16INTRIN_H | ||
lib/include/bmi2intrin.h+187-13
| ... | @@ -7,8 +7,8 @@ | ... | @@ -7,8 +7,8 @@ |
| 7 | *===-----------------------------------------------------------------------=== | 7 | *===-----------------------------------------------------------------------=== |
| 8 | */ | 8 | */ |
| 9 | 9 | ||
| 10 | #if !defined __X86INTRIN_H && !defined __IMMINTRIN_H | 10 | #ifndef __IMMINTRIN_H |
| 11 | #error "Never use <bmi2intrin.h> directly; include <x86intrin.h> instead." | 11 | #error "Never use <bmi2intrin.h> directly; include <immintrin.h> instead." |
| 12 | #endif | 12 | #endif |
| 13 | 13 | ||
| 14 | #ifndef __BMI2INTRIN_H | 14 | #ifndef __BMI2INTRIN_H |
| ... | @@ -17,44 +17,228 @@ | ... | @@ -17,44 +17,228 @@ |
| 17 | /* Define the default attributes for the functions in this file. */ | 17 | /* Define the default attributes for the functions in this file. */ |
| 18 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("bmi2"))) | 18 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("bmi2"))) |
| 19 | 19 | ||
| 20 | /// Copies the unsigned 32-bit integer \a __X and zeroes the upper bits | ||
| 21 | /// starting at bit number \a __Y. | ||
| 22 | /// | ||
| 23 | /// \code{.operation} | ||
| 24 | /// i := __Y[7:0] | ||
| 25 | /// result := __X | ||
| 26 | /// IF i < 32 | ||
| 27 | /// result[31:i] := 0 | ||
| 28 | /// FI | ||
| 29 | /// \endcode | ||
| 30 | /// | ||
| 31 | /// \headerfile <immintrin.h> | ||
| 32 | /// | ||
| 33 | /// This intrinsic corresponds to the \c BZHI instruction. | ||
| 34 | /// | ||
| 35 | /// \param __X | ||
| 36 | /// The 32-bit source value to copy. | ||
| 37 | /// \param __Y | ||
| 38 | /// The lower 8 bits specify the bit number of the lowest bit to zero. | ||
| 39 | /// \returns The partially zeroed 32-bit value. | ||
| 20 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | 40 | static __inline__ unsigned int __DEFAULT_FN_ATTRS |
| 21 | _bzhi_u32(unsigned int __X, unsigned int __Y) | 41 | _bzhi_u32(unsigned int __X, unsigned int __Y) |
| 22 | { | 42 | { |
| 23 | return __builtin_ia32_bzhi_si(__X, __Y); | 43 | return __builtin_ia32_bzhi_si(__X, __Y); |
| 24 | } | 44 | } |
| 25 | 45 | ||
| 46 | /// Deposit (scatter) low-order bits from the unsigned 32-bit integer \a __X | ||
| 47 | /// into the 32-bit result, according to the mask in the unsigned 32-bit | ||
| 48 | /// integer \a __Y. All other bits of the result are zero. | ||
| 49 | /// | ||
| 50 | /// \code{.operation} | ||
| 51 | /// i := 0 | ||
| 52 | /// result := 0 | ||
| 53 | /// FOR m := 0 TO 31 | ||
| 54 | /// IF __Y[m] == 1 | ||
| 55 | /// result[m] := __X[i] | ||
| 56 | /// i := i + 1 | ||
| 57 | /// ENDIF | ||
| 58 | /// ENDFOR | ||
| 59 | /// \endcode | ||
| 60 | /// | ||
| 61 | /// \headerfile <immintrin.h> | ||
| 62 | /// | ||
| 63 | /// This intrinsic corresponds to the \c PDEP instruction. | ||
| 64 | /// | ||
| 65 | /// \param __X | ||
| 66 | /// The 32-bit source value to copy. | ||
| 67 | /// \param __Y | ||
| 68 | /// The 32-bit mask specifying where to deposit source bits. | ||
| 69 | /// \returns The 32-bit result. | ||
| 26 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | 70 | static __inline__ unsigned int __DEFAULT_FN_ATTRS |
| 27 | _pdep_u32(unsigned int __X, unsigned int __Y) | 71 | _pdep_u32(unsigned int __X, unsigned int __Y) |
| 28 | { | 72 | { |
| 29 | return __builtin_ia32_pdep_si(__X, __Y); | 73 | return __builtin_ia32_pdep_si(__X, __Y); |
| 30 | } | 74 | } |
| 31 | 75 | ||
| 76 | /// Extract (gather) bits from the unsigned 32-bit integer \a __X into the | ||
| 77 | /// low-order bits of the 32-bit result, according to the mask in the | ||
| 78 | /// unsigned 32-bit integer \a __Y. All other bits of the result are zero. | ||
| 79 | /// | ||
| 80 | /// \code{.operation} | ||
| 81 | /// i := 0 | ||
| 82 | /// result := 0 | ||
| 83 | /// FOR m := 0 TO 31 | ||
| 84 | /// IF __Y[m] == 1 | ||
| 85 | /// result[i] := __X[m] | ||
| 86 | /// i := i + 1 | ||
| 87 | /// ENDIF | ||
| 88 | /// ENDFOR | ||
| 89 | /// \endcode | ||
| 90 | /// | ||
| 91 | /// \headerfile <immintrin.h> | ||
| 92 | /// | ||
| 93 | /// This intrinsic corresponds to the \c PEXT instruction. | ||
| 94 | /// | ||
| 95 | /// \param __X | ||
| 96 | /// The 32-bit source value to copy. | ||
| 97 | /// \param __Y | ||
| 98 | /// The 32-bit mask specifying which source bits to extract. | ||
| 99 | /// \returns The 32-bit result. | ||
| 32 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | 100 | static __inline__ unsigned int __DEFAULT_FN_ATTRS |
| 33 | _pext_u32(unsigned int __X, unsigned int __Y) | 101 | _pext_u32(unsigned int __X, unsigned int __Y) |
| 34 | { | 102 | { |
| 35 | return __builtin_ia32_pext_si(__X, __Y); | 103 | return __builtin_ia32_pext_si(__X, __Y); |
| 36 | } | 104 | } |
| 37 | 105 | ||
| 106 | /// Multiplies the unsigned 32-bit integers \a __X and \a __Y to form a | ||
| 107 | /// 64-bit product. Stores the upper 32 bits of the product in the | ||
| 108 | /// memory at \a __P and returns the lower 32 bits. | ||
| 109 | /// | ||
| 110 | /// \code{.operation} | ||
| 111 | /// Store32(__P, (__X * __Y)[63:32]) | ||
| 112 | /// result := (__X * __Y)[31:0] | ||
| 113 | /// \endcode | ||
| 114 | /// | ||
| 115 | /// \headerfile <immintrin.h> | ||
| 116 | /// | ||
| 117 | /// This intrinsic corresponds to the \c MULX instruction. | ||
| 118 | /// | ||
| 119 | /// \param __X | ||
| 120 | /// An unsigned 32-bit multiplicand. | ||
| 121 | /// \param __Y | ||
| 122 | /// An unsigned 32-bit multiplicand. | ||
| 123 | /// \param __P | ||
| 124 | /// A pointer to memory for storing the upper half of the product. | ||
| 125 | /// \returns The lower half of the product. | ||
| 126 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | ||
| 127 | _mulx_u32(unsigned int __X, unsigned int __Y, unsigned int *__P) | ||
| 128 | { | ||
| 129 | unsigned long long __res = (unsigned long long) __X * __Y; | ||
| 130 | *__P = (unsigned int)(__res >> 32); | ||
| 131 | return (unsigned int)__res; | ||
| 132 | } | ||
| 133 | |||
| 38 | #ifdef __x86_64__ | 134 | #ifdef __x86_64__ |
| 39 | 135 | ||
| 136 | /// Copies the unsigned 64-bit integer \a __X and zeroes the upper bits | ||
| 137 | /// starting at bit number \a __Y. | ||
| 138 | /// | ||
| 139 | /// \code{.operation} | ||
| 140 | /// i := __Y[7:0] | ||
| 141 | /// result := __X | ||
| 142 | /// IF i < 64 | ||
| 143 | /// result[63:i] := 0 | ||
| 144 | /// FI | ||
| 145 | /// \endcode | ||
| 146 | /// | ||
| 147 | /// \headerfile <immintrin.h> | ||
| 148 | /// | ||
| 149 | /// This intrinsic corresponds to the \c BZHI instruction. | ||
| 150 | /// | ||
| 151 | /// \param __X | ||
| 152 | /// The 64-bit source value to copy. | ||
| 153 | /// \param __Y | ||
| 154 | /// The lower 8 bits specify the bit number of the lowest bit to zero. | ||
| 155 | /// \returns The partially zeroed 64-bit value. | ||
| 40 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 156 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS |
| 41 | _bzhi_u64(unsigned long long __X, unsigned long long __Y) | 157 | _bzhi_u64(unsigned long long __X, unsigned long long __Y) |
| 42 | { | 158 | { |
| 43 | return __builtin_ia32_bzhi_di(__X, __Y); | 159 | return __builtin_ia32_bzhi_di(__X, __Y); |
| 44 | } | 160 | } |
| 45 | 161 | ||
| 162 | /// Deposit (scatter) low-order bits from the unsigned 64-bit integer \a __X | ||
| 163 | /// into the 64-bit result, according to the mask in the unsigned 64-bit | ||
| 164 | /// integer \a __Y. All other bits of the result are zero. | ||
| 165 | /// | ||
| 166 | /// \code{.operation} | ||
| 167 | /// i := 0 | ||
| 168 | /// result := 0 | ||
| 169 | /// FOR m := 0 TO 63 | ||
| 170 | /// IF __Y[m] == 1 | ||
| 171 | /// result[m] := __X[i] | ||
| 172 | /// i := i + 1 | ||
| 173 | /// ENDIF | ||
| 174 | /// ENDFOR | ||
| 175 | /// \endcode | ||
| 176 | /// | ||
| 177 | /// \headerfile <immintrin.h> | ||
| 178 | /// | ||
| 179 | /// This intrinsic corresponds to the \c PDEP instruction. | ||
| 180 | /// | ||
| 181 | /// \param __X | ||
| 182 | /// The 64-bit source value to copy. | ||
| 183 | /// \param __Y | ||
| 184 | /// The 64-bit mask specifying where to deposit source bits. | ||
| 185 | /// \returns The 64-bit result. | ||
| 46 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 186 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS |
| 47 | _pdep_u64(unsigned long long __X, unsigned long long __Y) | 187 | _pdep_u64(unsigned long long __X, unsigned long long __Y) |
| 48 | { | 188 | { |
| 49 | return __builtin_ia32_pdep_di(__X, __Y); | 189 | return __builtin_ia32_pdep_di(__X, __Y); |
| 50 | } | 190 | } |
| 51 | 191 | ||
| 192 | /// Extract (gather) bits from the unsigned 64-bit integer \a __X into the | ||
| 193 | /// low-order bits of the 64-bit result, according to the mask in the | ||
| 194 | /// unsigned 64-bit integer \a __Y. All other bits of the result are zero. | ||
| 195 | /// | ||
| 196 | /// \code{.operation} | ||
| 197 | /// i := 0 | ||
| 198 | /// result := 0 | ||
| 199 | /// FOR m := 0 TO 63 | ||
| 200 | /// IF __Y[m] == 1 | ||
| 201 | /// result[i] := __X[m] | ||
| 202 | /// i := i + 1 | ||
| 203 | /// ENDIF | ||
| 204 | /// ENDFOR | ||
| 205 | /// \endcode | ||
| 206 | /// | ||
| 207 | /// \headerfile <immintrin.h> | ||
| 208 | /// | ||
| 209 | /// This intrinsic corresponds to the \c PEXT instruction. | ||
| 210 | /// | ||
| 211 | /// \param __X | ||
| 212 | /// The 64-bit source value to copy. | ||
| 213 | /// \param __Y | ||
| 214 | /// The 64-bit mask specifying which source bits to extract. | ||
| 215 | /// \returns The 64-bit result. | ||
| 52 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 216 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS |
| 53 | _pext_u64(unsigned long long __X, unsigned long long __Y) | 217 | _pext_u64(unsigned long long __X, unsigned long long __Y) |
| 54 | { | 218 | { |
| 55 | return __builtin_ia32_pext_di(__X, __Y); | 219 | return __builtin_ia32_pext_di(__X, __Y); |
| 56 | } | 220 | } |
| 57 | 221 | ||
| 222 | /// Multiplies the unsigned 64-bit integers \a __X and \a __Y to form a | ||
| 223 | /// 128-bit product. Stores the upper 64 bits of the product to the | ||
| 224 | /// memory addressed by \a __P and returns the lower 64 bits. | ||
| 225 | /// | ||
| 226 | /// \code{.operation} | ||
| 227 | /// Store64(__P, (__X * __Y)[127:64]) | ||
| 228 | /// result := (__X * __Y)[63:0] | ||
| 229 | /// \endcode | ||
| 230 | /// | ||
| 231 | /// \headerfile <immintrin.h> | ||
| 232 | /// | ||
| 233 | /// This intrinsic corresponds to the \c MULX instruction. | ||
| 234 | /// | ||
| 235 | /// \param __X | ||
| 236 | /// An unsigned 64-bit multiplicand. | ||
| 237 | /// \param __Y | ||
| 238 | /// An unsigned 64-bit multiplicand. | ||
| 239 | /// \param __P | ||
| 240 | /// A pointer to memory for storing the upper half of the product. | ||
| 241 | /// \returns The lower half of the product. | ||
| 58 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS | 242 | static __inline__ unsigned long long __DEFAULT_FN_ATTRS |
| 59 | _mulx_u64 (unsigned long long __X, unsigned long long __Y, | 243 | _mulx_u64 (unsigned long long __X, unsigned long long __Y, |
| 60 | unsigned long long *__P) | 244 | unsigned long long *__P) |
| ... | @@ -64,17 +248,7 @@ _mulx_u64 (unsigned long long __X, unsigned long long __Y, | ... | @@ -64,17 +248,7 @@ _mulx_u64 (unsigned long long __X, unsigned long long __Y, |
| 64 | return (unsigned long long) __res; | 248 | return (unsigned long long) __res; |
| 65 | } | 249 | } |
| 66 | 250 | ||
| 67 | #else /* !__x86_64__ */ | 251 | #endif /* __x86_64__ */ |
| 68 | |||
| 69 | static __inline__ unsigned int __DEFAULT_FN_ATTRS | ||
| 70 | _mulx_u32 (unsigned int __X, unsigned int __Y, unsigned int *__P) | ||
| 71 | { | ||
| 72 | unsigned long long __res = (unsigned long long) __X * __Y; | ||
| 73 | *__P = (unsigned int) (__res >> 32); | ||
| 74 | return (unsigned int) __res; | ||
| 75 | } | ||
| 76 | |||
| 77 | #endif /* !__x86_64__ */ | ||
| 78 | 252 | ||
| 79 | #undef __DEFAULT_FN_ATTRS | 253 | #undef __DEFAULT_FN_ATTRS |
| 80 | 254 |
lib/include/clflushoptintrin.h+9
| ... | @@ -17,6 +17,15 @@ | ... | @@ -17,6 +17,15 @@ |
| 17 | /* Define the default attributes for the functions in this file. */ | 17 | /* Define the default attributes for the functions in this file. */ |
| 18 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("clflushopt"))) | 18 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("clflushopt"))) |
| 19 | 19 | ||
| 20 | /// Invalidates all levels of the cache hierarchy and flushes modified data to | ||
| 21 | /// memory for the cache line specified by the address \a __m. | ||
| 22 | /// | ||
| 23 | /// \headerfile <immintrin.h> | ||
| 24 | /// | ||
| 25 | /// This intrinsic corresponds to the \c CLFLUSHOPT instruction. | ||
| 26 | /// | ||
| 27 | /// \param __m | ||
| 28 | /// An address within the cache line to flush and invalidate. | ||
| 20 | static __inline__ void __DEFAULT_FN_ATTRS | 29 | static __inline__ void __DEFAULT_FN_ATTRS |
| 21 | _mm_clflushopt(void const * __m) { | 30 | _mm_clflushopt(void const * __m) { |
| 22 | __builtin_ia32_clflushopt(__m); | 31 | __builtin_ia32_clflushopt(__m); |
lib/include/clzerointrin.h+7-5
| ... | @@ -6,7 +6,7 @@ | ... | @@ -6,7 +6,7 @@ |
| 6 | * | 6 | * |
| 7 | *===-----------------------------------------------------------------------=== | 7 | *===-----------------------------------------------------------------------=== |
| 8 | */ | 8 | */ |
| 9 | #if !defined __X86INTRIN_H && !defined __IMMINTRIN_H | 9 | #ifndef __X86INTRIN_H |
| 10 | #error "Never use <clzerointrin.h> directly; include <x86intrin.h> instead." | 10 | #error "Never use <clzerointrin.h> directly; include <x86intrin.h> instead." |
| 11 | #endif | 11 | #endif |
| 12 | 12 | ||
| ... | @@ -17,14 +17,16 @@ | ... | @@ -17,14 +17,16 @@ |
| 17 | #define __DEFAULT_FN_ATTRS \ | 17 | #define __DEFAULT_FN_ATTRS \ |
| 18 | __attribute__((__always_inline__, __nodebug__, __target__("clzero"))) | 18 | __attribute__((__always_inline__, __nodebug__, __target__("clzero"))) |
| 19 | 19 | ||
| 20 | /// Loads the cache line address and zero's out the cacheline | 20 | /// Zeroes out the cache line for the address \a __line. This uses a |
| 21 | /// non-temporal store. Calling \c _mm_sfence() afterward might be needed | ||
| 22 | /// to enforce ordering. | ||
| 21 | /// | 23 | /// |
| 22 | /// \headerfile <clzerointrin.h> | 24 | /// \headerfile <x86intrin.h> |
| 23 | /// | 25 | /// |
| 24 | /// This intrinsic corresponds to the <c> CLZERO </c> instruction. | 26 | /// This intrinsic corresponds to the \c CLZERO instruction. |
| 25 | /// | 27 | /// |
| 26 | /// \param __line | 28 | /// \param __line |
| 27 | /// A pointer to a cacheline which needs to be zeroed out. | 29 | /// An address within the cache line to zero out. |
| 28 | static __inline__ void __DEFAULT_FN_ATTRS | 30 | static __inline__ void __DEFAULT_FN_ATTRS |
| 29 | _mm_clzero (void * __line) | 31 | _mm_clzero (void * __line) |
| 30 | { | 32 | { |
lib/include/cuda_wrappers/bits/shared_ptr_base.h created+9
| ... | @@ -0,0 +1,9 @@ | ||
| 1 | // CUDA headers define __noinline__ which interferes with libstdc++'s use of | ||
| 2 | // `__attribute((__noinline__))`. In order to avoid compilation error, | ||
| 3 | // temporarily unset __noinline__ when we include affected libstdc++ header. | ||
| 4 | |||
| 5 | #pragma push_macro("__noinline__") | ||
| 6 | #undef __noinline__ | ||
| 7 | #include_next "bits/shared_ptr_base.h" | ||
| 8 | |||
| 9 | #pragma pop_macro("__noinline__") | ||
lib/include/fmaintrin.h+564
| ... | @@ -18,192 +18,756 @@ | ... | @@ -18,192 +18,756 @@ |
| 18 | #define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("fma"), __min_vector_width__(128))) | 18 | #define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("fma"), __min_vector_width__(128))) |
| 19 | #define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("fma"), __min_vector_width__(256))) | 19 | #define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("fma"), __min_vector_width__(256))) |
| 20 | 20 | ||
| 21 | /// Computes a multiply-add of 128-bit vectors of [4 x float]. | ||
| 22 | /// For each element, computes <c> (__A * __B) + __C </c>. | ||
| 23 | /// | ||
| 24 | /// \headerfile <immintrin.h> | ||
| 25 | /// | ||
| 26 | /// This intrinsic corresponds to the \c VFMADD213PS instruction. | ||
| 27 | /// | ||
| 28 | /// \param __A | ||
| 29 | /// A 128-bit vector of [4 x float] containing the multiplicand. | ||
| 30 | /// \param __B | ||
| 31 | /// A 128-bit vector of [4 x float] containing the multiplier. | ||
| 32 | /// \param __C | ||
| 33 | /// A 128-bit vector of [4 x float] containing the addend. | ||
| 34 | /// \returns A 128-bit vector of [4 x float] containing the result. | ||
| 21 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 | 35 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 |
| 22 | _mm_fmadd_ps(__m128 __A, __m128 __B, __m128 __C) | 36 | _mm_fmadd_ps(__m128 __A, __m128 __B, __m128 __C) |
| 23 | { | 37 | { |
| 24 | return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, (__v4sf)__C); | 38 | return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, (__v4sf)__C); |
| 25 | } | 39 | } |
| 26 | 40 | ||
| 41 | /// Computes a multiply-add of 128-bit vectors of [2 x double]. | ||
| 42 | /// For each element, computes <c> (__A * __B) + __C </c>. | ||
| 43 | /// | ||
| 44 | /// \headerfile <immintrin.h> | ||
| 45 | /// | ||
| 46 | /// This intrinsic corresponds to the \c VFMADD213PD instruction. | ||
| 47 | /// | ||
| 48 | /// \param __A | ||
| 49 | /// A 128-bit vector of [2 x double] containing the multiplicand. | ||
| 50 | /// \param __B | ||
| 51 | /// A 128-bit vector of [2 x double] containing the multiplier. | ||
| 52 | /// \param __C | ||
| 53 | /// A 128-bit vector of [2 x double] containing the addend. | ||
| 54 | /// \returns A 128-bit [2 x double] vector containing the result. | ||
| 27 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 | 55 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 |
| 28 | _mm_fmadd_pd(__m128d __A, __m128d __B, __m128d __C) | 56 | _mm_fmadd_pd(__m128d __A, __m128d __B, __m128d __C) |
| 29 | { | 57 | { |
| 30 | return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, (__v2df)__C); | 58 | return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, (__v2df)__C); |
| 31 | } | 59 | } |
| 32 | 60 | ||
| 61 | /// Computes a scalar multiply-add of the single-precision values in the | ||
| 62 | /// low 32 bits of 128-bit vectors of [4 x float]. | ||
| 63 | /// \code | ||
| 64 | /// result[31:0] = (__A[31:0] * __B[31:0]) + __C[31:0] | ||
| 65 | /// result[127:32] = __A[127:32] | ||
| 66 | /// \endcode | ||
| 67 | /// | ||
| 68 | /// \headerfile <immintrin.h> | ||
| 69 | /// | ||
| 70 | /// This intrinsic corresponds to the \c VFMADD213SS instruction. | ||
| 71 | /// | ||
| 72 | /// \param __A | ||
| 73 | /// A 128-bit vector of [4 x float] containing the multiplicand in the low | ||
| 74 | /// 32 bits. | ||
| 75 | /// \param __B | ||
| 76 | /// A 128-bit vector of [4 x float] containing the multiplier in the low | ||
| 77 | /// 32 bits. | ||
| 78 | /// \param __C | ||
| 79 | /// A 128-bit vector of [4 x float] containing the addend in the low | ||
| 80 | /// 32 bits. | ||
| 81 | /// \returns A 128-bit vector of [4 x float] containing the result in the low | ||
| 82 | /// 32 bits and a copy of \a __A[127:32] in the upper 96 bits. | ||
| 33 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 | 83 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 |
| 34 | _mm_fmadd_ss(__m128 __A, __m128 __B, __m128 __C) | 84 | _mm_fmadd_ss(__m128 __A, __m128 __B, __m128 __C) |
| 35 | { | 85 | { |
| 36 | return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, (__v4sf)__B, (__v4sf)__C); | 86 | return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, (__v4sf)__B, (__v4sf)__C); |
| 37 | } | 87 | } |
| 38 | 88 | ||
| 89 | /// Computes a scalar multiply-add of the double-precision values in the | ||
| 90 | /// low 64 bits of 128-bit vectors of [2 x double]. | ||
| 91 | /// \code | ||
| 92 | /// result[63:0] = (__A[63:0] * __B[63:0]) + __C[63:0] | ||
| 93 | /// result[127:64] = __A[127:64] | ||
| 94 | /// \endcode | ||
| 95 | /// | ||
| 96 | /// \headerfile <immintrin.h> | ||
| 97 | /// | ||
| 98 | /// This intrinsic corresponds to the \c VFMADD213SD instruction. | ||
| 99 | /// | ||
| 100 | /// \param __A | ||
| 101 | /// A 128-bit vector of [2 x double] containing the multiplicand in the low | ||
| 102 | /// 64 bits. | ||
| 103 | /// \param __B | ||
| 104 | /// A 128-bit vector of [2 x double] containing the multiplier in the low | ||
| 105 | /// 64 bits. | ||
| 106 | /// \param __C | ||
| 107 | /// A 128-bit vector of [2 x double] containing the addend in the low | ||
| 108 | /// 64 bits. | ||
| 109 | /// \returns A 128-bit vector of [2 x double] containing the result in the low | ||
| 110 | /// 64 bits and a copy of \a __A[127:64] in the upper 64 bits. | ||
| 39 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 | 111 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 |
| 40 | _mm_fmadd_sd(__m128d __A, __m128d __B, __m128d __C) | 112 | _mm_fmadd_sd(__m128d __A, __m128d __B, __m128d __C) |
| 41 | { | 113 | { |
| 42 | return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, (__v2df)__B, (__v2df)__C); | 114 | return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, (__v2df)__B, (__v2df)__C); |
| 43 | } | 115 | } |
| 44 | 116 | ||
| 117 | /// Computes a multiply-subtract of 128-bit vectors of [4 x float]. | ||
| 118 | /// For each element, computes <c> (__A * __B) - __C </c>. | ||
| 119 | /// | ||
| 120 | /// \headerfile <immintrin.h> | ||
| 121 | /// | ||
| 122 | /// This intrinsic corresponds to the \c VFMSUB213PS instruction. | ||
| 123 | /// | ||
| 124 | /// \param __A | ||
| 125 | /// A 128-bit vector of [4 x float] containing the multiplicand. | ||
| 126 | /// \param __B | ||
| 127 | /// A 128-bit vector of [4 x float] containing the multiplier. | ||
| 128 | /// \param __C | ||
| 129 | /// A 128-bit vector of [4 x float] containing the subtrahend. | ||
| 130 | /// \returns A 128-bit vector of [4 x float] containing the result. | ||
| 45 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 | 131 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 |
| 46 | _mm_fmsub_ps(__m128 __A, __m128 __B, __m128 __C) | 132 | _mm_fmsub_ps(__m128 __A, __m128 __B, __m128 __C) |
| 47 | { | 133 | { |
| 48 | return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); | 134 | return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); |
| 49 | } | 135 | } |
| 50 | 136 | ||
| 137 | /// Computes a multiply-subtract of 128-bit vectors of [2 x double]. | ||
| 138 | /// For each element, computes <c> (__A * __B) - __C </c>. | ||
| 139 | /// | ||
| 140 | /// \headerfile <immintrin.h> | ||
| 141 | /// | ||
| 142 | /// This intrinsic corresponds to the \c VFMSUB213PD instruction. | ||
| 143 | /// | ||
| 144 | /// \param __A | ||
| 145 | /// A 128-bit vector of [2 x double] containing the multiplicand. | ||
| 146 | /// \param __B | ||
| 147 | /// A 128-bit vector of [2 x double] containing the multiplier. | ||
| 148 | /// \param __C | ||
| 149 | /// A 128-bit vector of [2 x double] containing the addend. | ||
| 150 | /// \returns A 128-bit vector of [2 x double] containing the result. | ||
| 51 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 | 151 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 |
| 52 | _mm_fmsub_pd(__m128d __A, __m128d __B, __m128d __C) | 152 | _mm_fmsub_pd(__m128d __A, __m128d __B, __m128d __C) |
| 53 | { | 153 | { |
| 54 | return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, -(__v2df)__C); | 154 | return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, -(__v2df)__C); |
| 55 | } | 155 | } |
| 56 | 156 | ||
| 157 | /// Computes a scalar multiply-subtract of the single-precision values in | ||
| 158 | /// the low 32 bits of 128-bit vectors of [4 x float]. | ||
| 159 | /// \code | ||
| 160 | /// result[31:0] = (__A[31:0] * __B[31:0]) - __C[31:0] | ||
| 161 | /// result[127:32] = __A[127:32] | ||
| 162 | /// \endcode | ||
| 163 | /// | ||
| 164 | /// \headerfile <immintrin.h> | ||
| 165 | /// | ||
| 166 | /// This intrinsic corresponds to the \c VFMSUB213SS instruction. | ||
| 167 | /// | ||
| 168 | /// \param __A | ||
| 169 | /// A 128-bit vector of [4 x float] containing the multiplicand in the low | ||
| 170 | /// 32 bits. | ||
| 171 | /// \param __B | ||
| 172 | /// A 128-bit vector of [4 x float] containing the multiplier in the low | ||
| 173 | /// 32 bits. | ||
| 174 | /// \param __C | ||
| 175 | /// A 128-bit vector of [4 x float] containing the subtrahend in the low | ||
| 176 | /// 32 bits. | ||
| 177 | /// \returns A 128-bit vector of [4 x float] containing the result in the low | ||
| 178 | /// 32 bits, and a copy of \a __A[127:32] in the upper 96 bits. | ||
| 57 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 | 179 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 |
| 58 | _mm_fmsub_ss(__m128 __A, __m128 __B, __m128 __C) | 180 | _mm_fmsub_ss(__m128 __A, __m128 __B, __m128 __C) |
| 59 | { | 181 | { |
| 60 | return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); | 182 | return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); |
| 61 | } | 183 | } |
| 62 | 184 | ||
| 185 | /// Computes a scalar multiply-subtract of the double-precision values in | ||
| 186 | /// the low 64 bits of 128-bit vectors of [2 x double]. | ||
| 187 | /// \code | ||
| 188 | /// result[63:0] = (__A[63:0] * __B[63:0]) - __C[63:0] | ||
| 189 | /// result[127:64] = __A[127:64] | ||
| 190 | /// \endcode | ||
| 191 | /// | ||
| 192 | /// \headerfile <immintrin.h> | ||
| 193 | /// | ||
| 194 | /// This intrinsic corresponds to the \c VFMSUB213SD instruction. | ||
| 195 | /// | ||
| 196 | /// \param __A | ||
| 197 | /// A 128-bit vector of [2 x double] containing the multiplicand in the low | ||
| 198 | /// 64 bits. | ||
| 199 | /// \param __B | ||
| 200 | /// A 128-bit vector of [2 x double] containing the multiplier in the low | ||
| 201 | /// 64 bits. | ||
| 202 | /// \param __C | ||
| 203 | /// A 128-bit vector of [2 x double] containing the subtrahend in the low | ||
| 204 | /// 64 bits. | ||
| 205 | /// \returns A 128-bit vector of [2 x double] containing the result in the low | ||
| 206 | /// 64 bits, and a copy of \a __A[127:64] in the upper 64 bits. | ||
| 63 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 | 207 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 |
| 64 | _mm_fmsub_sd(__m128d __A, __m128d __B, __m128d __C) | 208 | _mm_fmsub_sd(__m128d __A, __m128d __B, __m128d __C) |
| 65 | { | 209 | { |
| 66 | return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, (__v2df)__B, -(__v2df)__C); | 210 | return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, (__v2df)__B, -(__v2df)__C); |
| 67 | } | 211 | } |
| 68 | 212 | ||
| 213 | /// Computes a negated multiply-add of 128-bit vectors of [4 x float]. | ||
| 214 | /// For each element, computes <c> -(__A * __B) + __C </c>. | ||
| 215 | /// | ||
| 216 | /// \headerfile <immintrin.h> | ||
| 217 | /// | ||
| 218 | /// This intrinsic corresponds to the \c VFNMADD213DPS instruction. | ||
| 219 | /// | ||
| 220 | /// \param __A | ||
| 221 | /// A 128-bit vector of [4 x float] containing the multiplicand. | ||
| 222 | /// \param __B | ||
| 223 | /// A 128-bit vector of [4 x float] containing the multiplier. | ||
| 224 | /// \param __C | ||
| 225 | /// A 128-bit vector of [4 x float] containing the addend. | ||
| 226 | /// \returns A 128-bit [4 x float] vector containing the result. | ||
| 69 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 | 227 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 |
| 70 | _mm_fnmadd_ps(__m128 __A, __m128 __B, __m128 __C) | 228 | _mm_fnmadd_ps(__m128 __A, __m128 __B, __m128 __C) |
| 71 | { | 229 | { |
| 72 | return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C); | 230 | return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C); |
| 73 | } | 231 | } |
| 74 | 232 | ||
| 233 | /// Computes a negated multiply-add of 128-bit vectors of [2 x double]. | ||
| 234 | /// For each element, computes <c> -(__A * __B) + __C </c>. | ||
| 235 | /// | ||
| 236 | /// \headerfile <immintrin.h> | ||
| 237 | /// | ||
| 238 | /// This intrinsic corresponds to the \c VFNMADD213PD instruction. | ||
| 239 | /// | ||
| 240 | /// \param __A | ||
| 241 | /// A 128-bit vector of [2 x double] containing the multiplicand. | ||
| 242 | /// \param __B | ||
| 243 | /// A 128-bit vector of [2 x double] containing the multiplier. | ||
| 244 | /// \param __C | ||
| 245 | /// A 128-bit vector of [2 x double] containing the addend. | ||
| 246 | /// \returns A 128-bit vector of [2 x double] containing the result. | ||
| 75 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 | 247 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 |
| 76 | _mm_fnmadd_pd(__m128d __A, __m128d __B, __m128d __C) | 248 | _mm_fnmadd_pd(__m128d __A, __m128d __B, __m128d __C) |
| 77 | { | 249 | { |
| 78 | return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, (__v2df)__C); | 250 | return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, (__v2df)__C); |
| 79 | } | 251 | } |
| 80 | 252 | ||
| 253 | /// Computes a scalar negated multiply-add of the single-precision values in | ||
| 254 | /// the low 32 bits of 128-bit vectors of [4 x float]. | ||
| 255 | /// \code | ||
| 256 | /// result[31:0] = -(__A[31:0] * __B[31:0]) + __C[31:0] | ||
| 257 | /// result[127:32] = __A[127:32] | ||
| 258 | /// \endcode | ||
| 259 | /// | ||
| 260 | /// \headerfile <immintrin.h> | ||
| 261 | /// | ||
| 262 | /// This intrinsic corresponds to the \c VFNMADD213SS instruction. | ||
| 263 | /// | ||
| 264 | /// \param __A | ||
| 265 | /// A 128-bit vector of [4 x float] containing the multiplicand in the low | ||
| 266 | /// 32 bits. | ||
| 267 | /// \param __B | ||
| 268 | /// A 128-bit vector of [4 x float] containing the multiplier in the low | ||
| 269 | /// 32 bits. | ||
| 270 | /// \param __C | ||
| 271 | /// A 128-bit vector of [4 x float] containing the addend in the low | ||
| 272 | /// 32 bits. | ||
| 273 | /// \returns A 128-bit vector of [4 x float] containing the result in the low | ||
| 274 | /// 32 bits, and a copy of \a __A[127:32] in the upper 96 bits. | ||
| 81 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 | 275 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 |
| 82 | _mm_fnmadd_ss(__m128 __A, __m128 __B, __m128 __C) | 276 | _mm_fnmadd_ss(__m128 __A, __m128 __B, __m128 __C) |
| 83 | { | 277 | { |
| 84 | return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, -(__v4sf)__B, (__v4sf)__C); | 278 | return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, -(__v4sf)__B, (__v4sf)__C); |
| 85 | } | 279 | } |
| 86 | 280 | ||
| 281 | /// Computes a scalar negated multiply-add of the double-precision values | ||
| 282 | /// in the low 64 bits of 128-bit vectors of [2 x double]. | ||
| 283 | /// \code | ||
| 284 | /// result[63:0] = -(__A[63:0] * __B[63:0]) + __C[63:0] | ||
| 285 | /// result[127:64] = __A[127:64] | ||
| 286 | /// \endcode | ||
| 287 | /// | ||
| 288 | /// \headerfile <immintrin.h> | ||
| 289 | /// | ||
| 290 | /// This intrinsic corresponds to the \c VFNMADD213SD instruction. | ||
| 291 | /// | ||
| 292 | /// \param __A | ||
| 293 | /// A 128-bit vector of [2 x double] containing the multiplicand in the low | ||
| 294 | /// 64 bits. | ||
| 295 | /// \param __B | ||
| 296 | /// A 128-bit vector of [2 x double] containing the multiplier in the low | ||
| 297 | /// 64 bits. | ||
| 298 | /// \param __C | ||
| 299 | /// A 128-bit vector of [2 x double] containing the addend in the low | ||
| 300 | /// 64 bits. | ||
| 301 | /// \returns A 128-bit vector of [2 x double] containing the result in the low | ||
| 302 | /// 64 bits, and a copy of \a __A[127:64] in the upper 64 bits. | ||
| 87 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 | 303 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 |
| 88 | _mm_fnmadd_sd(__m128d __A, __m128d __B, __m128d __C) | 304 | _mm_fnmadd_sd(__m128d __A, __m128d __B, __m128d __C) |
| 89 | { | 305 | { |
| 90 | return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, -(__v2df)__B, (__v2df)__C); | 306 | return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, -(__v2df)__B, (__v2df)__C); |
| 91 | } | 307 | } |
| 92 | 308 | ||
| 309 | /// Computes a negated multiply-subtract of 128-bit vectors of [4 x float]. | ||
| 310 | /// For each element, computes <c> -(__A * __B) - __C </c>. | ||
| 311 | /// | ||
| 312 | /// \headerfile <immintrin.h> | ||
| 313 | /// | ||
| 314 | /// This intrinsic corresponds to the \c VFNMSUB213PS instruction. | ||
| 315 | /// | ||
| 316 | /// \param __A | ||
| 317 | /// A 128-bit vector of [4 x float] containing the multiplicand. | ||
| 318 | /// \param __B | ||
| 319 | /// A 128-bit vector of [4 x float] containing the multiplier. | ||
| 320 | /// \param __C | ||
| 321 | /// A 128-bit vector of [4 x float] containing the subtrahend. | ||
| 322 | /// \returns A 128-bit vector of [4 x float] containing the result. | ||
| 93 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 | 323 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 |
| 94 | _mm_fnmsub_ps(__m128 __A, __m128 __B, __m128 __C) | 324 | _mm_fnmsub_ps(__m128 __A, __m128 __B, __m128 __C) |
| 95 | { | 325 | { |
| 96 | return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); | 326 | return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); |
| 97 | } | 327 | } |
| 98 | 328 | ||
| 329 | /// Computes a negated multiply-subtract of 128-bit vectors of [2 x double]. | ||
| 330 | /// For each element, computes <c> -(__A * __B) - __C </c>. | ||
| 331 | /// | ||
| 332 | /// \headerfile <immintrin.h> | ||
| 333 | /// | ||
| 334 | /// This intrinsic corresponds to the \c VFNMSUB213PD instruction. | ||
| 335 | /// | ||
| 336 | /// \param __A | ||
| 337 | /// A 128-bit vector of [2 x double] containing the multiplicand. | ||
| 338 | /// \param __B | ||
| 339 | /// A 128-bit vector of [2 x double] containing the multiplier. | ||
| 340 | /// \param __C | ||
| 341 | /// A 128-bit vector of [2 x double] containing the subtrahend. | ||
| 342 | /// \returns A 128-bit vector of [2 x double] containing the result. | ||
| 99 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 | 343 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 |
| 100 | _mm_fnmsub_pd(__m128d __A, __m128d __B, __m128d __C) | 344 | _mm_fnmsub_pd(__m128d __A, __m128d __B, __m128d __C) |
| 101 | { | 345 | { |
| 102 | return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, -(__v2df)__C); | 346 | return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, -(__v2df)__C); |
| 103 | } | 347 | } |
| 104 | 348 | ||
| 349 | /// Computes a scalar negated multiply-subtract of the single-precision | ||
| 350 | /// values in the low 32 bits of 128-bit vectors of [4 x float]. | ||
| 351 | /// \code | ||
| 352 | /// result[31:0] = -(__A[31:0] * __B[31:0]) - __C[31:0] | ||
| 353 | /// result[127:32] = __A[127:32] | ||
| 354 | /// \endcode | ||
| 355 | /// | ||
| 356 | /// \headerfile <immintrin.h> | ||
| 357 | /// | ||
| 358 | /// This intrinsic corresponds to the \c VFNMSUB213SS instruction. | ||
| 359 | /// | ||
| 360 | /// \param __A | ||
| 361 | /// A 128-bit vector of [4 x float] containing the multiplicand in the low | ||
| 362 | /// 32 bits. | ||
| 363 | /// \param __B | ||
| 364 | /// A 128-bit vector of [4 x float] containing the multiplier in the low | ||
| 365 | /// 32 bits. | ||
| 366 | /// \param __C | ||
| 367 | /// A 128-bit vector of [4 x float] containing the subtrahend in the low | ||
| 368 | /// 32 bits. | ||
| 369 | /// \returns A 128-bit vector of [4 x float] containing the result in the low | ||
| 370 | /// 32 bits, and a copy of \a __A[127:32] in the upper 96 bits. | ||
| 105 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 | 371 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 |
| 106 | _mm_fnmsub_ss(__m128 __A, __m128 __B, __m128 __C) | 372 | _mm_fnmsub_ss(__m128 __A, __m128 __B, __m128 __C) |
| 107 | { | 373 | { |
| 108 | return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, -(__v4sf)__B, -(__v4sf)__C); | 374 | return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, -(__v4sf)__B, -(__v4sf)__C); |
| 109 | } | 375 | } |
| 110 | 376 | ||
| 377 | /// Computes a scalar negated multiply-subtract of the double-precision | ||
| 378 | /// values in the low 64 bits of 128-bit vectors of [2 x double]. | ||
| 379 | /// \code | ||
| 380 | /// result[63:0] = -(__A[63:0] * __B[63:0]) - __C[63:0] | ||
| 381 | /// result[127:64] = __A[127:64] | ||
| 382 | /// \endcode | ||
| 383 | /// | ||
| 384 | /// \headerfile <immintrin.h> | ||
| 385 | /// | ||
| 386 | /// This intrinsic corresponds to the \c VFNMSUB213SD instruction. | ||
| 387 | /// | ||
| 388 | /// \param __A | ||
| 389 | /// A 128-bit vector of [2 x double] containing the multiplicand in the low | ||
| 390 | /// 64 bits. | ||
| 391 | /// \param __B | ||
| 392 | /// A 128-bit vector of [2 x double] containing the multiplier in the low | ||
| 393 | /// 64 bits. | ||
| 394 | /// \param __C | ||
| 395 | /// A 128-bit vector of [2 x double] containing the subtrahend in the low | ||
| 396 | /// 64 bits. | ||
| 397 | /// \returns A 128-bit vector of [2 x double] containing the result in the low | ||
| 398 | /// 64 bits, and a copy of \a __A[127:64] in the upper 64 bits. | ||
| 111 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 | 399 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 |
| 112 | _mm_fnmsub_sd(__m128d __A, __m128d __B, __m128d __C) | 400 | _mm_fnmsub_sd(__m128d __A, __m128d __B, __m128d __C) |
| 113 | { | 401 | { |
| 114 | return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, -(__v2df)__B, -(__v2df)__C); | 402 | return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, -(__v2df)__B, -(__v2df)__C); |
| 115 | } | 403 | } |
| 116 | 404 | ||
| 405 | /// Computes a multiply with alternating add/subtract of 128-bit vectors of | ||
| 406 | /// [4 x float]. | ||
| 407 | /// \code | ||
| 408 | /// result[31:0] = (__A[31:0] * __B[31:0]) - __C[31:0] | ||
| 409 | /// result[63:32] = (__A[63:32] * __B[63:32]) + __C[63:32] | ||
| 410 | /// result[95:64] = (__A[95:64] * __B[95:64]) - __C[95:64] | ||
| 411 | /// result[127:96] = (__A[127:96] * __B[127:96]) + __C[127:96] | ||
| 412 | /// \endcode | ||
| 413 | /// | ||
| 414 | /// \headerfile <immintrin.h> | ||
| 415 | /// | ||
| 416 | /// This intrinsic corresponds to the \c VFMADDSUB213PS instruction. | ||
| 417 | /// | ||
| 418 | /// \param __A | ||
| 419 | /// A 128-bit vector of [4 x float] containing the multiplicand. | ||
| 420 | /// \param __B | ||
| 421 | /// A 128-bit vector of [4 x float] containing the multiplier. | ||
| 422 | /// \param __C | ||
| 423 | /// A 128-bit vector of [4 x float] containing the addend/subtrahend. | ||
| 424 | /// \returns A 128-bit vector of [4 x float] containing the result. | ||
| 117 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 | 425 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 |
| 118 | _mm_fmaddsub_ps(__m128 __A, __m128 __B, __m128 __C) | 426 | _mm_fmaddsub_ps(__m128 __A, __m128 __B, __m128 __C) |
| 119 | { | 427 | { |
| 120 | return (__m128)__builtin_ia32_vfmaddsubps((__v4sf)__A, (__v4sf)__B, (__v4sf)__C); | 428 | return (__m128)__builtin_ia32_vfmaddsubps((__v4sf)__A, (__v4sf)__B, (__v4sf)__C); |
| 121 | } | 429 | } |
| 122 | 430 | ||
| 431 | /// Computes a multiply with alternating add/subtract of 128-bit vectors of | ||
| 432 | /// [2 x double]. | ||
| 433 | /// \code | ||
| 434 | /// result[63:0] = (__A[63:0] * __B[63:0]) - __C[63:0] | ||
| 435 | /// result[127:64] = (__A[127:64] * __B[127:64]) + __C[127:64] | ||
| 436 | /// \endcode | ||
| 437 | /// | ||
| 438 | /// \headerfile <immintrin.h> | ||
| 439 | /// | ||
| 440 | /// This intrinsic corresponds to the \c VFMADDSUB213PD instruction. | ||
| 441 | /// | ||
| 442 | /// \param __A | ||
| 443 | /// A 128-bit vector of [2 x double] containing the multiplicand. | ||
| 444 | /// \param __B | ||
| 445 | /// A 128-bit vector of [2 x double] containing the multiplier. | ||
| 446 | /// \param __C | ||
| 447 | /// A 128-bit vector of [2 x double] containing the addend/subtrahend. | ||
| 448 | /// \returns A 128-bit vector of [2 x double] containing the result. | ||
| 123 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 | 449 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 |
| 124 | _mm_fmaddsub_pd(__m128d __A, __m128d __B, __m128d __C) | 450 | _mm_fmaddsub_pd(__m128d __A, __m128d __B, __m128d __C) |
| 125 | { | 451 | { |
| 126 | return (__m128d)__builtin_ia32_vfmaddsubpd((__v2df)__A, (__v2df)__B, (__v2df)__C); | 452 | return (__m128d)__builtin_ia32_vfmaddsubpd((__v2df)__A, (__v2df)__B, (__v2df)__C); |
| 127 | } | 453 | } |
| 128 | 454 | ||
| 455 | /// Computes a multiply with alternating add/subtract of 128-bit vectors of | ||
| 456 | /// [4 x float]. | ||
| 457 | /// \code | ||
| 458 | /// result[31:0] = (__A[31:0] * __B[31:0]) + __C[31:0] | ||
| 459 | /// result[63:32] = (__A[63:32] * __B[63:32]) - __C[63:32] | ||
| 460 | /// result[95:64] = (__A[95:64] * __B[95:64]) + __C[95:64] | ||
| 461 | /// result[127:96 = (__A[127:96] * __B[127:96]) - __C[127:96] | ||
| 462 | /// \endcode | ||
| 463 | /// | ||
| 464 | /// \headerfile <immintrin.h> | ||
| 465 | /// | ||
| 466 | /// This intrinsic corresponds to the \c VFMSUBADD213PS instruction. | ||
| 467 | /// | ||
| 468 | /// \param __A | ||
| 469 | /// A 128-bit vector of [4 x float] containing the multiplicand. | ||
| 470 | /// \param __B | ||
| 471 | /// A 128-bit vector of [4 x float] containing the multiplier. | ||
| 472 | /// \param __C | ||
| 473 | /// A 128-bit vector of [4 x float] containing the addend/subtrahend. | ||
| 474 | /// \returns A 128-bit vector of [4 x float] containing the result. | ||
| 129 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 | 475 | static __inline__ __m128 __DEFAULT_FN_ATTRS128 |
| 130 | _mm_fmsubadd_ps(__m128 __A, __m128 __B, __m128 __C) | 476 | _mm_fmsubadd_ps(__m128 __A, __m128 __B, __m128 __C) |
| 131 | { | 477 | { |
| 132 | return (__m128)__builtin_ia32_vfmaddsubps((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); | 478 | return (__m128)__builtin_ia32_vfmaddsubps((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); |
| 133 | } | 479 | } |
| 134 | 480 | ||
| 481 | /// Computes a multiply with alternating add/subtract of 128-bit vectors of | ||
| 482 | /// [2 x double]. | ||
| 483 | /// \code | ||
| 484 | /// result[63:0] = (__A[63:0] * __B[63:0]) + __C[63:0] | ||
| 485 | /// result[127:64] = (__A[127:64] * __B[127:64]) - __C[127:64] | ||
| 486 | /// \endcode | ||
| 487 | /// | ||
| 488 | /// \headerfile <immintrin.h> | ||
| 489 | /// | ||
| 490 | /// This intrinsic corresponds to the \c VFMADDSUB213PD instruction. | ||
| 491 | /// | ||
| 492 | /// \param __A | ||
| 493 | /// A 128-bit vector of [2 x double] containing the multiplicand. | ||
| 494 | /// \param __B | ||
| 495 | /// A 128-bit vector of [2 x double] containing the multiplier. | ||
| 496 | /// \param __C | ||
| 497 | /// A 128-bit vector of [2 x double] containing the addend/subtrahend. | ||
| 498 | /// \returns A 128-bit vector of [2 x double] containing the result. | ||
| 135 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 | 499 | static __inline__ __m128d __DEFAULT_FN_ATTRS128 |
| 136 | _mm_fmsubadd_pd(__m128d __A, __m128d __B, __m128d __C) | 500 | _mm_fmsubadd_pd(__m128d __A, __m128d __B, __m128d __C) |
| 137 | { | 501 | { |
| 138 | return (__m128d)__builtin_ia32_vfmaddsubpd((__v2df)__A, (__v2df)__B, -(__v2df)__C); | 502 | return (__m128d)__builtin_ia32_vfmaddsubpd((__v2df)__A, (__v2df)__B, -(__v2df)__C); |
| 139 | } | 503 | } |
| 140 | 504 | ||
| 505 | /// Computes a multiply-add of 256-bit vectors of [8 x float]. | ||
| 506 | /// For each element, computes <c> (__A * __B) + __C </c>. | ||
| 507 | /// | ||
| 508 | /// \headerfile <immintrin.h> | ||
| 509 | /// | ||
| 510 | /// This intrinsic corresponds to the \c VFMADD213PS instruction. | ||
| 511 | /// | ||
| 512 | /// \param __A | ||
| 513 | /// A 256-bit vector of [8 x float] containing the multiplicand. | ||
| 514 | /// \param __B | ||
| 515 | /// A 256-bit vector of [8 x float] containing the multiplier. | ||
| 516 | /// \param __C | ||
| 517 | /// A 256-bit vector of [8 x float] containing the addend. | ||
| 518 | /// \returns A 256-bit vector of [8 x float] containing the result. | ||
| 141 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 | 519 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 |
| 142 | _mm256_fmadd_ps(__m256 __A, __m256 __B, __m256 __C) | 520 | _mm256_fmadd_ps(__m256 __A, __m256 __B, __m256 __C) |
| 143 | { | 521 | { |
| 144 | return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, (__v8sf)__C); | 522 | return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, (__v8sf)__C); |
| 145 | } | 523 | } |
| 146 | 524 | ||
| 525 | /// Computes a multiply-add of 256-bit vectors of [4 x double]. | ||
| 526 | /// For each element, computes <c> (__A * __B) + __C </c>. | ||
| 527 | /// | ||
| 528 | /// \headerfile <immintrin.h> | ||
| 529 | /// | ||
| 530 | /// This intrinsic corresponds to the \c VFMADD213PD instruction. | ||
| 531 | /// | ||
| 532 | /// \param __A | ||
| 533 | /// A 256-bit vector of [4 x double] containing the multiplicand. | ||
| 534 | /// \param __B | ||
| 535 | /// A 256-bit vector of [4 x double] containing the multiplier. | ||
| 536 | /// \param __C | ||
| 537 | /// A 256-bit vector of [4 x double] containing the addend. | ||
| 538 | /// \returns A 256-bit vector of [4 x double] containing the result. | ||
| 147 | static __inline__ __m256d __DEFAULT_FN_ATTRS256 | 539 | static __inline__ __m256d __DEFAULT_FN_ATTRS256 |
| 148 | _mm256_fmadd_pd(__m256d __A, __m256d __B, __m256d __C) | 540 | _mm256_fmadd_pd(__m256d __A, __m256d __B, __m256d __C) |
| 149 | { | 541 | { |
| 150 | return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, (__v4df)__C); | 542 | return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, (__v4df)__C); |
| 151 | } | 543 | } |
| 152 | 544 | ||
| 545 | /// Computes a multiply-subtract of 256-bit vectors of [8 x float]. | ||
| 546 | /// For each element, computes <c> (__A * __B) - __C </c>. | ||
| 547 | /// | ||
| 548 | /// \headerfile <immintrin.h> | ||
| 549 | /// | ||
| 550 | /// This intrinsic corresponds to the \c VFMSUB213PS instruction. | ||
| 551 | /// | ||
| 552 | /// \param __A | ||
| 553 | /// A 256-bit vector of [8 x float] containing the multiplicand. | ||
| 554 | /// \param __B | ||
| 555 | /// A 256-bit vector of [8 x float] containing the multiplier. | ||
| 556 | /// \param __C | ||
| 557 | /// A 256-bit vector of [8 x float] containing the subtrahend. | ||
| 558 | /// \returns A 256-bit vector of [8 x float] containing the result. | ||
| 153 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 | 559 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 |
| 154 | _mm256_fmsub_ps(__m256 __A, __m256 __B, __m256 __C) | 560 | _mm256_fmsub_ps(__m256 __A, __m256 __B, __m256 __C) |
| 155 | { | 561 | { |
| 156 | return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C); | 562 | return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C); |
| 157 | } | 563 | } |
| 158 | 564 | ||
| 565 | /// Computes a multiply-subtract of 256-bit vectors of [4 x double]. | ||
| 566 | /// For each element, computes <c> (__A * __B) - __C </c>. | ||
| 567 | /// | ||
| 568 | /// \headerfile <immintrin.h> | ||
| 569 | /// | ||
| 570 | /// This intrinsic corresponds to the \c VFMSUB213PD instruction. | ||
| 571 | /// | ||
| 572 | /// \param __A | ||
| 573 | /// A 256-bit vector of [4 x double] containing the multiplicand. | ||
| 574 | /// \param __B | ||
| 575 | /// A 256-bit vector of [4 x double] containing the multiplier. | ||
| 576 | /// \param __C | ||
| 577 | /// A 256-bit vector of [4 x double] containing the subtrahend. | ||
| 578 | /// \returns A 256-bit vector of [4 x double] containing the result. | ||
| 159 | static __inline__ __m256d __DEFAULT_FN_ATTRS256 | 579 | static __inline__ __m256d __DEFAULT_FN_ATTRS256 |
| 160 | _mm256_fmsub_pd(__m256d __A, __m256d __B, __m256d __C) | 580 | _mm256_fmsub_pd(__m256d __A, __m256d __B, __m256d __C) |
| 161 | { | 581 | { |
| 162 | return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, -(__v4df)__C); | 582 | return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, -(__v4df)__C); |
| 163 | } | 583 | } |
| 164 | 584 | ||
| 585 | /// Computes a negated multiply-add of 256-bit vectors of [8 x float]. | ||
| 586 | /// For each element, computes <c> -(__A * __B) + __C </c>. | ||
| 587 | /// | ||
| 588 | /// \headerfile <immintrin.h> | ||
| 589 | /// | ||
| 590 | /// This intrinsic corresponds to the \c VFNMADD213PS instruction. | ||
| 591 | /// | ||
| 592 | /// \param __A | ||
| 593 | /// A 256-bit vector of [8 x float] containing the multiplicand. | ||
| 594 | /// \param __B | ||
| 595 | /// A 256-bit vector of [8 x float] containing the multiplier. | ||
| 596 | /// \param __C | ||
| 597 | /// A 256-bit vector of [8 x float] containing the addend. | ||
| 598 | /// \returns A 256-bit vector of [8 x float] containing the result. | ||
| 165 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 | 599 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 |
| 166 | _mm256_fnmadd_ps(__m256 __A, __m256 __B, __m256 __C) | 600 | _mm256_fnmadd_ps(__m256 __A, __m256 __B, __m256 __C) |
| 167 | { | 601 | { |
| 168 | return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, (__v8sf)__C); | 602 | return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, (__v8sf)__C); |
| 169 | } | 603 | } |
| 170 | 604 | ||
| 605 | /// Computes a negated multiply-add of 256-bit vectors of [4 x double]. | ||
| 606 | /// For each element, computes <c> -(__A * __B) + __C </c>. | ||
| 607 | /// | ||
| 608 | /// \headerfile <immintrin.h> | ||
| 609 | /// | ||
| 610 | /// This intrinsic corresponds to the \c VFNMADD213PD instruction. | ||
| 611 | /// | ||
| 612 | /// \param __A | ||
| 613 | /// A 256-bit vector of [4 x double] containing the multiplicand. | ||
| 614 | /// \param __B | ||
| 615 | /// A 256-bit vector of [4 x double] containing the multiplier. | ||
| 616 | /// \param __C | ||
| 617 | /// A 256-bit vector of [4 x double] containing the addend. | ||
| 618 | /// \returns A 256-bit vector of [4 x double] containing the result. | ||
| 171 | static __inline__ __m256d __DEFAULT_FN_ATTRS256 | 619 | static __inline__ __m256d __DEFAULT_FN_ATTRS256 |
| 172 | _mm256_fnmadd_pd(__m256d __A, __m256d __B, __m256d __C) | 620 | _mm256_fnmadd_pd(__m256d __A, __m256d __B, __m256d __C) |
| 173 | { | 621 | { |
| 174 | return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, (__v4df)__C); | 622 | return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, (__v4df)__C); |
| 175 | } | 623 | } |
| 176 | 624 | ||
| 625 | /// Computes a negated multiply-subtract of 256-bit vectors of [8 x float]. | ||
| 626 | /// For each element, computes <c> -(__A * __B) - __C </c>. | ||
| 627 | /// | ||
| 628 | /// \headerfile <immintrin.h> | ||
| 629 | /// | ||
| 630 | /// This intrinsic corresponds to the \c VFNMSUB213PS instruction. | ||
| 631 | /// | ||
| 632 | /// \param __A | ||
| 633 | /// A 256-bit vector of [8 x float] containing the multiplicand. | ||
| 634 | /// \param __B | ||
| 635 | /// A 256-bit vector of [8 x float] containing the multiplier. | ||
| 636 | /// \param __C | ||
| 637 | /// A 256-bit vector of [8 x float] containing the subtrahend. | ||
| 638 | /// \returns A 256-bit vector of [8 x float] containing the result. | ||
| 177 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 | 639 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 |
| 178 | _mm256_fnmsub_ps(__m256 __A, __m256 __B, __m256 __C) | 640 | _mm256_fnmsub_ps(__m256 __A, __m256 __B, __m256 __C) |
| 179 | { | 641 | { |
| 180 | return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, -(__v8sf)__C); | 642 | return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, -(__v8sf)__C); |
| 181 | } | 643 | } |
| 182 | 644 | ||
| 645 | /// Computes a negated multiply-subtract of 256-bit vectors of [4 x double]. | ||
| 646 | /// For each element, computes <c> -(__A * __B) - __C </c>. | ||
| 647 | /// | ||
| 648 | /// \headerfile <immintrin.h> | ||
| 649 | /// | ||
| 650 | /// This intrinsic corresponds to the \c VFNMSUB213PD instruction. | ||
| 651 | /// | ||
| 652 | /// \param __A | ||
| 653 | /// A 256-bit vector of [4 x double] containing the multiplicand. | ||
| 654 | /// \param __B | ||
| 655 | /// A 256-bit vector of [4 x double] containing the multiplier. | ||
| 656 | /// \param __C | ||
| 657 | /// A 256-bit vector of [4 x double] containing the subtrahend. | ||
| 658 | /// \returns A 256-bit vector of [4 x double] containing the result. | ||
| 183 | static __inline__ __m256d __DEFAULT_FN_ATTRS256 | 659 | static __inline__ __m256d __DEFAULT_FN_ATTRS256 |
| 184 | _mm256_fnmsub_pd(__m256d __A, __m256d __B, __m256d __C) | 660 | _mm256_fnmsub_pd(__m256d __A, __m256d __B, __m256d __C) |
| 185 | { | 661 | { |
| 186 | return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, -(__v4df)__C); | 662 | return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, -(__v4df)__C); |
| 187 | } | 663 | } |
| 188 | 664 | ||
| 665 | /// Computes a multiply with alternating add/subtract of 256-bit vectors of | ||
| 666 | /// [8 x float]. | ||
| 667 | /// \code | ||
| 668 | /// result[31:0] = (__A[31:0] * __B[31:0]) - __C[31:0] | ||
| 669 | /// result[63:32] = (__A[63:32] * __B[63:32]) + __C[63:32] | ||
| 670 | /// result[95:64] = (__A[95:64] * __B[95:64]) - __C[95:64] | ||
| 671 | /// result[127:96] = (__A[127:96] * __B[127:96]) + __C[127:96] | ||
| 672 | /// result[159:128] = (__A[159:128] * __B[159:128]) - __C[159:128] | ||
| 673 | /// result[191:160] = (__A[191:160] * __B[191:160]) + __C[191:160] | ||
| 674 | /// result[223:192] = (__A[223:192] * __B[223:192]) - __C[223:192] | ||
| 675 | /// result[255:224] = (__A[255:224] * __B[255:224]) + __C[255:224] | ||
| 676 | /// \endcode | ||
| 677 | /// | ||
| 678 | /// \headerfile <immintrin.h> | ||
| 679 | /// | ||
| 680 | /// This intrinsic corresponds to the \c VFMADDSUB213PS instruction. | ||
| 681 | /// | ||
| 682 | /// \param __A | ||
| 683 | /// A 256-bit vector of [8 x float] containing the multiplicand. | ||
| 684 | /// \param __B | ||
| 685 | /// A 256-bit vector of [8 x float] containing the multiplier. | ||
| 686 | /// \param __C | ||
| 687 | /// A 256-bit vector of [8 x float] containing the addend/subtrahend. | ||
| 688 | /// \returns A 256-bit vector of [8 x float] containing the result. | ||
| 189 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 | 689 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 |
| 190 | _mm256_fmaddsub_ps(__m256 __A, __m256 __B, __m256 __C) | 690 | _mm256_fmaddsub_ps(__m256 __A, __m256 __B, __m256 __C) |
| 191 | { | 691 | { |
| 192 | return (__m256)__builtin_ia32_vfmaddsubps256((__v8sf)__A, (__v8sf)__B, (__v8sf)__C); | 692 | return (__m256)__builtin_ia32_vfmaddsubps256((__v8sf)__A, (__v8sf)__B, (__v8sf)__C); |
| 193 | } | 693 | } |
| 194 | 694 | ||
| 695 | /// Computes a multiply with alternating add/subtract of 256-bit vectors of | ||
| 696 | /// [4 x double]. | ||
| 697 | /// \code | ||
| 698 | /// result[63:0] = (__A[63:0] * __B[63:0]) - __C[63:0] | ||
| 699 | /// result[127:64] = (__A[127:64] * __B[127:64]) + __C[127:64] | ||
| 700 | /// result[191:128] = (__A[191:128] * __B[191:128]) - __C[191:128] | ||
| 701 | /// result[255:192] = (__A[255:192] * __B[255:192]) + __C[255:192] | ||
| 702 | /// \endcode | ||
| 703 | /// | ||
| 704 | /// \headerfile <immintrin.h> | ||
| 705 | /// | ||
| 706 | /// This intrinsic corresponds to the \c VFMADDSUB213PD instruction. | ||
| 707 | /// | ||
| 708 | /// \param __A | ||
| 709 | /// A 256-bit vector of [4 x double] containing the multiplicand. | ||
| 710 | /// \param __B | ||
| 711 | /// A 256-bit vector of [4 x double] containing the multiplier. | ||
| 712 | /// \param __C | ||
| 713 | /// A 256-bit vector of [4 x double] containing the addend/subtrahend. | ||
| 714 | /// \returns A 256-bit vector of [4 x double] containing the result. | ||
| 195 | static __inline__ __m256d __DEFAULT_FN_ATTRS256 | 715 | static __inline__ __m256d __DEFAULT_FN_ATTRS256 |
| 196 | _mm256_fmaddsub_pd(__m256d __A, __m256d __B, __m256d __C) | 716 | _mm256_fmaddsub_pd(__m256d __A, __m256d __B, __m256d __C) |
| 197 | { | 717 | { |
| 198 | return (__m256d)__builtin_ia32_vfmaddsubpd256((__v4df)__A, (__v4df)__B, (__v4df)__C); | 718 | return (__m256d)__builtin_ia32_vfmaddsubpd256((__v4df)__A, (__v4df)__B, (__v4df)__C); |
| 199 | } | 719 | } |
| 200 | 720 | ||
| 721 | /// Computes a vector multiply with alternating add/subtract of 256-bit | ||
| 722 | /// vectors of [8 x float]. | ||
| 723 | /// \code | ||
| 724 | /// result[31:0] = (__A[31:0] * __B[31:0]) + __C[31:0] | ||
| 725 | /// result[63:32] = (__A[63:32] * __B[63:32]) - __C[63:32] | ||
| 726 | /// result[95:64] = (__A[95:64] * __B[95:64]) + __C[95:64] | ||
| 727 | /// result[127:96] = (__A[127:96] * __B[127:96]) - __C[127:96] | ||
| 728 | /// result[159:128] = (__A[159:128] * __B[159:128]) + __C[159:128] | ||
| 729 | /// result[191:160] = (__A[191:160] * __B[191:160]) - __C[191:160] | ||
| 730 | /// result[223:192] = (__A[223:192] * __B[223:192]) + __C[223:192] | ||
| 731 | /// result[255:224] = (__A[255:224] * __B[255:224]) - __C[255:224] | ||
| 732 | /// \endcode | ||
| 733 | /// | ||
| 734 | /// \headerfile <immintrin.h> | ||
| 735 | /// | ||
| 736 | /// This intrinsic corresponds to the \c VFMSUBADD213PS instruction. | ||
| 737 | /// | ||
| 738 | /// \param __A | ||
| 739 | /// A 256-bit vector of [8 x float] containing the multiplicand. | ||
| 740 | /// \param __B | ||
| 741 | /// A 256-bit vector of [8 x float] containing the multiplier. | ||
| 742 | /// \param __C | ||
| 743 | /// A 256-bit vector of [8 x float] containing the addend/subtrahend. | ||
| 744 | /// \returns A 256-bit vector of [8 x float] containing the result. | ||
| 201 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 | 745 | static __inline__ __m256 __DEFAULT_FN_ATTRS256 |
| 202 | _mm256_fmsubadd_ps(__m256 __A, __m256 __B, __m256 __C) | 746 | _mm256_fmsubadd_ps(__m256 __A, __m256 __B, __m256 __C) |
| 203 | { | 747 | { |
| 204 | return (__m256)__builtin_ia32_vfmaddsubps256((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C); | 748 | return (__m256)__builtin_ia32_vfmaddsubps256((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C); |
| 205 | } | 749 | } |
| 206 | 750 | ||
| 751 | /// Computes a vector multiply with alternating add/subtract of 256-bit | ||
| 752 | /// vectors of [4 x double]. | ||
| 753 | /// \code | ||
| 754 | /// result[63:0] = (__A[63:0] * __B[63:0]) + __C[63:0] | ||
| 755 | /// result[127:64] = (__A[127:64] * __B[127:64]) - __C[127:64] | ||
| 756 | /// result[191:128] = (__A[191:128] * __B[191:128]) + __C[191:128] | ||
| 757 | /// result[255:192] = (__A[255:192] * __B[255:192]) - __C[255:192] | ||
| 758 | /// \endcode | ||
| 759 | /// | ||
| 760 | /// \headerfile <immintrin.h> | ||
| 761 | /// | ||
| 762 | /// This intrinsic corresponds to the \c VFMSUBADD213PD instruction. | ||
| 763 | /// | ||
| 764 | /// \param __A | ||
| 765 | /// A 256-bit vector of [4 x double] containing the multiplicand. | ||
| 766 | /// \param __B | ||
| 767 | /// A 256-bit vector of [4 x double] containing the multiplier. | ||
| 768 | /// \param __C | ||
| 769 | /// A 256-bit vector of [4 x double] containing the addend/subtrahend. | ||
| 770 | /// \returns A 256-bit vector of [4 x double] containing the result. | ||
| 207 | static __inline__ __m256d __DEFAULT_FN_ATTRS256 | 771 | static __inline__ __m256d __DEFAULT_FN_ATTRS256 |
| 208 | _mm256_fmsubadd_pd(__m256d __A, __m256d __B, __m256d __C) | 772 | _mm256_fmsubadd_pd(__m256d __A, __m256d __B, __m256d __C) |
| 209 | { | 773 | { |
lib/include/immintrin.h+116-8
| ... | @@ -269,6 +269,26 @@ | ... | @@ -269,6 +269,26 @@ |
| 269 | #include <avxneconvertintrin.h> | 269 | #include <avxneconvertintrin.h> |
| 270 | #endif | 270 | #endif |
| 271 | 271 | ||
| 272 | #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ | ||
| 273 | defined(__SHA512__) | ||
| 274 | #include <sha512intrin.h> | ||
| 275 | #endif | ||
| 276 | |||
| 277 | #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ | ||
| 278 | defined(__SM3__) | ||
| 279 | #include <sm3intrin.h> | ||
| 280 | #endif | ||
| 281 | |||
| 282 | #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ | ||
| 283 | defined(__SM4__) | ||
| 284 | #include <sm4intrin.h> | ||
| 285 | #endif | ||
| 286 | |||
| 287 | #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ | ||
| 288 | defined(__AVXVNNIINT16__) | ||
| 289 | #include <avxvnniint16intrin.h> | ||
| 290 | #endif | ||
| 291 | |||
| 272 | #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ | 292 | #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ |
| 273 | defined(__RDPID__) | 293 | defined(__RDPID__) |
| 274 | /// Returns the value of the IA32_TSC_AUX MSR (0xc0000103). | 294 | /// Returns the value of the IA32_TSC_AUX MSR (0xc0000103). |
| ... | @@ -284,30 +304,53 @@ _rdpid_u32(void) { | ... | @@ -284,30 +304,53 @@ _rdpid_u32(void) { |
| 284 | 304 | ||
| 285 | #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ | 305 | #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ |
| 286 | defined(__RDRND__) | 306 | defined(__RDRND__) |
| 307 | /// Returns a 16-bit hardware-generated random value. | ||
| 308 | /// | ||
| 309 | /// \headerfile <immintrin.h> | ||
| 310 | /// | ||
| 311 | /// This intrinsic corresponds to the <c> RDRAND </c> instruction. | ||
| 312 | /// | ||
| 313 | /// \param __p | ||
| 314 | /// A pointer to a 16-bit memory location to place the random value. | ||
| 315 | /// \returns 1 if the value was successfully generated, 0 otherwise. | ||
| 287 | static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) | 316 | static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) |
| 288 | _rdrand16_step(unsigned short *__p) | 317 | _rdrand16_step(unsigned short *__p) |
| 289 | { | 318 | { |
| 290 | return (int)__builtin_ia32_rdrand16_step(__p); | 319 | return (int)__builtin_ia32_rdrand16_step(__p); |
| 291 | } | 320 | } |
| 292 | 321 | ||
| 322 | /// Returns a 32-bit hardware-generated random value. | ||
| 323 | /// | ||
| 324 | /// \headerfile <immintrin.h> | ||
| 325 | /// | ||
| 326 | /// This intrinsic corresponds to the <c> RDRAND </c> instruction. | ||
| 327 | /// | ||
| 328 | /// \param __p | ||
| 329 | /// A pointer to a 32-bit memory location to place the random value. | ||
| 330 | /// \returns 1 if the value was successfully generated, 0 otherwise. | ||
| 293 | static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) | 331 | static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) |
| 294 | _rdrand32_step(unsigned int *__p) | 332 | _rdrand32_step(unsigned int *__p) |
| 295 | { | 333 | { |
| 296 | return (int)__builtin_ia32_rdrand32_step(__p); | 334 | return (int)__builtin_ia32_rdrand32_step(__p); |
| 297 | } | 335 | } |
| 298 | 336 | ||
| 299 | #ifdef __x86_64__ | 337 | /// Returns a 64-bit hardware-generated random value. |
| 338 | /// | ||
| 339 | /// \headerfile <immintrin.h> | ||
| 340 | /// | ||
| 341 | /// This intrinsic corresponds to the <c> RDRAND </c> instruction. | ||
| 342 | /// | ||
| 343 | /// \param __p | ||
| 344 | /// A pointer to a 64-bit memory location to place the random value. | ||
| 345 | /// \returns 1 if the value was successfully generated, 0 otherwise. | ||
| 300 | static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) | 346 | static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) |
| 301 | _rdrand64_step(unsigned long long *__p) | 347 | _rdrand64_step(unsigned long long *__p) |
| 302 | { | 348 | { |
| 349 | #ifdef __x86_64__ | ||
| 303 | return (int)__builtin_ia32_rdrand64_step(__p); | 350 | return (int)__builtin_ia32_rdrand64_step(__p); |
| 304 | } | ||
| 305 | #else | 351 | #else |
| 306 | // We need to emulate the functionality of 64-bit rdrand with 2 32-bit | 352 | // We need to emulate the functionality of 64-bit rdrand with 2 32-bit |
| 307 | // rdrand instructions. | 353 | // rdrand instructions. |
| 308 | static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) | ||
| 309 | _rdrand64_step(unsigned long long *__p) | ||
| 310 | { | ||
| 311 | unsigned int __lo, __hi; | 354 | unsigned int __lo, __hi; |
| 312 | unsigned int __res_lo = __builtin_ia32_rdrand32_step(&__lo); | 355 | unsigned int __res_lo = __builtin_ia32_rdrand32_step(&__lo); |
| 313 | unsigned int __res_hi = __builtin_ia32_rdrand32_step(&__hi); | 356 | unsigned int __res_hi = __builtin_ia32_rdrand32_step(&__hi); |
| ... | @@ -318,55 +361,115 @@ _rdrand64_step(unsigned long long *__p) | ... | @@ -318,55 +361,115 @@ _rdrand64_step(unsigned long long *__p) |
| 318 | *__p = 0; | 361 | *__p = 0; |
| 319 | return 0; | 362 | return 0; |
| 320 | } | 363 | } |
| 321 | } | ||
| 322 | #endif | 364 | #endif |
| 365 | } | ||
| 323 | #endif /* __RDRND__ */ | 366 | #endif /* __RDRND__ */ |
| 324 | 367 | ||
| 325 | #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ | 368 | #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ |
| 326 | defined(__FSGSBASE__) | 369 | defined(__FSGSBASE__) |
| 327 | #ifdef __x86_64__ | 370 | #ifdef __x86_64__ |
| 371 | /// Reads the FS base register. | ||
| 372 | /// | ||
| 373 | /// \headerfile <immintrin.h> | ||
| 374 | /// | ||
| 375 | /// This intrinsic corresponds to the <c> RDFSBASE </c> instruction. | ||
| 376 | /// | ||
| 377 | /// \returns The lower 32 bits of the FS base register. | ||
| 328 | static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) | 378 | static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) |
| 329 | _readfsbase_u32(void) | 379 | _readfsbase_u32(void) |
| 330 | { | 380 | { |
| 331 | return __builtin_ia32_rdfsbase32(); | 381 | return __builtin_ia32_rdfsbase32(); |
| 332 | } | 382 | } |
| 333 | 383 | ||
| 384 | /// Reads the FS base register. | ||
| 385 | /// | ||
| 386 | /// \headerfile <immintrin.h> | ||
| 387 | /// | ||
| 388 | /// This intrinsic corresponds to the <c> RDFSBASE </c> instruction. | ||
| 389 | /// | ||
| 390 | /// \returns The contents of the FS base register. | ||
| 334 | static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) | 391 | static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) |
| 335 | _readfsbase_u64(void) | 392 | _readfsbase_u64(void) |
| 336 | { | 393 | { |
| 337 | return __builtin_ia32_rdfsbase64(); | 394 | return __builtin_ia32_rdfsbase64(); |
| 338 | } | 395 | } |
| 339 | 396 | ||
| 397 | /// Reads the GS base register. | ||
| 398 | /// | ||
| 399 | /// \headerfile <immintrin.h> | ||
| 400 | /// | ||
| 401 | /// This intrinsic corresponds to the <c> RDGSBASE </c> instruction. | ||
| 402 | /// | ||
| 403 | /// \returns The lower 32 bits of the GS base register. | ||
| 340 | static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) | 404 | static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) |
| 341 | _readgsbase_u32(void) | 405 | _readgsbase_u32(void) |
| 342 | { | 406 | { |
| 343 | return __builtin_ia32_rdgsbase32(); | 407 | return __builtin_ia32_rdgsbase32(); |
| 344 | } | 408 | } |
| 345 | 409 | ||
| 410 | /// Reads the GS base register. | ||
| 411 | /// | ||
| 412 | /// \headerfile <immintrin.h> | ||
| 413 | /// | ||
| 414 | /// This intrinsic corresponds to the <c> RDGSBASE </c> instruction. | ||
| 415 | /// | ||
| 416 | /// \returns The contents of the GS base register. | ||
| 346 | static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) | 417 | static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) |
| 347 | _readgsbase_u64(void) | 418 | _readgsbase_u64(void) |
| 348 | { | 419 | { |
| 349 | return __builtin_ia32_rdgsbase64(); | 420 | return __builtin_ia32_rdgsbase64(); |
| 350 | } | 421 | } |
| 351 | 422 | ||
| 423 | /// Modifies the FS base register. | ||
| 424 | /// | ||
| 425 | /// \headerfile <immintrin.h> | ||
| 426 | /// | ||
| 427 | /// This intrinsic corresponds to the <c> WRFSBASE </c> instruction. | ||
| 428 | /// | ||
| 429 | /// \param __V | ||
| 430 | /// Value to use for the lower 32 bits of the FS base register. | ||
| 352 | static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) | 431 | static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) |
| 353 | _writefsbase_u32(unsigned int __V) | 432 | _writefsbase_u32(unsigned int __V) |
| 354 | { | 433 | { |
| 355 | __builtin_ia32_wrfsbase32(__V); | 434 | __builtin_ia32_wrfsbase32(__V); |
| 356 | } | 435 | } |
| 357 | 436 | ||
| 437 | /// Modifies the FS base register. | ||
| 438 | /// | ||
| 439 | /// \headerfile <immintrin.h> | ||
| 440 | /// | ||
| 441 | /// This intrinsic corresponds to the <c> WRFSBASE </c> instruction. | ||
| 442 | /// | ||
| 443 | /// \param __V | ||
| 444 | /// Value to use for the FS base register. | ||
| 358 | static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) | 445 | static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) |
| 359 | _writefsbase_u64(unsigned long long __V) | 446 | _writefsbase_u64(unsigned long long __V) |
| 360 | { | 447 | { |
| 361 | __builtin_ia32_wrfsbase64(__V); | 448 | __builtin_ia32_wrfsbase64(__V); |
| 362 | } | 449 | } |
| 363 | 450 | ||
| 451 | /// Modifies the GS base register. | ||
| 452 | /// | ||
| 453 | /// \headerfile <immintrin.h> | ||
| 454 | /// | ||
| 455 | /// This intrinsic corresponds to the <c> WRGSBASE </c> instruction. | ||
| 456 | /// | ||
| 457 | /// \param __V | ||
| 458 | /// Value to use for the lower 32 bits of the GS base register. | ||
| 364 | static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) | 459 | static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) |
| 365 | _writegsbase_u32(unsigned int __V) | 460 | _writegsbase_u32(unsigned int __V) |
| 366 | { | 461 | { |
| 367 | __builtin_ia32_wrgsbase32(__V); | 462 | __builtin_ia32_wrgsbase32(__V); |
| 368 | } | 463 | } |
| 369 | 464 | ||
| 465 | /// Modifies the GS base register. | ||
| 466 | /// | ||
| 467 | /// \headerfile <immintrin.h> | ||
| 468 | /// | ||
| 469 | /// This intrinsic corresponds to the <c> WRFSBASE </c> instruction. | ||
| 470 | /// | ||
| 471 | /// \param __V | ||
| 472 | /// Value to use for GS base register. | ||
| 370 | static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) | 473 | static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) |
| 371 | _writegsbase_u64(unsigned long long __V) | 474 | _writegsbase_u64(unsigned long long __V) |
| 372 | { | 475 | { |
| ... | @@ -538,6 +641,11 @@ _storebe_i64(void * __P, long long __D) { | ... | @@ -538,6 +641,11 @@ _storebe_i64(void * __P, long long __D) { |
| 538 | #include <amxintrin.h> | 641 | #include <amxintrin.h> |
| 539 | #endif | 642 | #endif |
| 540 | 643 | ||
| 644 | #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ | ||
| 645 | defined(__AMX_COMPLEX__) | ||
| 646 | #include <amxcomplexintrin.h> | ||
| 647 | #endif | ||
| 648 | |||
| 541 | #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ | 649 | #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ |
| 542 | defined(__AVX512VP2INTERSECT__) | 650 | defined(__AVX512VP2INTERSECT__) |
| 543 | #include <avx512vp2intersectintrin.h> | 651 | #include <avx512vp2intersectintrin.h> |
lib/include/limits.h+5-1
| ... | @@ -52,7 +52,11 @@ | ... | @@ -52,7 +52,11 @@ |
| 52 | #define LONG_MIN (-__LONG_MAX__ -1L) | 52 | #define LONG_MIN (-__LONG_MAX__ -1L) |
| 53 | 53 | ||
| 54 | #define UCHAR_MAX (__SCHAR_MAX__*2 +1) | 54 | #define UCHAR_MAX (__SCHAR_MAX__*2 +1) |
| 55 | #define USHRT_MAX (__SHRT_MAX__ *2 +1) | 55 | #if __SHRT_WIDTH__ < __INT_WIDTH__ |
| 56 | #define USHRT_MAX (__SHRT_MAX__ * 2 + 1) | ||
| 57 | #else | ||
| 58 | #define USHRT_MAX (__SHRT_MAX__ * 2U + 1U) | ||
| 59 | #endif | ||
| 56 | #define UINT_MAX (__INT_MAX__ *2U +1U) | 60 | #define UINT_MAX (__INT_MAX__ *2U +1U) |
| 57 | #define ULONG_MAX (__LONG_MAX__ *2UL+1UL) | 61 | #define ULONG_MAX (__LONG_MAX__ *2UL+1UL) |
| 58 | 62 |
lib/include/llvm_libc_wrappers/ctype.h created+85
| ... | @@ -0,0 +1,85 @@ | ||
| 1 | //===-- Wrapper for C standard ctype.h declarations on the GPU ------------===// | ||
| 2 | // | ||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| 6 | // | ||
| 7 | //===----------------------------------------------------------------------===// | ||
| 8 | |||
| 9 | #ifndef __CLANG_LLVM_LIBC_WRAPPERS_CTYPE_H__ | ||
| 10 | #define __CLANG_LLVM_LIBC_WRAPPERS_CTYPE_H__ | ||
| 11 | |||
| 12 | #if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__) | ||
| 13 | #error "This file is for GPU offloading compilation only" | ||
| 14 | #endif | ||
| 15 | |||
| 16 | #include_next <ctype.h> | ||
| 17 | |||
| 18 | #if __has_include(<llvm-libc-decls/ctype.h>) | ||
| 19 | |||
| 20 | #if defined(__HIP__) || defined(__CUDA__) | ||
| 21 | #define __LIBC_ATTRS __attribute__((device)) | ||
| 22 | #endif | ||
| 23 | |||
| 24 | // The GNU headers like to provide these as macros, we need to undefine them so | ||
| 25 | // they do not conflict with the following definitions for the GPU. | ||
| 26 | |||
| 27 | #pragma push_macro("isalnum") | ||
| 28 | #pragma push_macro("isalpha") | ||
| 29 | #pragma push_macro("isblank") | ||
| 30 | #pragma push_macro("iscntrl") | ||
| 31 | #pragma push_macro("isdigit") | ||
| 32 | #pragma push_macro("isgraph") | ||
| 33 | #pragma push_macro("islower") | ||
| 34 | #pragma push_macro("isprint") | ||
| 35 | #pragma push_macro("ispunct") | ||
| 36 | #pragma push_macro("isspace") | ||
| 37 | #pragma push_macro("isupper") | ||
| 38 | #pragma push_macro("isxdigit") | ||
| 39 | #pragma push_macro("tolower") | ||
| 40 | #pragma push_macro("toupper") | ||
| 41 | |||
| 42 | #undef isalnum | ||
| 43 | #undef isalpha | ||
| 44 | #undef iscntrl | ||
| 45 | #undef isdigit | ||
| 46 | #undef islower | ||
| 47 | #undef isgraph | ||
| 48 | #undef isprint | ||
| 49 | #undef ispunct | ||
| 50 | #undef isspace | ||
| 51 | #undef isupper | ||
| 52 | #undef isblank | ||
| 53 | #undef isxdigit | ||
| 54 | #undef tolower | ||
| 55 | #undef toupper | ||
| 56 | |||
| 57 | #pragma omp begin declare target | ||
| 58 | |||
| 59 | #include <llvm-libc-decls/ctype.h> | ||
| 60 | |||
| 61 | #pragma omp end declare target | ||
| 62 | |||
| 63 | // Restore the original macros when compiling on the host. | ||
| 64 | #if !defined(__NVPTX__) && !defined(__AMDGPU__) | ||
| 65 | #pragma pop_macro("isalnum") | ||
| 66 | #pragma pop_macro("isalpha") | ||
| 67 | #pragma pop_macro("isblank") | ||
| 68 | #pragma pop_macro("iscntrl") | ||
| 69 | #pragma pop_macro("isdigit") | ||
| 70 | #pragma pop_macro("isgraph") | ||
| 71 | #pragma pop_macro("islower") | ||
| 72 | #pragma pop_macro("isprint") | ||
| 73 | #pragma pop_macro("ispunct") | ||
| 74 | #pragma pop_macro("isspace") | ||
| 75 | #pragma pop_macro("isupper") | ||
| 76 | #pragma pop_macro("isxdigit") | ||
| 77 | #pragma pop_macro("tolower") | ||
| 78 | #pragma pop_macro("toupper") | ||
| 79 | #endif | ||
| 80 | |||
| 81 | #undef __LIBC_ATTRS | ||
| 82 | |||
| 83 | #endif | ||
| 84 | |||
| 85 | #endif // __CLANG_LLVM_LIBC_WRAPPERS_CTYPE_H__ | ||
lib/include/llvm_libc_wrappers/inttypes.h created+34
| ... | @@ -0,0 +1,34 @@ | ||
| 1 | //===-- Wrapper for C standard inttypes.h declarations on the GPU ---------===// | ||
| 2 | // | ||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| 6 | // | ||
| 7 | //===----------------------------------------------------------------------===// | ||
| 8 | |||
| 9 | #ifndef __CLANG_LLVM_LIBC_WRAPPERS_INTTYPES_H__ | ||
| 10 | #define __CLANG_LLVM_LIBC_WRAPPERS_INTTYPES_H__ | ||
| 11 | |||
| 12 | #if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__) | ||
| 13 | #error "This file is for GPU offloading compilation only" | ||
| 14 | #endif | ||
| 15 | |||
| 16 | #include_next <inttypes.h> | ||
| 17 | |||
| 18 | #if __has_include(<llvm-libc-decls/inttypes.h>) | ||
| 19 | |||
| 20 | #if defined(__HIP__) || defined(__CUDA__) | ||
| 21 | #define __LIBC_ATTRS __attribute__((device)) | ||
| 22 | #endif | ||
| 23 | |||
| 24 | #pragma omp begin declare target | ||
| 25 | |||
| 26 | #include <llvm-libc-decls/inttypes.h> | ||
| 27 | |||
| 28 | #pragma omp end declare target | ||
| 29 | |||
| 30 | #undef __LIBC_ATTRS | ||
| 31 | |||
| 32 | #endif | ||
| 33 | |||
| 34 | #endif // __CLANG_LLVM_LIBC_WRAPPERS_INTTYPES_H__ | ||
lib/include/llvm_libc_wrappers/stdio.h created+34
| ... | @@ -0,0 +1,34 @@ | ||
| 1 | //===-- Wrapper for C standard stdio.h declarations on the GPU ------------===// | ||
| 2 | // | ||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| 6 | // | ||
| 7 | //===----------------------------------------------------------------------===// | ||
| 8 | |||
| 9 | #ifndef __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__ | ||
| 10 | #define __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__ | ||
| 11 | |||
| 12 | #if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__) | ||
| 13 | #error "This file is for GPU offloading compilation only" | ||
| 14 | #endif | ||
| 15 | |||
| 16 | #include_next <stdio.h> | ||
| 17 | |||
| 18 | #if __has_include(<llvm-libc-decls/stdio.h>) | ||
| 19 | |||
| 20 | #if defined(__HIP__) || defined(__CUDA__) | ||
| 21 | #define __LIBC_ATTRS __attribute__((device)) | ||
| 22 | #endif | ||
| 23 | |||
| 24 | #pragma omp begin declare target | ||
| 25 | |||
| 26 | #include <llvm-libc-decls/stdio.h> | ||
| 27 | |||
| 28 | #pragma omp end declare target | ||
| 29 | |||
| 30 | #undef __LIBC_ATTRS | ||
| 31 | |||
| 32 | #endif | ||
| 33 | |||
| 34 | #endif // __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__ | ||
lib/include/llvm_libc_wrappers/stdlib.h created+42
| ... | @@ -0,0 +1,42 @@ | ||
| 1 | //===-- Wrapper for C standard stdlib.h declarations on the GPU -----------===// | ||
| 2 | // | ||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| 6 | // | ||
| 7 | //===----------------------------------------------------------------------===// | ||
| 8 | |||
| 9 | #ifndef __CLANG_LLVM_LIBC_WRAPPERS_STDLIB_H__ | ||
| 10 | #define __CLANG_LLVM_LIBC_WRAPPERS_STDLIB_H__ | ||
| 11 | |||
| 12 | #if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__) | ||
| 13 | #error "This file is for GPU offloading compilation only" | ||
| 14 | #endif | ||
| 15 | |||
| 16 | #include_next <stdlib.h> | ||
| 17 | |||
| 18 | #if __has_include(<llvm-libc-decls/stdlib.h>) | ||
| 19 | |||
| 20 | #if defined(__HIP__) || defined(__CUDA__) | ||
| 21 | #define __LIBC_ATTRS __attribute__((device)) | ||
| 22 | #endif | ||
| 23 | |||
| 24 | #pragma omp begin declare target | ||
| 25 | |||
| 26 | // The LLVM C library uses this type so we forward declare it. | ||
| 27 | typedef void (*__atexithandler_t)(void); | ||
| 28 | |||
| 29 | // Enforce ABI compatibility with the structs used by the LLVM C library. | ||
| 30 | _Static_assert(__builtin_offsetof(div_t, quot) == 0, "ABI mismatch!"); | ||
| 31 | _Static_assert(__builtin_offsetof(ldiv_t, quot) == 0, "ABI mismatch!"); | ||
| 32 | _Static_assert(__builtin_offsetof(lldiv_t, quot) == 0, "ABI mismatch!"); | ||
| 33 | |||
| 34 | #include <llvm-libc-decls/stdlib.h> | ||
| 35 | |||
| 36 | #pragma omp end declare target | ||
| 37 | |||
| 38 | #undef __LIBC_ATTRS | ||
| 39 | |||
| 40 | #endif | ||
| 41 | |||
| 42 | #endif // __CLANG_LLVM_LIBC_WRAPPERS_STDLIB_H__ | ||
lib/include/llvm_libc_wrappers/string.h created+37
| ... | @@ -0,0 +1,37 @@ | ||
| 1 | //===-- Wrapper for C standard string.h declarations on the GPU -----------===// | ||
| 2 | // | ||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| 6 | // | ||
| 7 | //===----------------------------------------------------------------------===// | ||
| 8 | |||
| 9 | #ifndef __CLANG_LLVM_LIBC_WRAPPERS_STRING_H__ | ||
| 10 | #define __CLANG_LLVM_LIBC_WRAPPERS_STRING_H__ | ||
| 11 | |||
| 12 | #if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__) | ||
| 13 | #error "This file is for GPU offloading compilation only" | ||
| 14 | #endif | ||
| 15 | |||
| 16 | // FIXME: The GNU headers provide C++ standard compliant headers when in C++ | ||
| 17 | // mode and the LLVM libc does not. We cannot enable memchr, strchr, strchrnul, | ||
| 18 | // strpbrk, strrchr, strstr, or strcasestr until this is addressed. | ||
| 19 | #include_next <string.h> | ||
| 20 | |||
| 21 | #if __has_include(<llvm-libc-decls/string.h>) | ||
| 22 | |||
| 23 | #if defined(__HIP__) || defined(__CUDA__) | ||
| 24 | #define __LIBC_ATTRS __attribute__((device)) | ||
| 25 | #endif | ||
| 26 | |||
| 27 | #pragma omp begin declare target | ||
| 28 | |||
| 29 | #include <llvm-libc-decls/string.h> | ||
| 30 | |||
| 31 | #pragma omp end declare target | ||
| 32 | |||
| 33 | #undef __LIBC_ATTRS | ||
| 34 | |||
| 35 | #endif | ||
| 36 | |||
| 37 | #endif // __CLANG_LLVM_LIBC_WRAPPERS_STRING_H__ | ||
lib/include/mwaitxintrin.h+29
| ... | @@ -16,12 +16,41 @@ | ... | @@ -16,12 +16,41 @@ |
| 16 | 16 | ||
| 17 | /* Define the default attributes for the functions in this file. */ | 17 | /* Define the default attributes for the functions in this file. */ |
| 18 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("mwaitx"))) | 18 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("mwaitx"))) |
| 19 | |||
| 20 | /// Establishes a linear address memory range to be monitored and puts | ||
| 21 | /// the processor in the monitor event pending state. Data stored in the | ||
| 22 | /// monitored address range causes the processor to exit the pending state. | ||
| 23 | /// | ||
| 24 | /// \headerfile <x86intrin.h> | ||
| 25 | /// | ||
| 26 | /// This intrinsic corresponds to the \c MONITORX instruction. | ||
| 27 | /// | ||
| 28 | /// \param __p | ||
| 29 | /// The memory range to be monitored. The size of the range is determined by | ||
| 30 | /// CPUID function 0000_0005h. | ||
| 31 | /// \param __extensions | ||
| 32 | /// Optional extensions for the monitoring state. | ||
| 33 | /// \param __hints | ||
| 34 | /// Optional hints for the monitoring state. | ||
| 19 | static __inline__ void __DEFAULT_FN_ATTRS | 35 | static __inline__ void __DEFAULT_FN_ATTRS |
| 20 | _mm_monitorx(void * __p, unsigned __extensions, unsigned __hints) | 36 | _mm_monitorx(void * __p, unsigned __extensions, unsigned __hints) |
| 21 | { | 37 | { |
| 22 | __builtin_ia32_monitorx(__p, __extensions, __hints); | 38 | __builtin_ia32_monitorx(__p, __extensions, __hints); |
| 23 | } | 39 | } |
| 24 | 40 | ||
| 41 | /// Used with the \c MONITORX instruction to wait while the processor is in | ||
| 42 | /// the monitor event pending state. Data stored in the monitored address | ||
| 43 | /// range, or an interrupt, causes the processor to exit the pending state. | ||
| 44 | /// | ||
| 45 | /// \headerfile <x86intrin.h> | ||
| 46 | /// | ||
| 47 | /// This intrinsic corresponds to the \c MWAITX instruction. | ||
| 48 | /// | ||
| 49 | /// \param __extensions | ||
| 50 | /// Optional extensions for the monitoring state, which can vary by | ||
| 51 | /// processor. | ||
| 52 | /// \param __hints | ||
| 53 | /// Optional hints for the monitoring state, which can vary by processor. | ||
| 25 | static __inline__ void __DEFAULT_FN_ATTRS | 54 | static __inline__ void __DEFAULT_FN_ATTRS |
| 26 | _mm_mwaitx(unsigned __extensions, unsigned __hints, unsigned __clock) | 55 | _mm_mwaitx(unsigned __extensions, unsigned __hints, unsigned __clock) |
| 27 | { | 56 | { |
lib/include/opencl-c-base.h+3
| ... | @@ -474,6 +474,9 @@ typedef enum memory_order | ... | @@ -474,6 +474,9 @@ typedef enum memory_order |
| 474 | #define CLK_HALF_FLOAT 0x10DD | 474 | #define CLK_HALF_FLOAT 0x10DD |
| 475 | #define CLK_FLOAT 0x10DE | 475 | #define CLK_FLOAT 0x10DE |
| 476 | #define CLK_UNORM_INT24 0x10DF | 476 | #define CLK_UNORM_INT24 0x10DF |
| 477 | #if __OPENCL_C_VERSION__ >= CL_VERSION_3_0 | ||
| 478 | #define CLK_UNORM_INT_101010_2 0x10E0 | ||
| 479 | #endif // __OPENCL_C_VERSION__ >= CL_VERSION_3_0 | ||
| 477 | 480 | ||
| 478 | // Channel order, numbering must be aligned with cl_channel_order in cl.h | 481 | // Channel order, numbering must be aligned with cl_channel_order in cl.h |
| 479 | // | 482 | // |
lib/include/openmp_wrappers/__clang_openmp_device_functions.h-1
| ... | @@ -40,7 +40,6 @@ extern "C" { | ... | @@ -40,7 +40,6 @@ extern "C" { |
| 40 | 40 | ||
| 41 | // Import types which will be used by __clang_hip_libdevice_declares.h | 41 | // Import types which will be used by __clang_hip_libdevice_declares.h |
| 42 | #ifndef __cplusplus | 42 | #ifndef __cplusplus |
| 43 | #include <stdbool.h> | ||
| 44 | #include <stdint.h> | 43 | #include <stdint.h> |
| 45 | #endif | 44 | #endif |
| 46 | 45 |
lib/include/openmp_wrappers/new+1-1
| ... | @@ -13,7 +13,7 @@ | ... | @@ -13,7 +13,7 @@ |
| 13 | // which do not use nothrow_t are provided without the <new> header. | 13 | // which do not use nothrow_t are provided without the <new> header. |
| 14 | #include_next <new> | 14 | #include_next <new> |
| 15 | 15 | ||
| 16 | #if defined(__NVPTX__) && defined(_OPENMP) | 16 | #if (defined(__NVPTX__) || defined(__AMDGPU__)) && defined(_OPENMP) |
| 17 | 17 | ||
| 18 | #include <cstdlib> | 18 | #include <cstdlib> |
| 19 | 19 |
lib/include/pmmintrin.h+12-6
| ... | @@ -253,9 +253,12 @@ _mm_movedup_pd(__m128d __a) | ... | @@ -253,9 +253,12 @@ _mm_movedup_pd(__m128d __a) |
| 253 | /// the processor in the monitor event pending state. Data stored in the | 253 | /// the processor in the monitor event pending state. Data stored in the |
| 254 | /// monitored address range causes the processor to exit the pending state. | 254 | /// monitored address range causes the processor to exit the pending state. |
| 255 | /// | 255 | /// |
| 256 | /// The \c MONITOR instruction can be used in kernel mode, and in other modes | ||
| 257 | /// if MSR <c> C001_0015h[MonMwaitUserEn] </c> is set. | ||
| 258 | /// | ||
| 256 | /// \headerfile <x86intrin.h> | 259 | /// \headerfile <x86intrin.h> |
| 257 | /// | 260 | /// |
| 258 | /// This intrinsic corresponds to the <c> MONITOR </c> instruction. | 261 | /// This intrinsic corresponds to the \c MONITOR instruction. |
| 259 | /// | 262 | /// |
| 260 | /// \param __p | 263 | /// \param __p |
| 261 | /// The memory range to be monitored. The size of the range is determined by | 264 | /// The memory range to be monitored. The size of the range is determined by |
| ... | @@ -270,19 +273,22 @@ _mm_monitor(void const *__p, unsigned __extensions, unsigned __hints) | ... | @@ -270,19 +273,22 @@ _mm_monitor(void const *__p, unsigned __extensions, unsigned __hints) |
| 270 | __builtin_ia32_monitor(__p, __extensions, __hints); | 273 | __builtin_ia32_monitor(__p, __extensions, __hints); |
| 271 | } | 274 | } |
| 272 | 275 | ||
| 273 | /// Used with the MONITOR instruction to wait while the processor is in | 276 | /// Used with the \c MONITOR instruction to wait while the processor is in |
| 274 | /// the monitor event pending state. Data stored in the monitored address | 277 | /// the monitor event pending state. Data stored in the monitored address |
| 275 | /// range causes the processor to exit the pending state. | 278 | /// range, or an interrupt, causes the processor to exit the pending state. |
| 279 | /// | ||
| 280 | /// The \c MWAIT instruction can be used in kernel mode, and in other modes if | ||
| 281 | /// MSR <c> C001_0015h[MonMwaitUserEn] </c> is set. | ||
| 276 | /// | 282 | /// |
| 277 | /// \headerfile <x86intrin.h> | 283 | /// \headerfile <x86intrin.h> |
| 278 | /// | 284 | /// |
| 279 | /// This intrinsic corresponds to the <c> MWAIT </c> instruction. | 285 | /// This intrinsic corresponds to the \c MWAIT instruction. |
| 280 | /// | 286 | /// |
| 281 | /// \param __extensions | 287 | /// \param __extensions |
| 282 | /// Optional extensions for the monitoring state, which may vary by | 288 | /// Optional extensions for the monitoring state, which can vary by |
| 283 | /// processor. | 289 | /// processor. |
| 284 | /// \param __hints | 290 | /// \param __hints |
| 285 | /// Optional hints for the monitoring state, which may vary by processor. | 291 | /// Optional hints for the monitoring state, which can vary by processor. |
| 286 | static __inline__ void __DEFAULT_FN_ATTRS | 292 | static __inline__ void __DEFAULT_FN_ATTRS |
| 287 | _mm_mwait(unsigned __extensions, unsigned __hints) | 293 | _mm_mwait(unsigned __extensions, unsigned __hints) |
| 288 | { | 294 | { |
lib/include/ppc_wrappers/emmintrin.h+2-1
| ... | @@ -46,6 +46,7 @@ | ... | @@ -46,6 +46,7 @@ |
| 46 | 46 | ||
| 47 | /* SSE2 */ | 47 | /* SSE2 */ |
| 48 | typedef __vector double __v2df; | 48 | typedef __vector double __v2df; |
| 49 | typedef __vector float __v4f; | ||
| 49 | typedef __vector long long __v2di; | 50 | typedef __vector long long __v2di; |
| 50 | typedef __vector unsigned long long __v2du; | 51 | typedef __vector unsigned long long __v2du; |
| 51 | typedef __vector int __v4si; | 52 | typedef __vector int __v4si; |
| ... | @@ -951,7 +952,7 @@ extern __inline __m128d | ... | @@ -951,7 +952,7 @@ extern __inline __m128d |
| 951 | _mm_cvtpi32_pd(__m64 __A) { | 952 | _mm_cvtpi32_pd(__m64 __A) { |
| 952 | __v4si __temp; | 953 | __v4si __temp; |
| 953 | __v2di __tmp2; | 954 | __v2di __tmp2; |
| 954 | __v2df __result; | 955 | __v4f __result; |
| 955 | 956 | ||
| 956 | __temp = (__v4si)vec_splats(__A); | 957 | __temp = (__v4si)vec_splats(__A); |
| 957 | __tmp2 = (__v2di)vec_unpackl(__temp); | 958 | __tmp2 = (__v2di)vec_unpackl(__temp); |
lib/include/ppc_wrappers/smmintrin.h+2-2
| ... | @@ -305,9 +305,9 @@ extern __inline int | ... | @@ -305,9 +305,9 @@ extern __inline int |
| 305 | extern __inline __m128i | 305 | extern __inline __m128i |
| 306 | __attribute__((__gnu_inline__, __always_inline__, __artificial__)) | 306 | __attribute__((__gnu_inline__, __always_inline__, __artificial__)) |
| 307 | _mm_blend_epi16(__m128i __A, __m128i __B, const int __imm8) { | 307 | _mm_blend_epi16(__m128i __A, __m128i __B, const int __imm8) { |
| 308 | __v16qi __charmask = vec_splats((signed char)__imm8); | 308 | __v16qu __charmask = vec_splats((unsigned char)__imm8); |
| 309 | __charmask = vec_gb(__charmask); | 309 | __charmask = vec_gb(__charmask); |
| 310 | __v8hu __shortmask = (__v8hu)vec_unpackh(__charmask); | 310 | __v8hu __shortmask = (__v8hu)vec_unpackh((__v16qi)__charmask); |
| 311 | #ifdef __BIG_ENDIAN__ | 311 | #ifdef __BIG_ENDIAN__ |
| 312 | __shortmask = vec_reve(__shortmask); | 312 | __shortmask = vec_reve(__shortmask); |
| 313 | #endif | 313 | #endif |
lib/include/rdseedintrin.h+65-2
| ... | @@ -7,8 +7,8 @@ | ... | @@ -7,8 +7,8 @@ |
| 7 | *===-----------------------------------------------------------------------=== | 7 | *===-----------------------------------------------------------------------=== |
| 8 | */ | 8 | */ |
| 9 | 9 | ||
| 10 | #if !defined __X86INTRIN_H && !defined __IMMINTRIN_H | 10 | #ifndef __IMMINTRIN_H |
| 11 | #error "Never use <rdseedintrin.h> directly; include <x86intrin.h> instead." | 11 | #error "Never use <rdseedintrin.h> directly; include <immintrin.h> instead." |
| 12 | #endif | 12 | #endif |
| 13 | 13 | ||
| 14 | #ifndef __RDSEEDINTRIN_H | 14 | #ifndef __RDSEEDINTRIN_H |
| ... | @@ -17,12 +17,54 @@ | ... | @@ -17,12 +17,54 @@ |
| 17 | /* Define the default attributes for the functions in this file. */ | 17 | /* Define the default attributes for the functions in this file. */ |
| 18 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("rdseed"))) | 18 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("rdseed"))) |
| 19 | 19 | ||
| 20 | /// Stores a hardware-generated 16-bit random value in the memory at \a __p. | ||
| 21 | /// | ||
| 22 | /// The random number generator complies with NIST SP800-90B and SP800-90C. | ||
| 23 | /// | ||
| 24 | /// \code{.operation} | ||
| 25 | /// IF HW_NRND_GEN.ready == 1 | ||
| 26 | /// Store16(__p, HW_NRND_GEN.data) | ||
| 27 | /// result := 1 | ||
| 28 | /// ELSE | ||
| 29 | /// Store16(__p, 0) | ||
| 30 | /// result := 0 | ||
| 31 | /// END | ||
| 32 | /// \endcode | ||
| 33 | /// | ||
| 34 | /// \headerfile <immintrin.h> | ||
| 35 | /// | ||
| 36 | /// This intrinsic corresponds to the \c RDSEED instruction. | ||
| 37 | /// | ||
| 38 | /// \param __p | ||
| 39 | /// Pointer to memory for storing the 16-bit random number. | ||
| 40 | /// \returns 1 if a random number was generated, 0 if not. | ||
| 20 | static __inline__ int __DEFAULT_FN_ATTRS | 41 | static __inline__ int __DEFAULT_FN_ATTRS |
| 21 | _rdseed16_step(unsigned short *__p) | 42 | _rdseed16_step(unsigned short *__p) |
| 22 | { | 43 | { |
| 23 | return (int) __builtin_ia32_rdseed16_step(__p); | 44 | return (int) __builtin_ia32_rdseed16_step(__p); |
| 24 | } | 45 | } |
| 25 | 46 | ||
| 47 | /// Stores a hardware-generated 32-bit random value in the memory at \a __p. | ||
| 48 | /// | ||
| 49 | /// The random number generator complies with NIST SP800-90B and SP800-90C. | ||
| 50 | /// | ||
| 51 | /// \code{.operation} | ||
| 52 | /// IF HW_NRND_GEN.ready == 1 | ||
| 53 | /// Store32(__p, HW_NRND_GEN.data) | ||
| 54 | /// result := 1 | ||
| 55 | /// ELSE | ||
| 56 | /// Store32(__p, 0) | ||
| 57 | /// result := 0 | ||
| 58 | /// END | ||
| 59 | /// \endcode | ||
| 60 | /// | ||
| 61 | /// \headerfile <immintrin.h> | ||
| 62 | /// | ||
| 63 | /// This intrinsic corresponds to the \c RDSEED instruction. | ||
| 64 | /// | ||
| 65 | /// \param __p | ||
| 66 | /// Pointer to memory for storing the 32-bit random number. | ||
| 67 | /// \returns 1 if a random number was generated, 0 if not. | ||
| 26 | static __inline__ int __DEFAULT_FN_ATTRS | 68 | static __inline__ int __DEFAULT_FN_ATTRS |
| 27 | _rdseed32_step(unsigned int *__p) | 69 | _rdseed32_step(unsigned int *__p) |
| 28 | { | 70 | { |
| ... | @@ -30,6 +72,27 @@ _rdseed32_step(unsigned int *__p) | ... | @@ -30,6 +72,27 @@ _rdseed32_step(unsigned int *__p) |
| 30 | } | 72 | } |
| 31 | 73 | ||
| 32 | #ifdef __x86_64__ | 74 | #ifdef __x86_64__ |
| 75 | /// Stores a hardware-generated 64-bit random value in the memory at \a __p. | ||
| 76 | /// | ||
| 77 | /// The random number generator complies with NIST SP800-90B and SP800-90C. | ||
| 78 | /// | ||
| 79 | /// \code{.operation} | ||
| 80 | /// IF HW_NRND_GEN.ready == 1 | ||
| 81 | /// Store64(__p, HW_NRND_GEN.data) | ||
| 82 | /// result := 1 | ||
| 83 | /// ELSE | ||
| 84 | /// Store64(__p, 0) | ||
| 85 | /// result := 0 | ||
| 86 | /// END | ||
| 87 | /// \endcode | ||
| 88 | /// | ||
| 89 | /// \headerfile <immintrin.h> | ||
| 90 | /// | ||
| 91 | /// This intrinsic corresponds to the \c RDSEED instruction. | ||
| 92 | /// | ||
| 93 | /// \param __p | ||
| 94 | /// Pointer to memory for storing the 64-bit random number. | ||
| 95 | /// \returns 1 if a random number was generated, 0 if not. | ||
| 33 | static __inline__ int __DEFAULT_FN_ATTRS | 96 | static __inline__ int __DEFAULT_FN_ATTRS |
| 34 | _rdseed64_step(unsigned long long *__p) | 97 | _rdseed64_step(unsigned long long *__p) |
| 35 | { | 98 | { |
lib/include/riscv_ntlh.h created+28
| ... | @@ -0,0 +1,28 @@ | ||
| 1 | /*===---- riscv_ntlh.h - RISC-V NTLH intrinsics ----------------------------=== | ||
| 2 | * | ||
| 3 | * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| 4 | * See https://llvm.org/LICENSE.txt for license information. | ||
| 5 | * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| 6 | * | ||
| 7 | *===-----------------------------------------------------------------------=== | ||
| 8 | */ | ||
| 9 | |||
| 10 | #ifndef __RISCV_NTLH_H | ||
| 11 | #define __RISCV_NTLH_H | ||
| 12 | |||
| 13 | #ifndef __riscv_zihintntl | ||
| 14 | #error "NTLH intrinsics require the NTLH extension." | ||
| 15 | #endif | ||
| 16 | |||
| 17 | enum { | ||
| 18 | __RISCV_NTLH_INNERMOST_PRIVATE = 2, | ||
| 19 | __RISCV_NTLH_ALL_PRIVATE, | ||
| 20 | __RISCV_NTLH_INNERMOST_SHARED, | ||
| 21 | __RISCV_NTLH_ALL | ||
| 22 | }; | ||
| 23 | |||
| 24 | #define __riscv_ntl_load(PTR, DOMAIN) __builtin_riscv_ntl_load((PTR), (DOMAIN)) | ||
| 25 | #define __riscv_ntl_store(PTR, VAL, DOMAIN) \ | ||
| 26 | __builtin_riscv_ntl_store((PTR), (VAL), (DOMAIN)) | ||
| 27 | |||
| 28 | #endif | ||
| \ No newline at end of file | |||
lib/include/riscv_vector.h+240-51
| ... | @@ -25,52 +25,15 @@ extern "C" { | ... | @@ -25,52 +25,15 @@ extern "C" { |
| 25 | #pragma clang riscv intrinsic vector | 25 | #pragma clang riscv intrinsic vector |
| 26 | 26 | ||
| 27 | 27 | ||
| 28 | #define __riscv_vlenb() __builtin_rvv_vlenb() | 28 | enum __RISCV_FRM { |
| 29 | 29 | __RISCV_FRM_RNE = 0, | |
| 30 | enum RVV_CSR { | 30 | __RISCV_FRM_RTZ = 1, |
| 31 | RVV_VSTART = 0, | 31 | __RISCV_FRM_RDN = 2, |
| 32 | RVV_VXSAT, | 32 | __RISCV_FRM_RUP = 3, |
| 33 | RVV_VXRM, | 33 | __RISCV_FRM_RMM = 4, |
| 34 | RVV_VCSR, | ||
| 35 | }; | 34 | }; |
| 36 | 35 | ||
| 37 | static __inline__ __attribute__((__always_inline__, __nodebug__)) | 36 | #define __riscv_vlenb() __builtin_rvv_vlenb() |
| 38 | unsigned long __riscv_vread_csr(enum RVV_CSR __csr) { | ||
| 39 | unsigned long __rv = 0; | ||
| 40 | switch (__csr) { | ||
| 41 | case RVV_VSTART: | ||
| 42 | __asm__ __volatile__ ("csrr\t%0, vstart" : "=r"(__rv) : : "memory"); | ||
| 43 | break; | ||
| 44 | case RVV_VXSAT: | ||
| 45 | __asm__ __volatile__ ("csrr\t%0, vxsat" : "=r"(__rv) : : "memory"); | ||
| 46 | break; | ||
| 47 | case RVV_VXRM: | ||
| 48 | __asm__ __volatile__ ("csrr\t%0, vxrm" : "=r"(__rv) : : "memory"); | ||
| 49 | break; | ||
| 50 | case RVV_VCSR: | ||
| 51 | __asm__ __volatile__ ("csrr\t%0, vcsr" : "=r"(__rv) : : "memory"); | ||
| 52 | break; | ||
| 53 | } | ||
| 54 | return __rv; | ||
| 55 | } | ||
| 56 | |||
| 57 | static __inline__ __attribute__((__always_inline__, __nodebug__)) | ||
| 58 | void __riscv_vwrite_csr(enum RVV_CSR __csr, unsigned long __value) { | ||
| 59 | switch (__csr) { | ||
| 60 | case RVV_VSTART: | ||
| 61 | __asm__ __volatile__ ("csrw\tvstart, %z0" : : "rJ"(__value) : "memory"); | ||
| 62 | break; | ||
| 63 | case RVV_VXSAT: | ||
| 64 | __asm__ __volatile__ ("csrw\tvxsat, %z0" : : "rJ"(__value) : "memory"); | ||
| 65 | break; | ||
| 66 | case RVV_VXRM: | ||
| 67 | __asm__ __volatile__ ("csrw\tvxrm, %z0" : : "rJ"(__value) : "memory"); | ||
| 68 | break; | ||
| 69 | case RVV_VCSR: | ||
| 70 | __asm__ __volatile__ ("csrw\tvcsr, %z0" : : "rJ"(__value) : "memory"); | ||
| 71 | break; | ||
| 72 | } | ||
| 73 | } | ||
| 74 | 37 | ||
| 75 | #define __riscv_vsetvl_e8mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 6) | 38 | #define __riscv_vsetvl_e8mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 6) |
| 76 | #define __riscv_vsetvl_e8mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 7) | 39 | #define __riscv_vsetvl_e8mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 7) |
| ... | @@ -130,6 +93,13 @@ void __riscv_vwrite_csr(enum RVV_CSR __csr, unsigned long __value) { | ... | @@ -130,6 +93,13 @@ void __riscv_vwrite_csr(enum RVV_CSR __csr, unsigned long __value) { |
| 130 | #define __riscv_vsetvlmax_e64m8() __builtin_rvv_vsetvlimax(3, 3) | 93 | #define __riscv_vsetvlmax_e64m8() __builtin_rvv_vsetvlimax(3, 3) |
| 131 | #endif | 94 | #endif |
| 132 | 95 | ||
| 96 | |||
| 97 | enum __RISCV_VXRM { | ||
| 98 | __RISCV_VXRM_RNU = 0, | ||
| 99 | __RISCV_VXRM_RNE = 1, | ||
| 100 | __RISCV_VXRM_RDN = 2, | ||
| 101 | __RISCV_VXRM_ROD = 3, | ||
| 102 | }; | ||
| 133 | typedef __rvv_bool64_t vbool64_t; | 103 | typedef __rvv_bool64_t vbool64_t; |
| 134 | typedef __rvv_bool32_t vbool32_t; | 104 | typedef __rvv_bool32_t vbool32_t; |
| 135 | typedef __rvv_bool16_t vbool16_t; | 105 | typedef __rvv_bool16_t vbool16_t; |
| ... | @@ -139,70 +109,289 @@ typedef __rvv_bool2_t vbool2_t; | ... | @@ -139,70 +109,289 @@ typedef __rvv_bool2_t vbool2_t; |
| 139 | typedef __rvv_bool1_t vbool1_t; | 109 | typedef __rvv_bool1_t vbool1_t; |
| 140 | typedef __rvv_int8mf8_t vint8mf8_t; | 110 | typedef __rvv_int8mf8_t vint8mf8_t; |
| 141 | typedef __rvv_uint8mf8_t vuint8mf8_t; | 111 | typedef __rvv_uint8mf8_t vuint8mf8_t; |
| 112 | typedef __rvv_int8mf8x2_t vint8mf8x2_t; | ||
| 113 | typedef __rvv_uint8mf8x2_t vuint8mf8x2_t; | ||
| 114 | typedef __rvv_int8mf8x3_t vint8mf8x3_t; | ||
| 115 | typedef __rvv_uint8mf8x3_t vuint8mf8x3_t; | ||
| 116 | typedef __rvv_int8mf8x4_t vint8mf8x4_t; | ||
| 117 | typedef __rvv_uint8mf8x4_t vuint8mf8x4_t; | ||
| 118 | typedef __rvv_int8mf8x5_t vint8mf8x5_t; | ||
| 119 | typedef __rvv_uint8mf8x5_t vuint8mf8x5_t; | ||
| 120 | typedef __rvv_int8mf8x6_t vint8mf8x6_t; | ||
| 121 | typedef __rvv_uint8mf8x6_t vuint8mf8x6_t; | ||
| 122 | typedef __rvv_int8mf8x7_t vint8mf8x7_t; | ||
| 123 | typedef __rvv_uint8mf8x7_t vuint8mf8x7_t; | ||
| 124 | typedef __rvv_int8mf8x8_t vint8mf8x8_t; | ||
| 125 | typedef __rvv_uint8mf8x8_t vuint8mf8x8_t; | ||
| 142 | typedef __rvv_int8mf4_t vint8mf4_t; | 126 | typedef __rvv_int8mf4_t vint8mf4_t; |
| 143 | typedef __rvv_uint8mf4_t vuint8mf4_t; | 127 | typedef __rvv_uint8mf4_t vuint8mf4_t; |
| 128 | typedef __rvv_int8mf4x2_t vint8mf4x2_t; | ||
| 129 | typedef __rvv_uint8mf4x2_t vuint8mf4x2_t; | ||
| 130 | typedef __rvv_int8mf4x3_t vint8mf4x3_t; | ||
| 131 | typedef __rvv_uint8mf4x3_t vuint8mf4x3_t; | ||
| 132 | typedef __rvv_int8mf4x4_t vint8mf4x4_t; | ||
| 133 | typedef __rvv_uint8mf4x4_t vuint8mf4x4_t; | ||
| 134 | typedef __rvv_int8mf4x5_t vint8mf4x5_t; | ||
| 135 | typedef __rvv_uint8mf4x5_t vuint8mf4x5_t; | ||
| 136 | typedef __rvv_int8mf4x6_t vint8mf4x6_t; | ||
| 137 | typedef __rvv_uint8mf4x6_t vuint8mf4x6_t; | ||
| 138 | typedef __rvv_int8mf4x7_t vint8mf4x7_t; | ||
| 139 | typedef __rvv_uint8mf4x7_t vuint8mf4x7_t; | ||
| 140 | typedef __rvv_int8mf4x8_t vint8mf4x8_t; | ||
| 141 | typedef __rvv_uint8mf4x8_t vuint8mf4x8_t; | ||
| 144 | typedef __rvv_int8mf2_t vint8mf2_t; | 142 | typedef __rvv_int8mf2_t vint8mf2_t; |
| 145 | typedef __rvv_uint8mf2_t vuint8mf2_t; | 143 | typedef __rvv_uint8mf2_t vuint8mf2_t; |
| 144 | typedef __rvv_int8mf2x2_t vint8mf2x2_t; | ||
| 145 | typedef __rvv_uint8mf2x2_t vuint8mf2x2_t; | ||
| 146 | typedef __rvv_int8mf2x3_t vint8mf2x3_t; | ||
| 147 | typedef __rvv_uint8mf2x3_t vuint8mf2x3_t; | ||
| 148 | typedef __rvv_int8mf2x4_t vint8mf2x4_t; | ||
| 149 | typedef __rvv_uint8mf2x4_t vuint8mf2x4_t; | ||
| 150 | typedef __rvv_int8mf2x5_t vint8mf2x5_t; | ||
| 151 | typedef __rvv_uint8mf2x5_t vuint8mf2x5_t; | ||
| 152 | typedef __rvv_int8mf2x6_t vint8mf2x6_t; | ||
| 153 | typedef __rvv_uint8mf2x6_t vuint8mf2x6_t; | ||
| 154 | typedef __rvv_int8mf2x7_t vint8mf2x7_t; | ||
| 155 | typedef __rvv_uint8mf2x7_t vuint8mf2x7_t; | ||
| 156 | typedef __rvv_int8mf2x8_t vint8mf2x8_t; | ||
| 157 | typedef __rvv_uint8mf2x8_t vuint8mf2x8_t; | ||
| 146 | typedef __rvv_int8m1_t vint8m1_t; | 158 | typedef __rvv_int8m1_t vint8m1_t; |
| 147 | typedef __rvv_uint8m1_t vuint8m1_t; | 159 | typedef __rvv_uint8m1_t vuint8m1_t; |
| 160 | typedef __rvv_int8m1x2_t vint8m1x2_t; | ||
| 161 | typedef __rvv_uint8m1x2_t vuint8m1x2_t; | ||
| 162 | typedef __rvv_int8m1x3_t vint8m1x3_t; | ||
| 163 | typedef __rvv_uint8m1x3_t vuint8m1x3_t; | ||
| 164 | typedef __rvv_int8m1x4_t vint8m1x4_t; | ||
| 165 | typedef __rvv_uint8m1x4_t vuint8m1x4_t; | ||
| 166 | typedef __rvv_int8m1x5_t vint8m1x5_t; | ||
| 167 | typedef __rvv_uint8m1x5_t vuint8m1x5_t; | ||
| 168 | typedef __rvv_int8m1x6_t vint8m1x6_t; | ||
| 169 | typedef __rvv_uint8m1x6_t vuint8m1x6_t; | ||
| 170 | typedef __rvv_int8m1x7_t vint8m1x7_t; | ||
| 171 | typedef __rvv_uint8m1x7_t vuint8m1x7_t; | ||
| 172 | typedef __rvv_int8m1x8_t vint8m1x8_t; | ||
| 173 | typedef __rvv_uint8m1x8_t vuint8m1x8_t; | ||
| 148 | typedef __rvv_int8m2_t vint8m2_t; | 174 | typedef __rvv_int8m2_t vint8m2_t; |
| 149 | typedef __rvv_uint8m2_t vuint8m2_t; | 175 | typedef __rvv_uint8m2_t vuint8m2_t; |
| 176 | typedef __rvv_int8m2x2_t vint8m2x2_t; | ||
| 177 | typedef __rvv_uint8m2x2_t vuint8m2x2_t; | ||
| 178 | typedef __rvv_int8m2x3_t vint8m2x3_t; | ||
| 179 | typedef __rvv_uint8m2x3_t vuint8m2x3_t; | ||
| 180 | typedef __rvv_int8m2x4_t vint8m2x4_t; | ||
| 181 | typedef __rvv_uint8m2x4_t vuint8m2x4_t; | ||
| 150 | typedef __rvv_int8m4_t vint8m4_t; | 182 | typedef __rvv_int8m4_t vint8m4_t; |
| 151 | typedef __rvv_uint8m4_t vuint8m4_t; | 183 | typedef __rvv_uint8m4_t vuint8m4_t; |
| 184 | typedef __rvv_int8m4x2_t vint8m4x2_t; | ||
| 185 | typedef __rvv_uint8m4x2_t vuint8m4x2_t; | ||
| 152 | typedef __rvv_int8m8_t vint8m8_t; | 186 | typedef __rvv_int8m8_t vint8m8_t; |
| 153 | typedef __rvv_uint8m8_t vuint8m8_t; | 187 | typedef __rvv_uint8m8_t vuint8m8_t; |
| 154 | typedef __rvv_int16mf4_t vint16mf4_t; | 188 | typedef __rvv_int16mf4_t vint16mf4_t; |
| 155 | typedef __rvv_uint16mf4_t vuint16mf4_t; | 189 | typedef __rvv_uint16mf4_t vuint16mf4_t; |
| 190 | typedef __rvv_int16mf4x2_t vint16mf4x2_t; | ||
| 191 | typedef __rvv_uint16mf4x2_t vuint16mf4x2_t; | ||
| 192 | typedef __rvv_int16mf4x3_t vint16mf4x3_t; | ||
| 193 | typedef __rvv_uint16mf4x3_t vuint16mf4x3_t; | ||
| 194 | typedef __rvv_int16mf4x4_t vint16mf4x4_t; | ||
| 195 | typedef __rvv_uint16mf4x4_t vuint16mf4x4_t; | ||
| 196 | typedef __rvv_int16mf4x5_t vint16mf4x5_t; | ||
| 197 | typedef __rvv_uint16mf4x5_t vuint16mf4x5_t; | ||
| 198 | typedef __rvv_int16mf4x6_t vint16mf4x6_t; | ||
| 199 | typedef __rvv_uint16mf4x6_t vuint16mf4x6_t; | ||
| 200 | typedef __rvv_int16mf4x7_t vint16mf4x7_t; | ||
| 201 | typedef __rvv_uint16mf4x7_t vuint16mf4x7_t; | ||
| 202 | typedef __rvv_int16mf4x8_t vint16mf4x8_t; | ||
| 203 | typedef __rvv_uint16mf4x8_t vuint16mf4x8_t; | ||
| 156 | typedef __rvv_int16mf2_t vint16mf2_t; | 204 | typedef __rvv_int16mf2_t vint16mf2_t; |
| 157 | typedef __rvv_uint16mf2_t vuint16mf2_t; | 205 | typedef __rvv_uint16mf2_t vuint16mf2_t; |
| 206 | typedef __rvv_int16mf2x2_t vint16mf2x2_t; | ||
| 207 | typedef __rvv_uint16mf2x2_t vuint16mf2x2_t; | ||
| 208 | typedef __rvv_int16mf2x3_t vint16mf2x3_t; | ||
| 209 | typedef __rvv_uint16mf2x3_t vuint16mf2x3_t; | ||
| 210 | typedef __rvv_int16mf2x4_t vint16mf2x4_t; | ||
| 211 | typedef __rvv_uint16mf2x4_t vuint16mf2x4_t; | ||
| 212 | typedef __rvv_int16mf2x5_t vint16mf2x5_t; | ||
| 213 | typedef __rvv_uint16mf2x5_t vuint16mf2x5_t; | ||
| 214 | typedef __rvv_int16mf2x6_t vint16mf2x6_t; | ||
| 215 | typedef __rvv_uint16mf2x6_t vuint16mf2x6_t; | ||
| 216 | typedef __rvv_int16mf2x7_t vint16mf2x7_t; | ||
| 217 | typedef __rvv_uint16mf2x7_t vuint16mf2x7_t; | ||
| 218 | typedef __rvv_int16mf2x8_t vint16mf2x8_t; | ||
| 219 | typedef __rvv_uint16mf2x8_t vuint16mf2x8_t; | ||
| 158 | typedef __rvv_int16m1_t vint16m1_t; | 220 | typedef __rvv_int16m1_t vint16m1_t; |
| 159 | typedef __rvv_uint16m1_t vuint16m1_t; | 221 | typedef __rvv_uint16m1_t vuint16m1_t; |
| 222 | typedef __rvv_int16m1x2_t vint16m1x2_t; | ||
| 223 | typedef __rvv_uint16m1x2_t vuint16m1x2_t; | ||
| 224 | typedef __rvv_int16m1x3_t vint16m1x3_t; | ||
| 225 | typedef __rvv_uint16m1x3_t vuint16m1x3_t; | ||
| 226 | typedef __rvv_int16m1x4_t vint16m1x4_t; | ||
| 227 | typedef __rvv_uint16m1x4_t vuint16m1x4_t; | ||
| 228 | typedef __rvv_int16m1x5_t vint16m1x5_t; | ||
| 229 | typedef __rvv_uint16m1x5_t vuint16m1x5_t; | ||
| 230 | typedef __rvv_int16m1x6_t vint16m1x6_t; | ||
| 231 | typedef __rvv_uint16m1x6_t vuint16m1x6_t; | ||
| 232 | typedef __rvv_int16m1x7_t vint16m1x7_t; | ||
| 233 | typedef __rvv_uint16m1x7_t vuint16m1x7_t; | ||
| 234 | typedef __rvv_int16m1x8_t vint16m1x8_t; | ||
| 235 | typedef __rvv_uint16m1x8_t vuint16m1x8_t; | ||
| 160 | typedef __rvv_int16m2_t vint16m2_t; | 236 | typedef __rvv_int16m2_t vint16m2_t; |
| 161 | typedef __rvv_uint16m2_t vuint16m2_t; | 237 | typedef __rvv_uint16m2_t vuint16m2_t; |
| 238 | typedef __rvv_int16m2x2_t vint16m2x2_t; | ||
| 239 | typedef __rvv_uint16m2x2_t vuint16m2x2_t; | ||
| 240 | typedef __rvv_int16m2x3_t vint16m2x3_t; | ||
| 241 | typedef __rvv_uint16m2x3_t vuint16m2x3_t; | ||
| 242 | typedef __rvv_int16m2x4_t vint16m2x4_t; | ||
| 243 | typedef __rvv_uint16m2x4_t vuint16m2x4_t; | ||
| 162 | typedef __rvv_int16m4_t vint16m4_t; | 244 | typedef __rvv_int16m4_t vint16m4_t; |
| 163 | typedef __rvv_uint16m4_t vuint16m4_t; | 245 | typedef __rvv_uint16m4_t vuint16m4_t; |
| 246 | typedef __rvv_int16m4x2_t vint16m4x2_t; | ||
| 247 | typedef __rvv_uint16m4x2_t vuint16m4x2_t; | ||
| 164 | typedef __rvv_int16m8_t vint16m8_t; | 248 | typedef __rvv_int16m8_t vint16m8_t; |
| 165 | typedef __rvv_uint16m8_t vuint16m8_t; | 249 | typedef __rvv_uint16m8_t vuint16m8_t; |
| 166 | typedef __rvv_int32mf2_t vint32mf2_t; | 250 | typedef __rvv_int32mf2_t vint32mf2_t; |
| 167 | typedef __rvv_uint32mf2_t vuint32mf2_t; | 251 | typedef __rvv_uint32mf2_t vuint32mf2_t; |
| 252 | typedef __rvv_int32mf2x2_t vint32mf2x2_t; | ||
| 253 | typedef __rvv_uint32mf2x2_t vuint32mf2x2_t; | ||
| 254 | typedef __rvv_int32mf2x3_t vint32mf2x3_t; | ||
| 255 | typedef __rvv_uint32mf2x3_t vuint32mf2x3_t; | ||
| 256 | typedef __rvv_int32mf2x4_t vint32mf2x4_t; | ||
| 257 | typedef __rvv_uint32mf2x4_t vuint32mf2x4_t; | ||
| 258 | typedef __rvv_int32mf2x5_t vint32mf2x5_t; | ||
| 259 | typedef __rvv_uint32mf2x5_t vuint32mf2x5_t; | ||
| 260 | typedef __rvv_int32mf2x6_t vint32mf2x6_t; | ||
| 261 | typedef __rvv_uint32mf2x6_t vuint32mf2x6_t; | ||
| 262 | typedef __rvv_int32mf2x7_t vint32mf2x7_t; | ||
| 263 | typedef __rvv_uint32mf2x7_t vuint32mf2x7_t; | ||
| 264 | typedef __rvv_int32mf2x8_t vint32mf2x8_t; | ||
| 265 | typedef __rvv_uint32mf2x8_t vuint32mf2x8_t; | ||
| 168 | typedef __rvv_int32m1_t vint32m1_t; | 266 | typedef __rvv_int32m1_t vint32m1_t; |
| 169 | typedef __rvv_uint32m1_t vuint32m1_t; | 267 | typedef __rvv_uint32m1_t vuint32m1_t; |
| 268 | typedef __rvv_int32m1x2_t vint32m1x2_t; | ||
| 269 | typedef __rvv_uint32m1x2_t vuint32m1x2_t; | ||
| 270 | typedef __rvv_int32m1x3_t vint32m1x3_t; | ||
| 271 | typedef __rvv_uint32m1x3_t vuint32m1x3_t; | ||
| 272 | typedef __rvv_int32m1x4_t vint32m1x4_t; | ||
| 273 | typedef __rvv_uint32m1x4_t vuint32m1x4_t; | ||
| 274 | typedef __rvv_int32m1x5_t vint32m1x5_t; | ||
| 275 | typedef __rvv_uint32m1x5_t vuint32m1x5_t; | ||
| 276 | typedef __rvv_int32m1x6_t vint32m1x6_t; | ||
| 277 | typedef __rvv_uint32m1x6_t vuint32m1x6_t; | ||
| 278 | typedef __rvv_int32m1x7_t vint32m1x7_t; | ||
| 279 | typedef __rvv_uint32m1x7_t vuint32m1x7_t; | ||
| 280 | typedef __rvv_int32m1x8_t vint32m1x8_t; | ||
| 281 | typedef __rvv_uint32m1x8_t vuint32m1x8_t; | ||
| 170 | typedef __rvv_int32m2_t vint32m2_t; | 282 | typedef __rvv_int32m2_t vint32m2_t; |
| 171 | typedef __rvv_uint32m2_t vuint32m2_t; | 283 | typedef __rvv_uint32m2_t vuint32m2_t; |
| 284 | typedef __rvv_int32m2x2_t vint32m2x2_t; | ||
| 285 | typedef __rvv_uint32m2x2_t vuint32m2x2_t; | ||
| 286 | typedef __rvv_int32m2x3_t vint32m2x3_t; | ||
| 287 | typedef __rvv_uint32m2x3_t vuint32m2x3_t; | ||
| 288 | typedef __rvv_int32m2x4_t vint32m2x4_t; | ||
| 289 | typedef __rvv_uint32m2x4_t vuint32m2x4_t; | ||
| 172 | typedef __rvv_int32m4_t vint32m4_t; | 290 | typedef __rvv_int32m4_t vint32m4_t; |
| 173 | typedef __rvv_uint32m4_t vuint32m4_t; | 291 | typedef __rvv_uint32m4_t vuint32m4_t; |
| 292 | typedef __rvv_int32m4x2_t vint32m4x2_t; | ||
| 293 | typedef __rvv_uint32m4x2_t vuint32m4x2_t; | ||
| 174 | typedef __rvv_int32m8_t vint32m8_t; | 294 | typedef __rvv_int32m8_t vint32m8_t; |
| 175 | typedef __rvv_uint32m8_t vuint32m8_t; | 295 | typedef __rvv_uint32m8_t vuint32m8_t; |
| 176 | typedef __rvv_int64m1_t vint64m1_t; | 296 | typedef __rvv_int64m1_t vint64m1_t; |
| 177 | typedef __rvv_uint64m1_t vuint64m1_t; | 297 | typedef __rvv_uint64m1_t vuint64m1_t; |
| 298 | typedef __rvv_int64m1x2_t vint64m1x2_t; | ||
| 299 | typedef __rvv_uint64m1x2_t vuint64m1x2_t; | ||
| 300 | typedef __rvv_int64m1x3_t vint64m1x3_t; | ||
| 301 | typedef __rvv_uint64m1x3_t vuint64m1x3_t; | ||
| 302 | typedef __rvv_int64m1x4_t vint64m1x4_t; | ||
| 303 | typedef __rvv_uint64m1x4_t vuint64m1x4_t; | ||
| 304 | typedef __rvv_int64m1x5_t vint64m1x5_t; | ||
| 305 | typedef __rvv_uint64m1x5_t vuint64m1x5_t; | ||
| 306 | typedef __rvv_int64m1x6_t vint64m1x6_t; | ||
| 307 | typedef __rvv_uint64m1x6_t vuint64m1x6_t; | ||
| 308 | typedef __rvv_int64m1x7_t vint64m1x7_t; | ||
| 309 | typedef __rvv_uint64m1x7_t vuint64m1x7_t; | ||
| 310 | typedef __rvv_int64m1x8_t vint64m1x8_t; | ||
| 311 | typedef __rvv_uint64m1x8_t vuint64m1x8_t; | ||
| 178 | typedef __rvv_int64m2_t vint64m2_t; | 312 | typedef __rvv_int64m2_t vint64m2_t; |
| 179 | typedef __rvv_uint64m2_t vuint64m2_t; | 313 | typedef __rvv_uint64m2_t vuint64m2_t; |
| 314 | typedef __rvv_int64m2x2_t vint64m2x2_t; | ||
| 315 | typedef __rvv_uint64m2x2_t vuint64m2x2_t; | ||
| 316 | typedef __rvv_int64m2x3_t vint64m2x3_t; | ||
| 317 | typedef __rvv_uint64m2x3_t vuint64m2x3_t; | ||
| 318 | typedef __rvv_int64m2x4_t vint64m2x4_t; | ||
| 319 | typedef __rvv_uint64m2x4_t vuint64m2x4_t; | ||
| 180 | typedef __rvv_int64m4_t vint64m4_t; | 320 | typedef __rvv_int64m4_t vint64m4_t; |
| 181 | typedef __rvv_uint64m4_t vuint64m4_t; | 321 | typedef __rvv_uint64m4_t vuint64m4_t; |
| 322 | typedef __rvv_int64m4x2_t vint64m4x2_t; | ||
| 323 | typedef __rvv_uint64m4x2_t vuint64m4x2_t; | ||
| 182 | typedef __rvv_int64m8_t vint64m8_t; | 324 | typedef __rvv_int64m8_t vint64m8_t; |
| 183 | typedef __rvv_uint64m8_t vuint64m8_t; | 325 | typedef __rvv_uint64m8_t vuint64m8_t; |
| 184 | #if defined(__riscv_zvfh) | ||
| 185 | typedef __rvv_float16mf4_t vfloat16mf4_t; | 326 | typedef __rvv_float16mf4_t vfloat16mf4_t; |
| 327 | typedef __rvv_float16mf4x2_t vfloat16mf4x2_t; | ||
| 328 | typedef __rvv_float16mf4x3_t vfloat16mf4x3_t; | ||
| 329 | typedef __rvv_float16mf4x4_t vfloat16mf4x4_t; | ||
| 330 | typedef __rvv_float16mf4x5_t vfloat16mf4x5_t; | ||
| 331 | typedef __rvv_float16mf4x6_t vfloat16mf4x6_t; | ||
| 332 | typedef __rvv_float16mf4x7_t vfloat16mf4x7_t; | ||
| 333 | typedef __rvv_float16mf4x8_t vfloat16mf4x8_t; | ||
| 186 | typedef __rvv_float16mf2_t vfloat16mf2_t; | 334 | typedef __rvv_float16mf2_t vfloat16mf2_t; |
| 335 | typedef __rvv_float16mf2x2_t vfloat16mf2x2_t; | ||
| 336 | typedef __rvv_float16mf2x3_t vfloat16mf2x3_t; | ||
| 337 | typedef __rvv_float16mf2x4_t vfloat16mf2x4_t; | ||
| 338 | typedef __rvv_float16mf2x5_t vfloat16mf2x5_t; | ||
| 339 | typedef __rvv_float16mf2x6_t vfloat16mf2x6_t; | ||
| 340 | typedef __rvv_float16mf2x7_t vfloat16mf2x7_t; | ||
| 341 | typedef __rvv_float16mf2x8_t vfloat16mf2x8_t; | ||
| 187 | typedef __rvv_float16m1_t vfloat16m1_t; | 342 | typedef __rvv_float16m1_t vfloat16m1_t; |
| 343 | typedef __rvv_float16m1x2_t vfloat16m1x2_t; | ||
| 344 | typedef __rvv_float16m1x3_t vfloat16m1x3_t; | ||
| 345 | typedef __rvv_float16m1x4_t vfloat16m1x4_t; | ||
| 346 | typedef __rvv_float16m1x5_t vfloat16m1x5_t; | ||
| 347 | typedef __rvv_float16m1x6_t vfloat16m1x6_t; | ||
| 348 | typedef __rvv_float16m1x7_t vfloat16m1x7_t; | ||
| 349 | typedef __rvv_float16m1x8_t vfloat16m1x8_t; | ||
| 188 | typedef __rvv_float16m2_t vfloat16m2_t; | 350 | typedef __rvv_float16m2_t vfloat16m2_t; |
| 351 | typedef __rvv_float16m2x2_t vfloat16m2x2_t; | ||
| 352 | typedef __rvv_float16m2x3_t vfloat16m2x3_t; | ||
| 353 | typedef __rvv_float16m2x4_t vfloat16m2x4_t; | ||
| 189 | typedef __rvv_float16m4_t vfloat16m4_t; | 354 | typedef __rvv_float16m4_t vfloat16m4_t; |
| 355 | typedef __rvv_float16m4x2_t vfloat16m4x2_t; | ||
| 190 | typedef __rvv_float16m8_t vfloat16m8_t; | 356 | typedef __rvv_float16m8_t vfloat16m8_t; |
| 191 | #endif | ||
| 192 | #if (__riscv_v_elen_fp >= 32) | ||
| 193 | typedef __rvv_float32mf2_t vfloat32mf2_t; | 357 | typedef __rvv_float32mf2_t vfloat32mf2_t; |
| 358 | typedef __rvv_float32mf2x2_t vfloat32mf2x2_t; | ||
| 359 | typedef __rvv_float32mf2x3_t vfloat32mf2x3_t; | ||
| 360 | typedef __rvv_float32mf2x4_t vfloat32mf2x4_t; | ||
| 361 | typedef __rvv_float32mf2x5_t vfloat32mf2x5_t; | ||
| 362 | typedef __rvv_float32mf2x6_t vfloat32mf2x6_t; | ||
| 363 | typedef __rvv_float32mf2x7_t vfloat32mf2x7_t; | ||
| 364 | typedef __rvv_float32mf2x8_t vfloat32mf2x8_t; | ||
| 194 | typedef __rvv_float32m1_t vfloat32m1_t; | 365 | typedef __rvv_float32m1_t vfloat32m1_t; |
| 366 | typedef __rvv_float32m1x2_t vfloat32m1x2_t; | ||
| 367 | typedef __rvv_float32m1x3_t vfloat32m1x3_t; | ||
| 368 | typedef __rvv_float32m1x4_t vfloat32m1x4_t; | ||
| 369 | typedef __rvv_float32m1x5_t vfloat32m1x5_t; | ||
| 370 | typedef __rvv_float32m1x6_t vfloat32m1x6_t; | ||
| 371 | typedef __rvv_float32m1x7_t vfloat32m1x7_t; | ||
| 372 | typedef __rvv_float32m1x8_t vfloat32m1x8_t; | ||
| 195 | typedef __rvv_float32m2_t vfloat32m2_t; | 373 | typedef __rvv_float32m2_t vfloat32m2_t; |
| 374 | typedef __rvv_float32m2x2_t vfloat32m2x2_t; | ||
| 375 | typedef __rvv_float32m2x3_t vfloat32m2x3_t; | ||
| 376 | typedef __rvv_float32m2x4_t vfloat32m2x4_t; | ||
| 196 | typedef __rvv_float32m4_t vfloat32m4_t; | 377 | typedef __rvv_float32m4_t vfloat32m4_t; |
| 378 | typedef __rvv_float32m4x2_t vfloat32m4x2_t; | ||
| 197 | typedef __rvv_float32m8_t vfloat32m8_t; | 379 | typedef __rvv_float32m8_t vfloat32m8_t; |
| 198 | #endif | ||
| 199 | #if (__riscv_v_elen_fp >= 64) | ||
| 200 | typedef __rvv_float64m1_t vfloat64m1_t; | 380 | typedef __rvv_float64m1_t vfloat64m1_t; |
| 381 | typedef __rvv_float64m1x2_t vfloat64m1x2_t; | ||
| 382 | typedef __rvv_float64m1x3_t vfloat64m1x3_t; | ||
| 383 | typedef __rvv_float64m1x4_t vfloat64m1x4_t; | ||
| 384 | typedef __rvv_float64m1x5_t vfloat64m1x5_t; | ||
| 385 | typedef __rvv_float64m1x6_t vfloat64m1x6_t; | ||
| 386 | typedef __rvv_float64m1x7_t vfloat64m1x7_t; | ||
| 387 | typedef __rvv_float64m1x8_t vfloat64m1x8_t; | ||
| 201 | typedef __rvv_float64m2_t vfloat64m2_t; | 388 | typedef __rvv_float64m2_t vfloat64m2_t; |
| 389 | typedef __rvv_float64m2x2_t vfloat64m2x2_t; | ||
| 390 | typedef __rvv_float64m2x3_t vfloat64m2x3_t; | ||
| 391 | typedef __rvv_float64m2x4_t vfloat64m2x4_t; | ||
| 202 | typedef __rvv_float64m4_t vfloat64m4_t; | 392 | typedef __rvv_float64m4_t vfloat64m4_t; |
| 393 | typedef __rvv_float64m4x2_t vfloat64m4x2_t; | ||
| 203 | typedef __rvv_float64m8_t vfloat64m8_t; | 394 | typedef __rvv_float64m8_t vfloat64m8_t; |
| 204 | #endif | ||
| 205 | |||
| 206 | #define __riscv_v_intrinsic_overloading 1 | 395 | #define __riscv_v_intrinsic_overloading 1 |
| 207 | 396 | ||
| 208 | #ifdef __cplusplus | 397 | #ifdef __cplusplus |
lib/include/sha512intrin.h created+200
| ... | @@ -0,0 +1,200 @@ | ||
| 1 | /*===--------------- sha512intrin.h - SHA512 intrinsics -----------------=== | ||
| 2 | * | ||
| 3 | * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| 4 | * See https://llvm.org/LICENSE.txt for license information. | ||
| 5 | * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| 6 | * | ||
| 7 | *===-----------------------------------------------------------------------=== | ||
| 8 | */ | ||
| 9 | |||
| 10 | #ifndef __IMMINTRIN_H | ||
| 11 | #error "Never use <sha512intrin.h> directly; include <immintrin.h> instead." | ||
| 12 | #endif // __IMMINTRIN_H | ||
| 13 | |||
| 14 | #ifndef __SHA512INTRIN_H | ||
| 15 | #define __SHA512INTRIN_H | ||
| 16 | |||
| 17 | #define __DEFAULT_FN_ATTRS256 \ | ||
| 18 | __attribute__((__always_inline__, __nodebug__, __target__("sha512"), \ | ||
| 19 | __min_vector_width__(256))) | ||
| 20 | |||
| 21 | /// This intrinisc is one of the two SHA512 message scheduling instructions. | ||
| 22 | /// The intrinsic performs an intermediate calculation for the next four | ||
| 23 | /// SHA512 message qwords. The calculated results are stored in \a dst. | ||
| 24 | /// | ||
| 25 | /// \headerfile <immintrin.h> | ||
| 26 | /// | ||
| 27 | /// \code | ||
| 28 | /// __m256i _mm256_sha512msg1_epi64(__m256i __A, __m128i __B) | ||
| 29 | /// \endcode | ||
| 30 | /// | ||
| 31 | /// This intrinsic corresponds to the \c VSHA512MSG1 instruction. | ||
| 32 | /// | ||
| 33 | /// \param __A | ||
| 34 | /// A 256-bit vector of [4 x long long]. | ||
| 35 | /// \param __B | ||
| 36 | /// A 128-bit vector of [2 x long long]. | ||
| 37 | /// \returns | ||
| 38 | /// A 256-bit vector of [4 x long long]. | ||
| 39 | /// | ||
| 40 | /// \code{.operation} | ||
| 41 | /// DEFINE ROR64(qword, n) { | ||
| 42 | /// count := n % 64 | ||
| 43 | /// dest := (qword >> count) | (qword << (64 - count)) | ||
| 44 | /// RETURN dest | ||
| 45 | /// } | ||
| 46 | /// DEFINE SHR64(qword, n) { | ||
| 47 | /// RETURN qword >> n | ||
| 48 | /// } | ||
| 49 | /// DEFINE s0(qword): | ||
| 50 | /// RETURN ROR64(qword,1) ^ ROR64(qword, 8) ^ SHR64(qword, 7) | ||
| 51 | /// } | ||
| 52 | /// W[4] := __B.qword[0] | ||
| 53 | /// W[3] := __A.qword[3] | ||
| 54 | /// W[2] := __A.qword[2] | ||
| 55 | /// W[1] := __A.qword[1] | ||
| 56 | /// W[0] := __A.qword[0] | ||
| 57 | /// dst.qword[3] := W[3] + s0(W[4]) | ||
| 58 | /// dst.qword[2] := W[2] + s0(W[3]) | ||
| 59 | /// dst.qword[1] := W[1] + s0(W[2]) | ||
| 60 | /// dst.qword[0] := W[0] + s0(W[1]) | ||
| 61 | /// dst[MAX:256] := 0 | ||
| 62 | /// \endcode | ||
| 63 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 64 | _mm256_sha512msg1_epi64(__m256i __A, __m128i __B) { | ||
| 65 | return (__m256i)__builtin_ia32_vsha512msg1((__v4du)__A, (__v2du)__B); | ||
| 66 | } | ||
| 67 | |||
| 68 | /// This intrinisc is one of the two SHA512 message scheduling instructions. | ||
| 69 | /// The intrinsic performs the final calculation for the next four SHA512 | ||
| 70 | /// message qwords. The calculated results are stored in \a dst. | ||
| 71 | /// | ||
| 72 | /// \headerfile <immintrin.h> | ||
| 73 | /// | ||
| 74 | /// \code | ||
| 75 | /// __m256i _mm256_sha512msg2_epi64(__m256i __A, __m256i __B) | ||
| 76 | /// \endcode | ||
| 77 | /// | ||
| 78 | /// This intrinsic corresponds to the \c VSHA512MSG2 instruction. | ||
| 79 | /// | ||
| 80 | /// \param __A | ||
| 81 | /// A 256-bit vector of [4 x long long]. | ||
| 82 | /// \param __B | ||
| 83 | /// A 256-bit vector of [4 x long long]. | ||
| 84 | /// \returns | ||
| 85 | /// A 256-bit vector of [4 x long long]. | ||
| 86 | /// | ||
| 87 | /// \code{.operation} | ||
| 88 | /// DEFINE ROR64(qword, n) { | ||
| 89 | /// count := n % 64 | ||
| 90 | /// dest := (qword >> count) | (qword << (64 - count)) | ||
| 91 | /// RETURN dest | ||
| 92 | /// } | ||
| 93 | /// DEFINE SHR64(qword, n) { | ||
| 94 | /// RETURN qword >> n | ||
| 95 | /// } | ||
| 96 | /// DEFINE s1(qword) { | ||
| 97 | /// RETURN ROR64(qword,19) ^ ROR64(qword, 61) ^ SHR64(qword, 6) | ||
| 98 | /// } | ||
| 99 | /// W[14] := __B.qword[2] | ||
| 100 | /// W[15] := __B.qword[3] | ||
| 101 | /// W[16] := __A.qword[0] + s1(W[14]) | ||
| 102 | /// W[17] := __A.qword[1] + s1(W[15]) | ||
| 103 | /// W[18] := __A.qword[2] + s1(W[16]) | ||
| 104 | /// W[19] := __A.qword[3] + s1(W[17]) | ||
| 105 | /// dst.qword[3] := W[19] | ||
| 106 | /// dst.qword[2] := W[18] | ||
| 107 | /// dst.qword[1] := W[17] | ||
| 108 | /// dst.qword[0] := W[16] | ||
| 109 | /// dst[MAX:256] := 0 | ||
| 110 | /// \endcode | ||
| 111 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 112 | _mm256_sha512msg2_epi64(__m256i __A, __m256i __B) { | ||
| 113 | return (__m256i)__builtin_ia32_vsha512msg2((__v4du)__A, (__v4du)__B); | ||
| 114 | } | ||
| 115 | |||
| 116 | /// This intrinisc performs two rounds of SHA512 operation using initial SHA512 | ||
| 117 | /// state (C,D,G,H) from \a __A, an initial SHA512 state (A,B,E,F) from | ||
| 118 | /// \a __A, and a pre-computed sum of the next two round message qwords and | ||
| 119 | /// the corresponding round constants from \a __C (only the two lower qwords | ||
| 120 | /// of the third operand). The updated SHA512 state (A,B,E,F) is written to | ||
| 121 | /// \a __A, and \a __A can be used as the updated state (C,D,G,H) in later | ||
| 122 | /// rounds. | ||
| 123 | /// | ||
| 124 | /// \headerfile <immintrin.h> | ||
| 125 | /// | ||
| 126 | /// \code | ||
| 127 | /// __m256i _mm256_sha512rnds2_epi64(__m256i __A, __m256i __B, __m128i __C) | ||
| 128 | /// \endcode | ||
| 129 | /// | ||
| 130 | /// This intrinsic corresponds to the \c VSHA512RNDS2 instruction. | ||
| 131 | /// | ||
| 132 | /// \param __A | ||
| 133 | /// A 256-bit vector of [4 x long long]. | ||
| 134 | /// \param __B | ||
| 135 | /// A 256-bit vector of [4 x long long]. | ||
| 136 | /// \param __C | ||
| 137 | /// A 128-bit vector of [2 x long long]. | ||
| 138 | /// \returns | ||
| 139 | /// A 256-bit vector of [4 x long long]. | ||
| 140 | /// | ||
| 141 | /// \code{.operation} | ||
| 142 | /// DEFINE ROR64(qword, n) { | ||
| 143 | /// count := n % 64 | ||
| 144 | /// dest := (qword >> count) | (qword << (64 - count)) | ||
| 145 | /// RETURN dest | ||
| 146 | /// } | ||
| 147 | /// DEFINE SHR64(qword, n) { | ||
| 148 | /// RETURN qword >> n | ||
| 149 | /// } | ||
| 150 | /// DEFINE cap_sigma0(qword) { | ||
| 151 | /// RETURN ROR64(qword,28) ^ ROR64(qword, 34) ^ ROR64(qword, 39) | ||
| 152 | /// } | ||
| 153 | /// DEFINE cap_sigma1(qword) { | ||
| 154 | /// RETURN ROR64(qword,14) ^ ROR64(qword, 18) ^ ROR64(qword, 41) | ||
| 155 | /// } | ||
| 156 | /// DEFINE MAJ(a,b,c) { | ||
| 157 | /// RETURN (a & b) ^ (a & c) ^ (b & c) | ||
| 158 | /// } | ||
| 159 | /// DEFINE CH(e,f,g) { | ||
| 160 | /// RETURN (e & f) ^ (g & ~e) | ||
| 161 | /// } | ||
| 162 | /// A[0] := __B.qword[3] | ||
| 163 | /// B[0] := __B.qword[2] | ||
| 164 | /// C[0] := __C.qword[3] | ||
| 165 | /// D[0] := __C.qword[2] | ||
| 166 | /// E[0] := __B.qword[1] | ||
| 167 | /// F[0] := __B.qword[0] | ||
| 168 | /// G[0] := __C.qword[1] | ||
| 169 | /// H[0] := __C.qword[0] | ||
| 170 | /// WK[0]:= __A.qword[0] | ||
| 171 | /// WK[1]:= __A.qword[1] | ||
| 172 | /// FOR i := 0 to 1: | ||
| 173 | /// A[i+1] := CH(E[i], F[i], G[i]) + | ||
| 174 | /// cap_sigma1(E[i]) + WK[i] + H[i] + | ||
| 175 | /// MAJ(A[i], B[i], C[i]) + | ||
| 176 | /// cap_sigma0(A[i]) | ||
| 177 | /// B[i+1] := A[i] | ||
| 178 | /// C[i+1] := B[i] | ||
| 179 | /// D[i+1] := C[i] | ||
| 180 | /// E[i+1] := CH(E[i], F[i], G[i]) + | ||
| 181 | /// cap_sigma1(E[i]) + WK[i] + H[i] + D[i] | ||
| 182 | /// F[i+1] := E[i] | ||
| 183 | /// G[i+1] := F[i] | ||
| 184 | /// H[i+1] := G[i] | ||
| 185 | /// ENDFOR | ||
| 186 | /// dst.qword[3] := A[2] | ||
| 187 | /// dst.qword[2] := B[2] | ||
| 188 | /// dst.qword[1] := E[2] | ||
| 189 | /// dst.qword[0] := F[2] | ||
| 190 | /// dst[MAX:256] := 0 | ||
| 191 | /// \endcode | ||
| 192 | static __inline__ __m256i __DEFAULT_FN_ATTRS256 | ||
| 193 | _mm256_sha512rnds2_epi64(__m256i __A, __m256i __B, __m128i __C) { | ||
| 194 | return (__m256i)__builtin_ia32_vsha512rnds2((__v4du)__A, (__v4du)__B, | ||
| 195 | (__v2du)__C); | ||
| 196 | } | ||
| 197 | |||
| 198 | #undef __DEFAULT_FN_ATTRS256 | ||
| 199 | |||
| 200 | #endif // __SHA512INTRIN_H | ||
lib/include/shaintrin.h+128
| ... | @@ -17,39 +17,167 @@ | ... | @@ -17,39 +17,167 @@ |
| 17 | /* Define the default attributes for the functions in this file. */ | 17 | /* Define the default attributes for the functions in this file. */ |
| 18 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("sha"), __min_vector_width__(128))) | 18 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("sha"), __min_vector_width__(128))) |
| 19 | 19 | ||
| 20 | /// Performs four iterations of the inner loop of the SHA-1 message digest | ||
| 21 | /// algorithm using the starting SHA-1 state (A, B, C, D) from the 128-bit | ||
| 22 | /// vector of [4 x i32] in \a V1 and the next four 32-bit elements of the | ||
| 23 | /// message from the 128-bit vector of [4 x i32] in \a V2. Note that the | ||
| 24 | /// SHA-1 state variable E must have already been added to \a V2 | ||
| 25 | /// (\c _mm_sha1nexte_epu32() can perform this step). Returns the updated | ||
| 26 | /// SHA-1 state (A, B, C, D) as a 128-bit vector of [4 x i32]. | ||
| 27 | /// | ||
| 28 | /// The SHA-1 algorithm has an inner loop of 80 iterations, twenty each | ||
| 29 | /// with a different combining function and rounding constant. This | ||
| 30 | /// intrinsic performs four iterations using a combining function and | ||
| 31 | /// rounding constant selected by \a M[1:0]. | ||
| 32 | /// | ||
| 33 | /// \headerfile <immintrin.h> | ||
| 34 | /// | ||
| 35 | /// \code | ||
| 36 | /// __m128i _mm_sha1rnds4_epu32(__m128i V1, __m128i V2, const int M); | ||
| 37 | /// \endcode | ||
| 38 | /// | ||
| 39 | /// This intrinsic corresponds to the \c SHA1RNDS4 instruction. | ||
| 40 | /// | ||
| 41 | /// \param V1 | ||
| 42 | /// A 128-bit vector of [4 x i32] containing the initial SHA-1 state. | ||
| 43 | /// \param V2 | ||
| 44 | /// A 128-bit vector of [4 x i32] containing the next four elements of | ||
| 45 | /// the message, plus SHA-1 state variable E. | ||
| 46 | /// \param M | ||
| 47 | /// An immediate value where bits [1:0] select among four possible | ||
| 48 | /// combining functions and rounding constants (not specified here). | ||
| 49 | /// \returns A 128-bit vector of [4 x i32] containing the updated SHA-1 state. | ||
| 20 | #define _mm_sha1rnds4_epu32(V1, V2, M) \ | 50 | #define _mm_sha1rnds4_epu32(V1, V2, M) \ |
| 21 | __builtin_ia32_sha1rnds4((__v4si)(__m128i)(V1), (__v4si)(__m128i)(V2), (M)) | 51 | __builtin_ia32_sha1rnds4((__v4si)(__m128i)(V1), (__v4si)(__m128i)(V2), (M)) |
| 22 | 52 | ||
| 53 | /// Calculates the SHA-1 state variable E from the SHA-1 state variables in | ||
| 54 | /// the 128-bit vector of [4 x i32] in \a __X, adds that to the next set of | ||
| 55 | /// four message elements in the 128-bit vector of [4 x i32] in \a __Y, and | ||
| 56 | /// returns the result. | ||
| 57 | /// | ||
| 58 | /// \headerfile <immintrin.h> | ||
| 59 | /// | ||
| 60 | /// This intrinsic corresponds to the \c SHA1NEXTE instruction. | ||
| 61 | /// | ||
| 62 | /// \param __X | ||
| 63 | /// A 128-bit vector of [4 x i32] containing the current SHA-1 state. | ||
| 64 | /// \param __Y | ||
| 65 | /// A 128-bit vector of [4 x i32] containing the next four elements of the | ||
| 66 | /// message. | ||
| 67 | /// \returns A 128-bit vector of [4 x i32] containing the updated SHA-1 | ||
| 68 | /// values. | ||
| 23 | static __inline__ __m128i __DEFAULT_FN_ATTRS | 69 | static __inline__ __m128i __DEFAULT_FN_ATTRS |
| 24 | _mm_sha1nexte_epu32(__m128i __X, __m128i __Y) | 70 | _mm_sha1nexte_epu32(__m128i __X, __m128i __Y) |
| 25 | { | 71 | { |
| 26 | return (__m128i)__builtin_ia32_sha1nexte((__v4si)__X, (__v4si)__Y); | 72 | return (__m128i)__builtin_ia32_sha1nexte((__v4si)__X, (__v4si)__Y); |
| 27 | } | 73 | } |
| 28 | 74 | ||
| 75 | /// Performs an intermediate calculation for deriving the next four SHA-1 | ||
| 76 | /// message elements using previous message elements from the 128-bit | ||
| 77 | /// vectors of [4 x i32] in \a __X and \a __Y, and returns the result. | ||
| 78 | /// | ||
| 79 | /// \headerfile <immintrin.h> | ||
| 80 | /// | ||
| 81 | /// This intrinsic corresponds to the \c SHA1MSG1 instruction. | ||
| 82 | /// | ||
| 83 | /// \param __X | ||
| 84 | /// A 128-bit vector of [4 x i32] containing previous message elements. | ||
| 85 | /// \param __Y | ||
| 86 | /// A 128-bit vector of [4 x i32] containing previous message elements. | ||
| 87 | /// \returns A 128-bit vector of [4 x i32] containing the derived SHA-1 | ||
| 88 | /// elements. | ||
| 29 | static __inline__ __m128i __DEFAULT_FN_ATTRS | 89 | static __inline__ __m128i __DEFAULT_FN_ATTRS |
| 30 | _mm_sha1msg1_epu32(__m128i __X, __m128i __Y) | 90 | _mm_sha1msg1_epu32(__m128i __X, __m128i __Y) |
| 31 | { | 91 | { |
| 32 | return (__m128i)__builtin_ia32_sha1msg1((__v4si)__X, (__v4si)__Y); | 92 | return (__m128i)__builtin_ia32_sha1msg1((__v4si)__X, (__v4si)__Y); |
| 33 | } | 93 | } |
| 34 | 94 | ||
| 95 | /// Performs the final calculation for deriving the next four SHA-1 message | ||
| 96 | /// elements using previous message elements from the 128-bit vectors of | ||
| 97 | /// [4 x i32] in \a __X and \a __Y, and returns the result. | ||
| 98 | /// | ||
| 99 | /// \headerfile <immintrin.h> | ||
| 100 | /// | ||
| 101 | /// This intrinsic corresponds to the \c SHA1MSG2 instruction. | ||
| 102 | /// | ||
| 103 | /// \param __X | ||
| 104 | /// A 128-bit vector of [4 x i32] containing an intermediate result. | ||
| 105 | /// \param __Y | ||
| 106 | /// A 128-bit vector of [4 x i32] containing previous message values. | ||
| 107 | /// \returns A 128-bit vector of [4 x i32] containing the updated SHA-1 | ||
| 108 | /// values. | ||
| 35 | static __inline__ __m128i __DEFAULT_FN_ATTRS | 109 | static __inline__ __m128i __DEFAULT_FN_ATTRS |
| 36 | _mm_sha1msg2_epu32(__m128i __X, __m128i __Y) | 110 | _mm_sha1msg2_epu32(__m128i __X, __m128i __Y) |
| 37 | { | 111 | { |
| 38 | return (__m128i)__builtin_ia32_sha1msg2((__v4si)__X, (__v4si)__Y); | 112 | return (__m128i)__builtin_ia32_sha1msg2((__v4si)__X, (__v4si)__Y); |
| 39 | } | 113 | } |
| 40 | 114 | ||
| 115 | /// Performs two rounds of SHA-256 operation using the following inputs: a | ||
| 116 | /// starting SHA-256 state (C, D, G, H) from the 128-bit vector of | ||
| 117 | /// [4 x i32] in \a __X; a starting SHA-256 state (A, B, E, F) from the | ||
| 118 | /// 128-bit vector of [4 x i32] in \a __Y; and a pre-computed sum of the | ||
| 119 | /// next two message elements (unsigned 32-bit integers) and corresponding | ||
| 120 | /// rounding constants from the 128-bit vector of [4 x i32] in \a __Z. | ||
| 121 | /// Returns the updated SHA-256 state (A, B, E, F) as a 128-bit vector of | ||
| 122 | /// [4 x i32]. | ||
| 123 | /// | ||
| 124 | /// The SHA-256 algorithm has a core loop of 64 iterations. This intrinsic | ||
| 125 | /// performs two of those iterations. | ||
| 126 | /// | ||
| 127 | /// \headerfile <immintrin.h> | ||
| 128 | /// | ||
| 129 | /// This intrinsic corresponds to the \c SHA256RNDS2 instruction. | ||
| 130 | /// | ||
| 131 | /// \param __X | ||
| 132 | /// A 128-bit vector of [4 x i32] containing part of the initial SHA-256 | ||
| 133 | /// state. | ||
| 134 | /// \param __Y | ||
| 135 | /// A 128-bit vector of [4 x i32] containing part of the initial SHA-256 | ||
| 136 | /// state. | ||
| 137 | /// \param __Z | ||
| 138 | /// A 128-bit vector of [4 x i32] containing additional input to the | ||
| 139 | /// SHA-256 operation. | ||
| 140 | /// \returns A 128-bit vector of [4 x i32] containing the updated SHA-1 state. | ||
| 41 | static __inline__ __m128i __DEFAULT_FN_ATTRS | 141 | static __inline__ __m128i __DEFAULT_FN_ATTRS |
| 42 | _mm_sha256rnds2_epu32(__m128i __X, __m128i __Y, __m128i __Z) | 142 | _mm_sha256rnds2_epu32(__m128i __X, __m128i __Y, __m128i __Z) |
| 43 | { | 143 | { |
| 44 | return (__m128i)__builtin_ia32_sha256rnds2((__v4si)__X, (__v4si)__Y, (__v4si)__Z); | 144 | return (__m128i)__builtin_ia32_sha256rnds2((__v4si)__X, (__v4si)__Y, (__v4si)__Z); |
| 45 | } | 145 | } |
| 46 | 146 | ||
| 147 | /// Performs an intermediate calculation for deriving the next four SHA-256 | ||
| 148 | /// message elements using previous message elements from the 128-bit | ||
| 149 | /// vectors of [4 x i32] in \a __X and \a __Y, and returns the result. | ||
| 150 | /// | ||
| 151 | /// \headerfile <immintrin.h> | ||
| 152 | /// | ||
| 153 | /// This intrinsic corresponds to the \c SHA256MSG1 instruction. | ||
| 154 | /// | ||
| 155 | /// \param __X | ||
| 156 | /// A 128-bit vector of [4 x i32] containing previous message elements. | ||
| 157 | /// \param __Y | ||
| 158 | /// A 128-bit vector of [4 x i32] containing previous message elements. | ||
| 159 | /// \returns A 128-bit vector of [4 x i32] containing the updated SHA-256 | ||
| 160 | /// values. | ||
| 47 | static __inline__ __m128i __DEFAULT_FN_ATTRS | 161 | static __inline__ __m128i __DEFAULT_FN_ATTRS |
| 48 | _mm_sha256msg1_epu32(__m128i __X, __m128i __Y) | 162 | _mm_sha256msg1_epu32(__m128i __X, __m128i __Y) |
| 49 | { | 163 | { |
| 50 | return (__m128i)__builtin_ia32_sha256msg1((__v4si)__X, (__v4si)__Y); | 164 | return (__m128i)__builtin_ia32_sha256msg1((__v4si)__X, (__v4si)__Y); |
| 51 | } | 165 | } |
| 52 | 166 | ||
| 167 | /// Performs the final calculation for deriving the next four SHA-256 message | ||
| 168 | /// elements using previous message elements from the 128-bit vectors of | ||
| 169 | /// [4 x i32] in \a __X and \a __Y, and returns the result. | ||
| 170 | /// | ||
| 171 | /// \headerfile <immintrin.h> | ||
| 172 | /// | ||
| 173 | /// This intrinsic corresponds to the \c SHA256MSG2 instruction. | ||
| 174 | /// | ||
| 175 | /// \param __X | ||
| 176 | /// A 128-bit vector of [4 x i32] containing an intermediate result. | ||
| 177 | /// \param __Y | ||
| 178 | /// A 128-bit vector of [4 x i32] containing previous message values. | ||
| 179 | /// \returns A 128-bit vector of [4 x i32] containing the updated SHA-256 | ||
| 180 | /// values. | ||
| 53 | static __inline__ __m128i __DEFAULT_FN_ATTRS | 181 | static __inline__ __m128i __DEFAULT_FN_ATTRS |
| 54 | _mm_sha256msg2_epu32(__m128i __X, __m128i __Y) | 182 | _mm_sha256msg2_epu32(__m128i __X, __m128i __Y) |
| 55 | { | 183 | { |
lib/include/sifive_vector.h created+16
| ... | @@ -0,0 +1,16 @@ | ||
| 1 | //===----- sifive_vector.h - SiFive Vector definitions --------------------===// | ||
| 2 | // | ||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| 6 | // | ||
| 7 | //===----------------------------------------------------------------------===// | ||
| 8 | |||
| 9 | #ifndef _SIFIVE_VECTOR_H_ | ||
| 10 | #define _SIFIVE_VECTOR_H_ | ||
| 11 | |||
| 12 | #include "riscv_vector.h" | ||
| 13 | |||
| 14 | #pragma clang riscv intrinsic sifive_vector | ||
| 15 | |||
| 16 | #endif //_SIFIVE_VECTOR_H_ | ||
lib/include/sm3intrin.h created+238
| ... | @@ -0,0 +1,238 @@ | ||
| 1 | /*===-------------------- sm3intrin.h - SM3 intrinsics ---------------------=== | ||
| 2 | * | ||
| 3 | * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| 4 | * See https://llvm.org/LICENSE.txt for license information. | ||
| 5 | * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| 6 | * | ||
| 7 | *===-----------------------------------------------------------------------=== | ||
| 8 | */ | ||
| 9 | |||
| 10 | #ifndef __IMMINTRIN_H | ||
| 11 | #error "Never use <sm3intrin.h> directly; include <immintrin.h> instead." | ||
| 12 | #endif // __IMMINTRIN_H | ||
| 13 | |||
| 14 | #ifndef __SM3INTRIN_H | ||
| 15 | #define __SM3INTRIN_H | ||
| 16 | |||
| 17 | #define __DEFAULT_FN_ATTRS128 \ | ||
| 18 | __attribute__((__always_inline__, __nodebug__, __target__("sm3"), \ | ||
| 19 | __min_vector_width__(128))) | ||
| 20 | |||
| 21 | /// This intrinisc is one of the two SM3 message scheduling intrinsics. The | ||
| 22 | /// intrinsic performs an initial calculation for the next four SM3 message | ||
| 23 | /// words. The calculated results are stored in \a dst. | ||
| 24 | /// | ||
| 25 | /// \headerfile <immintrin.h> | ||
| 26 | /// | ||
| 27 | /// \code | ||
| 28 | /// __m128i _mm_sm3msg1_epi32(__m128i __A, __m128i __B, __m128i __C) | ||
| 29 | /// \endcode | ||
| 30 | /// | ||
| 31 | /// This intrinsic corresponds to the \c VSM3MSG1 instruction. | ||
| 32 | /// | ||
| 33 | /// \param __A | ||
| 34 | /// A 128-bit vector of [4 x int]. | ||
| 35 | /// \param __B | ||
| 36 | /// A 128-bit vector of [4 x int]. | ||
| 37 | /// \param __C | ||
| 38 | /// A 128-bit vector of [4 x int]. | ||
| 39 | /// \returns | ||
| 40 | /// A 128-bit vector of [4 x int]. | ||
| 41 | /// | ||
| 42 | /// \code{.operation} | ||
| 43 | /// DEFINE ROL32(dword, n) { | ||
| 44 | /// count := n % 32 | ||
| 45 | /// dest := (dword << count) | (dword >> (32 - count)) | ||
| 46 | /// RETURN dest | ||
| 47 | /// } | ||
| 48 | /// DEFINE P1(x) { | ||
| 49 | /// RETURN x ^ ROL32(x, 15) ^ ROL32(x, 23) | ||
| 50 | /// } | ||
| 51 | /// W[0] := __C.dword[0] | ||
| 52 | /// W[1] := __C.dword[1] | ||
| 53 | /// W[2] := __C.dword[2] | ||
| 54 | /// W[3] := __C.dword[3] | ||
| 55 | /// W[7] := __A.dword[0] | ||
| 56 | /// W[8] := __A.dword[1] | ||
| 57 | /// W[9] := __A.dword[2] | ||
| 58 | /// W[10] := __A.dword[3] | ||
| 59 | /// W[13] := __B.dword[0] | ||
| 60 | /// W[14] := __B.dword[1] | ||
| 61 | /// W[15] := __B.dword[2] | ||
| 62 | /// TMP0 := W[7] ^ W[0] ^ ROL32(W[13], 15) | ||
| 63 | /// TMP1 := W[8] ^ W[1] ^ ROL32(W[14], 15) | ||
| 64 | /// TMP2 := W[9] ^ W[2] ^ ROL32(W[15], 15) | ||
| 65 | /// TMP3 := W[10] ^ W[3] | ||
| 66 | /// dst.dword[0] := P1(TMP0) | ||
| 67 | /// dst.dword[1] := P1(TMP1) | ||
| 68 | /// dst.dword[2] := P1(TMP2) | ||
| 69 | /// dst.dword[3] := P1(TMP3) | ||
| 70 | /// dst[MAX:128] := 0 | ||
| 71 | /// \endcode | ||
| 72 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_sm3msg1_epi32(__m128i __A, | ||
| 73 | __m128i __B, | ||
| 74 | __m128i __C) { | ||
| 75 | return (__m128i)__builtin_ia32_vsm3msg1((__v4su)__A, (__v4su)__B, | ||
| 76 | (__v4su)__C); | ||
| 77 | } | ||
| 78 | |||
| 79 | /// This intrinisc is one of the two SM3 message scheduling intrinsics. The | ||
| 80 | /// intrinsic performs the final calculation for the next four SM3 message | ||
| 81 | /// words. The calculated results are stored in \a dst. | ||
| 82 | /// | ||
| 83 | /// \headerfile <immintrin.h> | ||
| 84 | /// | ||
| 85 | /// \code | ||
| 86 | /// __m128i _mm_sm3msg2_epi32(__m128i __A, __m128i __B, __m128i __C) | ||
| 87 | /// \endcode | ||
| 88 | /// | ||
| 89 | /// This intrinsic corresponds to the \c VSM3MSG2 instruction. | ||
| 90 | /// | ||
| 91 | /// \param __A | ||
| 92 | /// A 128-bit vector of [4 x int]. | ||
| 93 | /// \param __B | ||
| 94 | /// A 128-bit vector of [4 x int]. | ||
| 95 | /// \param __C | ||
| 96 | /// A 128-bit vector of [4 x int]. | ||
| 97 | /// \returns | ||
| 98 | /// A 128-bit vector of [4 x int]. | ||
| 99 | /// | ||
| 100 | /// \code{.operation} | ||
| 101 | /// DEFINE ROL32(dword, n) { | ||
| 102 | /// count := n % 32 | ||
| 103 | /// dest := (dword << count) | (dword >> (32-count)) | ||
| 104 | /// RETURN dest | ||
| 105 | /// } | ||
| 106 | /// WTMP[0] := __A.dword[0] | ||
| 107 | /// WTMP[1] := __A.dword[1] | ||
| 108 | /// WTMP[2] := __A.dword[2] | ||
| 109 | /// WTMP[3] := __A.dword[3] | ||
| 110 | /// W[3] := __B.dword[0] | ||
| 111 | /// W[4] := __B.dword[1] | ||
| 112 | /// W[5] := __B.dword[2] | ||
| 113 | /// W[6] := __B.dword[3] | ||
| 114 | /// W[10] := __C.dword[0] | ||
| 115 | /// W[11] := __C.dword[1] | ||
| 116 | /// W[12] := __C.dword[2] | ||
| 117 | /// W[13] := __C.dword[3] | ||
| 118 | /// W[16] := ROL32(W[3], 7) ^ W[10] ^ WTMP[0] | ||
| 119 | /// W[17] := ROL32(W[4], 7) ^ W[11] ^ WTMP[1] | ||
| 120 | /// W[18] := ROL32(W[5], 7) ^ W[12] ^ WTMP[2] | ||
| 121 | /// W[19] := ROL32(W[6], 7) ^ W[13] ^ WTMP[3] | ||
| 122 | /// W[19] := W[19] ^ ROL32(W[16], 6) ^ ROL32(W[16], 15) ^ ROL32(W[16], 30) | ||
| 123 | /// dst.dword[0] := W[16] | ||
| 124 | /// dst.dword[1] := W[17] | ||
| 125 | /// dst.dword[2] := W[18] | ||
| 126 | /// dst.dword[3] := W[19] | ||
| 127 | /// dst[MAX:128] := 0 | ||
| 128 | /// \endcode | ||
| 129 | static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_sm3msg2_epi32(__m128i __A, | ||
| 130 | __m128i __B, | ||
| 131 | __m128i __C) { | ||
| 132 | return (__m128i)__builtin_ia32_vsm3msg2((__v4su)__A, (__v4su)__B, | ||
| 133 | (__v4su)__C); | ||
| 134 | } | ||
| 135 | |||
| 136 | /// This intrinsic performs two rounds of SM3 operation using initial SM3 state | ||
| 137 | /// (C, D, G, H) from \a __A, an initial SM3 states (A, B, E, F) | ||
| 138 | /// from \a __B and a pre-computed words from the \a __C. \a __A with | ||
| 139 | /// initial SM3 state of (C, D, G, H) assumes input of non-rotated left | ||
| 140 | /// variables from previous state. The updated SM3 state (A, B, E, F) is | ||
| 141 | /// written to \a __A. The \a imm8 should contain the even round number | ||
| 142 | /// for the first of the two rounds computed by this instruction. The | ||
| 143 | /// computation masks the \a imm8 value by AND’ing it with 0x3E so that only | ||
| 144 | /// even round numbers from 0 through 62 are used for this operation. The | ||
| 145 | /// calculated results are stored in \a dst. | ||
| 146 | /// | ||
| 147 | /// \headerfile <immintrin.h> | ||
| 148 | /// | ||
| 149 | /// \code | ||
| 150 | /// __m128i _mm_sm3rnds2_epi32(__m128i __A, __m128i __B, __m128i __C, const int | ||
| 151 | /// imm8) \endcode | ||
| 152 | /// | ||
| 153 | /// This intrinsic corresponds to the \c VSM3RNDS2 instruction. | ||
| 154 | /// | ||
| 155 | /// \param __A | ||
| 156 | /// A 128-bit vector of [4 x int]. | ||
| 157 | /// \param __B | ||
| 158 | /// A 128-bit vector of [4 x int]. | ||
| 159 | /// \param __C | ||
| 160 | /// A 128-bit vector of [4 x int]. | ||
| 161 | /// \param imm8 | ||
| 162 | /// A 8-bit constant integer. | ||
| 163 | /// \returns | ||
| 164 | /// A 128-bit vector of [4 x int]. | ||
| 165 | /// | ||
| 166 | /// \code{.operation} | ||
| 167 | /// DEFINE ROL32(dword, n) { | ||
| 168 | /// count := n % 32 | ||
| 169 | /// dest := (dword << count) | (dword >> (32-count)) | ||
| 170 | /// RETURN dest | ||
| 171 | /// } | ||
| 172 | /// DEFINE P0(dword) { | ||
| 173 | /// RETURN dword ^ ROL32(dword, 9) ^ ROL32(dword, 17) | ||
| 174 | /// } | ||
| 175 | /// DEFINE FF(x,y,z, round){ | ||
| 176 | /// IF round < 16 | ||
| 177 | /// RETURN (x ^ y ^ z) | ||
| 178 | /// ELSE | ||
| 179 | /// RETURN (x & y) | (x & z) | (y & z) | ||
| 180 | /// FI | ||
| 181 | /// } | ||
| 182 | /// DEFINE GG(x, y, z, round){ | ||
| 183 | /// IF round < 16 | ||
| 184 | /// RETURN (x ^ y ^ z) | ||
| 185 | /// ELSE | ||
| 186 | /// RETURN (x & y) | (~x & z) | ||
| 187 | /// FI | ||
| 188 | /// } | ||
| 189 | /// A[0] := __B.dword[3] | ||
| 190 | /// B[0] := __B.dword[2] | ||
| 191 | /// C[0] := __A.dword[3] | ||
| 192 | /// D[0] := __A.dword[2] | ||
| 193 | /// E[0] := __B.dword[1] | ||
| 194 | /// F[0] := __B.dword[0] | ||
| 195 | /// G[0] := __A.dword[1] | ||
| 196 | /// H[0] := __A.dword[0] | ||
| 197 | /// W[0] := __C.dword[0] | ||
| 198 | /// W[1] := __C.dword[1] | ||
| 199 | /// W[4] := __C.dword[2] | ||
| 200 | /// W[5] := __C.dword[3] | ||
| 201 | /// C[0] := ROL32(C[0], 9) | ||
| 202 | /// D[0] := ROL32(D[0], 9) | ||
| 203 | /// G[0] := ROL32(G[0], 19) | ||
| 204 | /// H[0] := ROL32(H[0], 19) | ||
| 205 | /// ROUND := __D & 0x3E | ||
| 206 | /// IF ROUND < 16 | ||
| 207 | /// CONST := 0x79CC4519 | ||
| 208 | /// ELSE | ||
| 209 | /// CONST := 0x7A879D8A | ||
| 210 | /// FI | ||
| 211 | /// CONST := ROL32(CONST,ROUND) | ||
| 212 | /// FOR i:= 0 to 1 | ||
| 213 | /// S1 := ROL32((ROL32(A[i], 12) + E[i] + CONST), 7) | ||
| 214 | /// S2 := S1 ^ ROL32(A[i], 12) | ||
| 215 | /// T1 := FF(A[i], B[i], C[i], ROUND) + D[i] + S2 + (W[i] ^ W[i+4]) | ||
| 216 | /// T2 := GG(E[i], F[i], G[i], ROUND) + H[i] + S1 + W[i] | ||
| 217 | /// D[i+1] := C[i] | ||
| 218 | /// C[i+1] := ROL32(B[i],9) | ||
| 219 | /// B[i+1] := A[i] | ||
| 220 | /// A[i+1] := T1 | ||
| 221 | /// H[i+1] := G[i] | ||
| 222 | /// G[i+1] := ROL32(F[i], 19) | ||
| 223 | /// F[i+1] := E[i] | ||
| 224 | /// E[i+1] := P0(T2) | ||
| 225 | /// CONST := ROL32(CONST, 1) | ||
| 226 | /// ENDFOR | ||
| 227 | /// dst.dword[3] := A[2] | ||
| 228 | /// dst.dword[2] := B[2] | ||
| 229 | /// dst.dword[1] := E[2] | ||
| 230 | /// dst.dword[0] := F[2] | ||
| 231 | /// dst[MAX:128] := 0 | ||
| 232 | /// \endcode | ||
| 233 | #define _mm_sm3rnds2_epi32(A, B, C, D) \ | ||
| 234 | (__m128i) __builtin_ia32_vsm3rnds2((__v4su)A, (__v4su)B, (__v4su)C, (int)D) | ||
| 235 | |||
| 236 | #undef __DEFAULT_FN_ATTRS128 | ||
| 237 | |||
| 238 | #endif // __SM3INTRIN_H | ||
lib/include/sm4intrin.h created+269
| ... | @@ -0,0 +1,269 @@ | ||
| 1 | /*===--------------- sm4intrin.h - SM4 intrinsics -----------------=== | ||
| 2 | * | ||
| 3 | * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| 4 | * See https://llvm.org/LICENSE.txt for license information. | ||
| 5 | * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| 6 | * | ||
| 7 | *===-----------------------------------------------------------------------=== | ||
| 8 | */ | ||
| 9 | |||
| 10 | #ifndef __IMMINTRIN_H | ||
| 11 | #error "Never use <sm4intrin.h> directly; include <immintrin.h> instead." | ||
| 12 | #endif // __IMMINTRIN_H | ||
| 13 | |||
| 14 | #ifndef __SM4INTRIN_H | ||
| 15 | #define __SM4INTRIN_H | ||
| 16 | |||
| 17 | /// This intrinsic performs four rounds of SM4 key expansion. The intrinsic | ||
| 18 | /// operates on independent 128-bit lanes. The calculated results are | ||
| 19 | /// stored in \a dst. | ||
| 20 | /// \headerfile <immintrin.h> | ||
| 21 | /// | ||
| 22 | /// \code | ||
| 23 | /// __m128i _mm_sm4key4_epi32(__m128i __A, __m128i __B) | ||
| 24 | /// \endcode | ||
| 25 | /// | ||
| 26 | /// This intrinsic corresponds to the \c VSM4KEY4 instruction. | ||
| 27 | /// | ||
| 28 | /// \param __A | ||
| 29 | /// A 128-bit vector of [4 x int]. | ||
| 30 | /// \param __B | ||
| 31 | /// A 128-bit vector of [4 x int]. | ||
| 32 | /// \returns | ||
| 33 | /// A 128-bit vector of [4 x int]. | ||
| 34 | /// | ||
| 35 | /// \code{.operation} | ||
| 36 | /// DEFINE ROL32(dword, n) { | ||
| 37 | /// count := n % 32 | ||
| 38 | /// dest := (dword << count) | (dword >> (32-count)) | ||
| 39 | /// RETURN dest | ||
| 40 | /// } | ||
| 41 | /// DEFINE SBOX_BYTE(dword, i) { | ||
| 42 | /// RETURN sbox[dword.byte[i]] | ||
| 43 | /// } | ||
| 44 | /// DEFINE lower_t(dword) { | ||
| 45 | /// tmp.byte[0] := SBOX_BYTE(dword, 0) | ||
| 46 | /// tmp.byte[1] := SBOX_BYTE(dword, 1) | ||
| 47 | /// tmp.byte[2] := SBOX_BYTE(dword, 2) | ||
| 48 | /// tmp.byte[3] := SBOX_BYTE(dword, 3) | ||
| 49 | /// RETURN tmp | ||
| 50 | /// } | ||
| 51 | /// DEFINE L_KEY(dword) { | ||
| 52 | /// RETURN dword ^ ROL32(dword, 13) ^ ROL32(dword, 23) | ||
| 53 | /// } | ||
| 54 | /// DEFINE T_KEY(dword) { | ||
| 55 | /// RETURN L_KEY(lower_t(dword)) | ||
| 56 | /// } | ||
| 57 | /// DEFINE F_KEY(X0, X1, X2, X3, round_key) { | ||
| 58 | /// RETURN X0 ^ T_KEY(X1 ^ X2 ^ X3 ^ round_key) | ||
| 59 | /// } | ||
| 60 | /// FOR i:= 0 to 0 | ||
| 61 | /// P[0] := __B.xmm[i].dword[0] | ||
| 62 | /// P[1] := __B.xmm[i].dword[1] | ||
| 63 | /// P[2] := __B.xmm[i].dword[2] | ||
| 64 | /// P[3] := __B.xmm[i].dword[3] | ||
| 65 | /// C[0] := F_KEY(P[0], P[1], P[2], P[3], __A.xmm[i].dword[0]) | ||
| 66 | /// C[1] := F_KEY(P[1], P[2], P[3], C[0], __A.xmm[i].dword[1]) | ||
| 67 | /// C[2] := F_KEY(P[2], P[3], C[0], C[1], __A.xmm[i].dword[2]) | ||
| 68 | /// C[3] := F_KEY(P[3], C[0], C[1], C[2], __A.xmm[i].dword[3]) | ||
| 69 | /// DEST.xmm[i].dword[0] := C[0] | ||
| 70 | /// DEST.xmm[i].dword[1] := C[1] | ||
| 71 | /// DEST.xmm[i].dword[2] := C[2] | ||
| 72 | /// DEST.xmm[i].dword[3] := C[3] | ||
| 73 | /// ENDFOR | ||
| 74 | /// DEST[MAX:128] := 0 | ||
| 75 | /// \endcode | ||
| 76 | #define _mm_sm4key4_epi32(A, B) \ | ||
| 77 | (__m128i) __builtin_ia32_vsm4key4128((__v4su)A, (__v4su)B) | ||
| 78 | |||
| 79 | /// This intrinsic performs four rounds of SM4 key expansion. The intrinsic | ||
| 80 | /// operates on independent 128-bit lanes. The calculated results are | ||
| 81 | /// stored in \a dst. | ||
| 82 | /// \headerfile <immintrin.h> | ||
| 83 | /// | ||
| 84 | /// \code | ||
| 85 | /// __m256i _mm256_sm4key4_epi32(__m256i __A, __m256i __B) | ||
| 86 | /// \endcode | ||
| 87 | /// | ||
| 88 | /// This intrinsic corresponds to the \c VSM4KEY4 instruction. | ||
| 89 | /// | ||
| 90 | /// \param __A | ||
| 91 | /// A 256-bit vector of [8 x int]. | ||
| 92 | /// \param __B | ||
| 93 | /// A 256-bit vector of [8 x int]. | ||
| 94 | /// \returns | ||
| 95 | /// A 256-bit vector of [8 x int]. | ||
| 96 | /// | ||
| 97 | /// \code{.operation} | ||
| 98 | /// DEFINE ROL32(dword, n) { | ||
| 99 | /// count := n % 32 | ||
| 100 | /// dest := (dword << count) | (dword >> (32-count)) | ||
| 101 | /// RETURN dest | ||
| 102 | /// } | ||
| 103 | /// DEFINE SBOX_BYTE(dword, i) { | ||
| 104 | /// RETURN sbox[dword.byte[i]] | ||
| 105 | /// } | ||
| 106 | /// DEFINE lower_t(dword) { | ||
| 107 | /// tmp.byte[0] := SBOX_BYTE(dword, 0) | ||
| 108 | /// tmp.byte[1] := SBOX_BYTE(dword, 1) | ||
| 109 | /// tmp.byte[2] := SBOX_BYTE(dword, 2) | ||
| 110 | /// tmp.byte[3] := SBOX_BYTE(dword, 3) | ||
| 111 | /// RETURN tmp | ||
| 112 | /// } | ||
| 113 | /// DEFINE L_KEY(dword) { | ||
| 114 | /// RETURN dword ^ ROL32(dword, 13) ^ ROL32(dword, 23) | ||
| 115 | /// } | ||
| 116 | /// DEFINE T_KEY(dword) { | ||
| 117 | /// RETURN L_KEY(lower_t(dword)) | ||
| 118 | /// } | ||
| 119 | /// DEFINE F_KEY(X0, X1, X2, X3, round_key) { | ||
| 120 | /// RETURN X0 ^ T_KEY(X1 ^ X2 ^ X3 ^ round_key) | ||
| 121 | /// } | ||
| 122 | /// FOR i:= 0 to 1 | ||
| 123 | /// P[0] := __B.xmm[i].dword[0] | ||
| 124 | /// P[1] := __B.xmm[i].dword[1] | ||
| 125 | /// P[2] := __B.xmm[i].dword[2] | ||
| 126 | /// P[3] := __B.xmm[i].dword[3] | ||
| 127 | /// C[0] := F_KEY(P[0], P[1], P[2], P[3], __A.xmm[i].dword[0]) | ||
| 128 | /// C[1] := F_KEY(P[1], P[2], P[3], C[0], __A.xmm[i].dword[1]) | ||
| 129 | /// C[2] := F_KEY(P[2], P[3], C[0], C[1], __A.xmm[i].dword[2]) | ||
| 130 | /// C[3] := F_KEY(P[3], C[0], C[1], C[2], __A.xmm[i].dword[3]) | ||
| 131 | /// DEST.xmm[i].dword[0] := C[0] | ||
| 132 | /// DEST.xmm[i].dword[1] := C[1] | ||
| 133 | /// DEST.xmm[i].dword[2] := C[2] | ||
| 134 | /// DEST.xmm[i].dword[3] := C[3] | ||
| 135 | /// ENDFOR | ||
| 136 | /// DEST[MAX:256] := 0 | ||
| 137 | /// \endcode | ||
| 138 | #define _mm256_sm4key4_epi32(A, B) \ | ||
| 139 | (__m256i) __builtin_ia32_vsm4key4256((__v8su)A, (__v8su)B) | ||
| 140 | |||
| 141 | /// This intrinisc performs four rounds of SM4 encryption. The intrinisc | ||
| 142 | /// operates on independent 128-bit lanes. The calculated results are | ||
| 143 | /// stored in \a dst. | ||
| 144 | /// \headerfile <immintrin.h> | ||
| 145 | /// | ||
| 146 | /// \code | ||
| 147 | /// __m128i _mm_sm4rnds4_epi32(__m128i __A, __m128i __B) | ||
| 148 | /// \endcode | ||
| 149 | /// | ||
| 150 | /// This intrinsic corresponds to the \c VSM4RNDS4 instruction. | ||
| 151 | /// | ||
| 152 | /// \param __A | ||
| 153 | /// A 128-bit vector of [4 x int]. | ||
| 154 | /// \param __B | ||
| 155 | /// A 128-bit vector of [4 x int]. | ||
| 156 | /// \returns | ||
| 157 | /// A 128-bit vector of [4 x int]. | ||
| 158 | /// | ||
| 159 | /// \code{.operation} | ||
| 160 | /// DEFINE ROL32(dword, n) { | ||
| 161 | /// count := n % 32 | ||
| 162 | /// dest := (dword << count) | (dword >> (32-count)) | ||
| 163 | /// RETURN dest | ||
| 164 | /// } | ||
| 165 | /// DEFINE lower_t(dword) { | ||
| 166 | /// tmp.byte[0] := SBOX_BYTE(dword, 0) | ||
| 167 | /// tmp.byte[1] := SBOX_BYTE(dword, 1) | ||
| 168 | /// tmp.byte[2] := SBOX_BYTE(dword, 2) | ||
| 169 | /// tmp.byte[3] := SBOX_BYTE(dword, 3) | ||
| 170 | /// RETURN tmp | ||
| 171 | /// } | ||
| 172 | /// DEFINE L_RND(dword) { | ||
| 173 | /// tmp := dword | ||
| 174 | /// tmp := tmp ^ ROL32(dword, 2) | ||
| 175 | /// tmp := tmp ^ ROL32(dword, 10) | ||
| 176 | /// tmp := tmp ^ ROL32(dword, 18) | ||
| 177 | /// tmp := tmp ^ ROL32(dword, 24) | ||
| 178 | /// RETURN tmp | ||
| 179 | /// } | ||
| 180 | /// DEFINE T_RND(dword) { | ||
| 181 | /// RETURN L_RND(lower_t(dword)) | ||
| 182 | /// } | ||
| 183 | /// DEFINE F_RND(X0, X1, X2, X3, round_key) { | ||
| 184 | /// RETURN X0 ^ T_RND(X1 ^ X2 ^ X3 ^ round_key) | ||
| 185 | /// } | ||
| 186 | /// FOR i:= 0 to 0 | ||
| 187 | /// P[0] := __B.xmm[i].dword[0] | ||
| 188 | /// P[1] := __B.xmm[i].dword[1] | ||
| 189 | /// P[2] := __B.xmm[i].dword[2] | ||
| 190 | /// P[3] := __B.xmm[i].dword[3] | ||
| 191 | /// C[0] := F_RND(P[0], P[1], P[2], P[3], __A.xmm[i].dword[0]) | ||
| 192 | /// C[1] := F_RND(P[1], P[2], P[3], C[0], __A.xmm[i].dword[1]) | ||
| 193 | /// C[2] := F_RND(P[2], P[3], C[0], C[1], __A.xmm[i].dword[2]) | ||
| 194 | /// C[3] := F_RND(P[3], C[0], C[1], C[2], __A.xmm[i].dword[3]) | ||
| 195 | /// DEST.xmm[i].dword[0] := C[0] | ||
| 196 | /// DEST.xmm[i].dword[1] := C[1] | ||
| 197 | /// DEST.xmm[i].dword[2] := C[2] | ||
| 198 | /// DEST.xmm[i].dword[3] := C[3] | ||
| 199 | /// ENDFOR | ||
| 200 | /// DEST[MAX:128] := 0 | ||
| 201 | /// \endcode | ||
| 202 | #define _mm_sm4rnds4_epi32(A, B) \ | ||
| 203 | (__m128i) __builtin_ia32_vsm4rnds4128((__v4su)A, (__v4su)B) | ||
| 204 | |||
| 205 | /// This intrinisc performs four rounds of SM4 encryption. The intrinisc | ||
| 206 | /// operates on independent 128-bit lanes. The calculated results are | ||
| 207 | /// stored in \a dst. | ||
| 208 | /// \headerfile <immintrin.h> | ||
| 209 | /// | ||
| 210 | /// \code | ||
| 211 | /// __m256i _mm256_sm4rnds4_epi32(__m256i __A, __m256i __B) | ||
| 212 | /// \endcode | ||
| 213 | /// | ||
| 214 | /// This intrinsic corresponds to the \c VSM4RNDS4 instruction. | ||
| 215 | /// | ||
| 216 | /// \param __A | ||
| 217 | /// A 256-bit vector of [8 x int]. | ||
| 218 | /// \param __B | ||
| 219 | /// A 256-bit vector of [8 x int]. | ||
| 220 | /// \returns | ||
| 221 | /// A 256-bit vector of [8 x int]. | ||
| 222 | /// | ||
| 223 | /// \code{.operation} | ||
| 224 | /// DEFINE ROL32(dword, n) { | ||
| 225 | /// count := n % 32 | ||
| 226 | /// dest := (dword << count) | (dword >> (32-count)) | ||
| 227 | /// RETURN dest | ||
| 228 | /// } | ||
| 229 | /// DEFINE lower_t(dword) { | ||
| 230 | /// tmp.byte[0] := SBOX_BYTE(dword, 0) | ||
| 231 | /// tmp.byte[1] := SBOX_BYTE(dword, 1) | ||
| 232 | /// tmp.byte[2] := SBOX_BYTE(dword, 2) | ||
| 233 | /// tmp.byte[3] := SBOX_BYTE(dword, 3) | ||
| 234 | /// RETURN tmp | ||
| 235 | /// } | ||
| 236 | /// DEFINE L_RND(dword) { | ||
| 237 | /// tmp := dword | ||
| 238 | /// tmp := tmp ^ ROL32(dword, 2) | ||
| 239 | /// tmp := tmp ^ ROL32(dword, 10) | ||
| 240 | /// tmp := tmp ^ ROL32(dword, 18) | ||
| 241 | /// tmp := tmp ^ ROL32(dword, 24) | ||
| 242 | /// RETURN tmp | ||
| 243 | /// } | ||
| 244 | /// DEFINE T_RND(dword) { | ||
| 245 | /// RETURN L_RND(lower_t(dword)) | ||
| 246 | /// } | ||
| 247 | /// DEFINE F_RND(X0, X1, X2, X3, round_key) { | ||
| 248 | /// RETURN X0 ^ T_RND(X1 ^ X2 ^ X3 ^ round_key) | ||
| 249 | /// } | ||
| 250 | /// FOR i:= 0 to 0 | ||
| 251 | /// P[0] := __B.xmm[i].dword[0] | ||
| 252 | /// P[1] := __B.xmm[i].dword[1] | ||
| 253 | /// P[2] := __B.xmm[i].dword[2] | ||
| 254 | /// P[3] := __B.xmm[i].dword[3] | ||
| 255 | /// C[0] := F_RND(P[0], P[1], P[2], P[3], __A.xmm[i].dword[0]) | ||
| 256 | /// C[1] := F_RND(P[1], P[2], P[3], C[0], __A.xmm[i].dword[1]) | ||
| 257 | /// C[2] := F_RND(P[2], P[3], C[0], C[1], __A.xmm[i].dword[2]) | ||
| 258 | /// C[3] := F_RND(P[3], C[0], C[1], C[2], __A.xmm[i].dword[3]) | ||
| 259 | /// DEST.xmm[i].dword[0] := C[0] | ||
| 260 | /// DEST.xmm[i].dword[1] := C[1] | ||
| 261 | /// DEST.xmm[i].dword[2] := C[2] | ||
| 262 | /// DEST.xmm[i].dword[3] := C[3] | ||
| 263 | /// ENDFOR | ||
| 264 | /// DEST[MAX:256] := 0 | ||
| 265 | /// \endcode | ||
| 266 | #define _mm256_sm4rnds4_epi32(A, B) \ | ||
| 267 | (__m256i) __builtin_ia32_vsm4rnds4256((__v8su)A, (__v8su)B) | ||
| 268 | |||
| 269 | #endif // __SM4INTRIN_H | ||
lib/include/stdalign.h+5
| ... | @@ -10,6 +10,10 @@ | ... | @@ -10,6 +10,10 @@ |
| 10 | #ifndef __STDALIGN_H | 10 | #ifndef __STDALIGN_H |
| 11 | #define __STDALIGN_H | 11 | #define __STDALIGN_H |
| 12 | 12 | ||
| 13 | /* FIXME: This is using the placeholder dates Clang produces for these macros | ||
| 14 | in C2x mode; switch to the correct values once they've been published. */ | ||
| 15 | #if defined(__cplusplus) || \ | ||
| 16 | (defined(__STDC_VERSION__) && __STDC_VERSION__ < 202000L) | ||
| 13 | #ifndef __cplusplus | 17 | #ifndef __cplusplus |
| 14 | #define alignas _Alignas | 18 | #define alignas _Alignas |
| 15 | #define alignof _Alignof | 19 | #define alignof _Alignof |
| ... | @@ -17,5 +21,6 @@ | ... | @@ -17,5 +21,6 @@ |
| 17 | 21 | ||
| 18 | #define __alignas_is_defined 1 | 22 | #define __alignas_is_defined 1 |
| 19 | #define __alignof_is_defined 1 | 23 | #define __alignof_is_defined 1 |
| 24 | #endif /* __STDC_VERSION__ */ | ||
| 20 | 25 | ||
| 21 | #endif /* __STDALIGN_H */ | 26 | #endif /* __STDALIGN_H */ |
lib/include/stdatomic.h+9-2
| ... | @@ -45,9 +45,16 @@ extern "C" { | ... | @@ -45,9 +45,16 @@ extern "C" { |
| 45 | #define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE | 45 | #define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE |
| 46 | 46 | ||
| 47 | /* 7.17.2 Initialization */ | 47 | /* 7.17.2 Initialization */ |
| 48 | 48 | /* FIXME: This is using the placeholder dates Clang produces for these macros | |
| 49 | in C2x mode; switch to the correct values once they've been published. */ | ||
| 50 | #if (defined(__STDC_VERSION__) && __STDC_VERSION__ < 202000L) || \ | ||
| 51 | defined(__cplusplus) | ||
| 52 | /* ATOMIC_VAR_INIT was removed in C2x, but still remains in C++23. */ | ||
| 49 | #define ATOMIC_VAR_INIT(value) (value) | 53 | #define ATOMIC_VAR_INIT(value) (value) |
| 50 | #if ((defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201710L) || \ | 54 | #endif |
| 55 | |||
| 56 | #if ((defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201710L && \ | ||
| 57 | __STDC_VERSION__ < 202000L) || \ | ||
| 51 | (defined(__cplusplus) && __cplusplus >= 202002L)) && \ | 58 | (defined(__cplusplus) && __cplusplus >= 202002L)) && \ |
| 52 | !defined(_CLANG_DISABLE_CRT_DEPRECATION_WARNINGS) | 59 | !defined(_CLANG_DISABLE_CRT_DEPRECATION_WARNINGS) |
| 53 | /* ATOMIC_VAR_INIT was deprecated in C17 and C++20. */ | 60 | /* ATOMIC_VAR_INIT was deprecated in C17 and C++20. */ |
lib/include/stddef.h+5
| ... | @@ -103,6 +103,11 @@ using ::std::nullptr_t; | ... | @@ -103,6 +103,11 @@ using ::std::nullptr_t; |
| 103 | typedef typeof(nullptr) nullptr_t; | 103 | typedef typeof(nullptr) nullptr_t; |
| 104 | #endif /* defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202000L */ | 104 | #endif /* defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202000L */ |
| 105 | 105 | ||
| 106 | #if defined(__need_STDDEF_H_misc) && defined(__STDC_VERSION__) && \ | ||
| 107 | __STDC_VERSION__ >= 202000L | ||
| 108 | #define unreachable() __builtin_unreachable() | ||
| 109 | #endif /* defined(__need_STDDEF_H_misc) && >= C23 */ | ||
| 110 | |||
| 106 | #if defined(__need_STDDEF_H_misc) | 111 | #if defined(__need_STDDEF_H_misc) |
| 107 | #if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \ | 112 | #if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \ |
| 108 | (defined(__cplusplus) && __cplusplus >= 201103L) | 113 | (defined(__cplusplus) && __cplusplus >= 201103L) |
lib/include/wasm_simd128.h+132-12
| ... | @@ -961,17 +961,17 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_popcnt(v128_t __a) { | ... | @@ -961,17 +961,17 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_popcnt(v128_t __a) { |
| 961 | 961 | ||
| 962 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shl(v128_t __a, | 962 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shl(v128_t __a, |
| 963 | uint32_t __b) { | 963 | uint32_t __b) { |
| 964 | return (v128_t)((__i8x16)__a << __b); | 964 | return (v128_t)((__i8x16)__a << (__b & 0x7)); |
| 965 | } | 965 | } |
| 966 | 966 | ||
| 967 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shr(v128_t __a, | 967 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shr(v128_t __a, |
| 968 | uint32_t __b) { | 968 | uint32_t __b) { |
| 969 | return (v128_t)((__i8x16)__a >> __b); | 969 | return (v128_t)((__i8x16)__a >> (__b & 0x7)); |
| 970 | } | 970 | } |
| 971 | 971 | ||
| 972 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_shr(v128_t __a, | 972 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_shr(v128_t __a, |
| 973 | uint32_t __b) { | 973 | uint32_t __b) { |
| 974 | return (v128_t)((__u8x16)__a >> __b); | 974 | return (v128_t)((__u8x16)__a >> (__b & 0x7)); |
| 975 | } | 975 | } |
| 976 | 976 | ||
| 977 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_add(v128_t __a, | 977 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_add(v128_t __a, |
| ... | @@ -1047,17 +1047,17 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i16x8_bitmask(v128_t __a) { | ... | @@ -1047,17 +1047,17 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i16x8_bitmask(v128_t __a) { |
| 1047 | 1047 | ||
| 1048 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_shl(v128_t __a, | 1048 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_shl(v128_t __a, |
| 1049 | uint32_t __b) { | 1049 | uint32_t __b) { |
| 1050 | return (v128_t)((__i16x8)__a << __b); | 1050 | return (v128_t)((__i16x8)__a << (__b & 0xF)); |
| 1051 | } | 1051 | } |
| 1052 | 1052 | ||
| 1053 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_shr(v128_t __a, | 1053 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_shr(v128_t __a, |
| 1054 | uint32_t __b) { | 1054 | uint32_t __b) { |
| 1055 | return (v128_t)((__i16x8)__a >> __b); | 1055 | return (v128_t)((__i16x8)__a >> (__b & 0xF)); |
| 1056 | } | 1056 | } |
| 1057 | 1057 | ||
| 1058 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_shr(v128_t __a, | 1058 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_shr(v128_t __a, |
| 1059 | uint32_t __b) { | 1059 | uint32_t __b) { |
| 1060 | return (v128_t)((__u16x8)__a >> __b); | 1060 | return (v128_t)((__u16x8)__a >> (__b & 0xF)); |
| 1061 | } | 1061 | } |
| 1062 | 1062 | ||
| 1063 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_add(v128_t __a, | 1063 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_add(v128_t __a, |
| ... | @@ -1138,17 +1138,17 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i32x4_bitmask(v128_t __a) { | ... | @@ -1138,17 +1138,17 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i32x4_bitmask(v128_t __a) { |
| 1138 | 1138 | ||
| 1139 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_shl(v128_t __a, | 1139 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_shl(v128_t __a, |
| 1140 | uint32_t __b) { | 1140 | uint32_t __b) { |
| 1141 | return (v128_t)((__i32x4)__a << __b); | 1141 | return (v128_t)((__i32x4)__a << (__b & 0x1F)); |
| 1142 | } | 1142 | } |
| 1143 | 1143 | ||
| 1144 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_shr(v128_t __a, | 1144 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_shr(v128_t __a, |
| 1145 | uint32_t __b) { | 1145 | uint32_t __b) { |
| 1146 | return (v128_t)((__i32x4)__a >> __b); | 1146 | return (v128_t)((__i32x4)__a >> (__b & 0x1F)); |
| 1147 | } | 1147 | } |
| 1148 | 1148 | ||
| 1149 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_shr(v128_t __a, | 1149 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_shr(v128_t __a, |
| 1150 | uint32_t __b) { | 1150 | uint32_t __b) { |
| 1151 | return (v128_t)((__u32x4)__a >> __b); | 1151 | return (v128_t)((__u32x4)__a >> (__b & 0x1F)); |
| 1152 | } | 1152 | } |
| 1153 | 1153 | ||
| 1154 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_add(v128_t __a, | 1154 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_add(v128_t __a, |
| ... | @@ -1209,17 +1209,17 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i64x2_bitmask(v128_t __a) { | ... | @@ -1209,17 +1209,17 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i64x2_bitmask(v128_t __a) { |
| 1209 | 1209 | ||
| 1210 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_shl(v128_t __a, | 1210 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_shl(v128_t __a, |
| 1211 | uint32_t __b) { | 1211 | uint32_t __b) { |
| 1212 | return (v128_t)((__i64x2)__a << (int64_t)__b); | 1212 | return (v128_t)((__i64x2)__a << ((int64_t)__b & 0x3F)); |
| 1213 | } | 1213 | } |
| 1214 | 1214 | ||
| 1215 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_shr(v128_t __a, | 1215 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_shr(v128_t __a, |
| 1216 | uint32_t __b) { | 1216 | uint32_t __b) { |
| 1217 | return (v128_t)((__i64x2)__a >> (int64_t)__b); | 1217 | return (v128_t)((__i64x2)__a >> ((int64_t)__b & 0x3F)); |
| 1218 | } | 1218 | } |
| 1219 | 1219 | ||
| 1220 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u64x2_shr(v128_t __a, | 1220 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u64x2_shr(v128_t __a, |
| 1221 | uint32_t __b) { | 1221 | uint32_t __b) { |
| 1222 | return (v128_t)((__u64x2)__a >> (int64_t)__b); | 1222 | return (v128_t)((__u64x2)__a >> ((int64_t)__b & 0x3F)); |
| 1223 | } | 1223 | } |
| 1224 | 1224 | ||
| 1225 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_add(v128_t __a, | 1225 | static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_add(v128_t __a, |
| ... | @@ -1760,6 +1760,126 @@ wasm_u64x2_load_32x2(const void *__mem) { | ... | @@ -1760,6 +1760,126 @@ wasm_u64x2_load_32x2(const void *__mem) { |
| 1760 | __DEPRECATED_WASM_MACRO("wasm_v64x2_shuffle", "wasm_i64x2_shuffle") \ | 1760 | __DEPRECATED_WASM_MACRO("wasm_v64x2_shuffle", "wasm_i64x2_shuffle") \ |
| 1761 | wasm_i64x2_shuffle(__a, __b, __c0, __c1) | 1761 | wasm_i64x2_shuffle(__a, __b, __c0, __c1) |
| 1762 | 1762 | ||
| 1763 | // Relaxed SIMD intrinsics | ||
| 1764 | |||
| 1765 | #define __RELAXED_FN_ATTRS \ | ||
| 1766 | __attribute__((__always_inline__, __nodebug__, __target__("relaxed-simd"), \ | ||
| 1767 | __min_vector_width__(128))) | ||
| 1768 | |||
| 1769 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1770 | wasm_f32x4_relaxed_madd(v128_t __a, v128_t __b, v128_t __c) { | ||
| 1771 | return (v128_t)__builtin_wasm_relaxed_madd_f32x4((__f32x4)__a, (__f32x4)__b, | ||
| 1772 | (__f32x4)__c); | ||
| 1773 | } | ||
| 1774 | |||
| 1775 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1776 | wasm_f32x4_relaxed_nmadd(v128_t __a, v128_t __b, v128_t __c) { | ||
| 1777 | return (v128_t)__builtin_wasm_relaxed_nmadd_f32x4((__f32x4)__a, (__f32x4)__b, | ||
| 1778 | (__f32x4)__c); | ||
| 1779 | } | ||
| 1780 | |||
| 1781 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1782 | wasm_f64x2_relaxed_madd(v128_t __a, v128_t __b, v128_t __c) { | ||
| 1783 | return (v128_t)__builtin_wasm_relaxed_madd_f64x2((__f64x2)__a, (__f64x2)__b, | ||
| 1784 | (__f64x2)__c); | ||
| 1785 | } | ||
| 1786 | |||
| 1787 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1788 | wasm_f64x2_relaxed_nmadd(v128_t __a, v128_t __b, v128_t __c) { | ||
| 1789 | return (v128_t)__builtin_wasm_relaxed_nmadd_f64x2((__f64x2)__a, (__f64x2)__b, | ||
| 1790 | (__f64x2)__c); | ||
| 1791 | } | ||
| 1792 | |||
| 1793 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1794 | wasm_i8x16_relaxed_laneselect(v128_t __a, v128_t __b, v128_t __m) { | ||
| 1795 | return (v128_t)__builtin_wasm_relaxed_laneselect_i8x16( | ||
| 1796 | (__i8x16)__a, (__i8x16)__b, (__i8x16)__m); | ||
| 1797 | } | ||
| 1798 | |||
| 1799 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1800 | wasm_i16x8_relaxed_laneselect(v128_t __a, v128_t __b, v128_t __m) { | ||
| 1801 | return (v128_t)__builtin_wasm_relaxed_laneselect_i16x8( | ||
| 1802 | (__i16x8)__a, (__i16x8)__b, (__i16x8)__m); | ||
| 1803 | } | ||
| 1804 | |||
| 1805 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1806 | wasm_i32x4_relaxed_laneselect(v128_t __a, v128_t __b, v128_t __m) { | ||
| 1807 | return (v128_t)__builtin_wasm_relaxed_laneselect_i32x4( | ||
| 1808 | (__i32x4)__a, (__i32x4)__b, (__i32x4)__m); | ||
| 1809 | } | ||
| 1810 | |||
| 1811 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1812 | wasm_i64x2_relaxed_laneselect(v128_t __a, v128_t __b, v128_t __m) { | ||
| 1813 | return (v128_t)__builtin_wasm_relaxed_laneselect_i64x2( | ||
| 1814 | (__i64x2)__a, (__i64x2)__b, (__i64x2)__m); | ||
| 1815 | } | ||
| 1816 | |||
| 1817 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1818 | wasm_i8x16_relaxed_swizzle(v128_t __a, v128_t __s) { | ||
| 1819 | return (v128_t)__builtin_wasm_relaxed_swizzle_i8x16((__i8x16)__a, | ||
| 1820 | (__i8x16)__s); | ||
| 1821 | } | ||
| 1822 | |||
| 1823 | static __inline__ v128_t __RELAXED_FN_ATTRS wasm_f32x4_relaxed_min(v128_t __a, | ||
| 1824 | v128_t __b) { | ||
| 1825 | return (v128_t)__builtin_wasm_relaxed_min_f32x4((__f32x4)__a, (__f32x4)__b); | ||
| 1826 | } | ||
| 1827 | |||
| 1828 | static __inline__ v128_t __RELAXED_FN_ATTRS wasm_f32x4_relaxed_max(v128_t __a, | ||
| 1829 | v128_t __b) { | ||
| 1830 | return (v128_t)__builtin_wasm_relaxed_max_f32x4((__f32x4)__a, (__f32x4)__b); | ||
| 1831 | } | ||
| 1832 | |||
| 1833 | static __inline__ v128_t __RELAXED_FN_ATTRS wasm_f64x2_relaxed_min(v128_t __a, | ||
| 1834 | v128_t __b) { | ||
| 1835 | return (v128_t)__builtin_wasm_relaxed_min_f64x2((__f64x2)__a, (__f64x2)__b); | ||
| 1836 | } | ||
| 1837 | |||
| 1838 | static __inline__ v128_t __RELAXED_FN_ATTRS wasm_f64x2_relaxed_max(v128_t __a, | ||
| 1839 | v128_t __b) { | ||
| 1840 | return (v128_t)__builtin_wasm_relaxed_max_f64x2((__f64x2)__a, (__f64x2)__b); | ||
| 1841 | } | ||
| 1842 | |||
| 1843 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1844 | wasm_i32x4_relaxed_trunc_f32x4(v128_t __a) { | ||
| 1845 | return (v128_t)__builtin_wasm_relaxed_trunc_s_i32x4_f32x4((__f32x4)__a); | ||
| 1846 | } | ||
| 1847 | |||
| 1848 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1849 | wasm_u32x4_relaxed_trunc_f32x4(v128_t __a) { | ||
| 1850 | return (v128_t)__builtin_wasm_relaxed_trunc_u_i32x4_f32x4((__f32x4)__a); | ||
| 1851 | } | ||
| 1852 | |||
| 1853 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1854 | wasm_i32x4_relaxed_trunc_f64x2_zero(v128_t __a) { | ||
| 1855 | return (v128_t)__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2((__f64x2)__a); | ||
| 1856 | } | ||
| 1857 | |||
| 1858 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1859 | wasm_u32x4_relaxed_trunc_f64x2_zero(v128_t __a) { | ||
| 1860 | return (v128_t)__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2((__f64x2)__a); | ||
| 1861 | } | ||
| 1862 | |||
| 1863 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1864 | wasm_i16x8_relaxed_q15mulr(v128_t __a, v128_t __b) { | ||
| 1865 | return (v128_t)__builtin_wasm_relaxed_q15mulr_s_i16x8((__i16x8)__a, | ||
| 1866 | (__i16x8)__b); | ||
| 1867 | } | ||
| 1868 | |||
| 1869 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1870 | wasm_i16x8_relaxed_dot_i8x16_i7x16(v128_t __a, v128_t __b) { | ||
| 1871 | return (v128_t)__builtin_wasm_relaxed_dot_i8x16_i7x16_s_i16x8((__i8x16)__a, | ||
| 1872 | (__i8x16)__b); | ||
| 1873 | } | ||
| 1874 | |||
| 1875 | static __inline__ v128_t __RELAXED_FN_ATTRS | ||
| 1876 | wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v128_t __a, v128_t __b, v128_t __c) { | ||
| 1877 | return (v128_t)__builtin_wasm_relaxed_dot_i8x16_i7x16_add_s_i32x4( | ||
| 1878 | (__i8x16)__a, (__i8x16)__b, (__i32x4)__c); | ||
| 1879 | } | ||
| 1880 | |||
| 1881 | // Deprecated intrinsics | ||
| 1882 | |||
| 1763 | static __inline__ v128_t __DEPRECATED_FN_ATTRS("wasm_i8x16_swizzle") | 1883 | static __inline__ v128_t __DEPRECATED_FN_ATTRS("wasm_i8x16_swizzle") |
| 1764 | wasm_v8x16_swizzle(v128_t __a, v128_t __b) { | 1884 | wasm_v8x16_swizzle(v128_t __a, v128_t __b) { |
| 1765 | return wasm_i8x16_swizzle(__a, __b); | 1885 | return wasm_i8x16_swizzle(__a, __b); |
lib/include/xsavecintrin.h+50
| ... | @@ -17,12 +17,62 @@ | ... | @@ -17,12 +17,62 @@ |
| 17 | /* Define the default attributes for the functions in this file. */ | 17 | /* Define the default attributes for the functions in this file. */ |
| 18 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("xsavec"))) | 18 | #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("xsavec"))) |
| 19 | 19 | ||
| 20 | /// Performs a full or partial save of processor state to the memory at | ||
| 21 | /// \a __p. The exact state saved depends on the 64-bit mask \a __m and | ||
| 22 | /// processor control register \c XCR0. | ||
| 23 | /// | ||
| 24 | /// \code{.operation} | ||
| 25 | /// mask[62:0] := __m[62:0] AND XCR0[62:0] | ||
| 26 | /// FOR i := 0 TO 62 | ||
| 27 | /// IF mask[i] == 1 | ||
| 28 | /// CASE (i) OF | ||
| 29 | /// 0: save X87 FPU state | ||
| 30 | /// 1: save SSE state | ||
| 31 | /// DEFAULT: __p.Ext_Save_Area[i] := ProcessorState[i] | ||
| 32 | /// FI | ||
| 33 | /// ENDFOR | ||
| 34 | /// __p.Header.XSTATE_BV[62:0] := INIT_FUNCTION(mask[62:0]) | ||
| 35 | /// \endcode | ||
| 36 | /// | ||
| 37 | /// \headerfile <immintrin.h> | ||
| 38 | /// | ||
| 39 | /// This intrinsic corresponds to the \c XSAVEC instruction. | ||
| 40 | /// | ||
| 41 | /// \param __p | ||
| 42 | /// Pointer to the save area; must be 64-byte aligned. | ||
| 43 | /// \param __m | ||
| 44 | /// A 64-bit mask indicating what state should be saved. | ||
| 20 | static __inline__ void __DEFAULT_FN_ATTRS | 45 | static __inline__ void __DEFAULT_FN_ATTRS |
| 21 | _xsavec(void *__p, unsigned long long __m) { | 46 | _xsavec(void *__p, unsigned long long __m) { |
| 22 | __builtin_ia32_xsavec(__p, __m); | 47 | __builtin_ia32_xsavec(__p, __m); |
| 23 | } | 48 | } |
| 24 | 49 | ||
| 25 | #ifdef __x86_64__ | 50 | #ifdef __x86_64__ |
| 51 | /// Performs a full or partial save of processor state to the memory at | ||
| 52 | /// \a __p. The exact state saved depends on the 64-bit mask \a __m and | ||
| 53 | /// processor control register \c XCR0. | ||
| 54 | /// | ||
| 55 | /// \code{.operation} | ||
| 56 | /// mask[62:0] := __m[62:0] AND XCR0[62:0] | ||
| 57 | /// FOR i := 0 TO 62 | ||
| 58 | /// IF mask[i] == 1 | ||
| 59 | /// CASE (i) OF | ||
| 60 | /// 0: save X87 FPU state | ||
| 61 | /// 1: save SSE state | ||
| 62 | /// DEFAULT: __p.Ext_Save_Area[i] := ProcessorState[i] | ||
| 63 | /// FI | ||
| 64 | /// ENDFOR | ||
| 65 | /// __p.Header.XSTATE_BV[62:0] := INIT_FUNCTION(mask[62:0]) | ||
| 66 | /// \endcode | ||
| 67 | /// | ||
| 68 | /// \headerfile <immintrin.h> | ||
| 69 | /// | ||
| 70 | /// This intrinsic corresponds to the \c XSAVEC64 instruction. | ||
| 71 | /// | ||
| 72 | /// \param __p | ||
| 73 | /// Pointer to the save area; must be 64-byte aligned. | ||
| 74 | /// \param __m | ||
| 75 | /// A 64-bit mask indicating what state should be saved. | ||
| 26 | static __inline__ void __DEFAULT_FN_ATTRS | 76 | static __inline__ void __DEFAULT_FN_ATTRS |
| 27 | _xsavec64(void *__p, unsigned long long __m) { | 77 | _xsavec64(void *__p, unsigned long long __m) { |
| 28 | __builtin_ia32_xsavec64(__p, __m); | 78 | __builtin_ia32_xsavec64(__p, __m); |