Commit 1861036f3b61828d0b7641dcce65f37e516fd3f6 | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-08-11 16:20:14-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2023-09-19 09:37:31-07:00
log	1861036f3b61828d0b7641dcce65f37e516fd3f6
tree	30fb5a5dfec4e83c176d10296ee78d97318c29eb
parent	3ed40b114020ff774279a5af993851d74da64b52

update C language headers to LLVM 17

release/17.x branch, commit 8f4dd44097c9ae25dd203d5ac87f3b48f854bba8

49 files changed, 8865 insertions(+), 401 deletions(-)

lib/include/__clang_cuda_intrinsics.h+191

...	...	@@ -513,6 +513,197 @@ __device__ inline cuuint32_t __nvvm_get_smem_pointer(void *__ptr) {
513	513	return __nv_cvta_generic_to_shared_impl(__ptr);
514	514	}
515	515	} // extern "C"
	516
	517	#if !defined(__CUDA_ARCH__) \|\| __CUDA_ARCH__ >= 800
	518	__device__ inline unsigned __reduce_add_sync(unsigned __mask,
	519	unsigned __value) {
	520	return __nvvm_redux_sync_add(__mask, __value);
	521	}
	522	__device__ inline unsigned __reduce_min_sync(unsigned __mask,
	523	unsigned __value) {
	524	return __nvvm_redux_sync_umin(__mask, __value);
	525	}
	526	__device__ inline unsigned __reduce_max_sync(unsigned __mask,
	527	unsigned __value) {
	528	return __nvvm_redux_sync_umax(__mask, __value);
	529	}
	530	__device__ inline int __reduce_min_sync(unsigned __mask, int __value) {
	531	return __nvvm_redux_sync_min(__mask, __value);
	532	}
	533	__device__ inline int __reduce_max_sync(unsigned __mask, int __value) {
	534	return __nvvm_redux_sync_max(__mask, __value);
	535	}
	536	__device__ inline unsigned __reduce_or_sync(unsigned __mask, unsigned __value) {
	537	return __nvvm_redux_sync_or(__mask, __value);
	538	}
	539	__device__ inline unsigned __reduce_and_sync(unsigned __mask,
	540	unsigned __value) {
	541	return __nvvm_redux_sync_and(__mask, __value);
	542	}
	543	__device__ inline unsigned __reduce_xor_sync(unsigned __mask,
	544	unsigned __value) {
	545	return __nvvm_redux_sync_xor(__mask, __value);
	546	}
	547
	548	__device__ inline void __nv_memcpy_async_shared_global_4(void *__dst,
	549	const void *__src,
	550	unsigned __src_size) {
	551	__nvvm_cp_async_ca_shared_global_4(
	552	(void __attribute__((address_space(3))) *)__dst,
	553	(const void __attribute__((address_space(1))) *)__src, __src_size);
	554	}
	555	__device__ inline void __nv_memcpy_async_shared_global_8(void *__dst,
	556	const void *__src,
	557	unsigned __src_size) {
	558	__nvvm_cp_async_ca_shared_global_8(
	559	(void __attribute__((address_space(3))) *)__dst,
	560	(const void __attribute__((address_space(1))) *)__src, __src_size);
	561	}
	562	__device__ inline void __nv_memcpy_async_shared_global_16(void *__dst,
	563	const void *__src,
	564	unsigned __src_size) {
	565	__nvvm_cp_async_ca_shared_global_16(
	566	(void __attribute__((address_space(3))) *)__dst,
	567	(const void __attribute__((address_space(1))) *)__src, __src_size);
	568	}
	569
	570	__device__ inline void *
	571	__nv_associate_access_property(const void *__ptr, unsigned long long __prop) {
	572	// TODO: it appears to provide compiler with some sort of a hint. We do not
	573	// know what exactly it is supposed to do. However, CUDA headers suggest that
	574	// just passing through __ptr should not affect correctness. They do so on
	575	// pre-sm80 GPUs where this builtin is not available.
	576	return (void*)__ptr;
	577	}
	578	#endif // !defined(__CUDA_ARCH__) \|\| __CUDA_ARCH__ >= 800
	579
	580	#if !defined(__CUDA_ARCH__) \|\| __CUDA_ARCH__ >= 900
	581	__device__ inline unsigned __isCtaShared(const void *ptr) {
	582	return __isShared(ptr);
	583	}
	584
	585	__device__ inline unsigned __isClusterShared(const void *__ptr) {
	586	return __nvvm_isspacep_shared_cluster(__ptr);
	587	}
	588
	589	__device__ inline void __cluster_map_shared_rank(const void __ptr,
	590	unsigned __rank) {
	591	return __nvvm_mapa((void *)__ptr, __rank);
	592	}
	593
	594	__device__ inline unsigned __cluster_query_shared_rank(const void *__ptr) {
	595	return __nvvm_getctarank((void *)__ptr);
	596	}
	597
	598	__device__ inline uint2
	599	__cluster_map_shared_multicast(const void *__ptr,
	600	unsigned int __cluster_cta_mask) {
	601	return make_uint2((unsigned)__cvta_generic_to_shared(__ptr),
	602	__cluster_cta_mask);
	603	}
	604
	605	__device__ inline unsigned __clusterDimIsSpecified() {
	606	return __nvvm_is_explicit_cluster();
	607	}
	608
	609	__device__ inline dim3 __clusterDim() {
	610	return dim3(__nvvm_read_ptx_sreg_cluster_nctaid_x(),
	611	__nvvm_read_ptx_sreg_cluster_nctaid_y(),
	612	__nvvm_read_ptx_sreg_cluster_nctaid_z());
	613	}
	614
	615	__device__ inline dim3 __clusterRelativeBlockIdx() {
	616	return dim3(__nvvm_read_ptx_sreg_cluster_ctaid_x(),
	617	__nvvm_read_ptx_sreg_cluster_ctaid_y(),
	618	__nvvm_read_ptx_sreg_cluster_ctaid_z());
	619	}
	620
	621	__device__ inline dim3 __clusterGridDimInClusters() {
	622	return dim3(__nvvm_read_ptx_sreg_nclusterid_x(),
	623	__nvvm_read_ptx_sreg_nclusterid_y(),
	624	__nvvm_read_ptx_sreg_nclusterid_z());
	625	}
	626
	627	__device__ inline dim3 __clusterIdx() {
	628	return dim3(__nvvm_read_ptx_sreg_clusterid_x(),
	629	__nvvm_read_ptx_sreg_clusterid_y(),
	630	__nvvm_read_ptx_sreg_clusterid_z());
	631	}
	632
	633	__device__ inline unsigned __clusterRelativeBlockRank() {
	634	return __nvvm_read_ptx_sreg_cluster_ctarank();
	635	}
	636
	637	__device__ inline unsigned __clusterSizeInBlocks() {
	638	return __nvvm_read_ptx_sreg_cluster_nctarank();
	639	}
	640
	641	__device__ inline void __cluster_barrier_arrive() {
	642	__nvvm_barrier_cluster_arrive();
	643	}
	644
	645	__device__ inline void __cluster_barrier_arrive_relaxed() {
	646	__nvvm_barrier_cluster_arrive_relaxed();
	647	}
	648
	649	__device__ inline void __cluster_barrier_wait() {
	650	__nvvm_barrier_cluster_wait();
	651	}
	652
	653	__device__ inline void __threadfence_cluster() { __nvvm_fence_sc_cluster(); }
	654
	655	__device__ inline float2 atomicAdd(float2 *__ptr, float2 __val) {
	656	float2 __ret;
	657	__asm__("atom.add.v2.f32 {%0, %1}, [%2], {%3, %4};"
	658	: "=f"(__ret.x), "=f"(__ret.y)
	659	: "l"(__ptr), "f"(__val.x), "f"(__val.y));
	660	return __ret;
	661	}
	662
	663	__device__ inline float2 atomicAdd_block(float2 *__ptr, float2 __val) {
	664	float2 __ret;
	665	__asm__("atom.cta.add.v2.f32 {%0, %1}, [%2], {%3, %4};"
	666	: "=f"(__ret.x), "=f"(__ret.y)
	667	: "l"(__ptr), "f"(__val.x), "f"(__val.y));
	668	return __ret;
	669	}
	670
	671	__device__ inline float2 atomicAdd_system(float2 *__ptr, float2 __val) {
	672	float2 __ret;
	673	__asm__("atom.sys.add.v2.f32 {%0, %1}, [%2], {%3, %4};"
	674	: "=f"(__ret.x), "=f"(__ret.y)
	675	: "l"(__ptr), "f"(__val.x), "f"(__val.y));
	676	return __ret;
	677	}
	678
	679	__device__ inline float4 atomicAdd(float4 *__ptr, float4 __val) {
	680	float4 __ret;
	681	__asm__("atom.add.v4.f32 {%0, %1, %2, %3}, [%4], {%5, %6, %7, %8};"
	682	: "=f"(__ret.x), "=f"(__ret.y), "=f"(__ret.z), "=f"(__ret.w)
	683	: "l"(__ptr), "f"(__val.x), "f"(__val.y), "f"(__val.z), "f"(__val.w));
	684	return __ret;
	685	}
	686
	687	__device__ inline float4 atomicAdd_block(float4 *__ptr, float4 __val) {
	688	float4 __ret;
	689	__asm__(
	690	"atom.cta.add.v4.f32 {%0, %1, %2, %3}, [%4], {%5, %6, %7, %8};"
	691	: "=f"(__ret.x), "=f"(__ret.y), "=f"(__ret.z), "=f"(__ret.w)
	692	: "l"(__ptr), "f"(__val.x), "f"(__val.y), "f"(__val.z), "f"(__val.w));
	693	return __ret;
	694	}
	695
	696	__device__ inline float4 atomicAdd_system(float4 *__ptr, float4 __val) {
	697	float4 __ret;
	698	__asm__(
	699	"atom.sys.add.v4.f32 {%0, %1, %2, %3}, [%4], {%5, %6, %7, %8};"
	700	: "=f"(__ret.x), "=f"(__ret.y), "=f"(__ret.z), "=f"(__ret.w)
	701	: "l"(__ptr), "f"(__val.x), "f"(__val.y), "f"(__val.z), "f"(__val.w)
	702	:);
	703	return __ret;
	704	}
	705
	706	#endif // !defined(__CUDA_ARCH__) \|\| __CUDA_ARCH__ >= 900
516	707	#endif // CUDA_VERSION >= 11000
517	708
518	709	#endif // defined(__CLANG_CUDA_INTRINSICS_H__)

lib/include/__clang_hip_cmath.h+1-1

...	...	@@ -171,7 +171,7 @@ __DEVICE__ __CONSTEXPR__ bool signbit(double __x) { return ::__signbit(__x); }
171	171	// Other functions.
172	172	__DEVICE__ __CONSTEXPR__ _Float16 fma(_Float16 __x, _Float16 __y,
173	173	_Float16 __z) {
174		return __ocml_fma_f16(__x, __y, __z);
	174	return __builtin_fmaf16(__x, __y, __z);
175	175	}
176	176	__DEVICE__ __CONSTEXPR__ _Float16 pow(_Float16 __base, int __iexp) {
177	177	return __ocml_pown_f16(__base, __iexp);

lib/include/__clang_hip_libdevice_declares.h+32-30

...	...	@@ -10,6 +10,10 @@
10	10	#ifndef __CLANG_HIP_LIBDEVICE_DECLARES_H__
11	11	#define __CLANG_HIP_LIBDEVICE_DECLARES_H__
12	12
	13	#if !defined(__HIPCC_RTC__) && __has_include("hip/hip_version.h")
	14	#include "hip/hip_version.h"
	15	#endif // __has_include("hip/hip_version.h")
	16
13	17	#ifdef __cplusplus
14	18	extern "C" {
15	19	#endif
...	...	@@ -137,23 +141,6 @@ __device__ __attribute__((const)) float __ocml_fma_rte_f32(float, float, float);
137	141	__device__ __attribute__((const)) float __ocml_fma_rtn_f32(float, float, float);
138	142	__device__ __attribute__((const)) float __ocml_fma_rtp_f32(float, float, float);
139	143	__device__ __attribute__((const)) float __ocml_fma_rtz_f32(float, float, float);
140
141		__device__ inline __attribute__((const)) float
142		__llvm_amdgcn_cos_f32(float __x) {
143		return __builtin_amdgcn_cosf(__x);
144		}
145		__device__ inline __attribute__((const)) float
146		__llvm_amdgcn_rcp_f32(float __x) {
147		return __builtin_amdgcn_rcpf(__x);
148		}
149		__device__ inline __attribute__((const)) float
150		__llvm_amdgcn_rsq_f32(float __x) {
151		return __builtin_amdgcn_rsqf(__x);
152		}
153		__device__ inline __attribute__((const)) float
154		__llvm_amdgcn_sin_f32(float __x) {
155		return __builtin_amdgcn_sinf(__x);
156		}
157	144	// END INTRINSICS
158	145	// END FLOAT
159	146
...	...	@@ -277,15 +264,6 @@ __device__ __attribute__((const)) double __ocml_fma_rtp_f64(double, double,
277	264	__device__ __attribute__((const)) double __ocml_fma_rtz_f64(double, double,
278	265	double);
279	266
280		__device__ inline __attribute__((const)) double
281		__llvm_amdgcn_rcp_f64(double __x) {
282		return __builtin_amdgcn_rcp(__x);
283		}
284		__device__ inline __attribute__((const)) double
285		__llvm_amdgcn_rsq_f64(double __x) {
286		return __builtin_amdgcn_rsq(__x);
287		}
288
289	267	__device__ __attribute__((const)) _Float16 __ocml_ceil_f16(_Float16);
290	268	__device__ _Float16 __ocml_cos_f16(_Float16);
291	269	__device__ __attribute__((const)) _Float16 __ocml_cvtrtn_f16_f32(float);
...	...	@@ -305,7 +283,6 @@ __device__ __attribute__((const)) int __ocml_isnan_f16(_Float16);
305	283	__device__ __attribute__((pure)) _Float16 __ocml_log_f16(_Float16);
306	284	__device__ __attribute__((pure)) _Float16 __ocml_log10_f16(_Float16);
307	285	__device__ __attribute__((pure)) _Float16 __ocml_log2_f16(_Float16);
308		__device__ __attribute__((const)) _Float16 __llvm_amdgcn_rcp_f16(_Float16);
309	286	__device__ __attribute__((const)) _Float16 __ocml_rint_f16(_Float16);
310	287	__device__ __attribute__((const)) _Float16 __ocml_rsqrt_f16(_Float16);
311	288	__device__ _Float16 __ocml_sin_f16(_Float16);
...	...	@@ -316,8 +293,15 @@ __device__ __attribute__((pure)) _Float16 __ocml_pown_f16(_Float16, int);
316	293	typedef _Float16 __2f16 __attribute__((ext_vector_type(2)));
317	294	typedef short __2i16 __attribute__((ext_vector_type(2)));
318	295
	296	// We need to match C99's bool and get an i1 in the IR.
	297	#ifdef __cplusplus
	298	typedef bool __ockl_bool;
	299	#else
	300	typedef _Bool __ockl_bool;
	301	#endif
	302
319	303	__device__ __attribute__((const)) float __ockl_fdot2(__2f16 a, __2f16 b,
320		float c, bool s);
	304	float c, __ockl_bool s);
321	305	__device__ __attribute__((const)) __2f16 __ocml_ceil_2f16(__2f16);
322	306	__device__ __attribute__((const)) __2f16 __ocml_fabs_2f16(__2f16);
323	307	__device__ __2f16 __ocml_cos_2f16(__2f16);
...	...	@@ -332,11 +316,29 @@ __device__ __attribute__((const)) __2i16 __ocml_isnan_2f16(__2f16);
332	316	__device__ __attribute__((pure)) __2f16 __ocml_log_2f16(__2f16);
333	317	__device__ __attribute__((pure)) __2f16 __ocml_log10_2f16(__2f16);
334	318	__device__ __attribute__((pure)) __2f16 __ocml_log2_2f16(__2f16);
	319
	320	#if HIP_VERSION_MAJOR * 100 + HIP_VERSION_MINOR >= 560
	321	#define __DEPRECATED_SINCE_HIP_560(X) __attribute__((deprecated(X)))
	322	#else
	323	#define __DEPRECATED_SINCE_HIP_560(X)
	324	#endif
	325
	326	// Deprecated, should be removed when rocm releases using it are no longer
	327	// relevant.
	328	__DEPRECATED_SINCE_HIP_560("use ((_Float16)1.0) / ")
	329	__device__ inline _Float16 __llvm_amdgcn_rcp_f16(_Float16 x) {
	330	return ((_Float16)1.0f) / x;
	331	}
	332
	333	__DEPRECATED_SINCE_HIP_560("use ((__2f16)1.0) / ")
335	334	__device__ inline __2f16
336		__llvm_amdgcn_rcp_2f16(__2f16 __x) // Not currently exposed by ROCDL.
	335	__llvm_amdgcn_rcp_2f16(__2f16 __x)
337	336	{
338		return (__2f16)(__llvm_amdgcn_rcp_f16(__x.x), __llvm_amdgcn_rcp_f16(__x.y));
	337	return ((__2f16)1.0f) / __x;
339	338	}
	339
	340	#undef __DEPRECATED_SINCE_HIP_560
	341
340	342	__device__ __attribute__((const)) __2f16 __ocml_rint_2f16(__2f16);
341	343	__device__ __attribute__((const)) __2f16 __ocml_rsqrt_2f16(__2f16);
342	344	__device__ __2f16 __ocml_sin_2f16(__2f16);

lib/include/__clang_hip_math.h+56-71

...	...	@@ -182,10 +182,10 @@ __DEVICE__
182	182	float cbrtf(float __x) { return __ocml_cbrt_f32(__x); }
183	183
184	184	__DEVICE__
185		float ceilf(float __x) { return __ocml_ceil_f32(__x); }
	185	float ceilf(float __x) { return __builtin_ceilf(__x); }
186	186
187	187	__DEVICE__
188		float copysignf(float __x, float __y) { return __ocml_copysign_f32(__x, __y); }
	188	float copysignf(float __x, float __y) { return __builtin_copysignf(__x, __y); }
189	189
190	190	__DEVICE__
191	191	float cosf(float __x) { return __ocml_cos_f32(__x); }
...	...	@@ -221,10 +221,10 @@ __DEVICE__
221	221	float exp10f(float __x) { return __ocml_exp10_f32(__x); }
222	222
223	223	__DEVICE__
224		float exp2f(float __x) { return __ocml_exp2_f32(__x); }
	224	float exp2f(float __x) { return __builtin_exp2f(__x); }
225	225
226	226	__DEVICE__
227		float expf(float __x) { return __ocml_exp_f32(__x); }
	227	float expf(float __x) { return __builtin_expf(__x); }
228	228
229	229	__DEVICE__
230	230	float expm1f(float __x) { return __ocml_expm1_f32(__x); }
...	...	@@ -239,33 +239,25 @@ __DEVICE__
239	239	float fdividef(float __x, float __y) { return __x / __y; }
240	240
241	241	__DEVICE__
242		float floorf(float __x) { return __ocml_floor_f32(__x); }
	242	float floorf(float __x) { return __builtin_floorf(__x); }
243	243
244	244	__DEVICE__
245	245	float fmaf(float __x, float __y, float __z) {
246		return __ocml_fma_f32(__x, __y, __z);
	246	return __builtin_fmaf(__x, __y, __z);
247	247	}
248	248
249	249	__DEVICE__
250		float fmaxf(float __x, float __y) { return __ocml_fmax_f32(__x, __y); }
	250	float fmaxf(float __x, float __y) { return __builtin_fmaxf(__x, __y); }
251	251
252	252	__DEVICE__
253		float fminf(float __x, float __y) { return __ocml_fmin_f32(__x, __y); }
	253	float fminf(float __x, float __y) { return __builtin_fminf(__x, __y); }
254	254
255	255	__DEVICE__
256	256	float fmodf(float __x, float __y) { return __ocml_fmod_f32(__x, __y); }
257	257
258	258	__DEVICE__
259	259	float frexpf(float __x, int *__nptr) {
260		int __tmp;
261		#ifdef __OPENMP_AMDGCN__
262		#pragma omp allocate(__tmp) allocator(omp_thread_mem_alloc)
263		#endif
264		float __r =
265		__ocml_frexp_f32(__x, (__attribute__((address_space(5))) int *)&__tmp);
266		*__nptr = __tmp;
267
268		return __r;
	260	return __builtin_frexpf(__x, __nptr);
269	261	}
270	262
271	263	__DEVICE__
...	...	@@ -275,13 +267,13 @@ __DEVICE__
275	267	int ilogbf(float __x) { return __ocml_ilogb_f32(__x); }
276	268
277	269	__DEVICE__
278		__RETURN_TYPE __finitef(float __x) { return __ocml_isfinite_f32(__x); }
	270	__RETURN_TYPE __finitef(float __x) { return __builtin_isfinite(__x); }
279	271
280	272	__DEVICE__
281		__RETURN_TYPE __isinff(float __x) { return __ocml_isinf_f32(__x); }
	273	__RETURN_TYPE __isinff(float __x) { return __builtin_isinf(__x); }
282	274
283	275	__DEVICE__
284		__RETURN_TYPE __isnanf(float __x) { return __ocml_isnan_f32(__x); }
	276	__RETURN_TYPE __isnanf(float __x) { return __builtin_isnan(__x); }
285	277
286	278	__DEVICE__
287	279	float j0f(float __x) { return __ocml_j0_f32(__x); }
...	...	@@ -311,37 +303,37 @@ float jnf(int __n, float __x) { // TODO: we could use Ahmes multiplication
311	303	}
312	304
313	305	__DEVICE__
314		float ldexpf(float __x, int __e) { return __ocml_ldexp_f32(__x, __e); }
	306	float ldexpf(float __x, int __e) { return __builtin_amdgcn_ldexpf(__x, __e); }
315	307
316	308	__DEVICE__
317	309	float lgammaf(float __x) { return __ocml_lgamma_f32(__x); }
318	310
319	311	__DEVICE__
320		long long int llrintf(float __x) { return __ocml_rint_f32(__x); }
	312	long long int llrintf(float __x) { return __builtin_rintf(__x); }
321	313
322	314	__DEVICE__
323		long long int llroundf(float __x) { return __ocml_round_f32(__x); }
	315	long long int llroundf(float __x) { return __builtin_roundf(__x); }
324	316
325	317	__DEVICE__
326		float log10f(float __x) { return __ocml_log10_f32(__x); }
	318	float log10f(float __x) { return __builtin_log10f(__x); }
327	319
328	320	__DEVICE__
329	321	float log1pf(float __x) { return __ocml_log1p_f32(__x); }
330	322
331	323	__DEVICE__
332		float log2f(float __x) { return __ocml_log2_f32(__x); }
	324	float log2f(float __x) { return __builtin_log2f(__x); }
333	325
334	326	__DEVICE__
335	327	float logbf(float __x) { return __ocml_logb_f32(__x); }
336	328
337	329	__DEVICE__
338		float logf(float __x) { return __ocml_log_f32(__x); }
	330	float logf(float __x) { return __builtin_logf(__x); }
339	331
340	332	__DEVICE__
341		long int lrintf(float __x) { return __ocml_rint_f32(__x); }
	333	long int lrintf(float __x) { return __builtin_rintf(__x); }
342	334
343	335	__DEVICE__
344		long int lroundf(float __x) { return __ocml_round_f32(__x); }
	336	long int lroundf(float __x) { return __builtin_roundf(__x); }
345	337
346	338	__DEVICE__
347	339	float modff(float __x, float *__iptr) {
...	...	@@ -377,7 +369,7 @@ float nanf(const char *__tagp __attribute__((nonnull))) {
377	369	}
378	370
379	371	__DEVICE__
380		float nearbyintf(float __x) { return __ocml_nearbyint_f32(__x); }
	372	float nearbyintf(float __x) { return __builtin_nearbyintf(__x); }
381	373
382	374	__DEVICE__
383	375	float nextafterf(float __x, float __y) {
...	...	@@ -443,7 +435,7 @@ __DEVICE__
443	435	float rhypotf(float __x, float __y) { return __ocml_rhypot_f32(__x, __y); }
444	436
445	437	__DEVICE__
446		float rintf(float __x) { return __ocml_rint_f32(__x); }
	438	float rintf(float __x) { return __builtin_rintf(__x); }
447	439
448	440	__DEVICE__
449	441	float rnorm3df(float __x, float __y, float __z) {
...	...	@@ -468,22 +460,22 @@ float rnormf(int __dim,
468	460	}
469	461
470	462	__DEVICE__
471		float roundf(float __x) { return __ocml_round_f32(__x); }
	463	float roundf(float __x) { return __builtin_roundf(__x); }
472	464
473	465	__DEVICE__
474	466	float rsqrtf(float __x) { return __ocml_rsqrt_f32(__x); }
475	467
476	468	__DEVICE__
477	469	float scalblnf(float __x, long int __n) {
478		return (__n < INT_MAX) ? __ocml_scalbn_f32(__x, __n)
	470	return (__n < INT_MAX) ? __builtin_amdgcn_ldexpf(__x, __n)
479	471	: __ocml_scalb_f32(__x, __n);
480	472	}
481	473
482	474	__DEVICE__
483		float scalbnf(float __x, int __n) { return __ocml_scalbn_f32(__x, __n); }
	475	float scalbnf(float __x, int __n) { return __builtin_amdgcn_ldexpf(__x, __n); }
484	476
485	477	__DEVICE__
486		__RETURN_TYPE __signbitf(float __x) { return __ocml_signbit_f32(__x); }
	478	__RETURN_TYPE __signbitf(float __x) { return __builtin_signbitf(__x); }
487	479
488	480	__DEVICE__
489	481	void sincosf(float __x, float __sinptr, float __cosptr) {
...	...	@@ -529,7 +521,7 @@ __DEVICE__
529	521	float tgammaf(float __x) { return __ocml_tgamma_f32(__x); }
530	522
531	523	__DEVICE__
532		float truncf(float __x) { return __ocml_trunc_f32(__x); }
	524	float truncf(float __x) { return __builtin_truncf(__x); }
533	525
534	526	__DEVICE__
535	527	float y0f(float __x) { return __ocml_y0_f32(__x); }
...	...	@@ -621,7 +613,7 @@ float __fmaf_rz(float __x, float __y, float __z) {
621	613	#else
622	614	__DEVICE__
623	615	float __fmaf_rn(float __x, float __y, float __z) {
624		return __ocml_fma_f32(__x, __y, __z);
	616	return __builtin_fmaf(__x, __y, __z);
625	617	}
626	618	#endif
627	619
...	...	@@ -654,7 +646,7 @@ float __frcp_rn(float __x) { return 1.0f / __x; }
654	646	#endif
655	647
656	648	__DEVICE__
657		float __frsqrt_rn(float __x) { return __llvm_amdgcn_rsq_f32(__x); }
	649	float __frsqrt_rn(float __x) { return __builtin_amdgcn_rsqf(__x); }
658	650
659	651	#if defined OCML_BASIC_ROUNDED_OPERATIONS
660	652	__DEVICE__
...	...	@@ -739,11 +731,11 @@ __DEVICE__
739	731	double cbrt(double __x) { return __ocml_cbrt_f64(__x); }
740	732
741	733	__DEVICE__
742		double ceil(double __x) { return __ocml_ceil_f64(__x); }
	734	double ceil(double __x) { return __builtin_ceil(__x); }
743	735
744	736	__DEVICE__
745	737	double copysign(double __x, double __y) {
746		return __ocml_copysign_f64(__x, __y);
	738	return __builtin_copysign(__x, __y);
747	739	}
748	740
749	741	__DEVICE__
...	...	@@ -795,32 +787,25 @@ __DEVICE__
795	787	double fdim(double __x, double __y) { return __ocml_fdim_f64(__x, __y); }
796	788
797	789	__DEVICE__
798		double floor(double __x) { return __ocml_floor_f64(__x); }
	790	double floor(double __x) { return __builtin_floor(__x); }
799	791
800	792	__DEVICE__
801	793	double fma(double __x, double __y, double __z) {
802		return __ocml_fma_f64(__x, __y, __z);
	794	return __builtin_fma(__x, __y, __z);
803	795	}
804	796
805	797	__DEVICE__
806		double fmax(double __x, double __y) { return __ocml_fmax_f64(__x, __y); }
	798	double fmax(double __x, double __y) { return __builtin_fmax(__x, __y); }
807	799
808	800	__DEVICE__
809		double fmin(double __x, double __y) { return __ocml_fmin_f64(__x, __y); }
	801	double fmin(double __x, double __y) { return __builtin_fmin(__x, __y); }
810	802
811	803	__DEVICE__
812	804	double fmod(double __x, double __y) { return __ocml_fmod_f64(__x, __y); }
813	805
814	806	__DEVICE__
815	807	double frexp(double __x, int *__nptr) {
816		int __tmp;
817		#ifdef __OPENMP_AMDGCN__
818		#pragma omp allocate(__tmp) allocator(omp_thread_mem_alloc)
819		#endif
820		double __r =
821		__ocml_frexp_f64(__x, (__attribute__((address_space(5))) int *)&__tmp);
822		*__nptr = __tmp;
823		return __r;
	808	return __builtin_frexp(__x, __nptr);
824	809	}
825	810
826	811	__DEVICE__
...	...	@@ -830,13 +815,13 @@ __DEVICE__
830	815	int ilogb(double __x) { return __ocml_ilogb_f64(__x); }
831	816
832	817	__DEVICE__
833		__RETURN_TYPE __finite(double __x) { return __ocml_isfinite_f64(__x); }
	818	__RETURN_TYPE __finite(double __x) { return __builtin_isfinite(__x); }
834	819
835	820	__DEVICE__
836		__RETURN_TYPE __isinf(double __x) { return __ocml_isinf_f64(__x); }
	821	__RETURN_TYPE __isinf(double __x) { return __builtin_isinf(__x); }
837	822
838	823	__DEVICE__
839		__RETURN_TYPE __isnan(double __x) { return __ocml_isnan_f64(__x); }
	824	__RETURN_TYPE __isnan(double __x) { return __builtin_isnan(__x); }
840	825
841	826	__DEVICE__
842	827	double j0(double __x) { return __ocml_j0_f64(__x); }
...	...	@@ -866,16 +851,16 @@ double jn(int __n, double __x) { // TODO: we could use Ahmes multiplication
866	851	}
867	852
868	853	__DEVICE__
869		double ldexp(double __x, int __e) { return __ocml_ldexp_f64(__x, __e); }
	854	double ldexp(double __x, int __e) { return __builtin_amdgcn_ldexp(__x, __e); }
870	855
871	856	__DEVICE__
872	857	double lgamma(double __x) { return __ocml_lgamma_f64(__x); }
873	858
874	859	__DEVICE__
875		long long int llrint(double __x) { return __ocml_rint_f64(__x); }
	860	long long int llrint(double __x) { return __builtin_rint(__x); }
876	861
877	862	__DEVICE__
878		long long int llround(double __x) { return __ocml_round_f64(__x); }
	863	long long int llround(double __x) { return __builtin_round(__x); }
879	864
880	865	__DEVICE__
881	866	double log(double __x) { return __ocml_log_f64(__x); }
...	...	@@ -893,10 +878,10 @@ __DEVICE__
893	878	double logb(double __x) { return __ocml_logb_f64(__x); }
894	879
895	880	__DEVICE__
896		long int lrint(double __x) { return __ocml_rint_f64(__x); }
	881	long int lrint(double __x) { return __builtin_rint(__x); }
897	882
898	883	__DEVICE__
899		long int lround(double __x) { return __ocml_round_f64(__x); }
	884	long int lround(double __x) { return __builtin_round(__x); }
900	885
901	886	__DEVICE__
902	887	double modf(double __x, double *__iptr) {
...	...	@@ -940,7 +925,7 @@ double nan(const char *__tagp) {
940	925	}
941	926
942	927	__DEVICE__
943		double nearbyint(double __x) { return __ocml_nearbyint_f64(__x); }
	928	double nearbyint(double __x) { return __builtin_nearbyint(__x); }
944	929
945	930	__DEVICE__
946	931	double nextafter(double __x, double __y) {
...	...	@@ -1006,7 +991,7 @@ __DEVICE__
1006	991	double rhypot(double __x, double __y) { return __ocml_rhypot_f64(__x, __y); }
1007	992
1008	993	__DEVICE__
1009		double rint(double __x) { return __ocml_rint_f64(__x); }
	994	double rint(double __x) { return __builtin_rint(__x); }
1010	995
1011	996	__DEVICE__
1012	997	double rnorm(int __dim,
...	...	@@ -1031,21 +1016,21 @@ double rnorm4d(double __x, double __y, double __z, double __w) {
1031	1016	}
1032	1017
1033	1018	__DEVICE__
1034		double round(double __x) { return __ocml_round_f64(__x); }
	1019	double round(double __x) { return __builtin_round(__x); }
1035	1020
1036	1021	__DEVICE__
1037	1022	double rsqrt(double __x) { return __ocml_rsqrt_f64(__x); }
1038	1023
1039	1024	__DEVICE__
1040	1025	double scalbln(double __x, long int __n) {
1041		return (__n < INT_MAX) ? __ocml_scalbn_f64(__x, __n)
	1026	return (__n < INT_MAX) ? __builtin_amdgcn_ldexp(__x, __n)
1042	1027	: __ocml_scalb_f64(__x, __n);
1043	1028	}
1044	1029	__DEVICE__
1045		double scalbn(double __x, int __n) { return __ocml_scalbn_f64(__x, __n); }
	1030	double scalbn(double __x, int __n) { return __builtin_amdgcn_ldexp(__x, __n); }
1046	1031
1047	1032	__DEVICE__
1048		__RETURN_TYPE __signbit(double __x) { return __ocml_signbit_f64(__x); }
	1033	__RETURN_TYPE __signbit(double __x) { return __builtin_signbit(__x); }
1049	1034
1050	1035	__DEVICE__
1051	1036	double sin(double __x) { return __ocml_sin_f64(__x); }
...	...	@@ -1091,7 +1076,7 @@ __DEVICE__
1091	1076	double tgamma(double __x) { return __ocml_tgamma_f64(__x); }
1092	1077
1093	1078	__DEVICE__
1094		double trunc(double __x) { return __ocml_trunc_f64(__x); }
	1079	double trunc(double __x) { return __builtin_trunc(__x); }
1095	1080
1096	1081	__DEVICE__
1097	1082	double y0(double __x) { return __ocml_y0_f64(__x); }
...	...	@@ -1258,7 +1243,7 @@ double __fma_rz(double __x, double __y, double __z) {
1258	1243	#else
1259	1244	__DEVICE__
1260	1245	double __fma_rn(double __x, double __y, double __z) {
1261		return __ocml_fma_f64(__x, __y, __z);
	1246	return __builtin_fma(__x, __y, __z);
1262	1247	}
1263	1248	#endif
1264	1249	// END INTRINSICS
...	...	@@ -1290,16 +1275,16 @@ __DEVICE__ int max(int __arg1, int __arg2) {
1290	1275	}
1291	1276
1292	1277	__DEVICE__
1293		float max(float __x, float __y) { return fmaxf(__x, __y); }
	1278	float max(float __x, float __y) { return __builtin_fmaxf(__x, __y); }
1294	1279
1295	1280	__DEVICE__
1296		double max(double __x, double __y) { return fmax(__x, __y); }
	1281	double max(double __x, double __y) { return __builtin_fmax(__x, __y); }
1297	1282
1298	1283	__DEVICE__
1299		float min(float __x, float __y) { return fminf(__x, __y); }
	1284	float min(float __x, float __y) { return __builtin_fminf(__x, __y); }
1300	1285
1301	1286	__DEVICE__
1302		double min(double __x, double __y) { return fmin(__x, __y); }
	1287	double min(double __x, double __y) { return __builtin_fmin(__x, __y); }
1303	1288
1304	1289	#if !defined(__HIPCC_RTC__) && !defined(__OPENMP_AMDGCN__)
1305	1290	__host__ inline static int min(int __arg1, int __arg2) {

lib/include/__clang_hip_runtime_wrapper.h+13

...	...	@@ -80,12 +80,25 @@ extern "C" {
80	80	#if HIP_VERSION_MAJOR * 100 + HIP_VERSION_MINOR >= 405
81	81	extern "C" __device__ unsigned long long __ockl_dm_alloc(unsigned long long __size);
82	82	extern "C" __device__ void __ockl_dm_dealloc(unsigned long long __addr);
	83	#if __has_feature(address_sanitizer)
	84	extern "C" __device__ unsigned long long __asan_malloc_impl(unsigned long long __size, unsigned long long __pc);
	85	extern "C" __device__ void __asan_free_impl(unsigned long long __addr, unsigned long long __pc);
	86	__attribute__((noinline, weak)) __device__ void *malloc(__hip_size_t __size) {
	87	unsigned long long __pc = (unsigned long long)__builtin_return_address(0);
	88	return (void *)__asan_malloc_impl(__size, __pc);
	89	}
	90	__attribute__((noinline, weak)) __device__ void free(void *__ptr) {
	91	unsigned long long __pc = (unsigned long long)__builtin_return_address(0);
	92	__asan_free_impl((unsigned long long)__ptr, __pc);
	93	}
	94	#else
83	95	__attribute__((weak)) inline __device__ void *malloc(__hip_size_t __size) {
84	96	return (void *) __ockl_dm_alloc(__size);
85	97	}
86	98	__attribute__((weak)) inline __device__ void free(void *__ptr) {
87	99	__ockl_dm_dealloc((unsigned long long)__ptr);
88	100	}
	101	#endif // __has_feature(address_sanitizer)
89	102	#else // HIP version check
90	103	#if __HIP_ENABLE_DEVICE_MALLOC__
91	104	__device__ void *__hip_malloc(__hip_size_t __size);

lib/include/adxintrin.h+179-24

...	...	@@ -17,56 +17,211 @@
17	17	/* Define the default attributes for the functions in this file. */
18	18	#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
19	19
20		/* Intrinsics that are available only if __ADX__ defined */
21		static __inline unsigned char __attribute__((__always_inline__, __nodebug__, __target__("adx")))
22		_addcarryx_u32(unsigned char __cf, unsigned int __x, unsigned int __y,
23		unsigned int *__p)
24		{
	20	/* Use C++ inline semantics in C++, GNU inline for C mode. */
	21	#if defined(__cplusplus)
	22	#define __INLINE __inline
	23	#else
	24	#define __INLINE static __inline
	25	#endif
	26
	27	#if defined(__cplusplus)
	28	extern "C" {
	29	#endif
	30
	31	/* Intrinsics that are available only if __ADX__ is defined. */
	32
	33	/// Adds unsigned 32-bit integers \a __x and \a __y, plus 0 or 1 as indicated
	34	/// by the carry flag \a __cf. Stores the unsigned 32-bit sum in the memory
	35	/// at \a __p, and returns the 8-bit carry-out (carry flag).
	36	///
	37	/// \code{.operation}
	38	/// temp := (__cf == 0) ? 0 : 1
	39	/// Store32(__p, __x + __y + temp)
	40	/// result := CF
	41	/// \endcode
	42	///
	43	/// \headerfile <immintrin.h>
	44	///
	45	/// This intrinsic corresponds to the \c ADCX instruction.
	46	///
	47	/// \param __cf
	48	/// The 8-bit unsigned carry flag; any non-zero value indicates carry.
	49	/// \param __x
	50	/// A 32-bit unsigned addend.
	51	/// \param __y
	52	/// A 32-bit unsigned addend.
	53	/// \param __p
	54	/// Pointer to memory for storing the sum.
	55	/// \returns The 8-bit unsigned carry-out value.
	56	__INLINE unsigned char
	57	__attribute__((__always_inline__, __nodebug__, __target__("adx")))
	58	_addcarryx_u32(unsigned char __cf, unsigned int __x, unsigned int __y,
	59	unsigned int *__p) {
25	60	return __builtin_ia32_addcarryx_u32(__cf, __x, __y, __p);
26	61	}
27	62
28	63	#ifdef __x86_64__
29		static __inline unsigned char __attribute__((__always_inline__, __nodebug__, __target__("adx")))
30		_addcarryx_u64(unsigned char __cf, unsigned long long __x,
31		unsigned long long __y, unsigned long long *__p)
32		{
	64	/// Adds unsigned 64-bit integers \a __x and \a __y, plus 0 or 1 as indicated
	65	/// by the carry flag \a __cf. Stores the unsigned 64-bit sum in the memory
	66	/// at \a __p, and returns the 8-bit carry-out (carry flag).
	67	///
	68	/// \code{.operation}
	69	/// temp := (__cf == 0) ? 0 : 1
	70	/// Store64(__p, __x + __y + temp)
	71	/// result := CF
	72	/// \endcode
	73	///
	74	/// \headerfile <immintrin.h>
	75	///
	76	/// This intrinsic corresponds to the \c ADCX instruction.
	77	///
	78	/// \param __cf
	79	/// The 8-bit unsigned carry flag; any non-zero value indicates carry.
	80	/// \param __x
	81	/// A 64-bit unsigned addend.
	82	/// \param __y
	83	/// A 64-bit unsigned addend.
	84	/// \param __p
	85	/// Pointer to memory for storing the sum.
	86	/// \returns The 8-bit unsigned carry-out value.
	87	__INLINE unsigned char
	88	__attribute__((__always_inline__, __nodebug__, __target__("adx")))
	89	_addcarryx_u64(unsigned char __cf, unsigned long long __x,
	90	unsigned long long __y, unsigned long long *__p) {
33	91	return __builtin_ia32_addcarryx_u64(__cf, __x, __y, __p);
34	92	}
35	93	#endif
36	94
37		/* Intrinsics that are also available if __ADX__ undefined */
38		static __inline unsigned char __DEFAULT_FN_ATTRS
39		_addcarry_u32(unsigned char __cf, unsigned int __x, unsigned int __y,
40		unsigned int *__p)
41		{
	95	/* Intrinsics that are also available if __ADX__ is undefined. */
	96
	97	/// Adds unsigned 32-bit integers \a __x and \a __y, plus 0 or 1 as indicated
	98	/// by the carry flag \a __cf. Stores the unsigned 32-bit sum in the memory
	99	/// at \a __p, and returns the 8-bit carry-out (carry flag).
	100	///
	101	/// \code{.operation}
	102	/// temp := (__cf == 0) ? 0 : 1
	103	/// Store32(__p, __x + __y + temp)
	104	/// result := CF
	105	/// \endcode
	106	///
	107	/// \headerfile <immintrin.h>
	108	///
	109	/// This intrinsic corresponds to the \c ADC instruction.
	110	///
	111	/// \param __cf
	112	/// The 8-bit unsigned carry flag; any non-zero value indicates carry.
	113	/// \param __x
	114	/// A 32-bit unsigned addend.
	115	/// \param __y
	116	/// A 32-bit unsigned addend.
	117	/// \param __p
	118	/// Pointer to memory for storing the sum.
	119	/// \returns The 8-bit unsigned carry-out value.
	120	__INLINE unsigned char __DEFAULT_FN_ATTRS _addcarry_u32(unsigned char __cf,
	121	unsigned int __x,
	122	unsigned int __y,
	123	unsigned int *__p) {
42	124	return __builtin_ia32_addcarryx_u32(__cf, __x, __y, __p);
43	125	}
44	126
45	127	#ifdef __x86_64__
46		static __inline unsigned char __DEFAULT_FN_ATTRS
	128	/// Adds unsigned 64-bit integers \a __x and \a __y, plus 0 or 1 as indicated
	129	/// by the carry flag \a __cf. Stores the unsigned 64-bit sum in the memory
	130	/// at \a __p, and returns the 8-bit carry-out (carry flag).
	131	///
	132	/// \code{.operation}
	133	/// temp := (__cf == 0) ? 0 : 1
	134	/// Store64(__p, __x + __y + temp)
	135	/// result := CF
	136	/// \endcode
	137	///
	138	/// \headerfile <immintrin.h>
	139	///
	140	/// This intrinsic corresponds to the \c ADC instruction.
	141	///
	142	/// \param __cf
	143	/// The 8-bit unsigned carry flag; any non-zero value indicates carry.
	144	/// \param __x
	145	/// A 64-bit unsigned addend.
	146	/// \param __y
	147	/// A 64-bit unsigned addend.
	148	/// \param __p
	149	/// Pointer to memory for storing the sum.
	150	/// \returns The 8-bit unsigned carry-out value.
	151	__INLINE unsigned char __DEFAULT_FN_ATTRS
47	152	_addcarry_u64(unsigned char __cf, unsigned long long __x,
48		unsigned long long __y, unsigned long long *__p)
49		{
	153	unsigned long long __y, unsigned long long *__p) {
50	154	return __builtin_ia32_addcarryx_u64(__cf, __x, __y, __p);
51	155	}
52	156	#endif
53	157
54		static __inline unsigned char __DEFAULT_FN_ATTRS
55		_subborrow_u32(unsigned char __cf, unsigned int __x, unsigned int __y,
56		unsigned int *__p)
57		{
	158	/// Adds unsigned 32-bit integer \a __y to 0 or 1 as indicated by the carry
	159	/// flag \a __cf, and subtracts the result from unsigned 32-bit integer
	160	/// \a __x. Stores the unsigned 32-bit difference in the memory at \a __p,
	161	/// and returns the 8-bit carry-out (carry or overflow flag).
	162	///
	163	/// \code{.operation}
	164	/// temp := (__cf == 0) ? 0 : 1
	165	/// Store32(__p, __x - (__y + temp))
	166	/// result := CF
	167	/// \endcode
	168	///
	169	/// \headerfile <immintrin.h>
	170	///
	171	/// This intrinsic corresponds to the \c SBB instruction.
	172	///
	173	/// \param __cf
	174	/// The 8-bit unsigned carry flag; any non-zero value indicates carry.
	175	/// \param __x
	176	/// The 32-bit unsigned minuend.
	177	/// \param __y
	178	/// The 32-bit unsigned subtrahend.
	179	/// \param __p
	180	/// Pointer to memory for storing the difference.
	181	/// \returns The 8-bit unsigned carry-out value.
	182	__INLINE unsigned char __DEFAULT_FN_ATTRS _subborrow_u32(unsigned char __cf,
	183	unsigned int __x,
	184	unsigned int __y,
	185	unsigned int *__p) {
58	186	return __builtin_ia32_subborrow_u32(__cf, __x, __y, __p);
59	187	}
60	188
61	189	#ifdef __x86_64__
62		static __inline unsigned char __DEFAULT_FN_ATTRS
	190	/// Adds unsigned 64-bit integer \a __y to 0 or 1 as indicated by the carry
	191	/// flag \a __cf, and subtracts the result from unsigned 64-bit integer
	192	/// \a __x. Stores the unsigned 64-bit difference in the memory at \a __p,
	193	/// and returns the 8-bit carry-out (carry or overflow flag).
	194	///
	195	/// \code{.operation}
	196	/// temp := (__cf == 0) ? 0 : 1
	197	/// Store64(__p, __x - (__y + temp))
	198	/// result := CF
	199	/// \endcode
	200	///
	201	/// \headerfile <immintrin.h>
	202	///
	203	/// This intrinsic corresponds to the \c ADC instruction.
	204	///
	205	/// \param __cf
	206	/// The 8-bit unsigned carry flag; any non-zero value indicates carry.
	207	/// \param __x
	208	/// The 64-bit unsigned minuend.
	209	/// \param __y
	210	/// The 64-bit unsigned subtrahend.
	211	/// \param __p
	212	/// Pointer to memory for storing the difference.
	213	/// \returns The 8-bit unsigned carry-out value.
	214	__INLINE unsigned char __DEFAULT_FN_ATTRS
63	215	_subborrow_u64(unsigned char __cf, unsigned long long __x,
64		unsigned long long __y, unsigned long long *__p)
65		{
	216	unsigned long long __y, unsigned long long *__p) {
66	217	return __builtin_ia32_subborrow_u64(__cf, __x, __y, __p);
67	218	}
68	219	#endif
69	220
	221	#if defined(__cplusplus)
	222	}
	223	#endif
	224
70	225	#undef __DEFAULT_FN_ATTRS
71	226
72	227	#endif /* __ADXINTRIN_H */

lib/include/altivec.h+136-124

...	...	@@ -3202,71 +3202,79 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
3202	3202	// the XL-compatible signatures are used for those functions.
3203	3203	#ifdef __XL_COMPAT_ALTIVEC__
3204	3204	#define vec_ctf(__a, __b) \
3205		_Generic( \
3206		(__a), vector int \
3207		: (vector float)__builtin_altivec_vcfsx((vector int)(__a), (__b)), \
3208		vector unsigned int \
3209		: (vector float)__builtin_altivec_vcfux((vector unsigned int)(__a), \
3210		(__b)), \
3211		vector unsigned long long \
3212		: (vector float)(__builtin_vsx_xvcvuxdsp( \
3213		(vector unsigned long long)(__a)) * \
3214		(vector float)(vector unsigned)((0x7f - (__b)) << 23)), \
3215		vector signed long long \
3216		: (vector float)(__builtin_vsx_xvcvsxdsp( \
3217		(vector signed long long)(__a)) * \
3218		(vector float)(vector unsigned)((0x7f - (__b)) << 23)))
	3205	_Generic((__a), \
	3206	vector int: (vector float)__builtin_altivec_vcfsx((vector int)(__a), \
	3207	((__b)&0x1F)), \
	3208	vector unsigned int: (vector float)__builtin_altivec_vcfux( \
	3209	(vector unsigned int)(__a), ((__b)&0x1F)), \
	3210	vector unsigned long long: ( \
	3211	vector float)(__builtin_vsx_xvcvuxdsp( \
	3212	(vector unsigned long long)(__a)) * \
	3213	(vector float)(vector unsigned)((0x7f - \
	3214	((__b)&0x1F)) \
	3215	<< 23)), \
	3216	vector signed long long: ( \
	3217	vector float)(__builtin_vsx_xvcvsxdsp( \
	3218	(vector signed long long)(__a)) * \
	3219	(vector float)(vector unsigned)((0x7f - \
	3220	((__b)&0x1F)) \
	3221	<< 23)))
3219	3222	#else // __XL_COMPAT_ALTIVEC__
3220		#define vec_ctf(__a, __b) \
3221		_Generic( \
3222		(__a), vector int \
3223		: (vector float)__builtin_altivec_vcfsx((vector int)(__a), (__b)), \
3224		vector unsigned int \
3225		: (vector float)__builtin_altivec_vcfux((vector unsigned int)(__a), \
3226		(__b)), \
3227		vector unsigned long long \
3228		: (vector float)(__builtin_convertvector( \
3229		(vector unsigned long long)(__a), vector double) * \
3230		(vector double)(vector unsigned long long)((0x3ffULL - \
3231		(__b)) \
3232		<< 52)), \
3233		vector signed long long \
3234		: (vector float)(__builtin_convertvector((vector signed long long)(__a), \
3235		vector double) * \
3236		(vector double)(vector unsigned long long)((0x3ffULL - \
3237		(__b)) \
3238		<< 52)))
	3223	#define vec_ctf(__a, __b) \
	3224	_Generic( \
	3225	(__a), \
	3226	vector int: (vector float)__builtin_altivec_vcfsx((vector int)(__a), \
	3227	((__b)&0x1F)), \
	3228	vector unsigned int: (vector float)__builtin_altivec_vcfux( \
	3229	(vector unsigned int)(__a), ((__b)&0x1F)), \
	3230	vector unsigned long long: ( \
	3231	vector float)(__builtin_convertvector( \
	3232	(vector unsigned long long)(__a), vector double) * \
	3233	(vector double)(vector unsigned long long)((0x3ffULL - \
	3234	((__b)&0x1F)) \
	3235	<< 52)), \
	3236	vector signed long long: ( \
	3237	vector float)(__builtin_convertvector( \
	3238	(vector signed long long)(__a), vector double) * \
	3239	(vector double)(vector unsigned long long)((0x3ffULL - \
	3240	((__b)&0x1F)) \
	3241	<< 52)))
3239	3242	#endif // __XL_COMPAT_ALTIVEC__
3240	3243	#else
3241	3244	#define vec_ctf(__a, __b) \
3242		_Generic((__a), vector int \
3243		: (vector float)__builtin_altivec_vcfsx((vector int)(__a), (__b)), \
3244		vector unsigned int \
3245		: (vector float)__builtin_altivec_vcfux((vector unsigned int)(__a), \
3246		(__b)))
	3245	_Generic((__a), \
	3246	vector int: (vector float)__builtin_altivec_vcfsx((vector int)(__a), \
	3247	((__b)&0x1F)), \
	3248	vector unsigned int: (vector float)__builtin_altivec_vcfux( \
	3249	(vector unsigned int)(__a), ((__b)&0x1F)))
3247	3250	#endif
3248	3251
3249	3252	/* vec_ctd */
3250	3253	#ifdef __VSX__
3251	3254	#define vec_ctd(__a, __b) \
3252		_Generic((__a), vector signed int \
3253		: (vec_doublee((vector signed int)(__a)) * \
3254		(vector double)(vector unsigned long long)((0x3ffULL - (__b)) \
3255		<< 52)), \
3256		vector unsigned int \
3257		: (vec_doublee((vector unsigned int)(__a)) * \
3258		(vector double)(vector unsigned long long)((0x3ffULL - (__b)) \
3259		<< 52)), \
3260		vector unsigned long long \
3261		: (__builtin_convertvector((vector unsigned long long)(__a), \
3262		vector double) * \
3263		(vector double)(vector unsigned long long)((0x3ffULL - (__b)) \
3264		<< 52)), \
3265		vector signed long long \
3266		: (__builtin_convertvector((vector signed long long)(__a), \
3267		vector double) * \
3268		(vector double)(vector unsigned long long)((0x3ffULL - (__b)) \
3269		<< 52)))
	3255	_Generic((__a), \
	3256	vector signed int: ( \
	3257	vec_doublee((vector signed int)(__a)) * \
	3258	(vector double)(vector unsigned long long)((0x3ffULL - \
	3259	((__b)&0x1F)) \
	3260	<< 52)), \
	3261	vector unsigned int: ( \
	3262	vec_doublee((vector unsigned int)(__a)) * \
	3263	(vector double)(vector unsigned long long)((0x3ffULL - \
	3264	((__b)&0x1F)) \
	3265	<< 52)), \
	3266	vector unsigned long long: ( \
	3267	__builtin_convertvector((vector unsigned long long)(__a), \
	3268	vector double) * \
	3269	(vector double)(vector unsigned long long)((0x3ffULL - \
	3270	((__b)&0x1F)) \
	3271	<< 52)), \
	3272	vector signed long long: ( \
	3273	__builtin_convertvector((vector signed long long)(__a), \
	3274	vector double) * \
	3275	(vector double)(vector unsigned long long)((0x3ffULL - \
	3276	((__b)&0x1F)) \
	3277	<< 52)))
3270	3278	#endif // __VSX__
3271	3279
3272	3280	/* vec_vcfsx */
...	...	@@ -3281,27 +3289,27 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
3281	3289	#ifdef __VSX__
3282	3290	#ifdef __XL_COMPAT_ALTIVEC__
3283	3291	#define vec_cts(__a, __b) \
3284		_Generic((__a), vector float \
3285		: (vector signed int)__builtin_altivec_vctsxs((vector float)(__a), \
3286		(__b)), \
3287		vector double \
3288		: __extension__({ \
	3292	_Generic((__a), \
	3293	vector float: (vector signed int)__builtin_altivec_vctsxs( \
	3294	(vector float)(__a), ((__b)&0x1F)), \
	3295	vector double: __extension__({ \
3289	3296	vector double __ret = \
3290	3297	(vector double)(__a) * \
3291		(vector double)(vector unsigned long long)((0x3ffULL + (__b)) \
	3298	(vector double)(vector unsigned long long)((0x3ffULL + \
	3299	((__b)&0x1F)) \
3292	3300	<< 52); \
3293	3301	(vector signed long long)__builtin_vsx_xvcvdpsxws(__ret); \
3294	3302	}))
3295	3303	#else // __XL_COMPAT_ALTIVEC__
3296	3304	#define vec_cts(__a, __b) \
3297		_Generic((__a), vector float \
3298		: (vector signed int)__builtin_altivec_vctsxs((vector float)(__a), \
3299		(__b)), \
3300		vector double \
3301		: __extension__({ \
	3305	_Generic((__a), \
	3306	vector float: (vector signed int)__builtin_altivec_vctsxs( \
	3307	(vector float)(__a), ((__b)&0x1F)), \
	3308	vector double: __extension__({ \
3302	3309	vector double __ret = \
3303	3310	(vector double)(__a) * \
3304		(vector double)(vector unsigned long long)((0x3ffULL + (__b)) \
	3311	(vector double)(vector unsigned long long)((0x3ffULL + \
	3312	((__b)&0x1F)) \
3305	3313	<< 52); \
3306	3314	(vector signed long long)__builtin_convertvector( \
3307	3315	__ret, vector signed long long); \
...	...	@@ -3320,27 +3328,27 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
3320	3328	#ifdef __VSX__
3321	3329	#ifdef __XL_COMPAT_ALTIVEC__
3322	3330	#define vec_ctu(__a, __b) \
3323		_Generic((__a), vector float \
3324		: (vector unsigned int)__builtin_altivec_vctuxs( \
3325		(vector float)(__a), (__b)), \
3326		vector double \
3327		: __extension__({ \
	3331	_Generic((__a), \
	3332	vector float: (vector unsigned int)__builtin_altivec_vctuxs( \
	3333	(vector float)(__a), ((__b)&0x1F)), \
	3334	vector double: __extension__({ \
3328	3335	vector double __ret = \
3329	3336	(vector double)(__a) * \
3330		(vector double)(vector unsigned long long)((0x3ffULL + __b) \
	3337	(vector double)(vector unsigned long long)((0x3ffULL + \
	3338	((__b)&0x1F)) \
3331	3339	<< 52); \
3332	3340	(vector unsigned long long)__builtin_vsx_xvcvdpuxws(__ret); \
3333	3341	}))
3334	3342	#else // __XL_COMPAT_ALTIVEC__
3335	3343	#define vec_ctu(__a, __b) \
3336		_Generic((__a), vector float \
3337		: (vector unsigned int)__builtin_altivec_vctuxs( \
3338		(vector float)(__a), (__b)), \
3339		vector double \
3340		: __extension__({ \
	3344	_Generic((__a), \
	3345	vector float: (vector unsigned int)__builtin_altivec_vctuxs( \
	3346	(vector float)(__a), ((__b)&0x1F)), \
	3347	vector double: __extension__({ \
3341	3348	vector double __ret = \
3342	3349	(vector double)(__a) * \
3343		(vector double)(vector unsigned long long)((0x3ffULL + __b) \
	3350	(vector double)(vector unsigned long long)((0x3ffULL + \
	3351	((__b)&0x1F)) \
3344	3352	<< 52); \
3345	3353	(vector unsigned long long)__builtin_convertvector( \
3346	3354	__ret, vector unsigned long long); \
...	...	@@ -3355,60 +3363,62 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
3355	3363
3356	3364	#ifdef __VSX__
3357	3365	#define vec_ctsl(__a, __b) \
3358		_Generic((__a), vector float \
3359		: __extension__({ \
3360		vector float __ret = \
3361		(vector float)(__a) * \
3362		(vector float)(vector unsigned)((0x7f + (__b)) << 23); \
3363		__builtin_vsx_xvcvspsxds( \
3364		__builtin_vsx_xxsldwi(__ret, __ret, 1)); \
3365		}), \
3366		vector double \
3367		: __extension__({ \
3368		vector double __ret = \
3369		(vector double)(__a) * \
3370		(vector double)(vector unsigned long long)((0x3ffULL + __b) \
3371		<< 52); \
3372		__builtin_convertvector(__ret, vector signed long long); \
3373		}))
	3366	_Generic( \
	3367	(__a), vector float \
	3368	: __extension__({ \
	3369	vector float __ret = \
	3370	(vector float)(__a) * \
	3371	(vector float)(vector unsigned)((0x7f + ((__b)&0x1F)) << 23); \
	3372	__builtin_vsx_xvcvspsxds(__builtin_vsx_xxsldwi(__ret, __ret, 1)); \
	3373	}), \
	3374	vector double \
	3375	: __extension__({ \
	3376	vector double __ret = \
	3377	(vector double)(__a) * \
	3378	(vector double)(vector unsigned long long)((0x3ffULL + \
	3379	((__b)&0x1F)) \
	3380	<< 52); \
	3381	__builtin_convertvector(__ret, vector signed long long); \
	3382	}))
3374	3383
3375	3384	/* vec_ctul */
3376	3385
3377	3386	#define vec_ctul(__a, __b) \
3378		_Generic((__a), vector float \
3379		: __extension__({ \
3380		vector float __ret = \
3381		(vector float)(__a) * \
3382		(vector float)(vector unsigned)((0x7f + (__b)) << 23); \
3383		__builtin_vsx_xvcvspuxds( \
3384		__builtin_vsx_xxsldwi(__ret, __ret, 1)); \
3385		}), \
3386		vector double \
3387		: __extension__({ \
3388		vector double __ret = \
3389		(vector double)(__a) * \
3390		(vector double)(vector unsigned long long)((0x3ffULL + __b) \
3391		<< 52); \
3392		__builtin_convertvector(__ret, vector unsigned long long); \
3393		}))
	3387	_Generic( \
	3388	(__a), vector float \
	3389	: __extension__({ \
	3390	vector float __ret = \
	3391	(vector float)(__a) * \
	3392	(vector float)(vector unsigned)((0x7f + ((__b)&0x1F)) << 23); \
	3393	__builtin_vsx_xvcvspuxds(__builtin_vsx_xxsldwi(__ret, __ret, 1)); \
	3394	}), \
	3395	vector double \
	3396	: __extension__({ \
	3397	vector double __ret = \
	3398	(vector double)(__a) * \
	3399	(vector double)(vector unsigned long long)((0x3ffULL + \
	3400	((__b)&0x1F)) \
	3401	<< 52); \
	3402	__builtin_convertvector(__ret, vector unsigned long long); \
	3403	}))
3394	3404	#endif
3395	3405	#else // __LITTLE_ENDIAN__
3396	3406	/* vec_ctsl */
3397	3407
3398	3408	#ifdef __VSX__
3399	3409	#define vec_ctsl(__a, __b) \
3400		_Generic((__a), vector float \
3401		: __extension__({ \
3402		vector float __ret = \
3403		(vector float)(__a) * \
3404		(vector float)(vector unsigned)((0x7f + (__b)) << 23); \
3405		__builtin_vsx_xvcvspsxds(__ret); \
3406		}), \
3407		vector double \
3408		: __extension__({ \
	3410	_Generic((__a), \
	3411	vector float: __extension__({ \
	3412	vector float __ret = \
	3413	(vector float)(__a) * \
	3414	(vector float)(vector unsigned)((0x7f + ((__b)&0x1F)) << 23); \
	3415	__builtin_vsx_xvcvspsxds(__ret); \
	3416	}), \
	3417	vector double: __extension__({ \
3409	3418	vector double __ret = \
3410	3419	(vector double)(__a) * \
3411		(vector double)(vector unsigned long long)((0x3ffULL + __b) \
	3420	(vector double)(vector unsigned long long)((0x3ffULL + \
	3421	((__b)&0x1F)) \
3412	3422	<< 52); \
3413	3423	__builtin_convertvector(__ret, vector signed long long); \
3414	3424	}))
...	...	@@ -3420,14 +3430,16 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
3420	3430	: __extension__({ \
3421	3431	vector float __ret = \
3422	3432	(vector float)(__a) * \
3423		(vector float)(vector unsigned)((0x7f + (__b)) << 23); \
	3433	(vector float)(vector unsigned)((0x7f + ((__b)&0x1F)) \
	3434	<< 23); \
3424	3435	__builtin_vsx_xvcvspuxds(__ret); \
3425	3436	}), \
3426	3437	vector double \
3427	3438	: __extension__({ \
3428	3439	vector double __ret = \
3429	3440	(vector double)(__a) * \
3430		(vector double)(vector unsigned long long)((0x3ffULL + __b) \
	3441	(vector double)(vector unsigned long long)((0x3ffULL + \
	3442	((__b)&0x1F)) \
3431	3443	<< 52); \
3432	3444	__builtin_convertvector(__ret, vector unsigned long long); \
3433	3445	}))

lib/include/amxcomplexintrin.h created+169

...	...	@@ -0,0 +1,169 @@
	1	/===--------- amxcomplexintrin.h - AMXCOMPLEX intrinsics -- C++ -*---------===
	2	*
	3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	* See https://llvm.org/LICENSE.txt for license information.
	5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	*
	7	*===------------------------------------------------------------------------===
	8	*/
	9
	10	#ifndef __IMMINTRIN_H
	11	#error "Never use <amxcomplexintrin.h> directly; include <immintrin.h> instead."
	12	#endif // __IMMINTRIN_H
	13
	14	#ifndef __AMX_COMPLEXINTRIN_H
	15	#define __AMX_COMPLEXINTRIN_H
	16	#ifdef __x86_64__
	17
	18	#define __DEFAULT_FN_ATTRS_COMPLEX \
	19	__attribute__((__always_inline__, __nodebug__, __target__("amx-complex")))
	20
	21	/// Perform matrix multiplication of two tiles containing complex elements and
	22	/// accumulate the results into a packed single precision tile. Each dword
	23	/// element in input tiles \a a and \a b is interpreted as a complex number
	24	/// with FP16 real part and FP16 imaginary part.
	25	/// Calculates the imaginary part of the result. For each possible combination
	26	/// of (row of \a a, column of \a b), it performs a set of multiplication
	27	/// and accumulations on all corresponding complex numbers (one from \a a
	28	/// and one from \a b). The imaginary part of the \a a element is multiplied
	29	/// with the real part of the corresponding \a b element, and the real part
	30	/// of the \a a element is multiplied with the imaginary part of the
	31	/// corresponding \a b elements. The two accumulated results are added, and
	32	/// then accumulated into the corresponding row and column of \a dst.
	33	///
	34	/// \headerfile <x86intrin.h>
	35	///
	36	/// \code
	37	/// void _tile_cmmimfp16ps(__tile dst, __tile a, __tile b);
	38	/// \endcode
	39	///
	40	/// \code{.operation}
	41	/// FOR m := 0 TO dst.rows - 1
	42	/// tmp := dst.row[m]
	43	/// FOR k := 0 TO (a.colsb / 4) - 1
	44	/// FOR n := 0 TO (dst.colsb / 4) - 1
	45	/// tmp.fp32[n] += FP32(a.row[m].fp16[2k+0]) FP32(b.row[k].fp16[2*n+1])
	46	/// tmp.fp32[n] += FP32(a.row[m].fp16[2k+1]) FP32(b.row[k].fp16[2*n+0])
	47	/// ENDFOR
	48	/// ENDFOR
	49	/// write_row_and_zero(dst, m, tmp, dst.colsb)
	50	/// ENDFOR
	51	/// zero_upper_rows(dst, dst.rows)
	52	/// zero_tileconfig_start()
	53	/// \endcode
	54	///
	55	/// This intrinsic corresponds to the \c TCMMIMFP16PS instruction.
	56	///
	57	/// \param dst
	58	/// The destination tile. Max size is 1024 Bytes.
	59	/// \param a
	60	/// The 1st source tile. Max size is 1024 Bytes.
	61	/// \param b
	62	/// The 2nd source tile. Max size is 1024 Bytes.
	63	#define _tile_cmmimfp16ps(dst, a, b) __builtin_ia32_tcmmimfp16ps(dst, a, b)
	64
	65	/// Perform matrix multiplication of two tiles containing complex elements and
	66	/// accumulate the results into a packed single precision tile. Each dword
	67	/// element in input tiles \a a and \a b is interpreted as a complex number
	68	/// with FP16 real part and FP16 imaginary part.
	69	/// Calculates the real part of the result. For each possible combination
	70	/// of (row of \a a, column of \a b), it performs a set of multiplication
	71	/// and accumulations on all corresponding complex numbers (one from \a a
	72	/// and one from \a b). The real part of the \a a element is multiplied
	73	/// with the real part of the corresponding \a b element, and the negated
	74	/// imaginary part of the \a a element is multiplied with the imaginary
	75	/// part of the corresponding \a b elements. The two accumulated results
	76	/// are added, and then accumulated into the corresponding row and column
	77	/// of \a dst.
	78	///
	79	/// \headerfile <x86intrin.h>
	80	///
	81	/// \code
	82	/// void _tile_cmmrlfp16ps(__tile dst, __tile a, __tile b);
	83	/// \endcode
	84	///
	85	/// \code{.operation}
	86	/// FOR m := 0 TO dst.rows - 1
	87	/// tmp := dst.row[m]
	88	/// FOR k := 0 TO (a.colsb / 4) - 1
	89	/// FOR n := 0 TO (dst.colsb / 4) - 1
	90	/// tmp.fp32[n] += FP32(a.row[m].fp16[2k+0]) FP32(b.row[k].fp16[2*n+0])
	91	/// tmp.fp32[n] += FP32(-a.row[m].fp16[2k+1]) FP32(b.row[k].fp16[2*n+1])
	92	/// ENDFOR
	93	/// ENDFOR
	94	/// write_row_and_zero(dst, m, tmp, dst.colsb)
	95	/// ENDFOR
	96	/// zero_upper_rows(dst, dst.rows)
	97	/// zero_tileconfig_start()
	98	/// \endcode
	99	///
	100	/// This intrinsic corresponds to the \c TCMMIMFP16PS instruction.
	101	///
	102	/// \param dst
	103	/// The destination tile. Max size is 1024 Bytes.
	104	/// \param a
	105	/// The 1st source tile. Max size is 1024 Bytes.
	106	/// \param b
	107	/// The 2nd source tile. Max size is 1024 Bytes.
	108	#define _tile_cmmrlfp16ps(dst, a, b) __builtin_ia32_tcmmrlfp16ps(dst, a, b)
	109
	110	static __inline__ _tile1024i __DEFAULT_FN_ATTRS_COMPLEX
	111	_tile_cmmimfp16ps_internal(unsigned short m, unsigned short n, unsigned short k,
	112	_tile1024i dst, _tile1024i src1, _tile1024i src2) {
	113	return __builtin_ia32_tcmmimfp16ps_internal(m, n, k, dst, src1, src2);
	114	}
	115
	116	static __inline__ _tile1024i __DEFAULT_FN_ATTRS_COMPLEX
	117	_tile_cmmrlfp16ps_internal(unsigned short m, unsigned short n, unsigned short k,
	118	_tile1024i dst, _tile1024i src1, _tile1024i src2) {
	119	return __builtin_ia32_tcmmrlfp16ps_internal(m, n, k, dst, src1, src2);
	120	}
	121
	122	/// Perform matrix multiplication of two tiles containing complex elements and
	123	/// accumulate the results into a packed single precision tile. Each dword
	124	/// element in input tiles src0 and src1 is interpreted as a complex number with
	125	/// FP16 real part and FP16 imaginary part.
	126	/// This function calculates the imaginary part of the result.
	127	///
	128	/// \headerfile <immintrin.h>
	129	///
	130	/// This intrinsic corresponds to the <c> TCMMIMFP16PS </c> instruction.
	131	///
	132	/// \param dst
	133	/// The destination tile. Max size is 1024 Bytes.
	134	/// \param src0
	135	/// The 1st source tile. Max size is 1024 Bytes.
	136	/// \param src1
	137	/// The 2nd source tile. Max size is 1024 Bytes.
	138	__DEFAULT_FN_ATTRS_COMPLEX
	139	static void __tile_cmmimfp16ps(__tile1024i *dst, __tile1024i src0,
	140	__tile1024i src1) {
	141	dst->tile = _tile_cmmimfp16ps_internal(src0.row, src1.col, src0.col,
	142	dst->tile, src0.tile, src1.tile);
	143	}
	144
	145	/// Perform matrix multiplication of two tiles containing complex elements and
	146	/// accumulate the results into a packed single precision tile. Each dword
	147	/// element in input tiles src0 and src1 is interpreted as a complex number with
	148	/// FP16 real part and FP16 imaginary part.
	149	/// This function calculates the real part of the result.
	150	///
	151	/// \headerfile <immintrin.h>
	152	///
	153	/// This intrinsic corresponds to the <c> TCMMRLFP16PS </c> instruction.
	154	///
	155	/// \param dst
	156	/// The destination tile. Max size is 1024 Bytes.
	157	/// \param src0
	158	/// The 1st source tile. Max size is 1024 Bytes.
	159	/// \param src1
	160	/// The 2nd source tile. Max size is 1024 Bytes.
	161	__DEFAULT_FN_ATTRS_COMPLEX
	162	static void __tile_cmmrlfp16ps(__tile1024i *dst, __tile1024i src0,
	163	__tile1024i src1) {
	164	dst->tile = _tile_cmmrlfp16ps_internal(src0.row, src1.col, src0.col,
	165	dst->tile, src0.tile, src1.tile);
	166	}
	167
	168	#endif // __x86_64__
	169	#endif // __AMX_COMPLEXINTRIN_H

lib/include/arm_acle.h+13-9

...	...	@@ -138,28 +138,32 @@ __rorl(unsigned long __x, uint32_t __y) {
138	138
139	139
140	140	/* CLZ */
141		static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
	141	static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
142	142	__clz(uint32_t __t) {
143		return (uint32_t)__builtin_clz(__t);
	143	return __builtin_arm_clz(__t);
144	144	}
145	145
146		static __inline__ unsigned long __attribute__((__always_inline__, __nodebug__))
	146	static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
147	147	__clzl(unsigned long __t) {
148		return (unsigned long)__builtin_clzl(__t);
	148	#if __SIZEOF_LONG__ == 4
	149	return __builtin_arm_clz(__t);
	150	#else
	151	return __builtin_arm_clz64(__t);
	152	#endif
149	153	}
150	154
151		static __inline__ uint64_t __attribute__((__always_inline__, __nodebug__))
	155	static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
152	156	__clzll(uint64_t __t) {
153		return (uint64_t)__builtin_clzll(__t);
	157	return __builtin_arm_clz64(__t);
154	158	}
155	159
156	160	/* CLS */
157		static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
	161	static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
158	162	__cls(uint32_t __t) {
159	163	return __builtin_arm_cls(__t);
160	164	}
161	165
162		static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
	166	static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
163	167	__clsl(unsigned long __t) {
164	168	#if __SIZEOF_LONG__ == 4
165	169	return __builtin_arm_cls(__t);
...	...	@@ -168,7 +172,7 @@ __clsl(unsigned long __t) {
168	172	#endif
169	173	}
170	174
171		static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
	175	static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
172	176	__clsll(uint64_t __t) {
173	177	return __builtin_arm_cls64(__t);
174	178	}

lib/include/arm_neon.h+164-1

...	...	@@ -35,7 +35,6 @@
35	35	#include <stdint.h>
36	36
37	37	#include <arm_bf16.h>
38		typedef __bf16 bfloat16_t;
39	38	typedef float float32_t;
40	39	typedef __fp16 float16_t;
41	40	#ifdef __aarch64__
...	...	@@ -64938,6 +64937,170 @@ int8x16_t __reint_786 = __rev2_786; \
64938	64937	})
64939	64938	#endif
64940	64939
	64940	#define vldap1_lane_p64(__p0, __p1, __p2) __extension__ ({ \
	64941	poly64x1_t __ret; \
	64942	poly64x1_t __s1 = __p1; \
	64943	__ret = (poly64x1_t) __builtin_neon_vldap1_lane_p64(__p0, (int8x8_t)__s1, __p2, 6); \
	64944	__ret; \
	64945	})
	64946	#ifdef __LITTLE_ENDIAN__
	64947	#define vldap1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \
	64948	poly64x2_t __ret; \
	64949	poly64x2_t __s1 = __p1; \
	64950	__ret = (poly64x2_t) __builtin_neon_vldap1q_lane_p64(__p0, (int8x16_t)__s1, __p2, 38); \
	64951	__ret; \
	64952	})
	64953	#else
	64954	#define vldap1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \
	64955	poly64x2_t __ret; \
	64956	poly64x2_t __s1 = __p1; \
	64957	poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
	64958	__ret = (poly64x2_t) __builtin_neon_vldap1q_lane_p64(__p0, (int8x16_t)__rev1, __p2, 38); \
	64959	__ret = __builtin_shufflevector(__ret, __ret, 1, 0); \
	64960	__ret; \
	64961	})
	64962	#endif
	64963
	64964	#ifdef __LITTLE_ENDIAN__
	64965	#define vldap1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \
	64966	uint64x2_t __ret; \
	64967	uint64x2_t __s1 = __p1; \
	64968	__ret = (uint64x2_t) __builtin_neon_vldap1q_lane_u64(__p0, (int8x16_t)__s1, __p2, 51); \
	64969	__ret; \
	64970	})
	64971	#else
	64972	#define vldap1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \
	64973	uint64x2_t __ret; \
	64974	uint64x2_t __s1 = __p1; \
	64975	uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
	64976	__ret = (uint64x2_t) __builtin_neon_vldap1q_lane_u64(__p0, (int8x16_t)__rev1, __p2, 51); \
	64977	__ret = __builtin_shufflevector(__ret, __ret, 1, 0); \
	64978	__ret; \
	64979	})
	64980	#endif
	64981
	64982	#ifdef __LITTLE_ENDIAN__
	64983	#define vldap1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \
	64984	float64x2_t __ret; \
	64985	float64x2_t __s1 = __p1; \
	64986	__ret = (float64x2_t) __builtin_neon_vldap1q_lane_f64(__p0, (int8x16_t)__s1, __p2, 42); \
	64987	__ret; \
	64988	})
	64989	#else
	64990	#define vldap1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \
	64991	float64x2_t __ret; \
	64992	float64x2_t __s1 = __p1; \
	64993	float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
	64994	__ret = (float64x2_t) __builtin_neon_vldap1q_lane_f64(__p0, (int8x16_t)__rev1, __p2, 42); \
	64995	__ret = __builtin_shufflevector(__ret, __ret, 1, 0); \
	64996	__ret; \
	64997	})
	64998	#endif
	64999
	65000	#ifdef __LITTLE_ENDIAN__
	65001	#define vldap1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \
	65002	int64x2_t __ret; \
	65003	int64x2_t __s1 = __p1; \
	65004	__ret = (int64x2_t) __builtin_neon_vldap1q_lane_s64(__p0, (int8x16_t)__s1, __p2, 35); \
	65005	__ret; \
	65006	})
	65007	#else
	65008	#define vldap1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \
	65009	int64x2_t __ret; \
	65010	int64x2_t __s1 = __p1; \
	65011	int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
	65012	__ret = (int64x2_t) __builtin_neon_vldap1q_lane_s64(__p0, (int8x16_t)__rev1, __p2, 35); \
	65013	__ret = __builtin_shufflevector(__ret, __ret, 1, 0); \
	65014	__ret; \
	65015	})
	65016	#endif
	65017
	65018	#define vldap1_lane_u64(__p0, __p1, __p2) __extension__ ({ \
	65019	uint64x1_t __ret; \
	65020	uint64x1_t __s1 = __p1; \
	65021	__ret = (uint64x1_t) __builtin_neon_vldap1_lane_u64(__p0, (int8x8_t)__s1, __p2, 19); \
	65022	__ret; \
	65023	})
	65024	#define vldap1_lane_f64(__p0, __p1, __p2) __extension__ ({ \
	65025	float64x1_t __ret; \
	65026	float64x1_t __s1 = __p1; \
	65027	__ret = (float64x1_t) __builtin_neon_vldap1_lane_f64(__p0, (int8x8_t)__s1, __p2, 10); \
	65028	__ret; \
	65029	})
	65030	#define vldap1_lane_s64(__p0, __p1, __p2) __extension__ ({ \
	65031	int64x1_t __ret; \
	65032	int64x1_t __s1 = __p1; \
	65033	__ret = (int64x1_t) __builtin_neon_vldap1_lane_s64(__p0, (int8x8_t)__s1, __p2, 3); \
	65034	__ret; \
	65035	})
	65036	#define vstl1_lane_p64(__p0, __p1, __p2) __extension__ ({ \
	65037	poly64x1_t __s1 = __p1; \
	65038	__builtin_neon_vstl1_lane_p64(__p0, (int8x8_t)__s1, __p2, 6); \
	65039	})
	65040	#ifdef __LITTLE_ENDIAN__
	65041	#define vstl1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \
	65042	poly64x2_t __s1 = __p1; \
	65043	__builtin_neon_vstl1q_lane_p64(__p0, (int8x16_t)__s1, __p2, 38); \
	65044	})
	65045	#else
	65046	#define vstl1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \
	65047	poly64x2_t __s1 = __p1; \
	65048	poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
	65049	__builtin_neon_vstl1q_lane_p64(__p0, (int8x16_t)__rev1, __p2, 38); \
	65050	})
	65051	#endif
	65052
	65053	#ifdef __LITTLE_ENDIAN__
	65054	#define vstl1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \
	65055	uint64x2_t __s1 = __p1; \
	65056	__builtin_neon_vstl1q_lane_u64(__p0, (int8x16_t)__s1, __p2, 51); \
	65057	})
	65058	#else
	65059	#define vstl1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \
	65060	uint64x2_t __s1 = __p1; \
	65061	uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
	65062	__builtin_neon_vstl1q_lane_u64(__p0, (int8x16_t)__rev1, __p2, 51); \
	65063	})
	65064	#endif
	65065
	65066	#ifdef __LITTLE_ENDIAN__
	65067	#define vstl1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \
	65068	float64x2_t __s1 = __p1; \
	65069	__builtin_neon_vstl1q_lane_f64(__p0, (int8x16_t)__s1, __p2, 42); \
	65070	})
	65071	#else
	65072	#define vstl1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \
	65073	float64x2_t __s1 = __p1; \
	65074	float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
	65075	__builtin_neon_vstl1q_lane_f64(__p0, (int8x16_t)__rev1, __p2, 42); \
	65076	})
	65077	#endif
	65078
	65079	#ifdef __LITTLE_ENDIAN__
	65080	#define vstl1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \
	65081	int64x2_t __s1 = __p1; \
	65082	__builtin_neon_vstl1q_lane_s64(__p0, (int8x16_t)__s1, __p2, 35); \
	65083	})
	65084	#else
	65085	#define vstl1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \
	65086	int64x2_t __s1 = __p1; \
	65087	int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
	65088	__builtin_neon_vstl1q_lane_s64(__p0, (int8x16_t)__rev1, __p2, 35); \
	65089	})
	65090	#endif
	65091
	65092	#define vstl1_lane_u64(__p0, __p1, __p2) __extension__ ({ \
	65093	uint64x1_t __s1 = __p1; \
	65094	__builtin_neon_vstl1_lane_u64(__p0, (int8x8_t)__s1, __p2, 19); \
	65095	})
	65096	#define vstl1_lane_f64(__p0, __p1, __p2) __extension__ ({ \
	65097	float64x1_t __s1 = __p1; \
	65098	__builtin_neon_vstl1_lane_f64(__p0, (int8x8_t)__s1, __p2, 10); \
	65099	})
	65100	#define vstl1_lane_s64(__p0, __p1, __p2) __extension__ ({ \
	65101	int64x1_t __s1 = __p1; \
	65102	__builtin_neon_vstl1_lane_s64(__p0, (int8x8_t)__s1, __p2, 3); \
	65103	})
64941	65104	#ifdef __LITTLE_ENDIAN__
64942	65105	__ai __attribute__((target("sha3"))) uint8x16_t vbcaxq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) {
64943	65106	uint8x16_t __ret;

lib/include/arm_sme_draft_spec_subject_to_change.h created+642

...	...	@@ -0,0 +1,642 @@
	1	/*===---- arm_sme_draft_spec_subject_to_change.h - ARM SME intrinsics ------===
	2	*
	3	*
	4	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	5	* See https://llvm.org/LICENSE.txt for license information.
	6	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	7	*
	8	*===-----------------------------------------------------------------------===
	9	*/
	10
	11	#ifndef __ARM_SME_H
	12	#define __ARM_SME_H
	13
	14	#if !defined(__LITTLE_ENDIAN__)
	15	#error "Big endian is currently not supported for arm_sme_draft_spec_subject_to_change.h"
	16	#endif
	17	#include <arm_sve.h>
	18
	19	/* Function attributes */
	20	#define __ai static __inline__ __attribute__((__always_inline__, __nodebug__))
	21
	22	#define __aio static __inline__ __attribute__((__always_inline__, __nodebug__, __overloadable__))
	23
	24	#ifdef __cplusplus
	25	extern "C" {
	26	#endif
	27
	28	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_u32_m), arm_streaming, arm_shared_za))
	29	void svaddha_za32_u32_m(uint64_t, svbool_t, svbool_t, svuint32_t);
	30	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_s32_m), arm_streaming, arm_shared_za))
	31	void svaddha_za32_s32_m(uint64_t, svbool_t, svbool_t, svint32_t);
	32	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za32_u32_m), arm_streaming, arm_shared_za))
	33	void svaddva_za32_u32_m(uint64_t, svbool_t, svbool_t, svuint32_t);
	34	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za32_s32_m), arm_streaming, arm_shared_za))
	35	void svaddva_za32_s32_m(uint64_t, svbool_t, svbool_t, svint32_t);
	36	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svcntsb), arm_streaming_compatible, arm_preserves_za))
	37	uint64_t svcntsb(void);
	38	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svcntsd), arm_streaming_compatible, arm_preserves_za))
	39	uint64_t svcntsd(void);
	40	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svcntsh), arm_streaming_compatible, arm_preserves_za))
	41	uint64_t svcntsh(void);
	42	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svcntsw), arm_streaming_compatible, arm_preserves_za))
	43	uint64_t svcntsw(void);
	44	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za128), arm_streaming, arm_shared_za))
	45	void svld1_hor_vnum_za128(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
	46	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za16), arm_streaming, arm_shared_za))
	47	void svld1_hor_vnum_za16(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
	48	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za32), arm_streaming, arm_shared_za))
	49	void svld1_hor_vnum_za32(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
	50	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za64), arm_streaming, arm_shared_za))
	51	void svld1_hor_vnum_za64(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
	52	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za8), arm_streaming, arm_shared_za))
	53	void svld1_hor_vnum_za8(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
	54	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za128), arm_streaming, arm_shared_za))
	55	void svld1_hor_za128(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
	56	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za16), arm_streaming, arm_shared_za))
	57	void svld1_hor_za16(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
	58	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za32), arm_streaming, arm_shared_za))
	59	void svld1_hor_za32(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
	60	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za64), arm_streaming, arm_shared_za))
	61	void svld1_hor_za64(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
	62	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za8), arm_streaming, arm_shared_za))
	63	void svld1_hor_za8(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
	64	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za128), arm_streaming, arm_shared_za))
	65	void svld1_ver_vnum_za128(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
	66	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za16), arm_streaming, arm_shared_za))
	67	void svld1_ver_vnum_za16(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
	68	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za32), arm_streaming, arm_shared_za))
	69	void svld1_ver_vnum_za32(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
	70	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za64), arm_streaming, arm_shared_za))
	71	void svld1_ver_vnum_za64(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
	72	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za8), arm_streaming, arm_shared_za))
	73	void svld1_ver_vnum_za8(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
	74	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za128), arm_streaming, arm_shared_za))
	75	void svld1_ver_za128(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
	76	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za16), arm_streaming, arm_shared_za))
	77	void svld1_ver_za16(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
	78	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za32), arm_streaming, arm_shared_za))
	79	void svld1_ver_za32(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
	80	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za64), arm_streaming, arm_shared_za))
	81	void svld1_ver_za64(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
	82	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za8), arm_streaming, arm_shared_za))
	83	void svld1_ver_za8(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
	84	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_f16_m), arm_streaming, arm_shared_za))
	85	void svmopa_za32_f16_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t);
	86	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_bf16_m), arm_streaming, arm_shared_za))
	87	void svmopa_za32_bf16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
	88	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_f32_m), arm_streaming, arm_shared_za))
	89	void svmopa_za32_f32_m(uint64_t, svbool_t, svbool_t, svfloat32_t, svfloat32_t);
	90	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_s8_m), arm_streaming, arm_shared_za))
	91	void svmopa_za32_s8_m(uint64_t, svbool_t, svbool_t, svint8_t, svint8_t);
	92	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_u8_m), arm_streaming, arm_shared_za))
	93	void svmopa_za32_u8_m(uint64_t, svbool_t, svbool_t, svuint8_t, svuint8_t);
	94	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_f16_m), arm_streaming, arm_shared_za))
	95	void svmops_za32_f16_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t);
	96	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_bf16_m), arm_streaming, arm_shared_za))
	97	void svmops_za32_bf16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
	98	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_f32_m), arm_streaming, arm_shared_za))
	99	void svmops_za32_f32_m(uint64_t, svbool_t, svbool_t, svfloat32_t, svfloat32_t);
	100	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_s8_m), arm_streaming, arm_shared_za))
	101	void svmops_za32_s8_m(uint64_t, svbool_t, svbool_t, svint8_t, svint8_t);
	102	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_u8_m), arm_streaming, arm_shared_za))
	103	void svmops_za32_u8_m(uint64_t, svbool_t, svbool_t, svuint8_t, svuint8_t);
	104	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	105	svuint8_t svread_hor_za128_u8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	106	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	107	svuint32_t svread_hor_za128_u32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	108	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	109	svuint64_t svread_hor_za128_u64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	110	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	111	svuint16_t svread_hor_za128_u16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	112	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	113	svbfloat16_t svread_hor_za128_bf16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	114	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	115	svint8_t svread_hor_za128_s8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	116	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	117	svfloat64_t svread_hor_za128_f64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	118	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	119	svfloat32_t svread_hor_za128_f32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	120	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	121	svfloat16_t svread_hor_za128_f16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	122	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	123	svint32_t svread_hor_za128_s32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	124	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	125	svint64_t svread_hor_za128_s64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	126	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	127	svint16_t svread_hor_za128_s16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	128	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	129	svuint16_t svread_hor_za16_u16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	130	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	131	svbfloat16_t svread_hor_za16_bf16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	132	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	133	svfloat16_t svread_hor_za16_f16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	134	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	135	svint16_t svread_hor_za16_s16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	136	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	137	svuint32_t svread_hor_za32_u32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	138	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	139	svfloat32_t svread_hor_za32_f32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	140	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	141	svint32_t svread_hor_za32_s32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	142	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	143	svuint64_t svread_hor_za64_u64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	144	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	145	svfloat64_t svread_hor_za64_f64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	146	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	147	svint64_t svread_hor_za64_s64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	148	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za8_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	149	svuint8_t svread_hor_za8_u8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	150	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za8_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	151	svint8_t svread_hor_za8_s8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	152	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	153	svuint8_t svread_ver_za128_u8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	154	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	155	svuint32_t svread_ver_za128_u32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	156	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	157	svuint64_t svread_ver_za128_u64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	158	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	159	svuint16_t svread_ver_za128_u16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	160	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	161	svbfloat16_t svread_ver_za128_bf16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	162	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	163	svint8_t svread_ver_za128_s8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	164	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	165	svfloat64_t svread_ver_za128_f64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	166	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	167	svfloat32_t svread_ver_za128_f32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	168	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	169	svfloat16_t svread_ver_za128_f16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	170	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	171	svint32_t svread_ver_za128_s32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	172	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	173	svint64_t svread_ver_za128_s64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	174	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	175	svint16_t svread_ver_za128_s16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	176	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	177	svuint16_t svread_ver_za16_u16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	178	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	179	svbfloat16_t svread_ver_za16_bf16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	180	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	181	svfloat16_t svread_ver_za16_f16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	182	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	183	svint16_t svread_ver_za16_s16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	184	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	185	svuint32_t svread_ver_za32_u32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	186	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	187	svfloat32_t svread_ver_za32_f32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	188	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	189	svint32_t svread_ver_za32_s32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	190	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	191	svuint64_t svread_ver_za64_u64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	192	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	193	svfloat64_t svread_ver_za64_f64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	194	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	195	svint64_t svread_ver_za64_s64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	196	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za8_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	197	svuint8_t svread_ver_za8_u8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	198	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za8_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	199	svint8_t svread_ver_za8_s8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	200	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za128), arm_streaming, arm_shared_za, arm_preserves_za))
	201	void svst1_hor_vnum_za128(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
	202	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za16), arm_streaming, arm_shared_za, arm_preserves_za))
	203	void svst1_hor_vnum_za16(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
	204	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za32), arm_streaming, arm_shared_za, arm_preserves_za))
	205	void svst1_hor_vnum_za32(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
	206	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za64), arm_streaming, arm_shared_za, arm_preserves_za))
	207	void svst1_hor_vnum_za64(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
	208	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za8), arm_streaming, arm_shared_za, arm_preserves_za))
	209	void svst1_hor_vnum_za8(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
	210	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za128), arm_streaming, arm_shared_za, arm_preserves_za))
	211	void svst1_hor_za128(uint64_t, uint32_t, uint64_t, svbool_t, void *);
	212	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za16), arm_streaming, arm_shared_za, arm_preserves_za))
	213	void svst1_hor_za16(uint64_t, uint32_t, uint64_t, svbool_t, void *);
	214	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za32), arm_streaming, arm_shared_za, arm_preserves_za))
	215	void svst1_hor_za32(uint64_t, uint32_t, uint64_t, svbool_t, void *);
	216	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za64), arm_streaming, arm_shared_za, arm_preserves_za))
	217	void svst1_hor_za64(uint64_t, uint32_t, uint64_t, svbool_t, void *);
	218	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za8), arm_streaming, arm_shared_za, arm_preserves_za))
	219	void svst1_hor_za8(uint64_t, uint32_t, uint64_t, svbool_t, void *);
	220	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za128), arm_streaming, arm_shared_za, arm_preserves_za))
	221	void svst1_ver_vnum_za128(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
	222	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za16), arm_streaming, arm_shared_za, arm_preserves_za))
	223	void svst1_ver_vnum_za16(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
	224	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za32), arm_streaming, arm_shared_za, arm_preserves_za))
	225	void svst1_ver_vnum_za32(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
	226	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za64), arm_streaming, arm_shared_za, arm_preserves_za))
	227	void svst1_ver_vnum_za64(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
	228	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za8), arm_streaming, arm_shared_za, arm_preserves_za))
	229	void svst1_ver_vnum_za8(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
	230	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za128), arm_streaming, arm_shared_za, arm_preserves_za))
	231	void svst1_ver_za128(uint64_t, uint32_t, uint64_t, svbool_t, void *);
	232	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za16), arm_streaming, arm_shared_za, arm_preserves_za))
	233	void svst1_ver_za16(uint64_t, uint32_t, uint64_t, svbool_t, void *);
	234	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za32), arm_streaming, arm_shared_za, arm_preserves_za))
	235	void svst1_ver_za32(uint64_t, uint32_t, uint64_t, svbool_t, void *);
	236	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za64), arm_streaming, arm_shared_za, arm_preserves_za))
	237	void svst1_ver_za64(uint64_t, uint32_t, uint64_t, svbool_t, void *);
	238	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za8), arm_streaming, arm_shared_za, arm_preserves_za))
	239	void svst1_ver_za8(uint64_t, uint32_t, uint64_t, svbool_t, void *);
	240	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumopa_za32_s8_m), arm_streaming, arm_shared_za))
	241	void svsumopa_za32_s8_m(uint64_t, svbool_t, svbool_t, svint8_t, svuint8_t);
	242	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumops_za32_s8_m), arm_streaming, arm_shared_za))
	243	void svsumops_za32_s8_m(uint64_t, svbool_t, svbool_t, svint8_t, svuint8_t);
	244	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za32_u8_m), arm_streaming, arm_shared_za))
	245	void svusmopa_za32_u8_m(uint64_t, svbool_t, svbool_t, svuint8_t, svint8_t);
	246	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za32_u8_m), arm_streaming, arm_shared_za))
	247	void svusmops_za32_u8_m(uint64_t, svbool_t, svbool_t, svuint8_t, svint8_t);
	248	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u8_m), arm_streaming, arm_shared_za))
	249	void svwrite_hor_za128_u8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
	250	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u32_m), arm_streaming, arm_shared_za))
	251	void svwrite_hor_za128_u32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
	252	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u64_m), arm_streaming, arm_shared_za))
	253	void svwrite_hor_za128_u64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
	254	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u16_m), arm_streaming, arm_shared_za))
	255	void svwrite_hor_za128_u16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
	256	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_bf16_m), arm_streaming, arm_shared_za))
	257	void svwrite_hor_za128_bf16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
	258	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s8_m), arm_streaming, arm_shared_za))
	259	void svwrite_hor_za128_s8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
	260	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f64_m), arm_streaming, arm_shared_za))
	261	void svwrite_hor_za128_f64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
	262	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f32_m), arm_streaming, arm_shared_za))
	263	void svwrite_hor_za128_f32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
	264	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f16_m), arm_streaming, arm_shared_za))
	265	void svwrite_hor_za128_f16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
	266	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s32_m), arm_streaming, arm_shared_za))
	267	void svwrite_hor_za128_s32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
	268	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s64_m), arm_streaming, arm_shared_za))
	269	void svwrite_hor_za128_s64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
	270	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s16_m), arm_streaming, arm_shared_za))
	271	void svwrite_hor_za128_s16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
	272	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_u16_m), arm_streaming, arm_shared_za))
	273	void svwrite_hor_za16_u16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
	274	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_bf16_m), arm_streaming, arm_shared_za))
	275	void svwrite_hor_za16_bf16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
	276	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_f16_m), arm_streaming, arm_shared_za))
	277	void svwrite_hor_za16_f16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
	278	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_s16_m), arm_streaming, arm_shared_za))
	279	void svwrite_hor_za16_s16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
	280	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_u32_m), arm_streaming, arm_shared_za))
	281	void svwrite_hor_za32_u32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
	282	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_f32_m), arm_streaming, arm_shared_za))
	283	void svwrite_hor_za32_f32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
	284	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_s32_m), arm_streaming, arm_shared_za))
	285	void svwrite_hor_za32_s32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
	286	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_u64_m), arm_streaming, arm_shared_za))
	287	void svwrite_hor_za64_u64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
	288	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_f64_m), arm_streaming, arm_shared_za))
	289	void svwrite_hor_za64_f64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
	290	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_s64_m), arm_streaming, arm_shared_za))
	291	void svwrite_hor_za64_s64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
	292	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za8_u8_m), arm_streaming, arm_shared_za))
	293	void svwrite_hor_za8_u8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
	294	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za8_s8_m), arm_streaming, arm_shared_za))
	295	void svwrite_hor_za8_s8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
	296	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u8_m), arm_streaming, arm_shared_za))
	297	void svwrite_ver_za128_u8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
	298	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u32_m), arm_streaming, arm_shared_za))
	299	void svwrite_ver_za128_u32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
	300	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u64_m), arm_streaming, arm_shared_za))
	301	void svwrite_ver_za128_u64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
	302	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u16_m), arm_streaming, arm_shared_za))
	303	void svwrite_ver_za128_u16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
	304	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_bf16_m), arm_streaming, arm_shared_za))
	305	void svwrite_ver_za128_bf16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
	306	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s8_m), arm_streaming, arm_shared_za))
	307	void svwrite_ver_za128_s8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
	308	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f64_m), arm_streaming, arm_shared_za))
	309	void svwrite_ver_za128_f64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
	310	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f32_m), arm_streaming, arm_shared_za))
	311	void svwrite_ver_za128_f32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
	312	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f16_m), arm_streaming, arm_shared_za))
	313	void svwrite_ver_za128_f16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
	314	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s32_m), arm_streaming, arm_shared_za))
	315	void svwrite_ver_za128_s32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
	316	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s64_m), arm_streaming, arm_shared_za))
	317	void svwrite_ver_za128_s64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
	318	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s16_m), arm_streaming, arm_shared_za))
	319	void svwrite_ver_za128_s16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
	320	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_u16_m), arm_streaming, arm_shared_za))
	321	void svwrite_ver_za16_u16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
	322	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_bf16_m), arm_streaming, arm_shared_za))
	323	void svwrite_ver_za16_bf16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
	324	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_f16_m), arm_streaming, arm_shared_za))
	325	void svwrite_ver_za16_f16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
	326	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_s16_m), arm_streaming, arm_shared_za))
	327	void svwrite_ver_za16_s16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
	328	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_u32_m), arm_streaming, arm_shared_za))
	329	void svwrite_ver_za32_u32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
	330	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_f32_m), arm_streaming, arm_shared_za))
	331	void svwrite_ver_za32_f32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
	332	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_s32_m), arm_streaming, arm_shared_za))
	333	void svwrite_ver_za32_s32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
	334	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_u64_m), arm_streaming, arm_shared_za))
	335	void svwrite_ver_za64_u64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
	336	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_f64_m), arm_streaming, arm_shared_za))
	337	void svwrite_ver_za64_f64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
	338	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_s64_m), arm_streaming, arm_shared_za))
	339	void svwrite_ver_za64_s64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
	340	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_u8_m), arm_streaming, arm_shared_za))
	341	void svwrite_ver_za8_u8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
	342	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_s8_m), arm_streaming, arm_shared_za))
	343	void svwrite_ver_za8_s8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
	344	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svzero_mask_za), arm_streaming_compatible, arm_shared_za))
	345	void svzero_mask_za(uint64_t);
	346	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svzero_za), arm_streaming_compatible, arm_shared_za))
	347	void svzero_za();
	348	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_u32_m), arm_streaming, arm_shared_za))
	349	void svaddha_za32_m(uint64_t, svbool_t, svbool_t, svuint32_t);
	350	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_s32_m), arm_streaming, arm_shared_za))
	351	void svaddha_za32_m(uint64_t, svbool_t, svbool_t, svint32_t);
	352	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za32_u32_m), arm_streaming, arm_shared_za))
	353	void svaddva_za32_m(uint64_t, svbool_t, svbool_t, svuint32_t);
	354	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za32_s32_m), arm_streaming, arm_shared_za))
	355	void svaddva_za32_m(uint64_t, svbool_t, svbool_t, svint32_t);
	356	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_f16_m), arm_streaming, arm_shared_za))
	357	void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t);
	358	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_bf16_m), arm_streaming, arm_shared_za))
	359	void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
	360	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_f32_m), arm_streaming, arm_shared_za))
	361	void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svfloat32_t, svfloat32_t);
	362	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_s8_m), arm_streaming, arm_shared_za))
	363	void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svint8_t, svint8_t);
	364	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_u8_m), arm_streaming, arm_shared_za))
	365	void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svuint8_t, svuint8_t);
	366	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_f16_m), arm_streaming, arm_shared_za))
	367	void svmops_za32_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t);
	368	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_bf16_m), arm_streaming, arm_shared_za))
	369	void svmops_za32_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
	370	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_f32_m), arm_streaming, arm_shared_za))
	371	void svmops_za32_m(uint64_t, svbool_t, svbool_t, svfloat32_t, svfloat32_t);
	372	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_s8_m), arm_streaming, arm_shared_za))
	373	void svmops_za32_m(uint64_t, svbool_t, svbool_t, svint8_t, svint8_t);
	374	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_u8_m), arm_streaming, arm_shared_za))
	375	void svmops_za32_m(uint64_t, svbool_t, svbool_t, svuint8_t, svuint8_t);
	376	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	377	svuint8_t svread_hor_za128_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	378	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	379	svuint32_t svread_hor_za128_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	380	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	381	svuint64_t svread_hor_za128_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	382	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	383	svuint16_t svread_hor_za128_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	384	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	385	svbfloat16_t svread_hor_za128_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	386	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	387	svint8_t svread_hor_za128_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	388	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	389	svfloat64_t svread_hor_za128_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	390	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	391	svfloat32_t svread_hor_za128_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	392	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	393	svfloat16_t svread_hor_za128_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	394	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	395	svint32_t svread_hor_za128_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	396	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	397	svint64_t svread_hor_za128_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	398	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	399	svint16_t svread_hor_za128_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	400	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	401	svuint16_t svread_hor_za16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	402	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	403	svbfloat16_t svread_hor_za16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	404	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	405	svfloat16_t svread_hor_za16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	406	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	407	svint16_t svread_hor_za16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	408	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	409	svuint32_t svread_hor_za32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	410	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	411	svfloat32_t svread_hor_za32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	412	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	413	svint32_t svread_hor_za32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	414	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	415	svuint64_t svread_hor_za64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	416	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	417	svfloat64_t svread_hor_za64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	418	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	419	svint64_t svread_hor_za64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	420	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za8_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	421	svuint8_t svread_hor_za8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	422	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za8_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	423	svint8_t svread_hor_za8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	424	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	425	svuint8_t svread_ver_za128_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	426	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	427	svuint32_t svread_ver_za128_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	428	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	429	svuint64_t svread_ver_za128_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	430	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	431	svuint16_t svread_ver_za128_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	432	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	433	svbfloat16_t svread_ver_za128_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	434	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	435	svint8_t svread_ver_za128_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	436	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	437	svfloat64_t svread_ver_za128_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	438	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	439	svfloat32_t svread_ver_za128_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	440	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	441	svfloat16_t svread_ver_za128_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	442	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	443	svint32_t svread_ver_za128_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	444	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	445	svint64_t svread_ver_za128_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	446	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	447	svint16_t svread_ver_za128_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	448	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	449	svuint16_t svread_ver_za16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	450	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	451	svbfloat16_t svread_ver_za16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	452	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	453	svfloat16_t svread_ver_za16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	454	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
	455	svint16_t svread_ver_za16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
	456	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	457	svuint32_t svread_ver_za32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	458	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	459	svfloat32_t svread_ver_za32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	460	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
	461	svint32_t svread_ver_za32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
	462	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	463	svuint64_t svread_ver_za64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	464	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	465	svfloat64_t svread_ver_za64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	466	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
	467	svint64_t svread_ver_za64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
	468	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za8_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	469	svuint8_t svread_ver_za8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	470	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za8_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
	471	svint8_t svread_ver_za8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
	472	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumopa_za32_s8_m), arm_streaming, arm_shared_za))
	473	void svsumopa_za32_m(uint64_t, svbool_t, svbool_t, svint8_t, svuint8_t);
	474	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumops_za32_s8_m), arm_streaming, arm_shared_za))
	475	void svsumops_za32_m(uint64_t, svbool_t, svbool_t, svint8_t, svuint8_t);
	476	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za32_u8_m), arm_streaming, arm_shared_za))
	477	void svusmopa_za32_m(uint64_t, svbool_t, svbool_t, svuint8_t, svint8_t);
	478	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za32_u8_m), arm_streaming, arm_shared_za))
	479	void svusmops_za32_m(uint64_t, svbool_t, svbool_t, svuint8_t, svint8_t);
	480	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u8_m), arm_streaming, arm_shared_za))
	481	void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
	482	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u32_m), arm_streaming, arm_shared_za))
	483	void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
	484	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u64_m), arm_streaming, arm_shared_za))
	485	void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
	486	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u16_m), arm_streaming, arm_shared_za))
	487	void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
	488	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_bf16_m), arm_streaming, arm_shared_za))
	489	void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
	490	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s8_m), arm_streaming, arm_shared_za))
	491	void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
	492	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f64_m), arm_streaming, arm_shared_za))
	493	void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
	494	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f32_m), arm_streaming, arm_shared_za))
	495	void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
	496	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f16_m), arm_streaming, arm_shared_za))
	497	void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
	498	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s32_m), arm_streaming, arm_shared_za))
	499	void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
	500	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s64_m), arm_streaming, arm_shared_za))
	501	void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
	502	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s16_m), arm_streaming, arm_shared_za))
	503	void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
	504	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_u16_m), arm_streaming, arm_shared_za))
	505	void svwrite_hor_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
	506	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_bf16_m), arm_streaming, arm_shared_za))
	507	void svwrite_hor_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
	508	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_f16_m), arm_streaming, arm_shared_za))
	509	void svwrite_hor_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
	510	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_s16_m), arm_streaming, arm_shared_za))
	511	void svwrite_hor_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
	512	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_u32_m), arm_streaming, arm_shared_za))
	513	void svwrite_hor_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
	514	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_f32_m), arm_streaming, arm_shared_za))
	515	void svwrite_hor_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
	516	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_s32_m), arm_streaming, arm_shared_za))
	517	void svwrite_hor_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
	518	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_u64_m), arm_streaming, arm_shared_za))
	519	void svwrite_hor_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
	520	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_f64_m), arm_streaming, arm_shared_za))
	521	void svwrite_hor_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
	522	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_s64_m), arm_streaming, arm_shared_za))
	523	void svwrite_hor_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
	524	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za8_u8_m), arm_streaming, arm_shared_za))
	525	void svwrite_hor_za8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
	526	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za8_s8_m), arm_streaming, arm_shared_za))
	527	void svwrite_hor_za8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
	528	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u8_m), arm_streaming, arm_shared_za))
	529	void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
	530	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u32_m), arm_streaming, arm_shared_za))
	531	void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
	532	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u64_m), arm_streaming, arm_shared_za))
	533	void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
	534	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u16_m), arm_streaming, arm_shared_za))
	535	void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
	536	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_bf16_m), arm_streaming, arm_shared_za))
	537	void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
	538	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s8_m), arm_streaming, arm_shared_za))
	539	void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
	540	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f64_m), arm_streaming, arm_shared_za))
	541	void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
	542	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f32_m), arm_streaming, arm_shared_za))
	543	void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
	544	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f16_m), arm_streaming, arm_shared_za))
	545	void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
	546	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s32_m), arm_streaming, arm_shared_za))
	547	void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
	548	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s64_m), arm_streaming, arm_shared_za))
	549	void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
	550	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s16_m), arm_streaming, arm_shared_za))
	551	void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
	552	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_u16_m), arm_streaming, arm_shared_za))
	553	void svwrite_ver_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
	554	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_bf16_m), arm_streaming, arm_shared_za))
	555	void svwrite_ver_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
	556	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_f16_m), arm_streaming, arm_shared_za))
	557	void svwrite_ver_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
	558	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_s16_m), arm_streaming, arm_shared_za))
	559	void svwrite_ver_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
	560	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_u32_m), arm_streaming, arm_shared_za))
	561	void svwrite_ver_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
	562	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_f32_m), arm_streaming, arm_shared_za))
	563	void svwrite_ver_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
	564	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_s32_m), arm_streaming, arm_shared_za))
	565	void svwrite_ver_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
	566	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_u64_m), arm_streaming, arm_shared_za))
	567	void svwrite_ver_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
	568	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_f64_m), arm_streaming, arm_shared_za))
	569	void svwrite_ver_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
	570	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_s64_m), arm_streaming, arm_shared_za))
	571	void svwrite_ver_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
	572	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_u8_m), arm_streaming, arm_shared_za))
	573	void svwrite_ver_za8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
	574	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_s8_m), arm_streaming, arm_shared_za))
	575	void svwrite_ver_za8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
	576	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_f64_m), arm_streaming, arm_shared_za))
	577	void svmopa_za64_f64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);
	578	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_f64_m), arm_streaming, arm_shared_za))
	579	void svmops_za64_f64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);
	580	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_f64_m), arm_streaming, arm_shared_za))
	581	void svmopa_za64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);
	582	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_f64_m), arm_streaming, arm_shared_za))
	583	void svmops_za64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);
	584	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_u64_m), arm_streaming, arm_shared_za))
	585	void svaddha_za64_u64_m(uint64_t, svbool_t, svbool_t, svuint64_t);
	586	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_s64_m), arm_streaming, arm_shared_za))
	587	void svaddha_za64_s64_m(uint64_t, svbool_t, svbool_t, svint64_t);
	588	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za64_u64_m), arm_streaming, arm_shared_za))
	589	void svaddva_za64_u64_m(uint64_t, svbool_t, svbool_t, svuint64_t);
	590	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za64_s64_m), arm_streaming, arm_shared_za))
	591	void svaddva_za64_s64_m(uint64_t, svbool_t, svbool_t, svint64_t);
	592	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_s16_m), arm_streaming, arm_shared_za))
	593	void svmopa_za64_s16_m(uint64_t, svbool_t, svbool_t, svint16_t, svint16_t);
	594	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_u16_m), arm_streaming, arm_shared_za))
	595	void svmopa_za64_u16_m(uint64_t, svbool_t, svbool_t, svuint16_t, svuint16_t);
	596	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_s16_m), arm_streaming, arm_shared_za))
	597	void svmops_za64_s16_m(uint64_t, svbool_t, svbool_t, svint16_t, svint16_t);
	598	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_u16_m), arm_streaming, arm_shared_za))
	599	void svmops_za64_u16_m(uint64_t, svbool_t, svbool_t, svuint16_t, svuint16_t);
	600	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumopa_za64_s16_m), arm_streaming, arm_shared_za))
	601	void svsumopa_za64_s16_m(uint64_t, svbool_t, svbool_t, svint16_t, svuint16_t);
	602	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumops_za64_s16_m), arm_streaming, arm_shared_za))
	603	void svsumops_za64_s16_m(uint64_t, svbool_t, svbool_t, svint16_t, svuint16_t);
	604	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za64_u16_m), arm_streaming, arm_shared_za))
	605	void svusmopa_za64_u16_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t);
	606	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za64_u16_m), arm_streaming, arm_shared_za))
	607	void svusmops_za64_u16_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t);
	608	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_u64_m), arm_streaming, arm_shared_za))
	609	void svaddha_za64_m(uint64_t, svbool_t, svbool_t, svuint64_t);
	610	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_s64_m), arm_streaming, arm_shared_za))
	611	void svaddha_za64_m(uint64_t, svbool_t, svbool_t, svint64_t);
	612	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za64_u64_m), arm_streaming, arm_shared_za))
	613	void svaddva_za64_m(uint64_t, svbool_t, svbool_t, svuint64_t);
	614	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za64_s64_m), arm_streaming, arm_shared_za))
	615	void svaddva_za64_m(uint64_t, svbool_t, svbool_t, svint64_t);
	616	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_s16_m), arm_streaming, arm_shared_za))
	617	void svmopa_za64_m(uint64_t, svbool_t, svbool_t, svint16_t, svint16_t);
	618	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_u16_m), arm_streaming, arm_shared_za))
	619	void svmopa_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svuint16_t);
	620	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_s16_m), arm_streaming, arm_shared_za))
	621	void svmops_za64_m(uint64_t, svbool_t, svbool_t, svint16_t, svint16_t);
	622	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_u16_m), arm_streaming, arm_shared_za))
	623	void svmops_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svuint16_t);
	624	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumopa_za64_s16_m), arm_streaming, arm_shared_za))
	625	void svsumopa_za64_m(uint64_t, svbool_t, svbool_t, svint16_t, svuint16_t);
	626	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumops_za64_s16_m), arm_streaming, arm_shared_za))
	627	void svsumops_za64_m(uint64_t, svbool_t, svbool_t, svint16_t, svuint16_t);
	628	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za64_u16_m), arm_streaming, arm_shared_za))
	629	void svusmopa_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t);
	630	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za64_u16_m), arm_streaming, arm_shared_za))
	631	void svusmops_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t);
	632	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svldr_vnum_za), arm_streaming_compatible, arm_shared_za))
	633	void svldr_vnum_za(uint32_t, uint64_t, void const *);
	634	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svstr_vnum_za), arm_streaming_compatible, arm_shared_za, arm_preserves_za))
	635	void svstr_vnum_za(uint32_t, uint64_t, void *);
	636	#ifdef __cplusplus
	637	} // extern "C"
	638	#endif
	639
	640	#undef __ai
	641
	642	#endif /* __ARM_SME_H */

lib/include/arm_sve.h+68-13

...	...	@@ -37,7 +37,6 @@ typedef __SVFloat16_t svfloat16_t;
37	37
38	38	typedef __SVBFloat16_t svbfloat16_t;
39	39	#include <arm_bf16.h>
40		typedef __bf16 bfloat16_t;
41	40	typedef __SVFloat32_t svfloat32_t;
42	41	typedef __SVFloat64_t svfloat64_t;
43	42	typedef __clang_svint8x2_t svint8x2_t;
...	...	@@ -74,10 +73,14 @@ typedef __clang_svfloat16x4_t svfloat16x4_t;
74	73	typedef __clang_svfloat32x4_t svfloat32x4_t;
75	74	typedef __clang_svfloat64x4_t svfloat64x4_t;
76	75	typedef __SVBool_t svbool_t;
	76	typedef __clang_svboolx2_t svboolx2_t;
	77	typedef __clang_svboolx4_t svboolx4_t;
77	78
78	79	typedef __clang_svbfloat16x2_t svbfloat16x2_t;
79	80	typedef __clang_svbfloat16x3_t svbfloat16x3_t;
80	81	typedef __clang_svbfloat16x4_t svbfloat16x4_t;
	82	typedef __SVCount_t svcount_t;
	83
81	84	enum svpattern
82	85	{
83	86	SV_POW2 = 0,
...	...	@@ -2914,6 +2917,10 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s64)))
2914	2917	svint64_t svdup_lane_s64(svint64_t, uint64_t);
2915	2918	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s16)))
2916	2919	svint16_t svdup_lane_s16(svint16_t, uint16_t);
	2920	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u8)))
	2921	svuint8_t svdupq_n_u8(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t);
	2922	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s8)))
	2923	svint8_t svdupq_n_s8(int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t);
2917	2924	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u16)))
2918	2925	svuint16_t svdupq_n_u16(uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t);
2919	2926	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f16)))
...	...	@@ -2932,18 +2939,14 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f64)))
2932	2939	svfloat64_t svdupq_n_f64(float64_t, float64_t);
2933	2940	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s64)))
2934	2941	svint64_t svdupq_n_s64(int64_t, int64_t);
2935		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u8)))
2936		svuint8_t svdupq_n_u8(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t);
2937		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s8)))
2938		svint8_t svdupq_n_s8(int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t);
	2942	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b8)))
	2943	svbool_t svdupq_n_b8(bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool);
2939	2944	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b16)))
2940	2945	svbool_t svdupq_n_b16(bool, bool, bool, bool, bool, bool, bool, bool);
2941	2946	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b32)))
2942	2947	svbool_t svdupq_n_b32(bool, bool, bool, bool);
2943	2948	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b64)))
2944	2949	svbool_t svdupq_n_b64(bool, bool);
2945		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b8)))
2946		svbool_t svdupq_n_b8(bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool);
2947	2950	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u8)))
2948	2951	svuint8_t svdupq_lane_u8(svuint8_t, uint64_t);
2949	2952	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u32)))
...	...	@@ -10528,6 +10531,10 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s64)))
10528	10531	svint64_t svdup_lane(svint64_t, uint64_t);
10529	10532	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s16)))
10530	10533	svint16_t svdup_lane(svint16_t, uint16_t);
	10534	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u8)))
	10535	svuint8_t svdupq_u8(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t);
	10536	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s8)))
	10537	svint8_t svdupq_s8(int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t);
10531	10538	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u16)))
10532	10539	svuint16_t svdupq_u16(uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t);
10533	10540	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f16)))
...	...	@@ -10546,18 +10553,14 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f64)))
10546	10553	svfloat64_t svdupq_f64(float64_t, float64_t);
10547	10554	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s64)))
10548	10555	svint64_t svdupq_s64(int64_t, int64_t);
10549		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u8)))
10550		svuint8_t svdupq_u8(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t);
10551		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s8)))
10552		svint8_t svdupq_s8(int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t);
	10556	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b8)))
	10557	svbool_t svdupq_b8(bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool);
10553	10558	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b16)))
10554	10559	svbool_t svdupq_b16(bool, bool, bool, bool, bool, bool, bool, bool);
10555	10560	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b32)))
10556	10561	svbool_t svdupq_b32(bool, bool, bool, bool);
10557	10562	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b64)))
10558	10563	svbool_t svdupq_b64(bool, bool);
10559		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b8)))
10560		svbool_t svdupq_b8(bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool);
10561	10564	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u8)))
10562	10565	svuint8_t svdupq_lane(svuint8_t, uint64_t);
10563	10566	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u32)))
...	...	@@ -23874,6 +23877,58 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4e_u32)))
23874	23877	svuint32_t svsm4e(svuint32_t, svuint32_t);
23875	23878	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4ekey_u32)))
23876	23879	svuint32_t svsm4ekey(svuint32_t, svuint32_t);
	23880	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f64)))
	23881	svfloat64_t svclamp_f64(svfloat64_t, svfloat64_t, svfloat64_t);
	23882	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f32)))
	23883	svfloat32_t svclamp_f32(svfloat32_t, svfloat32_t, svfloat32_t);
	23884	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f16)))
	23885	svfloat16_t svclamp_f16(svfloat16_t, svfloat16_t, svfloat16_t);
	23886	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s8)))
	23887	svint8_t svclamp_s8(svint8_t, svint8_t, svint8_t);
	23888	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s32)))
	23889	svint32_t svclamp_s32(svint32_t, svint32_t, svint32_t);
	23890	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s64)))
	23891	svint64_t svclamp_s64(svint64_t, svint64_t, svint64_t);
	23892	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s16)))
	23893	svint16_t svclamp_s16(svint16_t, svint16_t, svint16_t);
	23894	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u8)))
	23895	svuint8_t svclamp_u8(svuint8_t, svuint8_t, svuint8_t);
	23896	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u32)))
	23897	svuint32_t svclamp_u32(svuint32_t, svuint32_t, svuint32_t);
	23898	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u64)))
	23899	svuint64_t svclamp_u64(svuint64_t, svuint64_t, svuint64_t);
	23900	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u16)))
	23901	svuint16_t svclamp_u16(svuint16_t, svuint16_t, svuint16_t);
	23902	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c8)))
	23903	svcount_t svptrue_c8(void);
	23904	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c32)))
	23905	svcount_t svptrue_c32(void);
	23906	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c64)))
	23907	svcount_t svptrue_c64(void);
	23908	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c16)))
	23909	svcount_t svptrue_c16(void);
	23910	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f64)))
	23911	svfloat64_t svclamp(svfloat64_t, svfloat64_t, svfloat64_t);
	23912	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f32)))
	23913	svfloat32_t svclamp(svfloat32_t, svfloat32_t, svfloat32_t);
	23914	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f16)))
	23915	svfloat16_t svclamp(svfloat16_t, svfloat16_t, svfloat16_t);
	23916	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s8)))
	23917	svint8_t svclamp(svint8_t, svint8_t, svint8_t);
	23918	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s32)))
	23919	svint32_t svclamp(svint32_t, svint32_t, svint32_t);
	23920	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s64)))
	23921	svint64_t svclamp(svint64_t, svint64_t, svint64_t);
	23922	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s16)))
	23923	svint16_t svclamp(svint16_t, svint16_t, svint16_t);
	23924	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u8)))
	23925	svuint8_t svclamp(svuint8_t, svuint8_t, svuint8_t);
	23926	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u32)))
	23927	svuint32_t svclamp(svuint32_t, svuint32_t, svuint32_t);
	23928	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u64)))
	23929	svuint64_t svclamp(svuint64_t, svuint64_t, svuint64_t);
	23930	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u16)))
	23931	svuint16_t svclamp(svuint16_t, svuint16_t, svuint16_t);
23877	23932	#define svcvtnt_bf16_x svcvtnt_bf16_m
23878	23933	#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m
23879	23934	#define svcvtnt_f16_x svcvtnt_f16_m

lib/include/avx2intrin.h+4116-1

...	...	@@ -19,128 +19,539 @@
19	19	#define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("avx2"), __min_vector_width__(128)))
20	20
21	21	/* SSE4 Multiple Packed Sums of Absolute Difference. */
	22	/// Computes sixteen sum of absolute difference (SAD) operations on sets of
	23	/// four unsigned 8-bit integers from the 256-bit integer vectors \a X and
	24	/// \a Y.
	25	///
	26	/// Eight SAD results are computed using the lower half of the input
	27	/// vectors, and another eight using the upper half. These 16-bit values
	28	/// are returned in the lower and upper halves of the 256-bit result,
	29	/// respectively.
	30	///
	31	/// A single SAD operation selects four bytes from \a X and four bytes from
	32	/// \a Y as input. It computes the differences between each \a X byte and
	33	/// the corresponding \a Y byte, takes the absolute value of each
	34	/// difference, and sums these four values to form one 16-bit result. The
	35	/// intrinsic computes 16 of these results with different sets of input
	36	/// bytes.
	37	///
	38	/// For each set of eight results, the SAD operations use the same four
	39	/// bytes from \a Y; the starting bit position for these four bytes is
	40	/// specified by \a M[1:0] times 32. The eight operations use successive
	41	/// sets of four bytes from \a X; the starting bit position for the first
	42	/// set of four bytes is specified by \a M[2] times 32. These bit positions
	43	/// are all relative to the 128-bit lane for each set of eight operations.
	44	///
	45	/// \code{.operation}
	46	/// r := 0
	47	/// FOR i := 0 TO 1
	48	/// j := i*3
	49	/// Ybase := M[j+1:j]32 + i128
	50	/// Xbase := M[j+2]32 + i128
	51	/// FOR k := 0 TO 3
	52	/// temp0 := ABS(X[Xbase+7:Xbase] - Y[Ybase+7:Ybase])
	53	/// temp1 := ABS(X[Xbase+15:Xbase+8] - Y[Ybase+15:Ybase+8])
	54	/// temp2 := ABS(X[Xbase+23:Xbase+16] - Y[Ybase+23:Ybase+16])
	55	/// temp3 := ABS(X[Xbase+31:Xbase+24] - Y[Ybase+31:Ybase+24])
	56	/// result[r+15:r] := temp0 + temp1 + temp2 + temp3
	57	/// Xbase := Xbase + 8
	58	/// r := r + 16
	59	/// ENDFOR
	60	/// ENDFOR
	61	/// \endcode
	62	///
	63	/// \headerfile <immintrin.h>
	64	///
	65	/// \code
	66	/// __m256i _mm256_mpsadbw_epu8(__m256i X, __m256i Y, const int M);
	67	/// \endcode
	68	///
	69	/// This intrinsic corresponds to the \c VMPSADBW instruction.
	70	///
	71	/// \param X
	72	/// A 256-bit integer vector containing one of the inputs.
	73	/// \param Y
	74	/// A 256-bit integer vector containing one of the inputs.
	75	/// \param M
	76	/// An unsigned immediate value specifying the starting positions of the
	77	/// bytes to operate on.
	78	/// \returns A 256-bit vector of [16 x i16] containing the result.
22	79	#define _mm256_mpsadbw_epu8(X, Y, M) \
23	80	((__m256i)__builtin_ia32_mpsadbw256((__v32qi)(__m256i)(X), \
24	81	(__v32qi)(__m256i)(Y), (int)(M)))
25	82
	83	/// Computes the absolute value of each signed byte in the 256-bit integer
	84	/// vector \a __a and returns each value in the corresponding byte of
	85	/// the result.
	86	///
	87	/// \headerfile <immintrin.h>
	88	///
	89	/// This intrinsic corresponds to the \c VPABSB instruction.
	90	///
	91	/// \param __a
	92	/// A 256-bit integer vector.
	93	/// \returns A 256-bit integer vector containing the result.
26	94	static __inline__ __m256i __DEFAULT_FN_ATTRS256
27	95	_mm256_abs_epi8(__m256i __a)
28	96	{
29	97	return (__m256i)__builtin_elementwise_abs((__v32qs)__a);
30	98	}
31	99
	100	/// Computes the absolute value of each signed 16-bit element in the 256-bit
	101	/// vector of [16 x i16] in \a __a and returns each value in the
	102	/// corresponding element of the result.
	103	///
	104	/// \headerfile <immintrin.h>
	105	///
	106	/// This intrinsic corresponds to the \c VPABSW instruction.
	107	///
	108	/// \param __a
	109	/// A 256-bit vector of [16 x i16].
	110	/// \returns A 256-bit vector of [16 x i16] containing the result.
32	111	static __inline__ __m256i __DEFAULT_FN_ATTRS256
33	112	_mm256_abs_epi16(__m256i __a)
34	113	{
35	114	return (__m256i)__builtin_elementwise_abs((__v16hi)__a);
36	115	}
37	116
	117	/// Computes the absolute value of each signed 32-bit element in the 256-bit
	118	/// vector of [8 x i32] in \a __a and returns each value in the
	119	/// corresponding element of the result.
	120	///
	121	/// \headerfile <immintrin.h>
	122	///
	123	/// This intrinsic corresponds to the \c VPABSD instruction.
	124	///
	125	/// \param __a
	126	/// A 256-bit vector of [8 x i32].
	127	/// \returns A 256-bit vector of [8 x i32] containing the result.
38	128	static __inline__ __m256i __DEFAULT_FN_ATTRS256
39	129	_mm256_abs_epi32(__m256i __a)
40	130	{
41	131	return (__m256i)__builtin_elementwise_abs((__v8si)__a);
42	132	}
43	133
	134	/// Converts the elements of two 256-bit vectors of [16 x i16] to 8-bit
	135	/// integers using signed saturation, and returns the 256-bit result.
	136	///
	137	/// \code{.operation}
	138	/// FOR i := 0 TO 7
	139	/// j := i*16
	140	/// k := i*8
	141	/// result[7+k:k] := SATURATE8(__a[15+j:j])
	142	/// result[71+k:64+k] := SATURATE8(__b[15+j:j])
	143	/// result[135+k:128+k] := SATURATE8(__a[143+j:128+j])
	144	/// result[199+k:192+k] := SATURATE8(__b[143+j:128+j])
	145	/// ENDFOR
	146	/// \endcode
	147	///
	148	/// \headerfile <immintrin.h>
	149	///
	150	/// This intrinsic corresponds to the \c VPACKSSWB instruction.
	151	///
	152	/// \param __a
	153	/// A 256-bit vector of [16 x i16] used to generate result[63:0] and
	154	/// result[191:128].
	155	/// \param __b
	156	/// A 256-bit vector of [16 x i16] used to generate result[127:64] and
	157	/// result[255:192].
	158	/// \returns A 256-bit integer vector containing the result.
44	159	static __inline__ __m256i __DEFAULT_FN_ATTRS256
45	160	_mm256_packs_epi16(__m256i __a, __m256i __b)
46	161	{
47	162	return (__m256i)__builtin_ia32_packsswb256((__v16hi)__a, (__v16hi)__b);
48	163	}
49	164
	165	/// Converts the elements of two 256-bit vectors of [8 x i32] to 16-bit
	166	/// integers using signed saturation, and returns the resulting 256-bit
	167	/// vector of [16 x i16].
	168	///
	169	/// \code{.operation}
	170	/// FOR i := 0 TO 3
	171	/// j := i*32
	172	/// k := i*16
	173	/// result[15+k:k] := SATURATE16(__a[31+j:j])
	174	/// result[79+k:64+k] := SATURATE16(__b[31+j:j])
	175	/// result[143+k:128+k] := SATURATE16(__a[159+j:128+j])
	176	/// result[207+k:192+k] := SATURATE16(__b[159+j:128+j])
	177	/// ENDFOR
	178	/// \endcode
	179	///
	180	/// \headerfile <immintrin.h>
	181	///
	182	/// This intrinsic corresponds to the \c VPACKSSDW instruction.
	183	///
	184	/// \param __a
	185	/// A 256-bit vector of [8 x i32] used to generate result[63:0] and
	186	/// result[191:128].
	187	/// \param __b
	188	/// A 256-bit vector of [8 x i32] used to generate result[127:64] and
	189	/// result[255:192].
	190	/// \returns A 256-bit vector of [16 x i16] containing the result.
50	191	static __inline__ __m256i __DEFAULT_FN_ATTRS256
51	192	_mm256_packs_epi32(__m256i __a, __m256i __b)
52	193	{
53	194	return (__m256i)__builtin_ia32_packssdw256((__v8si)__a, (__v8si)__b);
54	195	}
55	196
	197	/// Converts elements from two 256-bit vectors of [16 x i16] to 8-bit integers
	198	/// using unsigned saturation, and returns the 256-bit result.
	199	///
	200	/// \code{.operation}
	201	/// FOR i := 0 TO 7
	202	/// j := i*16
	203	/// k := i*8
	204	/// result[7+k:k] := SATURATE8U(__a[15+j:j])
	205	/// result[71+k:64+k] := SATURATE8U(__b[15+j:j])
	206	/// result[135+k:128+k] := SATURATE8U(__a[143+j:128+j])
	207	/// result[199+k:192+k] := SATURATE8U(__b[143+j:128+j])
	208	/// ENDFOR
	209	/// \endcode
	210	///
	211	/// \headerfile <immintrin.h>
	212	///
	213	/// This intrinsic corresponds to the \c VPACKUSWB instruction.
	214	///
	215	/// \param __a
	216	/// A 256-bit vector of [16 x i16] used to generate result[63:0] and
	217	/// result[191:128].
	218	/// \param __b
	219	/// A 256-bit vector of [16 x i16] used to generate result[127:64] and
	220	/// result[255:192].
	221	/// \returns A 256-bit integer vector containing the result.
56	222	static __inline__ __m256i __DEFAULT_FN_ATTRS256
57	223	_mm256_packus_epi16(__m256i __a, __m256i __b)
58	224	{
59	225	return (__m256i)__builtin_ia32_packuswb256((__v16hi)__a, (__v16hi)__b);
60	226	}
61	227
	228	/// Converts elements from two 256-bit vectors of [8 x i32] to 16-bit integers
	229	/// using unsigned saturation, and returns the resulting 256-bit vector of
	230	/// [16 x i16].
	231	///
	232	/// \code{.operation}
	233	/// FOR i := 0 TO 3
	234	/// j := i*32
	235	/// k := i*16
	236	/// result[15+k:k] := SATURATE16U(__V1[31+j:j])
	237	/// result[79+k:64+k] := SATURATE16U(__V2[31+j:j])
	238	/// result[143+k:128+k] := SATURATE16U(__V1[159+j:128+j])
	239	/// result[207+k:192+k] := SATURATE16U(__V2[159+j:128+j])
	240	/// ENDFOR
	241	/// \endcode
	242	///
	243	/// \headerfile <immintrin.h>
	244	///
	245	/// This intrinsic corresponds to the \c VPACKUSDW instruction.
	246	///
	247	/// \param __V1
	248	/// A 256-bit vector of [8 x i32] used to generate result[63:0] and
	249	/// result[191:128].
	250	/// \param __V2
	251	/// A 256-bit vector of [8 x i32] used to generate result[127:64] and
	252	/// result[255:192].
	253	/// \returns A 256-bit vector of [16 x i16] containing the result.
62	254	static __inline__ __m256i __DEFAULT_FN_ATTRS256
63	255	_mm256_packus_epi32(__m256i __V1, __m256i __V2)
64	256	{
65	257	return (__m256i) __builtin_ia32_packusdw256((__v8si)__V1, (__v8si)__V2);
66	258	}
67	259
	260	/// Adds 8-bit integers from corresponding bytes of two 256-bit integer
	261	/// vectors and returns the lower 8 bits of each sum in the corresponding
	262	/// byte of the 256-bit integer vector result (overflow is ignored).
	263	///
	264	/// \headerfile <immintrin.h>
	265	///
	266	/// This intrinsic corresponds to the \c VPADDB instruction.
	267	///
	268	/// \param __a
	269	/// A 256-bit integer vector containing one of the source operands.
	270	/// \param __b
	271	/// A 256-bit integer vector containing one of the source operands.
	272	/// \returns A 256-bit integer vector containing the sums.
68	273	static __inline__ __m256i __DEFAULT_FN_ATTRS256
69	274	_mm256_add_epi8(__m256i __a, __m256i __b)
70	275	{
71	276	return (__m256i)((__v32qu)__a + (__v32qu)__b);
72	277	}
73	278
	279	/// Adds 16-bit integers from corresponding elements of two 256-bit vectors of
	280	/// [16 x i16] and returns the lower 16 bits of each sum in the
	281	/// corresponding element of the [16 x i16] result (overflow is ignored).
	282	///
	283	/// \headerfile <immintrin.h>
	284	///
	285	/// This intrinsic corresponds to the \c VPADDW instruction.
	286	///
	287	/// \param __a
	288	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	289	/// \param __b
	290	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	291	/// \returns A 256-bit vector of [16 x i16] containing the sums.
74	292	static __inline__ __m256i __DEFAULT_FN_ATTRS256
75	293	_mm256_add_epi16(__m256i __a, __m256i __b)
76	294	{
77	295	return (__m256i)((__v16hu)__a + (__v16hu)__b);
78	296	}
79	297
	298	/// Adds 32-bit integers from corresponding elements of two 256-bit vectors of
	299	/// [8 x i32] and returns the lower 32 bits of each sum in the corresponding
	300	/// element of the [8 x i32] result (overflow is ignored).
	301	///
	302	/// \headerfile <immintrin.h>
	303	///
	304	/// This intrinsic corresponds to the \c VPADDD instruction.
	305	///
	306	/// \param __a
	307	/// A 256-bit vector of [8 x i32] containing one of the source operands.
	308	/// \param __b
	309	/// A 256-bit vector of [8 x i32] containing one of the source operands.
	310	/// \returns A 256-bit vector of [8 x i32] containing the sums.
80	311	static __inline__ __m256i __DEFAULT_FN_ATTRS256
81	312	_mm256_add_epi32(__m256i __a, __m256i __b)
82	313	{
83	314	return (__m256i)((__v8su)__a + (__v8su)__b);
84	315	}
85	316
	317	/// Adds 64-bit integers from corresponding elements of two 256-bit vectors of
	318	/// [4 x i64] and returns the lower 64 bits of each sum in the corresponding
	319	/// element of the [4 x i64] result (overflow is ignored).
	320	///
	321	/// \headerfile <immintrin.h>
	322	///
	323	/// This intrinsic corresponds to the \c VPADDQ instruction.
	324	///
	325	/// \param __a
	326	/// A 256-bit vector of [4 x i64] containing one of the source operands.
	327	/// \param __b
	328	/// A 256-bit vector of [4 x i64] containing one of the source operands.
	329	/// \returns A 256-bit vector of [4 x i64] containing the sums.
86	330	static __inline__ __m256i __DEFAULT_FN_ATTRS256
87	331	_mm256_add_epi64(__m256i __a, __m256i __b)
88	332	{
89	333	return (__m256i)((__v4du)__a + (__v4du)__b);
90	334	}
91	335
	336	/// Adds 8-bit integers from corresponding bytes of two 256-bit integer
	337	/// vectors using signed saturation, and returns each sum in the
	338	/// corresponding byte of the 256-bit integer vector result.
	339	///
	340	/// \headerfile <immintrin.h>
	341	///
	342	/// This intrinsic corresponds to the \c VPADDSB instruction.
	343	///
	344	/// \param __a
	345	/// A 256-bit integer vector containing one of the source operands.
	346	/// \param __b
	347	/// A 256-bit integer vector containing one of the source operands.
	348	/// \returns A 256-bit integer vector containing the sums.
92	349	static __inline__ __m256i __DEFAULT_FN_ATTRS256
93	350	_mm256_adds_epi8(__m256i __a, __m256i __b)
94	351	{
95	352	return (__m256i)__builtin_elementwise_add_sat((__v32qs)__a, (__v32qs)__b);
96	353	}
97	354
	355	/// Adds 16-bit integers from corresponding elements of two 256-bit vectors of
	356	/// [16 x i16] using signed saturation, and returns the [16 x i16] result.
	357	///
	358	/// \headerfile <immintrin.h>
	359	///
	360	/// This intrinsic corresponds to the \c VPADDSW instruction.
	361	///
	362	/// \param __a
	363	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	364	/// \param __b
	365	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	366	/// \returns A 256-bit vector of [16 x i16] containing the sums.
98	367	static __inline__ __m256i __DEFAULT_FN_ATTRS256
99	368	_mm256_adds_epi16(__m256i __a, __m256i __b)
100	369	{
101	370	return (__m256i)__builtin_elementwise_add_sat((__v16hi)__a, (__v16hi)__b);
102	371	}
103	372
	373	/// Adds 8-bit integers from corresponding bytes of two 256-bit integer
	374	/// vectors using unsigned saturation, and returns each sum in the
	375	/// corresponding byte of the 256-bit integer vector result.
	376	///
	377	/// \headerfile <immintrin.h>
	378	///
	379	/// This intrinsic corresponds to the \c VPADDUSB instruction.
	380	///
	381	/// \param __a
	382	/// A 256-bit integer vector containing one of the source operands.
	383	/// \param __b
	384	/// A 256-bit integer vector containing one of the source operands.
	385	/// \returns A 256-bit integer vector containing the sums.
104	386	static __inline__ __m256i __DEFAULT_FN_ATTRS256
105	387	_mm256_adds_epu8(__m256i __a, __m256i __b)
106	388	{
107	389	return (__m256i)__builtin_elementwise_add_sat((__v32qu)__a, (__v32qu)__b);
108	390	}
109	391
	392	/// Adds 16-bit integers from corresponding elements of two 256-bit vectors of
	393	/// [16 x i16] using unsigned saturation, and returns the [16 x i16] result.
	394	///
	395	/// \headerfile <immintrin.h>
	396	///
	397	/// This intrinsic corresponds to the \c VPADDUSW instruction.
	398	///
	399	/// \param __a
	400	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	401	/// \param __b
	402	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	403	/// \returns A 256-bit vector of [16 x i16] containing the sums.
110	404	static __inline__ __m256i __DEFAULT_FN_ATTRS256
111	405	_mm256_adds_epu16(__m256i __a, __m256i __b)
112	406	{
113	407	return (__m256i)__builtin_elementwise_add_sat((__v16hu)__a, (__v16hu)__b);
114	408	}
115	409
	410	/// Uses the lower half of the 256-bit vector \a a as the upper half of a
	411	/// temporary 256-bit value, and the lower half of the 256-bit vector \a b
	412	/// as the lower half of the temporary value. Right-shifts the temporary
	413	/// value by \a n bytes, and uses the lower 16 bytes of the shifted value
	414	/// as the lower 16 bytes of the result. Uses the upper halves of \a a and
	415	/// \a b to make another temporary value, right shifts by \a n, and uses
	416	/// the lower 16 bytes of the shifted value as the upper 16 bytes of the
	417	/// result.
	418	///
	419	/// \headerfile <immintrin.h>
	420	///
	421	/// \code
	422	/// __m256i _mm256_alignr_epi8(__m256i a, __m256i b, const int n);
	423	/// \endcode
	424	///
	425	/// This intrinsic corresponds to the \c VPALIGNR instruction.
	426	///
	427	/// \param a
	428	/// A 256-bit integer vector containing source values.
	429	/// \param b
	430	/// A 256-bit integer vector containing source values.
	431	/// \param n
	432	/// An immediate value specifying the number of bytes to shift.
	433	/// \returns A 256-bit integer vector containing the result.
116	434	#define _mm256_alignr_epi8(a, b, n) \
117	435	((__m256i)__builtin_ia32_palignr256((__v32qi)(__m256i)(a), \
118	436	(__v32qi)(__m256i)(b), (n)))
119	437
	438	/// Computes the bitwise AND of the 256-bit integer vectors in \a __a and
	439	/// \a __b.
	440	///
	441	/// \headerfile <immintrin.h>
	442	///
	443	/// This intrinsic corresponds to the \c VPAND instruction.
	444	///
	445	/// \param __a
	446	/// A 256-bit integer vector.
	447	/// \param __b
	448	/// A 256-bit integer vector.
	449	/// \returns A 256-bit integer vector containing the result.
120	450	static __inline__ __m256i __DEFAULT_FN_ATTRS256
121	451	_mm256_and_si256(__m256i __a, __m256i __b)
122	452	{
123	453	return (__m256i)((__v4du)__a & (__v4du)__b);
124	454	}
125	455
	456	/// Computes the bitwise AND of the 256-bit integer vector in \a __b with
	457	/// the bitwise NOT of the 256-bit integer vector in \a __a.
	458	///
	459	/// \headerfile <immintrin.h>
	460	///
	461	/// This intrinsic corresponds to the \c VPANDN instruction.
	462	///
	463	/// \param __a
	464	/// A 256-bit integer vector.
	465	/// \param __b
	466	/// A 256-bit integer vector.
	467	/// \returns A 256-bit integer vector containing the result.
126	468	static __inline__ __m256i __DEFAULT_FN_ATTRS256
127	469	_mm256_andnot_si256(__m256i __a, __m256i __b)
128	470	{
129	471	return (__m256i)(~(__v4du)__a & (__v4du)__b);
130	472	}
131	473
	474	/// Computes the averages of the corresponding unsigned bytes in the two
	475	/// 256-bit integer vectors in \a __a and \a __b and returns each
	476	/// average in the corresponding byte of the 256-bit result.
	477	///
	478	/// \code{.operation}
	479	/// FOR i := 0 TO 31
	480	/// j := i*8
	481	/// result[j+7:j] := (__a[j+7:j] + __b[j+7:j] + 1) >> 1
	482	/// ENDFOR
	483	/// \endcode
	484	///
	485	/// \headerfile <immintrin.h>
	486	///
	487	/// This intrinsic corresponds to the \c VPAVGB instruction.
	488	///
	489	/// \param __a
	490	/// A 256-bit integer vector.
	491	/// \param __b
	492	/// A 256-bit integer vector.
	493	/// \returns A 256-bit integer vector containing the result.
132	494	static __inline__ __m256i __DEFAULT_FN_ATTRS256
133	495	_mm256_avg_epu8(__m256i __a, __m256i __b)
134	496	{
135	497	return (__m256i)__builtin_ia32_pavgb256((__v32qi)__a, (__v32qi)__b);
136	498	}
137	499
	500	/// Computes the averages of the corresponding unsigned 16-bit integers in
	501	/// the two 256-bit vectors of [16 x i16] in \a __a and \a __b and returns
	502	/// each average in the corresponding element of the 256-bit result.
	503	///
	504	/// \code{.operation}
	505	/// FOR i := 0 TO 15
	506	/// j := i*16
	507	/// result[j+15:j] := (__a[j+15:j] + __b[j+15:j] + 1) >> 1
	508	/// ENDFOR
	509	/// \endcode
	510	///
	511	/// \headerfile <immintrin.h>
	512	///
	513	/// This intrinsic corresponds to the \c VPAVGW instruction.
	514	///
	515	/// \param __a
	516	/// A 256-bit vector of [16 x i16].
	517	/// \param __b
	518	/// A 256-bit vector of [16 x i16].
	519	/// \returns A 256-bit vector of [16 x i16] containing the result.
138	520	static __inline__ __m256i __DEFAULT_FN_ATTRS256
139	521	_mm256_avg_epu16(__m256i __a, __m256i __b)
140	522	{
141	523	return (__m256i)__builtin_ia32_pavgw256((__v16hi)__a, (__v16hi)__b);
142	524	}
143	525
	526	/// Merges 8-bit integer values from either of the two 256-bit vectors
	527	/// \a __V1 or \a __V2, as specified by the 256-bit mask \a __M and returns
	528	/// the resulting 256-bit integer vector.
	529	///
	530	/// \code{.operation}
	531	/// FOR i := 0 TO 31
	532	/// j := i*8
	533	/// IF __M[7+i] == 0
	534	/// result[7+j:j] := __V1[7+j:j]
	535	/// ELSE
	536	/// result[7+j:j] := __V2[7+j:j]
	537	/// FI
	538	/// ENDFOR
	539	/// \endcode
	540	///
	541	/// \headerfile <immintrin.h>
	542	///
	543	/// This intrinsic corresponds to the \c VPBLENDVB instruction.
	544	///
	545	/// \param __V1
	546	/// A 256-bit integer vector containing source values.
	547	/// \param __V2
	548	/// A 256-bit integer vector containing source values.
	549	/// \param __M
	550	/// A 256-bit integer vector, with bit [7] of each byte specifying the
	551	/// source for each corresponding byte of the result. When the mask bit
	552	/// is 0, the byte is copied from \a __V1; otherwise, it is copied from
	553	/// \a __V2.
	554	/// \returns A 256-bit integer vector containing the result.
144	555	static __inline__ __m256i __DEFAULT_FN_ATTRS256
145	556	_mm256_blendv_epi8(__m256i __V1, __m256i __V2, __m256i __M)
146	557	{
...	...	@@ -148,34 +559,171 @@ _mm256_blendv_epi8(__m256i __V1, __m256i __V2, __m256i __M)
148	559	(__v32qi)__M);
149	560	}
150	561
	562	/// Merges 16-bit integer values from either of the two 256-bit vectors
	563	/// \a V1 or \a V2, as specified by the immediate integer operand \a M,
	564	/// and returns the resulting 256-bit vector of [16 x i16].
	565	///
	566	/// \code{.operation}
	567	/// FOR i := 0 TO 7
	568	/// j := i*16
	569	/// IF M[i] == 0
	570	/// result[7+j:j] := V1[7+j:j]
	571	/// result[135+j:128+j] := V1[135+j:128+j]
	572	/// ELSE
	573	/// result[7+j:j] := V2[7+j:j]
	574	/// result[135+j:128+j] := V2[135+j:128+j]
	575	/// FI
	576	/// ENDFOR
	577	/// \endcode
	578	///
	579	/// \headerfile <immintrin.h>
	580	///
	581	/// \code
	582	/// __m256i _mm256_blend_epi16(__m256i V1, __m256i V2, const int M);
	583	/// \endcode
	584	///
	585	/// This intrinsic corresponds to the \c VPBLENDW instruction.
	586	///
	587	/// \param V1
	588	/// A 256-bit vector of [16 x i16] containing source values.
	589	/// \param V2
	590	/// A 256-bit vector of [16 x i16] containing source values.
	591	/// \param M
	592	/// An immediate 8-bit integer operand, with bits [7:0] specifying the
	593	/// source for each element of the result. The position of the mask bit
	594	/// corresponds to the index of a copied value. When a mask bit is 0, the
	595	/// element is copied from \a V1; otherwise, it is copied from \a V2.
	596	/// \a M[0] determines the source for elements 0 and 8, \a M[1] for
	597	/// elements 1 and 9, and so forth.
	598	/// \returns A 256-bit vector of [16 x i16] containing the result.
151	599	#define _mm256_blend_epi16(V1, V2, M) \
152	600	((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(V1), \
153	601	(__v16hi)(__m256i)(V2), (int)(M)))
154	602
	603	/// Compares corresponding bytes in the 256-bit integer vectors in \a __a and
	604	/// \a __b for equality and returns the outcomes in the corresponding
	605	/// bytes of the 256-bit result.
	606	///
	607	/// \code{.operation}
	608	/// FOR i := 0 TO 31
	609	/// j := i*8
	610	/// result[j+7:j] := (__a[j+7:j] == __b[j+7:j]) ? 0xFF : 0
	611	/// ENDFOR
	612	/// \endcode
	613	///
	614	/// \headerfile <immintrin.h>
	615	///
	616	/// This intrinsic corresponds to the \c VPCMPEQB instruction.
	617	///
	618	/// \param __a
	619	/// A 256-bit integer vector containing one of the inputs.
	620	/// \param __b
	621	/// A 256-bit integer vector containing one of the inputs.
	622	/// \returns A 256-bit integer vector containing the result.
155	623	static __inline__ __m256i __DEFAULT_FN_ATTRS256
156	624	_mm256_cmpeq_epi8(__m256i __a, __m256i __b)
157	625	{
158	626	return (__m256i)((__v32qi)__a == (__v32qi)__b);
159	627	}
160	628
	629	/// Compares corresponding elements in the 256-bit vectors of [16 x i16] in
	630	/// \a __a and \a __b for equality and returns the outcomes in the
	631	/// corresponding elements of the 256-bit result.
	632	///
	633	/// \code{.operation}
	634	/// FOR i := 0 TO 15
	635	/// j := i*16
	636	/// result[j+15:j] := (__a[j+15:j] == __b[j+15:j]) ? 0xFFFF : 0
	637	/// ENDFOR
	638	/// \endcode
	639	///
	640	/// \headerfile <immintrin.h>
	641	///
	642	/// This intrinsic corresponds to the \c VPCMPEQW instruction.
	643	///
	644	/// \param __a
	645	/// A 256-bit vector of [16 x i16] containing one of the inputs.
	646	/// \param __b
	647	/// A 256-bit vector of [16 x i16] containing one of the inputs.
	648	/// \returns A 256-bit vector of [16 x i16] containing the result.
161	649	static __inline__ __m256i __DEFAULT_FN_ATTRS256
162	650	_mm256_cmpeq_epi16(__m256i __a, __m256i __b)
163	651	{
164	652	return (__m256i)((__v16hi)__a == (__v16hi)__b);
165	653	}
166	654
	655	/// Compares corresponding elements in the 256-bit vectors of [8 x i32] in
	656	/// \a __a and \a __b for equality and returns the outcomes in the
	657	/// corresponding elements of the 256-bit result.
	658	///
	659	/// \code{.operation}
	660	/// FOR i := 0 TO 7
	661	/// j := i*32
	662	/// result[j+31:j] := (__a[j+31:j] == __b[j+31:j]) ? 0xFFFFFFFF : 0
	663	/// ENDFOR
	664	/// \endcode
	665	///
	666	/// \headerfile <immintrin.h>
	667	///
	668	/// This intrinsic corresponds to the \c VPCMPEQD instruction.
	669	///
	670	/// \param __a
	671	/// A 256-bit vector of [8 x i32] containing one of the inputs.
	672	/// \param __b
	673	/// A 256-bit vector of [8 x i32] containing one of the inputs.
	674	/// \returns A 256-bit vector of [8 x i32] containing the result.
167	675	static __inline__ __m256i __DEFAULT_FN_ATTRS256
168	676	_mm256_cmpeq_epi32(__m256i __a, __m256i __b)
169	677	{
170	678	return (__m256i)((__v8si)__a == (__v8si)__b);
171	679	}
172	680
	681	/// Compares corresponding elements in the 256-bit vectors of [4 x i64] in
	682	/// \a __a and \a __b for equality and returns the outcomes in the
	683	/// corresponding elements of the 256-bit result.
	684	///
	685	/// \code{.operation}
	686	/// FOR i := 0 TO 3
	687	/// j := i*64
	688	/// result[j+63:j] := (__a[j+63:j] == __b[j+63:j]) ? 0xFFFFFFFFFFFFFFFF : 0
	689	/// ENDFOR
	690	/// \endcode
	691	///
	692	/// \headerfile <immintrin.h>
	693	///
	694	/// This intrinsic corresponds to the \c VPCMPEQQ instruction.
	695	///
	696	/// \param __a
	697	/// A 256-bit vector of [4 x i64] containing one of the inputs.
	698	/// \param __b
	699	/// A 256-bit vector of [4 x i64] containing one of the inputs.
	700	/// \returns A 256-bit vector of [4 x i64] containing the result.
173	701	static __inline__ __m256i __DEFAULT_FN_ATTRS256
174	702	_mm256_cmpeq_epi64(__m256i __a, __m256i __b)
175	703	{
176	704	return (__m256i)((__v4di)__a == (__v4di)__b);
177	705	}
178	706
	707	/// Compares corresponding signed bytes in the 256-bit integer vectors in
	708	/// \a __a and \a __b for greater-than and returns the outcomes in the
	709	/// corresponding bytes of the 256-bit result.
	710	///
	711	/// \code{.operation}
	712	/// FOR i := 0 TO 31
	713	/// j := i*8
	714	/// result[j+7:j] := (__a[j+7:j] > __b[j+7:j]) ? 0xFF : 0
	715	/// ENDFOR
	716	/// \endcode
	717	///
	718	/// \headerfile <immintrin.h>
	719	///
	720	/// This intrinsic corresponds to the \c VPCMPGTB instruction.
	721	///
	722	/// \param __a
	723	/// A 256-bit integer vector containing one of the inputs.
	724	/// \param __b
	725	/// A 256-bit integer vector containing one of the inputs.
	726	/// \returns A 256-bit integer vector containing the result.
179	727	static __inline__ __m256i __DEFAULT_FN_ATTRS256
180	728	_mm256_cmpgt_epi8(__m256i __a, __m256i __b)
181	729	{
...	...	@@ -184,138 +732,575 @@ _mm256_cmpgt_epi8(__m256i __a, __m256i __b)
184	732	return (__m256i)((__v32qs)__a > (__v32qs)__b);
185	733	}
186	734
	735	/// Compares corresponding signed elements in the 256-bit vectors of
	736	/// [16 x i16] in \a __a and \a __b for greater-than and returns the
	737	/// outcomes in the corresponding elements of the 256-bit result.
	738	///
	739	/// \code{.operation}
	740	/// FOR i := 0 TO 15
	741	/// j := i*16
	742	/// result[j+15:j] := (__a[j+15:j] > __b[j+15:j]) ? 0xFFFF : 0
	743	/// ENDFOR
	744	/// \endcode
	745	///
	746	/// \headerfile <immintrin.h>
	747	///
	748	/// This intrinsic corresponds to the \c VPCMPGTW instruction.
	749	///
	750	/// \param __a
	751	/// A 256-bit vector of [16 x i16] containing one of the inputs.
	752	/// \param __b
	753	/// A 256-bit vector of [16 x i16] containing one of the inputs.
	754	/// \returns A 256-bit vector of [16 x i16] containing the result.
187	755	static __inline__ __m256i __DEFAULT_FN_ATTRS256
188	756	_mm256_cmpgt_epi16(__m256i __a, __m256i __b)
189	757	{
190	758	return (__m256i)((__v16hi)__a > (__v16hi)__b);
191	759	}
192	760
	761	/// Compares corresponding signed elements in the 256-bit vectors of
	762	/// [8 x i32] in \a __a and \a __b for greater-than and returns the
	763	/// outcomes in the corresponding elements of the 256-bit result.
	764	///
	765	/// \code{.operation}
	766	/// FOR i := 0 TO 7
	767	/// j := i*32
	768	/// result[j+31:j] := (__a[j+31:j] > __b[j+31:j]) ? 0xFFFFFFFF : 0
	769	/// ENDFOR
	770	/// \endcode
	771	///
	772	/// \headerfile <immintrin.h>
	773	///
	774	/// This intrinsic corresponds to the \c VPCMPGTD instruction.
	775	///
	776	/// \param __a
	777	/// A 256-bit vector of [8 x i32] containing one of the inputs.
	778	/// \param __b
	779	/// A 256-bit vector of [8 x i32] containing one of the inputs.
	780	/// \returns A 256-bit vector of [8 x i32] containing the result.
193	781	static __inline__ __m256i __DEFAULT_FN_ATTRS256
194	782	_mm256_cmpgt_epi32(__m256i __a, __m256i __b)
195	783	{
196	784	return (__m256i)((__v8si)__a > (__v8si)__b);
197	785	}
198	786
	787	/// Compares corresponding signed elements in the 256-bit vectors of
	788	/// [4 x i64] in \a __a and \a __b for greater-than and returns the
	789	/// outcomes in the corresponding elements of the 256-bit result.
	790	///
	791	/// \code{.operation}
	792	/// FOR i := 0 TO 3
	793	/// j := i*64
	794	/// result[j+63:j] := (__a[j+63:j] > __b[j+63:j]) ? 0xFFFFFFFFFFFFFFFF : 0
	795	/// ENDFOR
	796	/// \endcode
	797	///
	798	/// \headerfile <immintrin.h>
	799	///
	800	/// This intrinsic corresponds to the \c VPCMPGTQ instruction.
	801	///
	802	/// \param __a
	803	/// A 256-bit vector of [4 x i64] containing one of the inputs.
	804	/// \param __b
	805	/// A 256-bit vector of [4 x i64] containing one of the inputs.
	806	/// \returns A 256-bit vector of [4 x i64] containing the result.
199	807	static __inline__ __m256i __DEFAULT_FN_ATTRS256
200	808	_mm256_cmpgt_epi64(__m256i __a, __m256i __b)
201	809	{
202	810	return (__m256i)((__v4di)__a > (__v4di)__b);
203	811	}
204	812
	813	/// Horizontally adds the adjacent pairs of 16-bit integers from two 256-bit
	814	/// vectors of [16 x i16] and returns the lower 16 bits of each sum in an
	815	/// element of the [16 x i16] result (overflow is ignored). Sums from
	816	/// \a __a are returned in the lower 64 bits of each 128-bit half of the
	817	/// result; sums from \a __b are returned in the upper 64 bits of each
	818	/// 128-bit half of the result.
	819	///
	820	/// \code{.operation}
	821	/// FOR i := 0 TO 1
	822	/// j := i*128
	823	/// result[j+15:j] := __a[j+15:j] + __a[j+31:j+16]
	824	/// result[j+31:j+16] := __a[j+47:j+32] + __a[j+63:j+48]
	825	/// result[j+47:j+32] := __a[j+79:j+64] + __a[j+95:j+80]
	826	/// result[j+63:j+48] := __a[j+111:j+96] + __a[j+127:j+112]
	827	/// result[j+79:j+64] := __b[j+15:j] + __b[j+31:j+16]
	828	/// result[j+95:j+80] := __b[j+47:j+32] + __b[j+63:j+48]
	829	/// result[j+111:j+96] := __b[j+79:j+64] + __b[j+95:j+80]
	830	/// result[j+127:j+112] := __b[j+111:j+96] + __b[j+127:j+112]
	831	/// ENDFOR
	832	/// \endcode
	833	///
	834	/// \headerfile <immintrin.h>
	835	///
	836	/// This intrinsic corresponds to the \c VPHADDW instruction.
	837	///
	838	/// \param __a
	839	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	840	/// \param __b
	841	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	842	/// \returns A 256-bit vector of [16 x i16] containing the sums.
205	843	static __inline__ __m256i __DEFAULT_FN_ATTRS256
206	844	_mm256_hadd_epi16(__m256i __a, __m256i __b)
207	845	{
208	846	return (__m256i)__builtin_ia32_phaddw256((__v16hi)__a, (__v16hi)__b);
209	847	}
210	848
	849	/// Horizontally adds the adjacent pairs of 32-bit integers from two 256-bit
	850	/// vectors of [8 x i32] and returns the lower 32 bits of each sum in an
	851	/// element of the [8 x i32] result (overflow is ignored). Sums from \a __a
	852	/// are returned in the lower 64 bits of each 128-bit half of the result;
	853	/// sums from \a __b are returned in the upper 64 bits of each 128-bit half
	854	/// of the result.
	855	///
	856	/// \code{.operation}
	857	/// FOR i := 0 TO 1
	858	/// j := i*128
	859	/// result[j+31:j] := __a[j+31:j] + __a[j+63:j+32]
	860	/// result[j+63:j+32] := __a[j+95:j+64] + __a[j+127:j+96]
	861	/// result[j+95:j+64] := __b[j+31:j] + __b[j+63:j+32]
	862	/// result[j+127:j+96] := __b[j+95:j+64] + __b[j+127:j+96]
	863	/// ENDFOR
	864	/// \endcode
	865	///
	866	/// \headerfile <immintrin.h>
	867	///
	868	/// This intrinsic corresponds to the \c VPHADDD instruction.
	869	///
	870	/// \param __a
	871	/// A 256-bit vector of [8 x i32] containing one of the source operands.
	872	/// \param __b
	873	/// A 256-bit vector of [8 x i32] containing one of the source operands.
	874	/// \returns A 256-bit vector of [8 x i32] containing the sums.
211	875	static __inline__ __m256i __DEFAULT_FN_ATTRS256
212	876	_mm256_hadd_epi32(__m256i __a, __m256i __b)
213	877	{
214	878	return (__m256i)__builtin_ia32_phaddd256((__v8si)__a, (__v8si)__b);
215	879	}
216	880
	881	/// Horizontally adds the adjacent pairs of 16-bit integers from two 256-bit
	882	/// vectors of [16 x i16] using signed saturation and returns each sum in
	883	/// an element of the [16 x i16] result. Sums from \a __a are returned in
	884	/// the lower 64 bits of each 128-bit half of the result; sums from \a __b
	885	/// are returned in the upper 64 bits of each 128-bit half of the result.
	886	///
	887	/// \code{.operation}
	888	/// FOR i := 0 TO 1
	889	/// j := i*128
	890	/// result[j+15:j] := SATURATE16(__a[j+15:j] + __a[j+31:j+16])
	891	/// result[j+31:j+16] := SATURATE16(__a[j+47:j+32] + __a[j+63:j+48])
	892	/// result[j+47:j+32] := SATURATE16(__a[j+79:j+64] + __a[j+95:j+80])
	893	/// result[j+63:j+48] := SATURATE16(__a[j+111:j+96] + __a[j+127:j+112])
	894	/// result[j+79:j+64] := SATURATE16(__b[j+15:j] + __b[j+31:j+16])
	895	/// result[j+95:j+80] := SATURATE16(__b[j+47:j+32] + __b[j+63:j+48])
	896	/// result[j+111:j+96] := SATURATE16(__b[j+79:j+64] + __b[j+95:j+80])
	897	/// result[j+127:j+112] := SATURATE16(__b[j+111:j+96] + __b[j+127:j+112])
	898	/// ENDFOR
	899	/// \endcode
	900	///
	901	/// \headerfile <immintrin.h>
	902	///
	903	/// This intrinsic corresponds to the \c VPHADDSW instruction.
	904	///
	905	/// \param __a
	906	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	907	/// \param __b
	908	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	909	/// \returns A 256-bit vector of [16 x i16] containing the sums.
217	910	static __inline__ __m256i __DEFAULT_FN_ATTRS256
218	911	_mm256_hadds_epi16(__m256i __a, __m256i __b)
219	912	{
220	913	return (__m256i)__builtin_ia32_phaddsw256((__v16hi)__a, (__v16hi)__b);
221	914	}
222	915
	916	/// Horizontally subtracts adjacent pairs of 16-bit integers from two 256-bit
	917	/// vectors of [16 x i16] and returns the lower 16 bits of each difference
	918	/// in an element of the [16 x i16] result (overflow is ignored).
	919	/// Differences from \a __a are returned in the lower 64 bits of each
	920	/// 128-bit half of the result; differences from \a __b are returned in the
	921	/// upper 64 bits of each 128-bit half of the result.
	922	///
	923	/// \code{.operation}
	924	/// FOR i := 0 TO 1
	925	/// j := i*128
	926	/// result[j+15:j] := __a[j+15:j] - __a[j+31:j+16]
	927	/// result[j+31:j+16] := __a[j+47:j+32] - __a[j+63:j+48]
	928	/// result[j+47:j+32] := __a[j+79:j+64] - __a[j+95:j+80]
	929	/// result[j+63:j+48] := __a[j+111:j+96] - __a[j+127:j+112]
	930	/// result[j+79:j+64] := __b[j+15:j] - __b[j+31:j+16]
	931	/// result[j+95:j+80] := __b[j+47:j+32] - __b[j+63:j+48]
	932	/// result[j+111:j+96] := __b[j+79:j+64] - __b[j+95:j+80]
	933	/// result[j+127:j+112] := __b[j+111:j+96] - __b[j+127:j+112]
	934	/// ENDFOR
	935	/// \endcode
	936	///
	937	/// \headerfile <immintrin.h>
	938	///
	939	/// This intrinsic corresponds to the \c VPHSUBW instruction.
	940	///
	941	/// \param __a
	942	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	943	/// \param __b
	944	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	945	/// \returns A 256-bit vector of [16 x i16] containing the differences.
223	946	static __inline__ __m256i __DEFAULT_FN_ATTRS256
224	947	_mm256_hsub_epi16(__m256i __a, __m256i __b)
225	948	{
226	949	return (__m256i)__builtin_ia32_phsubw256((__v16hi)__a, (__v16hi)__b);
227	950	}
228	951
	952	/// Horizontally subtracts adjacent pairs of 32-bit integers from two 256-bit
	953	/// vectors of [8 x i32] and returns the lower 32 bits of each difference in
	954	/// an element of the [8 x i32] result (overflow is ignored). Differences
	955	/// from \a __a are returned in the lower 64 bits of each 128-bit half of
	956	/// the result; differences from \a __b are returned in the upper 64 bits
	957	/// of each 128-bit half of the result.
	958	///
	959	/// \code{.operation}
	960	/// FOR i := 0 TO 1
	961	/// j := i*128
	962	/// result[j+31:j] := __a[j+31:j] - __a[j+63:j+32]
	963	/// result[j+63:j+32] := __a[j+95:j+64] - __a[j+127:j+96]
	964	/// result[j+95:j+64] := __b[j+31:j] - __b[j+63:j+32]
	965	/// result[j+127:j+96] := __b[j+95:j+64] - __b[j+127:j+96]
	966	/// ENDFOR
	967	/// \endcode
	968	///
	969	/// \headerfile <immintrin.h>
	970	///
	971	/// This intrinsic corresponds to the \c VPHSUBD instruction.
	972	///
	973	/// \param __a
	974	/// A 256-bit vector of [8 x i32] containing one of the source operands.
	975	/// \param __b
	976	/// A 256-bit vector of [8 x i32] containing one of the source operands.
	977	/// \returns A 256-bit vector of [8 x i32] containing the differences.
229	978	static __inline__ __m256i __DEFAULT_FN_ATTRS256
230	979	_mm256_hsub_epi32(__m256i __a, __m256i __b)
231	980	{
232	981	return (__m256i)__builtin_ia32_phsubd256((__v8si)__a, (__v8si)__b);
233	982	}
234	983
	984	/// Horizontally subtracts adjacent pairs of 16-bit integers from two 256-bit
	985	/// vectors of [16 x i16] using signed saturation and returns each sum in
	986	/// an element of the [16 x i16] result. Differences from \a __a are
	987	/// returned in the lower 64 bits of each 128-bit half of the result;
	988	/// differences from \a __b are returned in the upper 64 bits of each
	989	/// 128-bit half of the result.
	990	///
	991	/// \code{.operation}
	992	/// FOR i := 0 TO 1
	993	/// j := i*128
	994	/// result[j+15:j] := SATURATE16(__a[j+15:j] - __a[j+31:j+16])
	995	/// result[j+31:j+16] := SATURATE16(__a[j+47:j+32] - __a[j+63:j+48])
	996	/// result[j+47:j+32] := SATURATE16(__a[j+79:j+64] - __a[j+95:j+80])
	997	/// result[j+63:j+48] := SATURATE16(__a[j+111:j+96] - __a[j+127:j+112])
	998	/// result[j+79:j+64] := SATURATE16(__b[j+15:j] - __b[j+31:j+16])
	999	/// result[j+95:j+80] := SATURATE16(__b[j+47:j+32] - __b[j+63:j+48])
	1000	/// result[j+111:j+96] := SATURATE16(__b[j+79:j+64] - __b[j+95:j+80])
	1001	/// result[j+127:j+112] := SATURATE16(__b[j+111:j+96] - __b[j+127:j+112])
	1002	/// ENDFOR
	1003	/// \endcode
	1004	///
	1005	/// \headerfile <immintrin.h>
	1006	///
	1007	/// This intrinsic corresponds to the \c VPHSUBSW instruction.
	1008	///
	1009	/// \param __a
	1010	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	1011	/// \param __b
	1012	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	1013	/// \returns A 256-bit vector of [16 x i16] containing the differences.
235	1014	static __inline__ __m256i __DEFAULT_FN_ATTRS256
236	1015	_mm256_hsubs_epi16(__m256i __a, __m256i __b)
237	1016	{
238	1017	return (__m256i)__builtin_ia32_phsubsw256((__v16hi)__a, (__v16hi)__b);
239	1018	}
240	1019
	1020	/// Multiplies each unsigned byte from the 256-bit integer vector in \a __a
	1021	/// with the corresponding signed byte from the 256-bit integer vector in
	1022	/// \a __b, forming signed 16-bit intermediate products. Adds adjacent
	1023	/// pairs of those products using signed saturation to form 16-bit sums
	1024	/// returned as elements of the [16 x i16] result.
	1025	///
	1026	/// \code{.operation}
	1027	/// FOR i := 0 TO 15
	1028	/// j := i*16
	1029	/// temp1 := __a[j+7:j] * __b[j+7:j]
	1030	/// temp2 := __a[j+15:j+8] * __b[j+15:j+8]
	1031	/// result[j+15:j] := SATURATE16(temp1 + temp2)
	1032	/// ENDFOR
	1033	/// \endcode
	1034	///
	1035	/// \headerfile <immintrin.h>
	1036	///
	1037	/// This intrinsic corresponds to the \c VPMADDUBSW instruction.
	1038	///
	1039	/// \param __a
	1040	/// A 256-bit vector containing one of the source operands.
	1041	/// \param __b
	1042	/// A 256-bit vector containing one of the source operands.
	1043	/// \returns A 256-bit vector of [16 x i16] containing the result.
241	1044	static __inline__ __m256i __DEFAULT_FN_ATTRS256
242	1045	_mm256_maddubs_epi16(__m256i __a, __m256i __b)
243	1046	{
244	1047	return (__m256i)__builtin_ia32_pmaddubsw256((__v32qi)__a, (__v32qi)__b);
245	1048	}
246	1049
	1050	/// Multiplies corresponding 16-bit elements of two 256-bit vectors of
	1051	/// [16 x i16], forming 32-bit intermediate products, and adds pairs of
	1052	/// those products to form 32-bit sums returned as elements of the
	1053	/// [8 x i32] result.
	1054	///
	1055	/// There is only one wraparound case: when all four of the 16-bit sources
	1056	/// are \c 0x8000, the result will be \c 0x80000000.
	1057	///
	1058	/// \code{.operation}
	1059	/// FOR i := 0 TO 7
	1060	/// j := i*32
	1061	/// temp1 := __a[j+15:j] * __b[j+15:j]
	1062	/// temp2 := __a[j+31:j+16] * __b[j+31:j+16]
	1063	/// result[j+31:j] := temp1 + temp2
	1064	/// ENDFOR
	1065	/// \endcode
	1066	///
	1067	/// \headerfile <immintrin.h>
	1068	///
	1069	/// This intrinsic corresponds to the \c VPMADDWD instruction.
	1070	///
	1071	/// \param __a
	1072	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	1073	/// \param __b
	1074	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	1075	/// \returns A 256-bit vector of [8 x i32] containing the result.
247	1076	static __inline__ __m256i __DEFAULT_FN_ATTRS256
248	1077	_mm256_madd_epi16(__m256i __a, __m256i __b)
249	1078	{
250	1079	return (__m256i)__builtin_ia32_pmaddwd256((__v16hi)__a, (__v16hi)__b);
251	1080	}
252	1081
	1082	/// Compares the corresponding signed bytes in the two 256-bit integer vectors
	1083	/// in \a __a and \a __b and returns the larger of each pair in the
	1084	/// corresponding byte of the 256-bit result.
	1085	///
	1086	/// \headerfile <immintrin.h>
	1087	///
	1088	/// This intrinsic corresponds to the \c VPMAXSB instruction.
	1089	///
	1090	/// \param __a
	1091	/// A 256-bit integer vector.
	1092	/// \param __b
	1093	/// A 256-bit integer vector.
	1094	/// \returns A 256-bit integer vector containing the result.
253	1095	static __inline__ __m256i __DEFAULT_FN_ATTRS256
254	1096	_mm256_max_epi8(__m256i __a, __m256i __b)
255	1097	{
256	1098	return (__m256i)__builtin_elementwise_max((__v32qs)__a, (__v32qs)__b);
257	1099	}
258	1100
	1101	/// Compares the corresponding signed 16-bit integers in the two 256-bit
	1102	/// vectors of [16 x i16] in \a __a and \a __b and returns the larger of
	1103	/// each pair in the corresponding element of the 256-bit result.
	1104	///
	1105	/// \headerfile <immintrin.h>
	1106	///
	1107	/// This intrinsic corresponds to the \c VPMAXSW instruction.
	1108	///
	1109	/// \param __a
	1110	/// A 256-bit vector of [16 x i16].
	1111	/// \param __b
	1112	/// A 256-bit vector of [16 x i16].
	1113	/// \returns A 256-bit vector of [16 x i16] containing the result.
259	1114	static __inline__ __m256i __DEFAULT_FN_ATTRS256
260	1115	_mm256_max_epi16(__m256i __a, __m256i __b)
261	1116	{
262	1117	return (__m256i)__builtin_elementwise_max((__v16hi)__a, (__v16hi)__b);
263	1118	}
264	1119
	1120	/// Compares the corresponding signed 32-bit integers in the two 256-bit
	1121	/// vectors of [8 x i32] in \a __a and \a __b and returns the larger of
	1122	/// each pair in the corresponding element of the 256-bit result.
	1123	///
	1124	/// \headerfile <immintrin.h>
	1125	///
	1126	/// This intrinsic corresponds to the \c VPMAXSD instruction.
	1127	///
	1128	/// \param __a
	1129	/// A 256-bit vector of [8 x i32].
	1130	/// \param __b
	1131	/// A 256-bit vector of [8 x i32].
	1132	/// \returns A 256-bit vector of [8 x i32] containing the result.
265	1133	static __inline__ __m256i __DEFAULT_FN_ATTRS256
266	1134	_mm256_max_epi32(__m256i __a, __m256i __b)
267	1135	{
268	1136	return (__m256i)__builtin_elementwise_max((__v8si)__a, (__v8si)__b);
269	1137	}
270	1138
	1139	/// Compares the corresponding unsigned bytes in the two 256-bit integer
	1140	/// vectors in \a __a and \a __b and returns the larger of each pair in
	1141	/// the corresponding byte of the 256-bit result.
	1142	///
	1143	/// \headerfile <immintrin.h>
	1144	///
	1145	/// This intrinsic corresponds to the \c VPMAXUB instruction.
	1146	///
	1147	/// \param __a
	1148	/// A 256-bit integer vector.
	1149	/// \param __b
	1150	/// A 256-bit integer vector.
	1151	/// \returns A 256-bit integer vector containing the result.
271	1152	static __inline__ __m256i __DEFAULT_FN_ATTRS256
272	1153	_mm256_max_epu8(__m256i __a, __m256i __b)
273	1154	{
274	1155	return (__m256i)__builtin_elementwise_max((__v32qu)__a, (__v32qu)__b);
275	1156	}
276	1157
	1158	/// Compares the corresponding unsigned 16-bit integers in the two 256-bit
	1159	/// vectors of [16 x i16] in \a __a and \a __b and returns the larger of
	1160	/// each pair in the corresponding element of the 256-bit result.
	1161	///
	1162	/// \headerfile <immintrin.h>
	1163	///
	1164	/// This intrinsic corresponds to the \c VPMAXUW instruction.
	1165	///
	1166	/// \param __a
	1167	/// A 256-bit vector of [16 x i16].
	1168	/// \param __b
	1169	/// A 256-bit vector of [16 x i16].
	1170	/// \returns A 256-bit vector of [16 x i16] containing the result.
277	1171	static __inline__ __m256i __DEFAULT_FN_ATTRS256
278	1172	_mm256_max_epu16(__m256i __a, __m256i __b)
279	1173	{
280	1174	return (__m256i)__builtin_elementwise_max((__v16hu)__a, (__v16hu)__b);
281	1175	}
282	1176
	1177	/// Compares the corresponding unsigned 32-bit integers in the two 256-bit
	1178	/// vectors of [8 x i32] in \a __a and \a __b and returns the larger of
	1179	/// each pair in the corresponding element of the 256-bit result.
	1180	///
	1181	/// \headerfile <immintrin.h>
	1182	///
	1183	/// This intrinsic corresponds to the \c VPMAXUD instruction.
	1184	///
	1185	/// \param __a
	1186	/// A 256-bit vector of [8 x i32].
	1187	/// \param __b
	1188	/// A 256-bit vector of [8 x i32].
	1189	/// \returns A 256-bit vector of [8 x i32] containing the result.
283	1190	static __inline__ __m256i __DEFAULT_FN_ATTRS256
284	1191	_mm256_max_epu32(__m256i __a, __m256i __b)
285	1192	{
286	1193	return (__m256i)__builtin_elementwise_max((__v8su)__a, (__v8su)__b);
287	1194	}
288	1195
	1196	/// Compares the corresponding signed bytes in the two 256-bit integer vectors
	1197	/// in \a __a and \a __b and returns the smaller of each pair in the
	1198	/// corresponding byte of the 256-bit result.
	1199	///
	1200	/// \headerfile <immintrin.h>
	1201	///
	1202	/// This intrinsic corresponds to the \c VPMINSB instruction.
	1203	///
	1204	/// \param __a
	1205	/// A 256-bit integer vector.
	1206	/// \param __b
	1207	/// A 256-bit integer vector.
	1208	/// \returns A 256-bit integer vector containing the result.
289	1209	static __inline__ __m256i __DEFAULT_FN_ATTRS256
290	1210	_mm256_min_epi8(__m256i __a, __m256i __b)
291	1211	{
292	1212	return (__m256i)__builtin_elementwise_min((__v32qs)__a, (__v32qs)__b);
293	1213	}
294	1214
	1215	/// Compares the corresponding signed 16-bit integers in the two 256-bit
	1216	/// vectors of [16 x i16] in \a __a and \a __b and returns the smaller of
	1217	/// each pair in the corresponding element of the 256-bit result.
	1218	///
	1219	/// \headerfile <immintrin.h>
	1220	///
	1221	/// This intrinsic corresponds to the \c VPMINSW instruction.
	1222	///
	1223	/// \param __a
	1224	/// A 256-bit vector of [16 x i16].
	1225	/// \param __b
	1226	/// A 256-bit vector of [16 x i16].
	1227	/// \returns A 256-bit vector of [16 x i16] containing the result.
295	1228	static __inline__ __m256i __DEFAULT_FN_ATTRS256
296	1229	_mm256_min_epi16(__m256i __a, __m256i __b)
297	1230	{
298	1231	return (__m256i)__builtin_elementwise_min((__v16hi)__a, (__v16hi)__b);
299	1232	}
300	1233
	1234	/// Compares the corresponding signed 32-bit integers in the two 256-bit
	1235	/// vectors of [8 x i32] in \a __a and \a __b and returns the smaller of
	1236	/// each pair in the corresponding element of the 256-bit result.
	1237	///
	1238	/// \headerfile <immintrin.h>
	1239	///
	1240	/// This intrinsic corresponds to the \c VPMINSD instruction.
	1241	///
	1242	/// \param __a
	1243	/// A 256-bit vector of [8 x i32].
	1244	/// \param __b
	1245	/// A 256-bit vector of [8 x i32].
	1246	/// \returns A 256-bit vector of [8 x i32] containing the result.
301	1247	static __inline__ __m256i __DEFAULT_FN_ATTRS256
302	1248	_mm256_min_epi32(__m256i __a, __m256i __b)
303	1249	{
304	1250	return (__m256i)__builtin_elementwise_min((__v8si)__a, (__v8si)__b);
305	1251	}
306	1252
	1253	/// Compares the corresponding unsigned bytes in the two 256-bit integer
	1254	/// vectors in \a __a and \a __b and returns the smaller of each pair in
	1255	/// the corresponding byte of the 256-bit result.
	1256	///
	1257	/// \headerfile <immintrin.h>
	1258	///
	1259	/// This intrinsic corresponds to the \c VPMINUB instruction.
	1260	///
	1261	/// \param __a
	1262	/// A 256-bit integer vector.
	1263	/// \param __b
	1264	/// A 256-bit integer vector.
	1265	/// \returns A 256-bit integer vector containing the result.
307	1266	static __inline__ __m256i __DEFAULT_FN_ATTRS256
308	1267	_mm256_min_epu8(__m256i __a, __m256i __b)
309	1268	{
310	1269	return (__m256i)__builtin_elementwise_min((__v32qu)__a, (__v32qu)__b);
311	1270	}
312	1271
	1272	/// Compares the corresponding unsigned 16-bit integers in the two 256-bit
	1273	/// vectors of [16 x i16] in \a __a and \a __b and returns the smaller of
	1274	/// each pair in the corresponding element of the 256-bit result.
	1275	///
	1276	/// \headerfile <immintrin.h>
	1277	///
	1278	/// This intrinsic corresponds to the \c VPMINUW instruction.
	1279	///
	1280	/// \param __a
	1281	/// A 256-bit vector of [16 x i16].
	1282	/// \param __b
	1283	/// A 256-bit vector of [16 x i16].
	1284	/// \returns A 256-bit vector of [16 x i16] containing the result.
313	1285	static __inline__ __m256i __DEFAULT_FN_ATTRS256
314	1286	_mm256_min_epu16(__m256i __a, __m256i __b)
315	1287	{
316	1288	return (__m256i)__builtin_elementwise_min((__v16hu)__a, (__v16hu)__b);
317	1289	}
318	1290
	1291	/// Compares the corresponding unsigned 32-bit integers in the two 256-bit
	1292	/// vectors of [8 x i32] in \a __a and \a __b and returns the smaller of
	1293	/// each pair in the corresponding element of the 256-bit result.
	1294	///
	1295	/// \headerfile <immintrin.h>
	1296	///
	1297	/// This intrinsic corresponds to the \c VPMINUD instruction.
	1298	///
	1299	/// \param __a
	1300	/// A 256-bit vector of [8 x i32].
	1301	/// \param __b
	1302	/// A 256-bit vector of [8 x i32].
	1303	/// \returns A 256-bit vector of [8 x i32] containing the result.
319	1304	static __inline__ __m256i __DEFAULT_FN_ATTRS256
320	1305	_mm256_min_epu32(__m256i __a, __m256i __b)
321	1306	{
...	...	@@ -328,6 +1313,26 @@ _mm256_movemask_epi8(__m256i __a)
328	1313	return __builtin_ia32_pmovmskb256((__v32qi)__a);
329	1314	}
330	1315
	1316	/// Sign-extends bytes from the 128-bit integer vector in \a __V and returns
	1317	/// the 16-bit values in the corresponding elements of a 256-bit vector
	1318	/// of [16 x i16].
	1319	///
	1320	/// \code{.operation}
	1321	/// FOR i := 0 TO 15
	1322	/// j := i*8
	1323	/// k := i*16
	1324	/// result[k+15:k] := SignExtend(__V[j+7:j])
	1325	/// ENDFOR
	1326	/// \endcode
	1327	///
	1328	/// \headerfile <immintrin.h>
	1329	///
	1330	/// This intrinsic corresponds to the \c VPMOVSXBW instruction.
	1331	///
	1332	/// \param __V
	1333	/// A 128-bit integer vector containing the source bytes.
	1334	/// \returns A 256-bit vector of [16 x i16] containing the sign-extended
	1335	/// values.
331	1336	static __inline__ __m256i __DEFAULT_FN_ATTRS256
332	1337	_mm256_cvtepi8_epi16(__m128i __V)
333	1338	{
...	...	@@ -336,6 +1341,26 @@ _mm256_cvtepi8_epi16(__m128i __V)
336	1341	return (__m256i)__builtin_convertvector((__v16qs)__V, __v16hi);
337	1342	}
338	1343
	1344	/// Sign-extends bytes from the lower half of the 128-bit integer vector in
	1345	/// \a __V and returns the 32-bit values in the corresponding elements of a
	1346	/// 256-bit vector of [8 x i32].
	1347	///
	1348	/// \code{.operation}
	1349	/// FOR i := 0 TO 7
	1350	/// j := i*8
	1351	/// k := i*32
	1352	/// result[k+31:k] := SignExtend(__V[j+7:j])
	1353	/// ENDFOR
	1354	/// \endcode
	1355	///
	1356	/// \headerfile <immintrin.h>
	1357	///
	1358	/// This intrinsic corresponds to the \c VPMOVSXBD instruction.
	1359	///
	1360	/// \param __V
	1361	/// A 128-bit integer vector containing the source bytes.
	1362	/// \returns A 256-bit vector of [8 x i32] containing the sign-extended
	1363	/// values.
339	1364	static __inline__ __m256i __DEFAULT_FN_ATTRS256
340	1365	_mm256_cvtepi8_epi32(__m128i __V)
341	1366	{
...	...	@@ -344,6 +1369,25 @@ _mm256_cvtepi8_epi32(__m128i __V)
344	1369	return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8si);
345	1370	}
346	1371
	1372	/// Sign-extends the first four bytes from the 128-bit integer vector in
	1373	/// \a __V and returns the 64-bit values in the corresponding elements of a
	1374	/// 256-bit vector of [4 x i64].
	1375	///
	1376	/// \code{.operation}
	1377	/// result[63:0] := SignExtend(__V[7:0])
	1378	/// result[127:64] := SignExtend(__V[15:8])
	1379	/// result[191:128] := SignExtend(__V[23:16])
	1380	/// result[255:192] := SignExtend(__V[31:24])
	1381	/// \endcode
	1382	///
	1383	/// \headerfile <immintrin.h>
	1384	///
	1385	/// This intrinsic corresponds to the \c VPMOVSXBQ instruction.
	1386	///
	1387	/// \param __V
	1388	/// A 128-bit integer vector containing the source bytes.
	1389	/// \returns A 256-bit vector of [4 x i64] containing the sign-extended
	1390	/// values.
347	1391	static __inline__ __m256i __DEFAULT_FN_ATTRS256
348	1392	_mm256_cvtepi8_epi64(__m128i __V)
349	1393	{
...	...	@@ -352,357 +1396,1571 @@ _mm256_cvtepi8_epi64(__m128i __V)
352	1396	return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3), __v4di);
353	1397	}
354	1398
	1399	/// Sign-extends 16-bit elements from the 128-bit vector of [8 x i16] in
	1400	/// \a __V and returns the 32-bit values in the corresponding elements of a
	1401	/// 256-bit vector of [8 x i32].
	1402	///
	1403	/// \code{.operation}
	1404	/// FOR i := 0 TO 7
	1405	/// j := i*16
	1406	/// k := i*32
	1407	/// result[k+31:k] := SignExtend(__V[j+15:j])
	1408	/// ENDFOR
	1409	/// \endcode
	1410	///
	1411	/// \headerfile <immintrin.h>
	1412	///
	1413	/// This intrinsic corresponds to the \c VPMOVSXWD instruction.
	1414	///
	1415	/// \param __V
	1416	/// A 128-bit vector of [8 x i16] containing the source values.
	1417	/// \returns A 256-bit vector of [8 x i32] containing the sign-extended
	1418	/// values.
355	1419	static __inline__ __m256i __DEFAULT_FN_ATTRS256
356	1420	_mm256_cvtepi16_epi32(__m128i __V)
357	1421	{
358	1422	return (__m256i)__builtin_convertvector((__v8hi)__V, __v8si);
359	1423	}
360	1424
	1425	/// Sign-extends 16-bit elements from the lower half of the 128-bit vector of
	1426	/// [8 x i16] in \a __V and returns the 64-bit values in the corresponding
	1427	/// elements of a 256-bit vector of [4 x i64].
	1428	///
	1429	/// \code{.operation}
	1430	/// result[63:0] := SignExtend(__V[15:0])
	1431	/// result[127:64] := SignExtend(__V[31:16])
	1432	/// result[191:128] := SignExtend(__V[47:32])
	1433	/// result[255:192] := SignExtend(__V[64:48])
	1434	/// \endcode
	1435	///
	1436	/// \headerfile <immintrin.h>
	1437	///
	1438	/// This intrinsic corresponds to the \c VPMOVSXWQ instruction.
	1439	///
	1440	/// \param __V
	1441	/// A 128-bit vector of [8 x i16] containing the source values.
	1442	/// \returns A 256-bit vector of [4 x i64] containing the sign-extended
	1443	/// values.
361	1444	static __inline__ __m256i __DEFAULT_FN_ATTRS256
362	1445	_mm256_cvtepi16_epi64(__m128i __V)
363	1446	{
364	1447	return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1, 2, 3), __v4di);
365	1448	}
366	1449
	1450	/// Sign-extends 32-bit elements from the 128-bit vector of [4 x i32] in
	1451	/// \a __V and returns the 64-bit values in the corresponding elements of a
	1452	/// 256-bit vector of [4 x i64].
	1453	///
	1454	/// \code{.operation}
	1455	/// result[63:0] := SignExtend(__V[31:0])
	1456	/// result[127:64] := SignExtend(__V[63:32])
	1457	/// result[191:128] := SignExtend(__V[95:64])
	1458	/// result[255:192] := SignExtend(__V[127:96])
	1459	/// \endcode
	1460	///
	1461	/// \headerfile <immintrin.h>
	1462	///
	1463	/// This intrinsic corresponds to the \c VPMOVSXDQ instruction.
	1464	///
	1465	/// \param __V
	1466	/// A 128-bit vector of [4 x i32] containing the source values.
	1467	/// \returns A 256-bit vector of [4 x i64] containing the sign-extended
	1468	/// values.
367	1469	static __inline__ __m256i __DEFAULT_FN_ATTRS256
368	1470	_mm256_cvtepi32_epi64(__m128i __V)
369	1471	{
370	1472	return (__m256i)__builtin_convertvector((__v4si)__V, __v4di);
371	1473	}
372	1474
	1475	/// Zero-extends bytes from the 128-bit integer vector in \a __V and returns
	1476	/// the 16-bit values in the corresponding elements of a 256-bit vector
	1477	/// of [16 x i16].
	1478	///
	1479	/// \code{.operation}
	1480	/// FOR i := 0 TO 15
	1481	/// j := i*8
	1482	/// k := i*16
	1483	/// result[k+15:k] := ZeroExtend(__V[j+7:j])
	1484	/// ENDFOR
	1485	/// \endcode
	1486	///
	1487	/// \headerfile <immintrin.h>
	1488	///
	1489	/// This intrinsic corresponds to the \c VPMOVZXBW instruction.
	1490	///
	1491	/// \param __V
	1492	/// A 128-bit integer vector containing the source bytes.
	1493	/// \returns A 256-bit vector of [16 x i16] containing the zero-extended
	1494	/// values.
373	1495	static __inline__ __m256i __DEFAULT_FN_ATTRS256
374	1496	_mm256_cvtepu8_epi16(__m128i __V)
375	1497	{
376	1498	return (__m256i)__builtin_convertvector((__v16qu)__V, __v16hi);
377	1499	}
378	1500
	1501	/// Zero-extends bytes from the lower half of the 128-bit integer vector in
	1502	/// \a __V and returns the 32-bit values in the corresponding elements of a
	1503	/// 256-bit vector of [8 x i32].
	1504	///
	1505	/// \code{.operation}
	1506	/// FOR i := 0 TO 7
	1507	/// j := i*8
	1508	/// k := i*32
	1509	/// result[k+31:k] := ZeroExtend(__V[j+7:j])
	1510	/// ENDFOR
	1511	/// \endcode
	1512	///
	1513	/// \headerfile <immintrin.h>
	1514	///
	1515	/// This intrinsic corresponds to the \c VPMOVZXBD instruction.
	1516	///
	1517	/// \param __V
	1518	/// A 128-bit integer vector containing the source bytes.
	1519	/// \returns A 256-bit vector of [8 x i32] containing the zero-extended
	1520	/// values.
379	1521	static __inline__ __m256i __DEFAULT_FN_ATTRS256
380	1522	_mm256_cvtepu8_epi32(__m128i __V)
381	1523	{
382	1524	return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8si);
383	1525	}
384	1526
	1527	/// Zero-extends the first four bytes from the 128-bit integer vector in
	1528	/// \a __V and returns the 64-bit values in the corresponding elements of a
	1529	/// 256-bit vector of [4 x i64].
	1530	///
	1531	/// \code{.operation}
	1532	/// result[63:0] := ZeroExtend(__V[7:0])
	1533	/// result[127:64] := ZeroExtend(__V[15:8])
	1534	/// result[191:128] := ZeroExtend(__V[23:16])
	1535	/// result[255:192] := ZeroExtend(__V[31:24])
	1536	/// \endcode
	1537	///
	1538	/// \headerfile <immintrin.h>
	1539	///
	1540	/// This intrinsic corresponds to the \c VPMOVZXBQ instruction.
	1541	///
	1542	/// \param __V
	1543	/// A 128-bit integer vector containing the source bytes.
	1544	/// \returns A 256-bit vector of [4 x i64] containing the zero-extended
	1545	/// values.
385	1546	static __inline__ __m256i __DEFAULT_FN_ATTRS256
386	1547	_mm256_cvtepu8_epi64(__m128i __V)
387	1548	{
388	1549	return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3), __v4di);
389	1550	}
390	1551
	1552	/// Zero-extends 16-bit elements from the 128-bit vector of [8 x i16] in
	1553	/// \a __V and returns the 32-bit values in the corresponding elements of a
	1554	/// 256-bit vector of [8 x i32].
	1555	///
	1556	/// \code{.operation}
	1557	/// FOR i := 0 TO 7
	1558	/// j := i*16
	1559	/// k := i*32
	1560	/// result[k+31:k] := ZeroExtend(__V[j+15:j])
	1561	/// ENDFOR
	1562	/// \endcode
	1563	///
	1564	/// \headerfile <immintrin.h>
	1565	///
	1566	/// This intrinsic corresponds to the \c VPMOVZXWD instruction.
	1567	///
	1568	/// \param __V
	1569	/// A 128-bit vector of [8 x i16] containing the source values.
	1570	/// \returns A 256-bit vector of [8 x i32] containing the zero-extended
	1571	/// values.
391	1572	static __inline__ __m256i __DEFAULT_FN_ATTRS256
392	1573	_mm256_cvtepu16_epi32(__m128i __V)
393	1574	{
394	1575	return (__m256i)__builtin_convertvector((__v8hu)__V, __v8si);
395	1576	}
396	1577
	1578	/// Zero-extends 16-bit elements from the lower half of the 128-bit vector of
	1579	/// [8 x i16] in \a __V and returns the 64-bit values in the corresponding
	1580	/// elements of a 256-bit vector of [4 x i64].
	1581	///
	1582	/// \code{.operation}
	1583	/// result[63:0] := ZeroExtend(__V[15:0])
	1584	/// result[127:64] := ZeroExtend(__V[31:16])
	1585	/// result[191:128] := ZeroExtend(__V[47:32])
	1586	/// result[255:192] := ZeroExtend(__V[64:48])
	1587	/// \endcode
	1588	///
	1589	/// \headerfile <immintrin.h>
	1590	///
	1591	/// This intrinsic corresponds to the \c VPMOVSXWQ instruction.
	1592	///
	1593	/// \param __V
	1594	/// A 128-bit vector of [8 x i16] containing the source values.
	1595	/// \returns A 256-bit vector of [4 x i64] containing the zero-extended
	1596	/// values.
397	1597	static __inline__ __m256i __DEFAULT_FN_ATTRS256
398	1598	_mm256_cvtepu16_epi64(__m128i __V)
399	1599	{
400	1600	return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v8hu)__V, (__v8hu)__V, 0, 1, 2, 3), __v4di);
401	1601	}
402	1602
	1603	/// Zero-extends 32-bit elements from the 128-bit vector of [4 x i32] in
	1604	/// \a __V and returns the 64-bit values in the corresponding elements of a
	1605	/// 256-bit vector of [4 x i64].
	1606	///
	1607	/// \code{.operation}
	1608	/// result[63:0] := ZeroExtend(__V[31:0])
	1609	/// result[127:64] := ZeroExtend(__V[63:32])
	1610	/// result[191:128] := ZeroExtend(__V[95:64])
	1611	/// result[255:192] := ZeroExtend(__V[127:96])
	1612	/// \endcode
	1613	///
	1614	/// \headerfile <immintrin.h>
	1615	///
	1616	/// This intrinsic corresponds to the \c VPMOVZXDQ instruction.
	1617	///
	1618	/// \param __V
	1619	/// A 128-bit vector of [4 x i32] containing the source values.
	1620	/// \returns A 256-bit vector of [4 x i64] containing the zero-extended
	1621	/// values.
403	1622	static __inline__ __m256i __DEFAULT_FN_ATTRS256
404	1623	_mm256_cvtepu32_epi64(__m128i __V)
405	1624	{
406	1625	return (__m256i)__builtin_convertvector((__v4su)__V, __v4di);
407	1626	}
408	1627
	1628	/// Multiplies signed 32-bit integers from even-numbered elements of two
	1629	/// 256-bit vectors of [8 x i32] and returns the 64-bit products in the
	1630	/// [4 x i64] result.
	1631	///
	1632	/// \code{.operation}
	1633	/// result[63:0] := __a[31:0] * __b[31:0]
	1634	/// result[127:64] := __a[95:64] * __b[95:64]
	1635	/// result[191:128] := __a[159:128] * __b[159:128]
	1636	/// result[255:192] := __a[223:192] * __b[223:192]
	1637	/// \endcode
	1638	///
	1639	/// \headerfile <immintrin.h>
	1640	///
	1641	/// This intrinsic corresponds to the \c VPMULDQ instruction.
	1642	///
	1643	/// \param __a
	1644	/// A 256-bit vector of [8 x i32] containing one of the source operands.
	1645	/// \param __b
	1646	/// A 256-bit vector of [8 x i32] containing one of the source operands.
	1647	/// \returns A 256-bit vector of [4 x i64] containing the products.
409	1648	static __inline__ __m256i __DEFAULT_FN_ATTRS256
410	1649	_mm256_mul_epi32(__m256i __a, __m256i __b)
411	1650	{
412	1651	return (__m256i)__builtin_ia32_pmuldq256((__v8si)__a, (__v8si)__b);
413	1652	}
414	1653
	1654	/// Multiplies signed 16-bit integer elements of two 256-bit vectors of
	1655	/// [16 x i16], truncates the 32-bit results to the most significant 18
	1656	/// bits, rounds by adding 1, and returns bits [16:1] of each rounded
	1657	/// product in the [16 x i16] result.
	1658	///
	1659	/// \code{.operation}
	1660	/// FOR i := 0 TO 15
	1661	/// j := i*16
	1662	/// temp := ((__a[j+15:j] * __b[j+15:j]) >> 14) + 1
	1663	/// result[j+15:j] := temp[16:1]
	1664	/// \endcode
	1665	///
	1666	/// \headerfile <immintrin.h>
	1667	///
	1668	/// This intrinsic corresponds to the \c VPMULHRSW instruction.
	1669	///
	1670	/// \param __a
	1671	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	1672	/// \param __b
	1673	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	1674	/// \returns A 256-bit vector of [16 x i16] containing the rounded products.
415	1675	static __inline__ __m256i __DEFAULT_FN_ATTRS256
416	1676	_mm256_mulhrs_epi16(__m256i __a, __m256i __b)
417	1677	{
418	1678	return (__m256i)__builtin_ia32_pmulhrsw256((__v16hi)__a, (__v16hi)__b);
419	1679	}
420	1680
	1681	/// Multiplies unsigned 16-bit integer elements of two 256-bit vectors of
	1682	/// [16 x i16], and returns the upper 16 bits of each 32-bit product in the
	1683	/// [16 x i16] result.
	1684	///
	1685	/// \headerfile <immintrin.h>
	1686	///
	1687	/// This intrinsic corresponds to the \c VPMULHUW instruction.
	1688	///
	1689	/// \param __a
	1690	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	1691	/// \param __b
	1692	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	1693	/// \returns A 256-bit vector of [16 x i16] containing the products.
421	1694	static __inline__ __m256i __DEFAULT_FN_ATTRS256
422	1695	_mm256_mulhi_epu16(__m256i __a, __m256i __b)
423	1696	{
424	1697	return (__m256i)__builtin_ia32_pmulhuw256((__v16hi)__a, (__v16hi)__b);
425	1698	}
426	1699
	1700	/// Multiplies signed 16-bit integer elements of two 256-bit vectors of
	1701	/// [16 x i16], and returns the upper 16 bits of each 32-bit product in the
	1702	/// [16 x i16] result.
	1703	///
	1704	/// \headerfile <immintrin.h>
	1705	///
	1706	/// This intrinsic corresponds to the \c VPMULHW instruction.
	1707	///
	1708	/// \param __a
	1709	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	1710	/// \param __b
	1711	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	1712	/// \returns A 256-bit vector of [16 x i16] containing the products.
427	1713	static __inline__ __m256i __DEFAULT_FN_ATTRS256
428	1714	_mm256_mulhi_epi16(__m256i __a, __m256i __b)
429	1715	{
430	1716	return (__m256i)__builtin_ia32_pmulhw256((__v16hi)__a, (__v16hi)__b);
431	1717	}
432	1718
	1719	/// Multiplies signed 16-bit integer elements of two 256-bit vectors of
	1720	/// [16 x i16], and returns the lower 16 bits of each 32-bit product in the
	1721	/// [16 x i16] result.
	1722	///
	1723	/// \headerfile <immintrin.h>
	1724	///
	1725	/// This intrinsic corresponds to the \c VPMULLW instruction.
	1726	///
	1727	/// \param __a
	1728	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	1729	/// \param __b
	1730	/// A 256-bit vector of [16 x i16] containing one of the source operands.
	1731	/// \returns A 256-bit vector of [16 x i16] containing the products.
433	1732	static __inline__ __m256i __DEFAULT_FN_ATTRS256
434	1733	_mm256_mullo_epi16(__m256i __a, __m256i __b)
435	1734	{
436	1735	return (__m256i)((__v16hu)__a * (__v16hu)__b);
437	1736	}
438	1737
	1738	/// Multiplies signed 32-bit integer elements of two 256-bit vectors of
	1739	/// [8 x i32], and returns the lower 32 bits of each 64-bit product in the
	1740	/// [8 x i32] result.
	1741	///
	1742	/// \headerfile <immintrin.h>
	1743	///
	1744	/// This intrinsic corresponds to the \c VPMULLD instruction.
	1745	///
	1746	/// \param __a
	1747	/// A 256-bit vector of [8 x i32] containing one of the source operands.
	1748	/// \param __b
	1749	/// A 256-bit vector of [8 x i32] containing one of the source operands.
	1750	/// \returns A 256-bit vector of [8 x i32] containing the products.
439	1751	static __inline__ __m256i __DEFAULT_FN_ATTRS256
440	1752	_mm256_mullo_epi32 (__m256i __a, __m256i __b)
441	1753	{
442	1754	return (__m256i)((__v8su)__a * (__v8su)__b);
443	1755	}
444	1756
	1757	/// Multiplies unsigned 32-bit integers from even-numered elements of two
	1758	/// 256-bit vectors of [8 x i32] and returns the 64-bit products in the
	1759	/// [4 x i64] result.
	1760	///
	1761	/// \code{.operation}
	1762	/// result[63:0] := __a[31:0] * __b[31:0]
	1763	/// result[127:64] := __a[95:64] * __b[95:64]
	1764	/// result[191:128] := __a[159:128] * __b[159:128]
	1765	/// result[255:192] := __a[223:192] * __b[223:192]
	1766	/// \endcode
	1767	///
	1768	/// \headerfile <immintrin.h>
	1769	///
	1770	/// This intrinsic corresponds to the \c VPMULUDQ instruction.
	1771	///
	1772	/// \param __a
	1773	/// A 256-bit vector of [8 x i32] containing one of the source operands.
	1774	/// \param __b
	1775	/// A 256-bit vector of [8 x i32] containing one of the source operands.
	1776	/// \returns A 256-bit vector of [4 x i64] containing the products.
445	1777	static __inline__ __m256i __DEFAULT_FN_ATTRS256
446	1778	_mm256_mul_epu32(__m256i __a, __m256i __b)
447	1779	{
448	1780	return __builtin_ia32_pmuludq256((__v8si)__a, (__v8si)__b);
449	1781	}
450	1782
	1783	/// Computes the bitwise OR of the 256-bit integer vectors in \a __a and
	1784	/// \a __b.
	1785	///
	1786	/// \headerfile <immintrin.h>
	1787	///
	1788	/// This intrinsic corresponds to the \c VPOR instruction.
	1789	///
	1790	/// \param __a
	1791	/// A 256-bit integer vector.
	1792	/// \param __b
	1793	/// A 256-bit integer vector.
	1794	/// \returns A 256-bit integer vector containing the result.
451	1795	static __inline__ __m256i __DEFAULT_FN_ATTRS256
452	1796	_mm256_or_si256(__m256i __a, __m256i __b)
453	1797	{
454	1798	return (__m256i)((__v4du)__a \| (__v4du)__b);
455	1799	}
456	1800
	1801	/// Computes four sum of absolute difference (SAD) operations on sets of eight
	1802	/// unsigned 8-bit integers from the 256-bit integer vectors \a __a and
	1803	/// \a __b.
	1804	///
	1805	/// One SAD result is computed for each set of eight bytes from \a __a and
	1806	/// eight bytes from \a __b. The zero-extended SAD value is returned in the
	1807	/// corresponding 64-bit element of the result.
	1808	///
	1809	/// A single SAD operation takes the differences between the corresponding
	1810	/// bytes of \a __a and \a __b, takes the absolute value of each difference,
	1811	/// and sums these eight values to form one 16-bit result. This operation
	1812	/// is repeated four times with successive sets of eight bytes.
	1813	///
	1814	/// \code{.operation}
	1815	/// FOR i := 0 TO 3
	1816	/// j := i*64
	1817	/// temp0 := ABS(__a[j+7:j] - __b[j+7:j])
	1818	/// temp1 := ABS(__a[j+15:j+8] - __b[j+15:j+8])
	1819	/// temp2 := ABS(__a[j+23:j+16] - __b[j+23:j+16])
	1820	/// temp3 := ABS(__a[j+31:j+24] - __b[j+31:j+24])
	1821	/// temp4 := ABS(__a[j+39:j+32] - __b[j+39:j+32])
	1822	/// temp5 := ABS(__a[j+47:j+40] - __b[j+47:j+40])
	1823	/// temp6 := ABS(__a[j+55:j+48] - __b[j+55:j+48])
	1824	/// temp7 := ABS(__a[j+63:j+56] - __b[j+63:j+56])
	1825	/// result[j+15:j] := temp0 + temp1 + temp2 + temp3 +
	1826	/// temp4 + temp5 + temp6 + temp7
	1827	/// result[j+63:j+16] := 0
	1828	/// ENDFOR
	1829	/// \endcode
	1830	///
	1831	/// \headerfile <immintrin.h>
	1832	///
	1833	/// This intrinsic corresponds to the \c VPSADBW instruction.
	1834	///
	1835	/// \param __a
	1836	/// A 256-bit integer vector.
	1837	/// \param __b
	1838	/// A 256-bit integer vector.
	1839	/// \returns A 256-bit integer vector containing the result.
457	1840	static __inline__ __m256i __DEFAULT_FN_ATTRS256
458	1841	_mm256_sad_epu8(__m256i __a, __m256i __b)
459	1842	{
460	1843	return __builtin_ia32_psadbw256((__v32qi)__a, (__v32qi)__b);
461	1844	}
462	1845
	1846	/// Shuffles 8-bit integers in the 256-bit integer vector \a __a according
	1847	/// to control information in the 256-bit integer vector \a __b, and
	1848	/// returns the 256-bit result. In effect there are two separate 128-bit
	1849	/// shuffles in the lower and upper halves.
	1850	///
	1851	/// \code{.operation}
	1852	/// FOR i := 0 TO 31
	1853	/// j := i*8
	1854	/// IF __b[j+7] == 1
	1855	/// result[j+7:j] := 0
	1856	/// ELSE
	1857	/// k := __b[j+3:j] * 8
	1858	/// IF i > 15
	1859	/// k := k + 128
	1860	/// FI
	1861	/// result[j+7:j] := __a[k+7:k]
	1862	/// FI
	1863	/// ENDFOR
	1864	/// \endcode
	1865	///
	1866	/// \headerfile <immintrin.h>
	1867	///
	1868	/// This intrinsic corresponds to the \c VPSHUFB instruction.
	1869	///
	1870	/// \param __a
	1871	/// A 256-bit integer vector containing source values.
	1872	/// \param __b
	1873	/// A 256-bit integer vector containing control information to determine
	1874	/// what goes into the corresponding byte of the result. If bit 7 of the
	1875	/// control byte is 1, the result byte is 0; otherwise, bits 3:0 of the
	1876	/// control byte specify the index (within the same 128-bit half) of \a __a
	1877	/// to copy to the result byte.
	1878	/// \returns A 256-bit integer vector containing the result.
463	1879	static __inline__ __m256i __DEFAULT_FN_ATTRS256
464	1880	_mm256_shuffle_epi8(__m256i __a, __m256i __b)
465	1881	{
466	1882	return (__m256i)__builtin_ia32_pshufb256((__v32qi)__a, (__v32qi)__b);
467	1883	}
468	1884
	1885	/// Shuffles 32-bit integers from the 256-bit vector of [8 x i32] in \a a
	1886	/// according to control information in the integer literal \a imm, and
	1887	/// returns the 256-bit result. In effect there are two parallel 128-bit
	1888	/// shuffles in the lower and upper halves.
	1889	///
	1890	/// \code{.operation}
	1891	/// FOR i := 0 to 3
	1892	/// j := i*32
	1893	/// k := (imm >> i2)[1:0] 32
	1894	/// result[j+31:j] := a[k+31:k]
	1895	/// result[128+j+31:128+j] := a[128+k+31:128+k]
	1896	/// ENDFOR
	1897	/// \endcode
	1898	///
	1899	/// \headerfile <immintrin.h>
	1900	///
	1901	/// \code
	1902	/// __m256i _mm256_shuffle_epi32(__m256i a, const int imm);
	1903	/// \endcode
	1904	///
	1905	/// This intrinsic corresponds to the \c VPSHUFB instruction.
	1906	///
	1907	/// \param a
	1908	/// A 256-bit vector of [8 x i32] containing source values.
	1909	/// \param imm
	1910	/// An immediate 8-bit value specifying which elements to copy from \a a.
	1911	/// \a imm[1:0] specifies the index in \a a for elements 0 and 4 of the
	1912	/// result, \a imm[3:2] specifies the index for elements 1 and 5, and so
	1913	/// forth.
	1914	/// \returns A 256-bit vector of [8 x i32] containing the result.
469	1915	#define _mm256_shuffle_epi32(a, imm) \
470	1916	((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(a), (int)(imm)))
471	1917
	1918	/// Shuffles 16-bit integers from the 256-bit vector of [16 x i16] in \a a
	1919	/// according to control information in the integer literal \a imm, and
	1920	/// returns the 256-bit result. The upper 64 bits of each 128-bit half
	1921	/// are shuffled in parallel; the lower 64 bits of each 128-bit half are
	1922	/// copied from \a a unchanged.
	1923	///
	1924	/// \code{.operation}
	1925	/// result[63:0] := a[63:0]
	1926	/// result[191:128] := a[191:128]
	1927	/// FOR i := 0 TO 3
	1928	/// j := i * 16 + 64
	1929	/// k := (imm >> i2)[1:0] 16 + 64
	1930	/// result[j+15:j] := a[k+15:k]
	1931	/// result[128+j+15:128+j] := a[128+k+15:128+k]
	1932	/// ENDFOR
	1933	/// \endcode
	1934	///
	1935	/// \headerfile <immintrin.h>
	1936	///
	1937	/// \code
	1938	/// __m256i _mm256_shufflehi_epi16(__m256i a, const int imm);
	1939	/// \endcode
	1940	///
	1941	/// This intrinsic corresponds to the \c VPSHUFHW instruction.
	1942	///
	1943	/// \param a
	1944	/// A 256-bit vector of [16 x i16] containing source values.
	1945	/// \param imm
	1946	/// An immediate 8-bit value specifying which elements to copy from \a a.
	1947	/// \a imm[1:0] specifies the index in \a a for elements 4 and 8 of the
	1948	/// result, \a imm[3:2] specifies the index for elements 5 and 9, and so
	1949	/// forth. Indexes are offset by 4 (so 0 means index 4, and so forth).
	1950	/// \returns A 256-bit vector of [16 x i16] containing the result.
472	1951	#define _mm256_shufflehi_epi16(a, imm) \
473	1952	((__m256i)__builtin_ia32_pshufhw256((__v16hi)(__m256i)(a), (int)(imm)))
474	1953
	1954	/// Shuffles 16-bit integers from the 256-bit vector of [16 x i16] \a a
	1955	/// according to control information in the integer literal \a imm, and
	1956	/// returns the 256-bit [16 x i16] result. The lower 64 bits of each
	1957	/// 128-bit half are shuffled; the upper 64 bits of each 128-bit half are
	1958	/// copied from \a a unchanged.
	1959	///
	1960	/// \code{.operation}
	1961	/// result[127:64] := a[127:64]
	1962	/// result[255:192] := a[255:192]
	1963	/// FOR i := 0 TO 3
	1964	/// j := i * 16
	1965	/// k := (imm >> i2)[1:0] 16
	1966	/// result[j+15:j] := a[k+15:k]
	1967	/// result[128+j+15:128+j] := a[128+k+15:128+k]
	1968	/// ENDFOR
	1969	/// \endcode
	1970	///
	1971	/// \headerfile <immintrin.h>
	1972	///
	1973	/// \code
	1974	/// __m256i _mm256_shufflelo_epi16(__m256i a, const int imm);
	1975	/// \endcode
	1976	///
	1977	/// This intrinsic corresponds to the \c VPSHUFLW instruction.
	1978	///
	1979	/// \param a
	1980	/// A 256-bit vector of [16 x i16] to use as a source of data for the
	1981	/// result.
	1982	/// \param imm
	1983	/// An immediate 8-bit value specifying which elements to copy from \a a.
	1984	/// \a imm[1:0] specifies the index in \a a for elements 0 and 8 of the
	1985	/// result, \a imm[3:2] specifies the index for elements 1 and 9, and so
	1986	/// forth.
	1987	/// \returns A 256-bit vector of [16 x i16] containing the result.
475	1988	#define _mm256_shufflelo_epi16(a, imm) \
476	1989	((__m256i)__builtin_ia32_pshuflw256((__v16hi)(__m256i)(a), (int)(imm)))
477	1990
	1991	/// Sets each byte of the result to the corresponding byte of the 256-bit
	1992	/// integer vector in \a __a, the negative of that byte, or zero, depending
	1993	/// on whether the corresponding byte of the 256-bit integer vector in
	1994	/// \a __b is greater than zero, less than zero, or equal to zero,
	1995	/// respectively.
	1996	///
	1997	/// \headerfile <immintrin.h>
	1998	///
	1999	/// This intrinsic corresponds to the \c VPSIGNB instruction.
	2000	///
	2001	/// \param __a
	2002	/// A 256-bit integer vector.
	2003	/// \param __b
	2004	/// A 256-bit integer vector].
	2005	/// \returns A 256-bit integer vector containing the result.
478	2006	static __inline__ __m256i __DEFAULT_FN_ATTRS256
479	2007	_mm256_sign_epi8(__m256i __a, __m256i __b)
480	2008	{
481	2009	return (__m256i)__builtin_ia32_psignb256((__v32qi)__a, (__v32qi)__b);
482	2010	}
483	2011
	2012	/// Sets each element of the result to the corresponding element of the
	2013	/// 256-bit vector of [16 x i16] in \a __a, the negative of that element,
	2014	/// or zero, depending on whether the corresponding element of the 256-bit
	2015	/// vector of [16 x i16] in \a __b is greater than zero, less than zero, or
	2016	/// equal to zero, respectively.
	2017	///
	2018	/// \headerfile <immintrin.h>
	2019	///
	2020	/// This intrinsic corresponds to the \c VPSIGNW instruction.
	2021	///
	2022	/// \param __a
	2023	/// A 256-bit vector of [16 x i16].
	2024	/// \param __b
	2025	/// A 256-bit vector of [16 x i16].
	2026	/// \returns A 256-bit vector of [16 x i16] containing the result.
484	2027	static __inline__ __m256i __DEFAULT_FN_ATTRS256
485	2028	_mm256_sign_epi16(__m256i __a, __m256i __b)
486	2029	{
487	2030	return (__m256i)__builtin_ia32_psignw256((__v16hi)__a, (__v16hi)__b);
488	2031	}
489	2032
	2033	/// Sets each element of the result to the corresponding element of the
	2034	/// 256-bit vector of [8 x i32] in \a __a, the negative of that element, or
	2035	/// zero, depending on whether the corresponding element of the 256-bit
	2036	/// vector of [8 x i32] in \a __b is greater than zero, less than zero, or
	2037	/// equal to zero, respectively.
	2038	///
	2039	/// \headerfile <immintrin.h>
	2040	///
	2041	/// This intrinsic corresponds to the \c VPSIGND instruction.
	2042	///
	2043	/// \param __a
	2044	/// A 256-bit vector of [8 x i32].
	2045	/// \param __b
	2046	/// A 256-bit vector of [8 x i32].
	2047	/// \returns A 256-bit vector of [8 x i32] containing the result.
490	2048	static __inline__ __m256i __DEFAULT_FN_ATTRS256
491	2049	_mm256_sign_epi32(__m256i __a, __m256i __b)
492	2050	{
493	2051	return (__m256i)__builtin_ia32_psignd256((__v8si)__a, (__v8si)__b);
494	2052	}
495	2053
	2054	/// Shifts each 128-bit half of the 256-bit integer vector \a a left by
	2055	/// \a imm bytes, shifting in zero bytes, and returns the result. If \a imm
	2056	/// is greater than 15, the returned result is all zeroes.
	2057	///
	2058	/// \headerfile <immintrin.h>
	2059	///
	2060	/// \code
	2061	/// __m256i _mm256_slli_si256(__m256i a, const int imm);
	2062	/// \endcode
	2063	///
	2064	/// This intrinsic corresponds to the \c VPSLLDQ instruction.
	2065	///
	2066	/// \param a
	2067	/// A 256-bit integer vector to be shifted.
	2068	/// \param imm
	2069	/// An unsigned immediate value specifying the shift count (in bytes).
	2070	/// \returns A 256-bit integer vector containing the result.
496	2071	#define _mm256_slli_si256(a, imm) \
497	2072	((__m256i)__builtin_ia32_pslldqi256_byteshift((__v4di)(__m256i)(a), (int)(imm)))
498	2073
	2074	/// Shifts each 128-bit half of the 256-bit integer vector \a a left by
	2075	/// \a imm bytes, shifting in zero bytes, and returns the result. If \a imm
	2076	/// is greater than 15, the returned result is all zeroes.
	2077	///
	2078	/// \headerfile <immintrin.h>
	2079	///
	2080	/// \code
	2081	/// __m256i _mm256_bslli_epi128(__m256i a, const int imm);
	2082	/// \endcode
	2083	///
	2084	/// This intrinsic corresponds to the \c VPSLLDQ instruction.
	2085	///
	2086	/// \param a
	2087	/// A 256-bit integer vector to be shifted.
	2088	/// \param imm
	2089	/// An unsigned immediate value specifying the shift count (in bytes).
	2090	/// \returns A 256-bit integer vector containing the result.
499	2091	#define _mm256_bslli_epi128(a, imm) \
500	2092	((__m256i)__builtin_ia32_pslldqi256_byteshift((__v4di)(__m256i)(a), (int)(imm)))
501	2093
	2094	/// Shifts each 16-bit element of the 256-bit vector of [16 x i16] in \a __a
	2095	/// left by \a __count bits, shifting in zero bits, and returns the result.
	2096	/// If \a __count is greater than 15, the returned result is all zeroes.
	2097	///
	2098	/// \headerfile <immintrin.h>
	2099	///
	2100	/// This intrinsic corresponds to the \c VPSLLW instruction.
	2101	///
	2102	/// \param __a
	2103	/// A 256-bit vector of [16 x i16] to be shifted.
	2104	/// \param __count
	2105	/// An unsigned integer value specifying the shift count (in bits).
	2106	/// \returns A 256-bit vector of [16 x i16] containing the result.
502	2107	static __inline__ __m256i __DEFAULT_FN_ATTRS256
503	2108	_mm256_slli_epi16(__m256i __a, int __count)
504	2109	{
505	2110	return (__m256i)__builtin_ia32_psllwi256((__v16hi)__a, __count);
506	2111	}
507	2112
	2113	/// Shifts each 16-bit element of the 256-bit vector of [16 x i16] in \a __a
	2114	/// left by the number of bits specified by the lower 64 bits of \a __count,
	2115	/// shifting in zero bits, and returns the result. If \a __count is greater
	2116	/// than 15, the returned result is all zeroes.
	2117	///
	2118	/// \headerfile <immintrin.h>
	2119	///
	2120	/// This intrinsic corresponds to the \c VPSLLW instruction.
	2121	///
	2122	/// \param __a
	2123	/// A 256-bit vector of [16 x i16] to be shifted.
	2124	/// \param __count
	2125	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
	2126	/// shift count (in bits). The upper element is ignored.
	2127	/// \returns A 256-bit vector of [16 x i16] containing the result.
508	2128	static __inline__ __m256i __DEFAULT_FN_ATTRS256
509	2129	_mm256_sll_epi16(__m256i __a, __m128i __count)
510	2130	{
511	2131	return (__m256i)__builtin_ia32_psllw256((__v16hi)__a, (__v8hi)__count);
512	2132	}
513	2133
	2134	/// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __a
	2135	/// left by \a __count bits, shifting in zero bits, and returns the result.
	2136	/// If \a __count is greater than 31, the returned result is all zeroes.
	2137	///
	2138	/// \headerfile <immintrin.h>
	2139	///
	2140	/// This intrinsic corresponds to the \c VPSLLD instruction.
	2141	///
	2142	/// \param __a
	2143	/// A 256-bit vector of [8 x i32] to be shifted.
	2144	/// \param __count
	2145	/// An unsigned integer value specifying the shift count (in bits).
	2146	/// \returns A 256-bit vector of [8 x i32] containing the result.
514	2147	static __inline__ __m256i __DEFAULT_FN_ATTRS256
515	2148	_mm256_slli_epi32(__m256i __a, int __count)
516	2149	{
517	2150	return (__m256i)__builtin_ia32_pslldi256((__v8si)__a, __count);
518	2151	}
519	2152
	2153	/// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __a
	2154	/// left by the number of bits given in the lower 64 bits of \a __count,
	2155	/// shifting in zero bits, and returns the result. If \a __count is greater
	2156	/// than 31, the returned result is all zeroes.
	2157	///
	2158	/// \headerfile <immintrin.h>
	2159	///
	2160	/// This intrinsic corresponds to the \c VPSLLD instruction.
	2161	///
	2162	/// \param __a
	2163	/// A 256-bit vector of [8 x i32] to be shifted.
	2164	/// \param __count
	2165	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
	2166	/// shift count (in bits). The upper element is ignored.
	2167	/// \returns A 256-bit vector of [8 x i32] containing the result.
520	2168	static __inline__ __m256i __DEFAULT_FN_ATTRS256
521	2169	_mm256_sll_epi32(__m256i __a, __m128i __count)
522	2170	{
523	2171	return (__m256i)__builtin_ia32_pslld256((__v8si)__a, (__v4si)__count);
524	2172	}
525	2173
	2174	/// Shifts each 64-bit element of the 256-bit vector of [4 x i64] in \a __a
	2175	/// left by \a __count bits, shifting in zero bits, and returns the result.
	2176	/// If \a __count is greater than 63, the returned result is all zeroes.
	2177	///
	2178	/// \headerfile <immintrin.h>
	2179	///
	2180	/// This intrinsic corresponds to the \c VPSLLQ instruction.
	2181	///
	2182	/// \param __a
	2183	/// A 256-bit vector of [4 x i64] to be shifted.
	2184	/// \param __count
	2185	/// An unsigned integer value specifying the shift count (in bits).
	2186	/// \returns A 256-bit vector of [4 x i64] containing the result.
526	2187	static __inline__ __m256i __DEFAULT_FN_ATTRS256
527	2188	_mm256_slli_epi64(__m256i __a, int __count)
528	2189	{
529	2190	return __builtin_ia32_psllqi256((__v4di)__a, __count);
530	2191	}
531	2192
	2193	/// Shifts each 64-bit element of the 256-bit vector of [4 x i64] in \a __a
	2194	/// left by the number of bits given in the lower 64 bits of \a __count,
	2195	/// shifting in zero bits, and returns the result. If \a __count is greater
	2196	/// than 63, the returned result is all zeroes.
	2197	///
	2198	/// \headerfile <immintrin.h>
	2199	///
	2200	/// This intrinsic corresponds to the \c VPSLLQ instruction.
	2201	///
	2202	/// \param __a
	2203	/// A 256-bit vector of [4 x i64] to be shifted.
	2204	/// \param __count
	2205	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
	2206	/// shift count (in bits). The upper element is ignored.
	2207	/// \returns A 256-bit vector of [4 x i64] containing the result.
532	2208	static __inline__ __m256i __DEFAULT_FN_ATTRS256
533	2209	_mm256_sll_epi64(__m256i __a, __m128i __count)
534	2210	{
535	2211	return __builtin_ia32_psllq256((__v4di)__a, __count);
536	2212	}
537	2213
	2214	/// Shifts each 16-bit element of the 256-bit vector of [16 x i16] in \a __a
	2215	/// right by \a __count bits, shifting in sign bits, and returns the result.
	2216	/// If \a __count is greater than 15, each element of the result is either
	2217	/// 0 or -1 according to the corresponding input sign bit.
	2218	///
	2219	/// \headerfile <immintrin.h>
	2220	///
	2221	/// This intrinsic corresponds to the \c VPSRAW instruction.
	2222	///
	2223	/// \param __a
	2224	/// A 256-bit vector of [16 x i16] to be shifted.
	2225	/// \param __count
	2226	/// An unsigned integer value specifying the shift count (in bits).
	2227	/// \returns A 256-bit vector of [16 x i16] containing the result.
538	2228	static __inline__ __m256i __DEFAULT_FN_ATTRS256
539	2229	_mm256_srai_epi16(__m256i __a, int __count)
540	2230	{
541	2231	return (__m256i)__builtin_ia32_psrawi256((__v16hi)__a, __count);
542	2232	}
543	2233
	2234	/// Shifts each 16-bit element of the 256-bit vector of [16 x i16] in \a __a
	2235	/// right by the number of bits given in the lower 64 bits of \a __count,
	2236	/// shifting in sign bits, and returns the result. If \a __count is greater
	2237	/// than 15, each element of the result is either 0 or -1 according to the
	2238	/// corresponding input sign bit.
	2239	///
	2240	/// \headerfile <immintrin.h>
	2241	///
	2242	/// This intrinsic corresponds to the \c VPSRAW instruction.
	2243	///
	2244	/// \param __a
	2245	/// A 256-bit vector of [16 x i16] to be shifted.
	2246	/// \param __count
	2247	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
	2248	/// shift count (in bits). The upper element is ignored.
	2249	/// \returns A 256-bit vector of [16 x i16] containing the result.
544	2250	static __inline__ __m256i __DEFAULT_FN_ATTRS256
545	2251	_mm256_sra_epi16(__m256i __a, __m128i __count)
546	2252	{
547	2253	return (__m256i)__builtin_ia32_psraw256((__v16hi)__a, (__v8hi)__count);
548	2254	}
549	2255
	2256	/// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __a
	2257	/// right by \a __count bits, shifting in sign bits, and returns the result.
	2258	/// If \a __count is greater than 31, each element of the result is either
	2259	/// 0 or -1 according to the corresponding input sign bit.
	2260	///
	2261	/// \headerfile <immintrin.h>
	2262	///
	2263	/// This intrinsic corresponds to the \c VPSRAD instruction.
	2264	///
	2265	/// \param __a
	2266	/// A 256-bit vector of [8 x i32] to be shifted.
	2267	/// \param __count
	2268	/// An unsigned integer value specifying the shift count (in bits).
	2269	/// \returns A 256-bit vector of [8 x i32] containing the result.
550	2270	static __inline__ __m256i __DEFAULT_FN_ATTRS256
551	2271	_mm256_srai_epi32(__m256i __a, int __count)
552	2272	{
553	2273	return (__m256i)__builtin_ia32_psradi256((__v8si)__a, __count);
554	2274	}
555	2275
	2276	/// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __a
	2277	/// right by the number of bits given in the lower 64 bits of \a __count,
	2278	/// shifting in sign bits, and returns the result. If \a __count is greater
	2279	/// than 31, each element of the result is either 0 or -1 according to the
	2280	/// corresponding input sign bit.
	2281	///
	2282	/// \headerfile <immintrin.h>
	2283	///
	2284	/// This intrinsic corresponds to the \c VPSRAD instruction.
	2285	///
	2286	/// \param __a
	2287	/// A 256-bit vector of [8 x i32] to be shifted.
	2288	/// \param __count
	2289	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
	2290	/// shift count (in bits). The upper element is ignored.
	2291	/// \returns A 256-bit vector of [8 x i32] containing the result.
556	2292	static __inline__ __m256i __DEFAULT_FN_ATTRS256
557	2293	_mm256_sra_epi32(__m256i __a, __m128i __count)
558	2294	{
559	2295	return (__m256i)__builtin_ia32_psrad256((__v8si)__a, (__v4si)__count);
560	2296	}
561	2297
	2298	/// Shifts each 128-bit half of the 256-bit integer vector in \a a right by
	2299	/// \a imm bytes, shifting in zero bytes, and returns the result. If
	2300	/// \a imm is greater than 15, the returned result is all zeroes.
	2301	///
	2302	/// \headerfile <immintrin.h>
	2303	///
	2304	/// \code
	2305	/// __m256i _mm256_srli_si256(__m256i a, const int imm);
	2306	/// \endcode
	2307	///
	2308	/// This intrinsic corresponds to the \c VPSRLDQ instruction.
	2309	///
	2310	/// \param a
	2311	/// A 256-bit integer vector to be shifted.
	2312	/// \param imm
	2313	/// An unsigned immediate value specifying the shift count (in bytes).
	2314	/// \returns A 256-bit integer vector containing the result.
562	2315	#define _mm256_srli_si256(a, imm) \
563	2316	((__m256i)__builtin_ia32_psrldqi256_byteshift((__m256i)(a), (int)(imm)))
564	2317
	2318	/// Shifts each 128-bit half of the 256-bit integer vector in \a a right by
	2319	/// \a imm bytes, shifting in zero bytes, and returns the result. If
	2320	/// \a imm is greater than 15, the returned result is all zeroes.
	2321	///
	2322	/// \headerfile <immintrin.h>
	2323	///
	2324	/// \code
	2325	/// __m256i _mm256_bsrli_epi128(__m256i a, const int imm);
	2326	/// \endcode
	2327	///
	2328	/// This intrinsic corresponds to the \c VPSRLDQ instruction.
	2329	///
	2330	/// \param a
	2331	/// A 256-bit integer vector to be shifted.
	2332	/// \param imm
	2333	/// An unsigned immediate value specifying the shift count (in bytes).
	2334	/// \returns A 256-bit integer vector containing the result.
565	2335	#define _mm256_bsrli_epi128(a, imm) \
566	2336	((__m256i)__builtin_ia32_psrldqi256_byteshift((__m256i)(a), (int)(imm)))
567	2337
	2338	/// Shifts each 16-bit element of the 256-bit vector of [16 x i16] in \a __a
	2339	/// right by \a __count bits, shifting in zero bits, and returns the result.
	2340	/// If \a __count is greater than 15, the returned result is all zeroes.
	2341	///
	2342	/// \headerfile <immintrin.h>
	2343	///
	2344	/// This intrinsic corresponds to the \c VPSRLW instruction.
	2345	///
	2346	/// \param __a
	2347	/// A 256-bit vector of [16 x i16] to be shifted.
	2348	/// \param __count
	2349	/// An unsigned integer value specifying the shift count (in bits).
	2350	/// \returns A 256-bit vector of [16 x i16] containing the result.
568	2351	static __inline__ __m256i __DEFAULT_FN_ATTRS256
569	2352	_mm256_srli_epi16(__m256i __a, int __count)
570	2353	{
571	2354	return (__m256i)__builtin_ia32_psrlwi256((__v16hi)__a, __count);
572	2355	}
573	2356
	2357	/// Shifts each 16-bit element of the 256-bit vector of [16 x i16] in \a __a
	2358	/// right by the number of bits given in the lower 64 bits of \a __count,
	2359	/// shifting in zero bits, and returns the result. If \a __count is greater
	2360	/// than 15, the returned result is all zeroes.
	2361	///
	2362	/// \headerfile <immintrin.h>
	2363	///
	2364	/// This intrinsic corresponds to the \c VPSRLW instruction.
	2365	///
	2366	/// \param __a
	2367	/// A 256-bit vector of [16 x i16] to be shifted.
	2368	/// \param __count
	2369	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
	2370	/// shift count (in bits). The upper element is ignored.
	2371	/// \returns A 256-bit vector of [16 x i16] containing the result.
574	2372	static __inline__ __m256i __DEFAULT_FN_ATTRS256
575	2373	_mm256_srl_epi16(__m256i __a, __m128i __count)
576	2374	{
577	2375	return (__m256i)__builtin_ia32_psrlw256((__v16hi)__a, (__v8hi)__count);
578	2376	}
579	2377
	2378	/// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __a
	2379	/// right by \a __count bits, shifting in zero bits, and returns the result.
	2380	/// If \a __count is greater than 31, the returned result is all zeroes.
	2381	///
	2382	/// \headerfile <immintrin.h>
	2383	///
	2384	/// This intrinsic corresponds to the \c VPSRLD instruction.
	2385	///
	2386	/// \param __a
	2387	/// A 256-bit vector of [8 x i32] to be shifted.
	2388	/// \param __count
	2389	/// An unsigned integer value specifying the shift count (in bits).
	2390	/// \returns A 256-bit vector of [8 x i32] containing the result.
580	2391	static __inline__ __m256i __DEFAULT_FN_ATTRS256
581	2392	_mm256_srli_epi32(__m256i __a, int __count)
582	2393	{
583	2394	return (__m256i)__builtin_ia32_psrldi256((__v8si)__a, __count);
584	2395	}
585	2396
	2397	/// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __a
	2398	/// right by the number of bits given in the lower 64 bits of \a __count,
	2399	/// shifting in zero bits, and returns the result. If \a __count is greater
	2400	/// than 31, the returned result is all zeroes.
	2401	///
	2402	/// \headerfile <immintrin.h>
	2403	///
	2404	/// This intrinsic corresponds to the \c VPSRLD instruction.
	2405	///
	2406	/// \param __a
	2407	/// A 256-bit vector of [8 x i32] to be shifted.
	2408	/// \param __count
	2409	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
	2410	/// shift count (in bits). The upper element is ignored.
	2411	/// \returns A 256-bit vector of [8 x i32] containing the result.
586	2412	static __inline__ __m256i __DEFAULT_FN_ATTRS256
587	2413	_mm256_srl_epi32(__m256i __a, __m128i __count)
588	2414	{
589	2415	return (__m256i)__builtin_ia32_psrld256((__v8si)__a, (__v4si)__count);
590	2416	}
591	2417
	2418	/// Shifts each 64-bit element of the 256-bit vector of [4 x i64] in \a __a
	2419	/// right by \a __count bits, shifting in zero bits, and returns the result.
	2420	/// If \a __count is greater than 63, the returned result is all zeroes.
	2421	///
	2422	/// \headerfile <immintrin.h>
	2423	///
	2424	/// This intrinsic corresponds to the \c VPSRLQ instruction.
	2425	///
	2426	/// \param __a
	2427	/// A 256-bit vector of [4 x i64] to be shifted.
	2428	/// \param __count
	2429	/// An unsigned integer value specifying the shift count (in bits).
	2430	/// \returns A 256-bit vector of [4 x i64] containing the result.
592	2431	static __inline__ __m256i __DEFAULT_FN_ATTRS256
593	2432	_mm256_srli_epi64(__m256i __a, int __count)
594	2433	{
595	2434	return __builtin_ia32_psrlqi256((__v4di)__a, __count);
596	2435	}
597	2436
	2437	/// Shifts each 64-bit element of the 256-bit vector of [4 x i64] in \a __a
	2438	/// right by the number of bits given in the lower 64 bits of \a __count,
	2439	/// shifting in zero bits, and returns the result. If \a __count is greater
	2440	/// than 63, the returned result is all zeroes.
	2441	///
	2442	/// \headerfile <immintrin.h>
	2443	///
	2444	/// This intrinsic corresponds to the \c VPSRLQ instruction.
	2445	///
	2446	/// \param __a
	2447	/// A 256-bit vector of [4 x i64] to be shifted.
	2448	/// \param __count
	2449	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
	2450	/// shift count (in bits). The upper element is ignored.
	2451	/// \returns A 256-bit vector of [4 x i64] containing the result.
598	2452	static __inline__ __m256i __DEFAULT_FN_ATTRS256
599	2453	_mm256_srl_epi64(__m256i __a, __m128i __count)
600	2454	{
601	2455	return __builtin_ia32_psrlq256((__v4di)__a, __count);
602	2456	}
603	2457
	2458	/// Subtracts 8-bit integers from corresponding bytes of two 256-bit integer
	2459	/// vectors. Returns the lower 8 bits of each difference in the
	2460	/// corresponding byte of the 256-bit integer vector result (overflow is
	2461	/// ignored).
	2462	///
	2463	/// \code{.operation}
	2464	/// FOR i := 0 TO 31
	2465	/// j := i*8
	2466	/// result[j+7:j] := __a[j+7:j] - __b[j+7:j]
	2467	/// ENDFOR
	2468	/// \endcode
	2469	///
	2470	/// \headerfile <immintrin.h>
	2471	///
	2472	/// This intrinsic corresponds to the \c VPSUBB instruction.
	2473	///
	2474	/// \param __a
	2475	/// A 256-bit integer vector containing the minuends.
	2476	/// \param __b
	2477	/// A 256-bit integer vector containing the subtrahends.
	2478	/// \returns A 256-bit integer vector containing the differences.
604	2479	static __inline__ __m256i __DEFAULT_FN_ATTRS256
605	2480	_mm256_sub_epi8(__m256i __a, __m256i __b)
606	2481	{
607	2482	return (__m256i)((__v32qu)__a - (__v32qu)__b);
608	2483	}
609	2484
	2485	/// Subtracts 16-bit integers from corresponding elements of two 256-bit
	2486	/// vectors of [16 x i16]. Returns the lower 16 bits of each difference in
	2487	/// the corresponding element of the [16 x i16] result (overflow is
	2488	/// ignored).
	2489	///
	2490	/// \code{.operation}
	2491	/// FOR i := 0 TO 15
	2492	/// j := i*16
	2493	/// result[j+15:j] := __a[j+15:j] - __b[j+15:j]
	2494	/// ENDFOR
	2495	/// \endcode
	2496	///
	2497	/// \headerfile <immintrin.h>
	2498	///
	2499	/// This intrinsic corresponds to the \c VPSUBW instruction.
	2500	///
	2501	/// \param __a
	2502	/// A 256-bit vector of [16 x i16] containing the minuends.
	2503	/// \param __b
	2504	/// A 256-bit vector of [16 x i16] containing the subtrahends.
	2505	/// \returns A 256-bit vector of [16 x i16] containing the differences.
610	2506	static __inline__ __m256i __DEFAULT_FN_ATTRS256
611	2507	_mm256_sub_epi16(__m256i __a, __m256i __b)
612	2508	{
613	2509	return (__m256i)((__v16hu)__a - (__v16hu)__b);
614	2510	}
615	2511
	2512	/// Subtracts 32-bit integers from corresponding elements of two 256-bit
	2513	/// vectors of [8 x i32]. Returns the lower 32 bits of each difference in
	2514	/// the corresponding element of the [8 x i32] result (overflow is ignored).
	2515	///
	2516	/// \code{.operation}
	2517	/// FOR i := 0 TO 7
	2518	/// j := i*32
	2519	/// result[j+31:j] := __a[j+31:j] - __b[j+31:j]
	2520	/// ENDFOR
	2521	/// \endcode
	2522	///
	2523	/// \headerfile <immintrin.h>
	2524	///
	2525	/// This intrinsic corresponds to the \c VPSUBD instruction.
	2526	///
	2527	/// \param __a
	2528	/// A 256-bit vector of [8 x i32] containing the minuends.
	2529	/// \param __b
	2530	/// A 256-bit vector of [8 x i32] containing the subtrahends.
	2531	/// \returns A 256-bit vector of [8 x i32] containing the differences.
616	2532	static __inline__ __m256i __DEFAULT_FN_ATTRS256
617	2533	_mm256_sub_epi32(__m256i __a, __m256i __b)
618	2534	{
619	2535	return (__m256i)((__v8su)__a - (__v8su)__b);
620	2536	}
621	2537
	2538	/// Subtracts 64-bit integers from corresponding elements of two 256-bit
	2539	/// vectors of [4 x i64]. Returns the lower 64 bits of each difference in
	2540	/// the corresponding element of the [4 x i64] result (overflow is ignored).
	2541	///
	2542	/// \code{.operation}
	2543	/// FOR i := 0 TO 3
	2544	/// j := i*64
	2545	/// result[j+63:j] := __a[j+63:j] - __b[j+63:j]
	2546	/// ENDFOR
	2547	/// \endcode
	2548	///
	2549	/// \headerfile <immintrin.h>
	2550	///
	2551	/// This intrinsic corresponds to the \c VPSUBQ instruction.
	2552	///
	2553	/// \param __a
	2554	/// A 256-bit vector of [4 x i64] containing the minuends.
	2555	/// \param __b
	2556	/// A 256-bit vector of [4 x i64] containing the subtrahends.
	2557	/// \returns A 256-bit vector of [4 x i64] containing the differences.
622	2558	static __inline__ __m256i __DEFAULT_FN_ATTRS256
623	2559	_mm256_sub_epi64(__m256i __a, __m256i __b)
624	2560	{
625	2561	return (__m256i)((__v4du)__a - (__v4du)__b);
626	2562	}
627	2563
	2564	/// Subtracts 8-bit integers from corresponding bytes of two 256-bit integer
	2565	/// vectors using signed saturation, and returns each differences in the
	2566	/// corresponding byte of the 256-bit integer vector result.
	2567	///
	2568	/// \code{.operation}
	2569	/// FOR i := 0 TO 31
	2570	/// j := i*8
	2571	/// result[j+7:j] := SATURATE8(__a[j+7:j] - __b[j+7:j])
	2572	/// ENDFOR
	2573	/// \endcode
	2574	///
	2575	/// \headerfile <immintrin.h>
	2576	///
	2577	/// This intrinsic corresponds to the \c VPSUBSB instruction.
	2578	///
	2579	/// \param __a
	2580	/// A 256-bit integer vector containing the minuends.
	2581	/// \param __b
	2582	/// A 256-bit integer vector containing the subtrahends.
	2583	/// \returns A 256-bit integer vector containing the differences.
628	2584	static __inline__ __m256i __DEFAULT_FN_ATTRS256
629	2585	_mm256_subs_epi8(__m256i __a, __m256i __b)
630	2586	{
631	2587	return (__m256i)__builtin_elementwise_sub_sat((__v32qs)__a, (__v32qs)__b);
632	2588	}
633	2589
	2590	/// Subtracts 16-bit integers from corresponding elements of two 256-bit
	2591	/// vectors of [16 x i16] using signed saturation, and returns each
	2592	/// difference in the corresponding element of the [16 x i16] result.
	2593	///
	2594	/// \code{.operation}
	2595	/// FOR i := 0 TO 15
	2596	/// j := i*16
	2597	/// result[j+7:j] := SATURATE16(__a[j+7:j] - __b[j+7:j])
	2598	/// ENDFOR
	2599	/// \endcode
	2600	///
	2601	/// \headerfile <immintrin.h>
	2602	///
	2603	/// This intrinsic corresponds to the \c VPSUBSW instruction.
	2604	///
	2605	/// \param __a
	2606	/// A 256-bit vector of [16 x i16] containing the minuends.
	2607	/// \param __b
	2608	/// A 256-bit vector of [16 x i16] containing the subtrahends.
	2609	/// \returns A 256-bit vector of [16 x i16] containing the differences.
634	2610	static __inline__ __m256i __DEFAULT_FN_ATTRS256
635	2611	_mm256_subs_epi16(__m256i __a, __m256i __b)
636	2612	{
637	2613	return (__m256i)__builtin_elementwise_sub_sat((__v16hi)__a, (__v16hi)__b);
638	2614	}
639	2615
	2616	/// Subtracts 8-bit integers from corresponding bytes of two 256-bit integer
	2617	/// vectors using unsigned saturation, and returns each difference in the
	2618	/// corresponding byte of the 256-bit integer vector result. For each byte,
	2619	/// computes <c> result = __a - __b </c>.
	2620	///
	2621	/// \code{.operation}
	2622	/// FOR i := 0 TO 31
	2623	/// j := i*8
	2624	/// result[j+7:j] := SATURATE8U(__a[j+7:j] - __b[j+7:j])
	2625	/// ENDFOR
	2626	/// \endcode
	2627	///
	2628	/// \headerfile <immintrin.h>
	2629	///
	2630	/// This intrinsic corresponds to the \c VPSUBUSB instruction.
	2631	///
	2632	/// \param __a
	2633	/// A 256-bit integer vector containing the minuends.
	2634	/// \param __b
	2635	/// A 256-bit integer vector containing the subtrahends.
	2636	/// \returns A 256-bit integer vector containing the differences.
640	2637	static __inline__ __m256i __DEFAULT_FN_ATTRS256
641	2638	_mm256_subs_epu8(__m256i __a, __m256i __b)
642	2639	{
643	2640	return (__m256i)__builtin_elementwise_sub_sat((__v32qu)__a, (__v32qu)__b);
644	2641	}
645	2642
	2643	/// Subtracts 16-bit integers from corresponding elements of two 256-bit
	2644	/// vectors of [16 x i16] using unsigned saturation, and returns each
	2645	/// difference in the corresponding element of the [16 x i16] result.
	2646	///
	2647	/// \code{.operation}
	2648	/// FOR i := 0 TO 15
	2649	/// j := i*16
	2650	/// result[j+15:j] := SATURATE16U(__a[j+15:j] - __b[j+15:j])
	2651	/// ENDFOR
	2652	/// \endcode
	2653	///
	2654	/// \headerfile <immintrin.h>
	2655	///
	2656	/// This intrinsic corresponds to the \c VPSUBUSW instruction.
	2657	///
	2658	/// \param __a
	2659	/// A 256-bit vector of [16 x i16] containing the minuends.
	2660	/// \param __b
	2661	/// A 256-bit vector of [16 x i16] containing the subtrahends.
	2662	/// \returns A 256-bit vector of [16 x i16] containing the differences.
646	2663	static __inline__ __m256i __DEFAULT_FN_ATTRS256
647	2664	_mm256_subs_epu16(__m256i __a, __m256i __b)
648	2665	{
649	2666	return (__m256i)__builtin_elementwise_sub_sat((__v16hu)__a, (__v16hu)__b);
650	2667	}
651	2668
	2669	/// Unpacks and interleaves 8-bit integers from parts of the 256-bit integer
	2670	/// vectors in \a __a and \a __b to form the 256-bit result. Specifically,
	2671	/// uses the upper 64 bits of each 128-bit half of \a __a and \a __b as
	2672	/// input; other bits in these parameters are ignored.
	2673	///
	2674	/// \code{.operation}
	2675	/// result[7:0] := __a[71:64]
	2676	/// result[15:8] := __b[71:64]
	2677	/// result[23:16] := __a[79:72]
	2678	/// result[31:24] := __b[79:72]
	2679	/// . . .
	2680	/// result[127:120] := __b[127:120]
	2681	/// result[135:128] := __a[199:192]
	2682	/// . . .
	2683	/// result[255:248] := __b[255:248]
	2684	/// \endcode
	2685	///
	2686	/// \headerfile <immintrin.h>
	2687	///
	2688	/// This intrinsic corresponds to the \c VPUNPCKHBW instruction.
	2689	///
	2690	/// \param __a
	2691	/// A 256-bit integer vector used as the source for the even-numbered bytes
	2692	/// of the result.
	2693	/// \param __b
	2694	/// A 256-bit integer vector used as the source for the odd-numbered bytes
	2695	/// of the result.
	2696	/// \returns A 256-bit integer vector containing the result.
652	2697	static __inline__ __m256i __DEFAULT_FN_ATTRS256
653	2698	_mm256_unpackhi_epi8(__m256i __a, __m256i __b)
654	2699	{
655	2700	return (__m256i)__builtin_shufflevector((__v32qi)__a, (__v32qi)__b, 8, 32+8, 9, 32+9, 10, 32+10, 11, 32+11, 12, 32+12, 13, 32+13, 14, 32+14, 15, 32+15, 24, 32+24, 25, 32+25, 26, 32+26, 27, 32+27, 28, 32+28, 29, 32+29, 30, 32+30, 31, 32+31);
656	2701	}
657	2702
	2703	/// Unpacks and interleaves 16-bit integers from parts of the 256-bit vectors
	2704	/// of [16 x i16] in \a __a and \a __b to return the resulting 256-bit
	2705	/// vector of [16 x i16]. Specifically, uses the upper 64 bits of each
	2706	/// 128-bit half of \a __a and \a __b as input; other bits in these
	2707	/// parameters are ignored.
	2708	///
	2709	/// \code{.operation}
	2710	/// result[15:0] := __a[79:64]
	2711	/// result[31:16] := __b[79:64]
	2712	/// result[47:32] := __a[95:80]
	2713	/// result[63:48] := __b[95:80]
	2714	/// . . .
	2715	/// result[127:112] := __b[127:112]
	2716	/// result[143:128] := __a[211:196]
	2717	/// . . .
	2718	/// result[255:240] := __b[255:240]
	2719	/// \endcode
	2720	///
	2721	/// \headerfile <immintrin.h>
	2722	///
	2723	/// This intrinsic corresponds to the \c VPUNPCKHWD instruction.
	2724	///
	2725	/// \param __a
	2726	/// A 256-bit vector of [16 x i16] used as the source for the even-numbered
	2727	/// elements of the result.
	2728	/// \param __b
	2729	/// A 256-bit vector of [16 x i16] used as the source for the odd-numbered
	2730	/// elements of the result.
	2731	/// \returns A 256-bit vector of [16 x i16] containing the result.
658	2732	static __inline__ __m256i __DEFAULT_FN_ATTRS256
659	2733	_mm256_unpackhi_epi16(__m256i __a, __m256i __b)
660	2734	{
661	2735	return (__m256i)__builtin_shufflevector((__v16hi)__a, (__v16hi)__b, 4, 16+4, 5, 16+5, 6, 16+6, 7, 16+7, 12, 16+12, 13, 16+13, 14, 16+14, 15, 16+15);
662	2736	}
663	2737
	2738	/// Unpacks and interleaves 32-bit integers from parts of the 256-bit vectors
	2739	/// of [8 x i32] in \a __a and \a __b to return the resulting 256-bit vector
	2740	/// of [8 x i32]. Specifically, uses the upper 64 bits of each 128-bit half
	2741	/// of \a __a and \a __b as input; other bits in these parameters are
	2742	/// ignored.
	2743	///
	2744	/// \code{.operation}
	2745	/// result[31:0] := __a[95:64]
	2746	/// result[63:32] := __b[95:64]
	2747	/// result[95:64] := __a[127:96]
	2748	/// result[127:96] := __b[127:96]
	2749	/// result[159:128] := __a[223:192]
	2750	/// result[191:160] := __b[223:192]
	2751	/// result[223:192] := __a[255:224]
	2752	/// result[255:224] := __b[255:224]
	2753	/// \endcode
	2754	///
	2755	/// \headerfile <immintrin.h>
	2756	///
	2757	/// This intrinsic corresponds to the \c VPUNPCKHDQ instruction.
	2758	///
	2759	/// \param __a
	2760	/// A 256-bit vector of [8 x i32] used as the source for the even-numbered
	2761	/// elements of the result.
	2762	/// \param __b
	2763	/// A 256-bit vector of [8 x i32] used as the source for the odd-numbered
	2764	/// elements of the result.
	2765	/// \returns A 256-bit vector of [8 x i32] containing the result.
664	2766	static __inline__ __m256i __DEFAULT_FN_ATTRS256
665	2767	_mm256_unpackhi_epi32(__m256i __a, __m256i __b)
666	2768	{
667	2769	return (__m256i)__builtin_shufflevector((__v8si)__a, (__v8si)__b, 2, 8+2, 3, 8+3, 6, 8+6, 7, 8+7);
668	2770	}
669	2771
	2772	/// Unpacks and interleaves 64-bit integers from parts of the 256-bit vectors
	2773	/// of [4 x i64] in \a __a and \a __b to return the resulting 256-bit vector
	2774	/// of [4 x i64]. Specifically, uses the upper 64 bits of each 128-bit half
	2775	/// of \a __a and \a __b as input; other bits in these parameters are
	2776	/// ignored.
	2777	///
	2778	/// \code{.operation}
	2779	/// result[63:0] := __a[127:64]
	2780	/// result[127:64] := __b[127:64]
	2781	/// result[191:128] := __a[255:192]
	2782	/// result[255:192] := __b[255:192]
	2783	/// \endcode
	2784	///
	2785	/// \headerfile <immintrin.h>
	2786	///
	2787	/// This intrinsic corresponds to the \c VPUNPCKHQDQ instruction.
	2788	///
	2789	/// \param __a
	2790	/// A 256-bit vector of [4 x i64] used as the source for the even-numbered
	2791	/// elements of the result.
	2792	/// \param __b
	2793	/// A 256-bit vector of [4 x i64] used as the source for the odd-numbered
	2794	/// elements of the result.
	2795	/// \returns A 256-bit vector of [4 x i64] containing the result.
670	2796	static __inline__ __m256i __DEFAULT_FN_ATTRS256
671	2797	_mm256_unpackhi_epi64(__m256i __a, __m256i __b)
672	2798	{
673	2799	return (__m256i)__builtin_shufflevector((__v4di)__a, (__v4di)__b, 1, 4+1, 3, 4+3);
674	2800	}
675	2801
	2802	/// Unpacks and interleaves 8-bit integers from parts of the 256-bit integer
	2803	/// vectors in \a __a and \a __b to form the 256-bit result. Specifically,
	2804	/// uses the lower 64 bits of each 128-bit half of \a __a and \a __b as
	2805	/// input; other bits in these parameters are ignored.
	2806	///
	2807	/// \code{.operation}
	2808	/// result[7:0] := __a[7:0]
	2809	/// result[15:8] := __b[7:0]
	2810	/// result[23:16] := __a[15:8]
	2811	/// result[31:24] := __b[15:8]
	2812	/// . . .
	2813	/// result[127:120] := __b[63:56]
	2814	/// result[135:128] := __a[135:128]
	2815	/// . . .
	2816	/// result[255:248] := __b[191:184]
	2817	/// \endcode
	2818	///
	2819	/// \headerfile <immintrin.h>
	2820	///
	2821	/// This intrinsic corresponds to the \c VPUNPCKLBW instruction.
	2822	///
	2823	/// \param __a
	2824	/// A 256-bit integer vector used as the source for the even-numbered bytes
	2825	/// of the result.
	2826	/// \param __b
	2827	/// A 256-bit integer vector used as the source for the odd-numbered bytes
	2828	/// of the result.
	2829	/// \returns A 256-bit integer vector containing the result.
676	2830	static __inline__ __m256i __DEFAULT_FN_ATTRS256
677	2831	_mm256_unpacklo_epi8(__m256i __a, __m256i __b)
678	2832	{
679	2833	return (__m256i)__builtin_shufflevector((__v32qi)__a, (__v32qi)__b, 0, 32+0, 1, 32+1, 2, 32+2, 3, 32+3, 4, 32+4, 5, 32+5, 6, 32+6, 7, 32+7, 16, 32+16, 17, 32+17, 18, 32+18, 19, 32+19, 20, 32+20, 21, 32+21, 22, 32+22, 23, 32+23);
680	2834	}
681	2835
	2836	/// Unpacks and interleaves 16-bit integers from parts of the 256-bit vectors
	2837	/// of [16 x i16] in \a __a and \a __b to return the resulting 256-bit
	2838	/// vector of [16 x i16]. Specifically, uses the lower 64 bits of each
	2839	/// 128-bit half of \a __a and \a __b as input; other bits in these
	2840	/// parameters are ignored.
	2841	///
	2842	/// \code{.operation}
	2843	/// result[15:0] := __a[15:0]
	2844	/// result[31:16] := __b[15:0]
	2845	/// result[47:32] := __a[31:16]
	2846	/// result[63:48] := __b[31:16]
	2847	/// . . .
	2848	/// result[127:112] := __b[63:48]
	2849	/// result[143:128] := __a[143:128]
	2850	/// . . .
	2851	/// result[255:239] := __b[191:176]
	2852	/// \endcode
	2853	///
	2854	/// \headerfile <immintrin.h>
	2855	///
	2856	/// This intrinsic corresponds to the \c VPUNPCKLWD instruction.
	2857	///
	2858	/// \param __a
	2859	/// A 256-bit vector of [16 x i16] used as the source for the even-numbered
	2860	/// elements of the result.
	2861	/// \param __b
	2862	/// A 256-bit vector of [16 x i16] used as the source for the odd-numbered
	2863	/// elements of the result.
	2864	/// \returns A 256-bit vector of [16 x i16] containing the result.
682	2865	static __inline__ __m256i __DEFAULT_FN_ATTRS256
683	2866	_mm256_unpacklo_epi16(__m256i __a, __m256i __b)
684	2867	{
685	2868	return (__m256i)__builtin_shufflevector((__v16hi)__a, (__v16hi)__b, 0, 16+0, 1, 16+1, 2, 16+2, 3, 16+3, 8, 16+8, 9, 16+9, 10, 16+10, 11, 16+11);
686	2869	}
687	2870
	2871	/// Unpacks and interleaves 32-bit integers from parts of the 256-bit vectors
	2872	/// of [8 x i32] in \a __a and \a __b to return the resulting 256-bit vector
	2873	/// of [8 x i32]. Specifically, uses the lower 64 bits of each 128-bit half
	2874	/// of \a __a and \a __b as input; other bits in these parameters are
	2875	/// ignored.
	2876	///
	2877	/// \code{.operation}
	2878	/// result[31:0] := __a[31:0]
	2879	/// result[63:32] := __b[31:0]
	2880	/// result[95:64] := __a[63:32]
	2881	/// result[127:96] := __b[63:32]
	2882	/// result[159:128] := __a[159:128]
	2883	/// result[191:160] := __b[159:128]
	2884	/// result[223:192] := __a[191:160]
	2885	/// result[255:224] := __b[191:190]
	2886	/// \endcode
	2887	///
	2888	/// \headerfile <immintrin.h>
	2889	///
	2890	/// This intrinsic corresponds to the \c VPUNPCKLDQ instruction.
	2891	///
	2892	/// \param __a
	2893	/// A 256-bit vector of [8 x i32] used as the source for the even-numbered
	2894	/// elements of the result.
	2895	/// \param __b
	2896	/// A 256-bit vector of [8 x i32] used as the source for the odd-numbered
	2897	/// elements of the result.
	2898	/// \returns A 256-bit vector of [8 x i32] containing the result.
688	2899	static __inline__ __m256i __DEFAULT_FN_ATTRS256
689	2900	_mm256_unpacklo_epi32(__m256i __a, __m256i __b)
690	2901	{
691	2902	return (__m256i)__builtin_shufflevector((__v8si)__a, (__v8si)__b, 0, 8+0, 1, 8+1, 4, 8+4, 5, 8+5);
692	2903	}
693	2904
	2905	/// Unpacks and interleaves 64-bit integers from parts of the 256-bit vectors
	2906	/// of [4 x i64] in \a __a and \a __b to return the resulting 256-bit vector
	2907	/// of [4 x i64]. Specifically, uses the lower 64 bits of each 128-bit half
	2908	/// of \a __a and \a __b as input; other bits in these parameters are
	2909	/// ignored.
	2910	///
	2911	/// \code{.operation}
	2912	/// result[63:0] := __a[63:0]
	2913	/// result[127:64] := __b[63:0]
	2914	/// result[191:128] := __a[191:128]
	2915	/// result[255:192] := __b[191:128]
	2916	/// \endcode
	2917	///
	2918	/// \headerfile <immintrin.h>
	2919	///
	2920	/// This intrinsic corresponds to the \c VPUNPCKLQDQ instruction.
	2921	///
	2922	/// \param __a
	2923	/// A 256-bit vector of [4 x i64] used as the source for the even-numbered
	2924	/// elements of the result.
	2925	/// \param __b
	2926	/// A 256-bit vector of [4 x i64] used as the source for the odd-numbered
	2927	/// elements of the result.
	2928	/// \returns A 256-bit vector of [4 x i64] containing the result.
694	2929	static __inline__ __m256i __DEFAULT_FN_ATTRS256
695	2930	_mm256_unpacklo_epi64(__m256i __a, __m256i __b)
696	2931	{
697	2932	return (__m256i)__builtin_shufflevector((__v4di)__a, (__v4di)__b, 0, 4+0, 2, 4+2);
698	2933	}
699	2934
	2935	/// Computes the bitwise XOR of the 256-bit integer vectors in \a __a and
	2936	/// \a __b.
	2937	///
	2938	/// \headerfile <immintrin.h>
	2939	///
	2940	/// This intrinsic corresponds to the \c VPXOR instruction.
	2941	///
	2942	/// \param __a
	2943	/// A 256-bit integer vector.
	2944	/// \param __b
	2945	/// A 256-bit integer vector.
	2946	/// \returns A 256-bit integer vector containing the result.
700	2947	static __inline__ __m256i __DEFAULT_FN_ATTRS256
701	2948	_mm256_xor_si256(__m256i __a, __m256i __b)
702	2949	{
703	2950	return (__m256i)((__v4du)__a ^ (__v4du)__b);
704	2951	}
705	2952
	2953	/// Loads the 256-bit integer vector from memory \a __V using a non-temporal
	2954	/// memory hint and returns the vector. \a __V must be aligned on a 32-byte
	2955	/// boundary.
	2956	///
	2957	/// \headerfile <immintrin.h>
	2958	///
	2959	/// This intrinsic corresponds to the \c VMOVNTDQA instruction.
	2960	///
	2961	/// \param __V
	2962	/// A pointer to the 32-byte aligned memory containing the vector to load.
	2963	/// \returns A 256-bit integer vector loaded from memory.
706	2964	static __inline__ __m256i __DEFAULT_FN_ATTRS256
707	2965	_mm256_stream_load_si256(__m256i const *__V)
708	2966	{
...	...	@@ -710,30 +2968,84 @@ _mm256_stream_load_si256(__m256i const *__V)
710	2968	return (__m256i)__builtin_nontemporal_load((const __v4di_aligned *)__V);
711	2969	}
712	2970
	2971	/// Broadcasts the 32-bit floating-point value from the low element of the
	2972	/// 128-bit vector of [4 x float] in \a __X to all elements of the result's
	2973	/// 128-bit vector of [4 x float].
	2974	///
	2975	/// \headerfile <immintrin.h>
	2976	///
	2977	/// This intrinsic corresponds to the \c VBROADCASTSS instruction.
	2978	///
	2979	/// \param __X
	2980	/// A 128-bit vector of [4 x float] whose low element will be broadcast.
	2981	/// \returns A 128-bit vector of [4 x float] containing the result.
713	2982	static __inline__ __m128 __DEFAULT_FN_ATTRS128
714	2983	_mm_broadcastss_ps(__m128 __X)
715	2984	{
716	2985	return (__m128)__builtin_shufflevector((__v4sf)__X, (__v4sf)__X, 0, 0, 0, 0);
717	2986	}
718	2987
	2988	/// Broadcasts the 64-bit floating-point value from the low element of the
	2989	/// 128-bit vector of [2 x double] in \a __a to both elements of the
	2990	/// result's 128-bit vector of [2 x double].
	2991	///
	2992	/// \headerfile <immintrin.h>
	2993	///
	2994	/// This intrinsic corresponds to the \c MOVDDUP instruction.
	2995	///
	2996	/// \param __a
	2997	/// A 128-bit vector of [2 x double] whose low element will be broadcast.
	2998	/// \returns A 128-bit vector of [2 x double] containing the result.
719	2999	static __inline__ __m128d __DEFAULT_FN_ATTRS128
720	3000	_mm_broadcastsd_pd(__m128d __a)
721	3001	{
722	3002	return __builtin_shufflevector((__v2df)__a, (__v2df)__a, 0, 0);
723	3003	}
724	3004
	3005	/// Broadcasts the 32-bit floating-point value from the low element of the
	3006	/// 128-bit vector of [4 x float] in \a __X to all elements of the
	3007	/// result's 256-bit vector of [8 x float].
	3008	///
	3009	/// \headerfile <immintrin.h>
	3010	///
	3011	/// This intrinsic corresponds to the \c VBROADCASTSS instruction.
	3012	///
	3013	/// \param __X
	3014	/// A 128-bit vector of [4 x float] whose low element will be broadcast.
	3015	/// \returns A 256-bit vector of [8 x float] containing the result.
725	3016	static __inline__ __m256 __DEFAULT_FN_ATTRS256
726	3017	_mm256_broadcastss_ps(__m128 __X)
727	3018	{
728	3019	return (__m256)__builtin_shufflevector((__v4sf)__X, (__v4sf)__X, 0, 0, 0, 0, 0, 0, 0, 0);
729	3020	}
730	3021
	3022	/// Broadcasts the 64-bit floating-point value from the low element of the
	3023	/// 128-bit vector of [2 x double] in \a __X to all elements of the
	3024	/// result's 256-bit vector of [4 x double].
	3025	///
	3026	/// \headerfile <immintrin.h>
	3027	///
	3028	/// This intrinsic corresponds to the \c VBROADCASTSD instruction.
	3029	///
	3030	/// \param __X
	3031	/// A 128-bit vector of [2 x double] whose low element will be broadcast.
	3032	/// \returns A 256-bit vector of [4 x double] containing the result.
731	3033	static __inline__ __m256d __DEFAULT_FN_ATTRS256
732	3034	_mm256_broadcastsd_pd(__m128d __X)
733	3035	{
734	3036	return (__m256d)__builtin_shufflevector((__v2df)__X, (__v2df)__X, 0, 0, 0, 0);
735	3037	}
736	3038
	3039	/// Broadcasts the 128-bit integer data from \a __X to both the lower and
	3040	/// upper halves of the 256-bit result.
	3041	///
	3042	/// \headerfile <immintrin.h>
	3043	///
	3044	/// This intrinsic corresponds to the \c VBROADCASTI128 instruction.
	3045	///
	3046	/// \param __X
	3047	/// A 128-bit integer vector to be broadcast.
	3048	/// \returns A 256-bit integer vector containing the result.
737	3049	static __inline__ __m256i __DEFAULT_FN_ATTRS256
738	3050	_mm256_broadcastsi128_si256(__m128i __X)
739	3051	{
...	...	@@ -742,295 +3054,1688 @@ _mm256_broadcastsi128_si256(__m128i __X)
742	3054
743	3055	#define _mm_broadcastsi128_si256(X) _mm256_broadcastsi128_si256(X)
744	3056
	3057	/// Merges 32-bit integer elements from either of the two 128-bit vectors of
	3058	/// [4 x i32] in \a V1 or \a V2 to the result's 128-bit vector of [4 x i32],
	3059	/// as specified by the immediate integer operand \a M.
	3060	///
	3061	/// \code{.operation}
	3062	/// FOR i := 0 TO 3
	3063	/// j := i*32
	3064	/// IF M[i] == 0
	3065	/// result[31+j:j] := V1[31+j:j]
	3066	/// ELSE
	3067	/// result[31+j:j] := V2[32+j:j]
	3068	/// FI
	3069	/// ENDFOR
	3070	/// \endcode
	3071	///
	3072	/// \headerfile <immintrin.h>
	3073	///
	3074	/// \code
	3075	/// __m128i _mm_blend_epi32(__m128i V1, __m128i V2, const int M);
	3076	/// \endcode
	3077	///
	3078	/// This intrinsic corresponds to the \c VPBLENDDD instruction.
	3079	///
	3080	/// \param V1
	3081	/// A 128-bit vector of [4 x i32] containing source values.
	3082	/// \param V2
	3083	/// A 128-bit vector of [4 x i32] containing source values.
	3084	/// \param M
	3085	/// An immediate 8-bit integer operand, with bits [3:0] specifying the
	3086	/// source for each element of the result. The position of the mask bit
	3087	/// corresponds to the index of a copied value. When a mask bit is 0, the
	3088	/// element is copied from \a V1; otherwise, it is copied from \a V2.
	3089	/// \returns A 128-bit vector of [4 x i32] containing the result.
745	3090	#define _mm_blend_epi32(V1, V2, M) \
746	3091	((__m128i)__builtin_ia32_pblendd128((__v4si)(__m128i)(V1), \
747	3092	(__v4si)(__m128i)(V2), (int)(M)))
748	3093
	3094	/// Merges 32-bit integer elements from either of the two 256-bit vectors of
	3095	/// [8 x i32] in \a V1 or \a V2 to return a 256-bit vector of [8 x i32],
	3096	/// as specified by the immediate integer operand \a M.
	3097	///
	3098	/// \code{.operation}
	3099	/// FOR i := 0 TO 7
	3100	/// j := i*32
	3101	/// IF M[i] == 0
	3102	/// result[31+j:j] := V1[31+j:j]
	3103	/// ELSE
	3104	/// result[31+j:j] := V2[32+j:j]
	3105	/// FI
	3106	/// ENDFOR
	3107	/// \endcode
	3108	///
	3109	/// \headerfile <immintrin.h>
	3110	///
	3111	/// \code
	3112	/// __m256i _mm256_blend_epi32(__m256i V1, __m256i V2, const int M);
	3113	/// \endcode
	3114	///
	3115	/// This intrinsic corresponds to the \c VPBLENDDD instruction.
	3116	///
	3117	/// \param V1
	3118	/// A 256-bit vector of [8 x i32] containing source values.
	3119	/// \param V2
	3120	/// A 256-bit vector of [8 x i32] containing source values.
	3121	/// \param M
	3122	/// An immediate 8-bit integer operand, with bits [7:0] specifying the
	3123	/// source for each element of the result. The position of the mask bit
	3124	/// corresponds to the index of a copied value. When a mask bit is 0, the
	3125	/// element is copied from \a V1; otherwise, it is is copied from \a V2.
	3126	/// \returns A 256-bit vector of [8 x i32] containing the result.
749	3127	#define _mm256_blend_epi32(V1, V2, M) \
750	3128	((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(V1), \
751	3129	(__v8si)(__m256i)(V2), (int)(M)))
752	3130
	3131	/// Broadcasts the low byte from the 128-bit integer vector in \a __X to all
	3132	/// bytes of the 256-bit result.
	3133	///
	3134	/// \headerfile <immintrin.h>
	3135	///
	3136	/// This intrinsic corresponds to the \c VPBROADCASTB instruction.
	3137	///
	3138	/// \param __X
	3139	/// A 128-bit integer vector whose low byte will be broadcast.
	3140	/// \returns A 256-bit integer vector containing the result.
753	3141	static __inline__ __m256i __DEFAULT_FN_ATTRS256
754	3142	_mm256_broadcastb_epi8(__m128i __X)
755	3143	{
756	3144	return (__m256i)__builtin_shufflevector((__v16qi)__X, (__v16qi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
757	3145	}
758	3146
	3147	/// Broadcasts the low element from the 128-bit vector of [8 x i16] in \a __X
	3148	/// to all elements of the result's 256-bit vector of [16 x i16].
	3149	///
	3150	/// \headerfile <immintrin.h>
	3151	///
	3152	/// This intrinsic corresponds to the \c VPBROADCASTW instruction.
	3153	///
	3154	/// \param __X
	3155	/// A 128-bit vector of [8 x i16] whose low element will be broadcast.
	3156	/// \returns A 256-bit vector of [16 x i16] containing the result.
759	3157	static __inline__ __m256i __DEFAULT_FN_ATTRS256
760	3158	_mm256_broadcastw_epi16(__m128i __X)
761	3159	{
762	3160	return (__m256i)__builtin_shufflevector((__v8hi)__X, (__v8hi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
763	3161	}
764	3162
	3163	/// Broadcasts the low element from the 128-bit vector of [4 x i32] in \a __X
	3164	/// to all elements of the result's 256-bit vector of [8 x i32].
	3165	///
	3166	/// \headerfile <immintrin.h>
	3167	///
	3168	/// This intrinsic corresponds to the \c VPBROADCASTD instruction.
	3169	///
	3170	/// \param __X
	3171	/// A 128-bit vector of [4 x i32] whose low element will be broadcast.
	3172	/// \returns A 256-bit vector of [8 x i32] containing the result.
765	3173	static __inline__ __m256i __DEFAULT_FN_ATTRS256
766	3174	_mm256_broadcastd_epi32(__m128i __X)
767	3175	{
768	3176	return (__m256i)__builtin_shufflevector((__v4si)__X, (__v4si)__X, 0, 0, 0, 0, 0, 0, 0, 0);
769	3177	}
770	3178
	3179	/// Broadcasts the low element from the 128-bit vector of [2 x i64] in \a __X
	3180	/// to all elements of the result's 256-bit vector of [4 x i64].
	3181	///
	3182	/// \headerfile <immintrin.h>
	3183	///
	3184	/// This intrinsic corresponds to the \c VPBROADCASTQ instruction.
	3185	///
	3186	/// \param __X
	3187	/// A 128-bit vector of [2 x i64] whose low element will be broadcast.
	3188	/// \returns A 256-bit vector of [4 x i64] containing the result.
771	3189	static __inline__ __m256i __DEFAULT_FN_ATTRS256
772	3190	_mm256_broadcastq_epi64(__m128i __X)
773	3191	{
774	3192	return (__m256i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 0, 0, 0);
775	3193	}
776	3194
	3195	/// Broadcasts the low byte from the 128-bit integer vector in \a __X to all
	3196	/// bytes of the 128-bit result.
	3197	///
	3198	/// \headerfile <immintrin.h>
	3199	///
	3200	/// This intrinsic corresponds to the \c VPBROADCASTB instruction.
	3201	///
	3202	/// \param __X
	3203	/// A 128-bit integer vector whose low byte will be broadcast.
	3204	/// \returns A 128-bit integer vector containing the result.
777	3205	static __inline__ __m128i __DEFAULT_FN_ATTRS128
778	3206	_mm_broadcastb_epi8(__m128i __X)
779	3207	{
780	3208	return (__m128i)__builtin_shufflevector((__v16qi)__X, (__v16qi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
781	3209	}
782	3210
	3211	/// Broadcasts the low element from the 128-bit vector of [8 x i16] in
	3212	/// \a __X to all elements of the result's 128-bit vector of [8 x i16].
	3213	///
	3214	/// \headerfile <immintrin.h>
	3215	///
	3216	/// This intrinsic corresponds to the \c VPBROADCASTW instruction.
	3217	///
	3218	/// \param __X
	3219	/// A 128-bit vector of [8 x i16] whose low element will be broadcast.
	3220	/// \returns A 128-bit vector of [8 x i16] containing the result.
783	3221	static __inline__ __m128i __DEFAULT_FN_ATTRS128
784	3222	_mm_broadcastw_epi16(__m128i __X)
785	3223	{
786	3224	return (__m128i)__builtin_shufflevector((__v8hi)__X, (__v8hi)__X, 0, 0, 0, 0, 0, 0, 0, 0);
787	3225	}
788	3226
789
	3227	/// Broadcasts the low element from the 128-bit vector of [4 x i32] in \a __X
	3228	/// to all elements of the result's vector of [4 x i32].
	3229	///
	3230	/// \headerfile <immintrin.h>
	3231	///
	3232	/// This intrinsic corresponds to the \c VPBROADCASTD instruction.
	3233	///
	3234	/// \param __X
	3235	/// A 128-bit vector of [4 x i32] whose low element will be broadcast.
	3236	/// \returns A 128-bit vector of [4 x i32] containing the result.
790	3237	static __inline__ __m128i __DEFAULT_FN_ATTRS128
791	3238	_mm_broadcastd_epi32(__m128i __X)
792	3239	{
793	3240	return (__m128i)__builtin_shufflevector((__v4si)__X, (__v4si)__X, 0, 0, 0, 0);
794	3241	}
795	3242
	3243	/// Broadcasts the low element from the 128-bit vector of [2 x i64] in \a __X
	3244	/// to both elements of the result's 128-bit vector of [2 x i64].
	3245	///
	3246	/// \headerfile <immintrin.h>
	3247	///
	3248	/// This intrinsic corresponds to the \c VPBROADCASTQ instruction.
	3249	///
	3250	/// \param __X
	3251	/// A 128-bit vector of [2 x i64] whose low element will be broadcast.
	3252	/// \returns A 128-bit vector of [2 x i64] containing the result.
796	3253	static __inline__ __m128i __DEFAULT_FN_ATTRS128
797	3254	_mm_broadcastq_epi64(__m128i __X)
798	3255	{
799	3256	return (__m128i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 0);
800	3257	}
801	3258
	3259	/// Sets the result's 256-bit vector of [8 x i32] to copies of elements of the
	3260	/// 256-bit vector of [8 x i32] in \a __a as specified by indexes in the
	3261	/// elements of the 256-bit vector of [8 x i32] in \a __b.
	3262	///
	3263	/// \code{.operation}
	3264	/// FOR i := 0 TO 7
	3265	/// j := i*32
	3266	/// k := __b[j+2:j] * 32
	3267	/// result[j+31:j] := __a[k+31:k]
	3268	/// ENDFOR
	3269	/// \endcode
	3270	///
	3271	/// \headerfile <immintrin.h>
	3272	///
	3273	/// This intrinsic corresponds to the \c VPERMD instruction.
	3274	///
	3275	/// \param __a
	3276	/// A 256-bit vector of [8 x i32] containing the source values.
	3277	/// \param __b
	3278	/// A 256-bit vector of [8 x i32] containing indexes of values to use from
	3279	/// \a __a.
	3280	/// \returns A 256-bit vector of [8 x i32] containing the result.
802	3281	static __inline__ __m256i __DEFAULT_FN_ATTRS256
803	3282	_mm256_permutevar8x32_epi32(__m256i __a, __m256i __b)
804	3283	{
805	3284	return (__m256i)__builtin_ia32_permvarsi256((__v8si)__a, (__v8si)__b);
806	3285	}
807	3286
	3287	/// Sets the result's 256-bit vector of [4 x double] to copies of elements of
	3288	/// the 256-bit vector of [4 x double] in \a V as specified by the
	3289	/// immediate value \a M.
	3290	///
	3291	/// \code{.operation}
	3292	/// FOR i := 0 TO 3
	3293	/// j := i*64
	3294	/// k := (M >> i2)[1:0] 64
	3295	/// result[j+63:j] := V[k+63:k]
	3296	/// ENDFOR
	3297	/// \endcode
	3298	///
	3299	/// \headerfile <immintrin.h>
	3300	///
	3301	/// \code
	3302	/// __m256d _mm256_permute4x64_pd(__m256d V, const int M);
	3303	/// \endcode
	3304	///
	3305	/// This intrinsic corresponds to the \c VPERMPD instruction.
	3306	///
	3307	/// \param V
	3308	/// A 256-bit vector of [4 x double] containing the source values.
	3309	/// \param M
	3310	/// An immediate 8-bit value specifying which elements to copy from \a V.
	3311	/// \a M[1:0] specifies the index in \a a for element 0 of the result,
	3312	/// \a M[3:2] specifies the index for element 1, and so forth.
	3313	/// \returns A 256-bit vector of [4 x double] containing the result.
808	3314	#define _mm256_permute4x64_pd(V, M) \
809	3315	((__m256d)__builtin_ia32_permdf256((__v4df)(__m256d)(V), (int)(M)))
810	3316
	3317	/// Sets the result's 256-bit vector of [8 x float] to copies of elements of
	3318	/// the 256-bit vector of [8 x float] in \a __a as specified by indexes in
	3319	/// the elements of the 256-bit vector of [8 x i32] in \a __b.
	3320	///
	3321	/// \code{.operation}
	3322	/// FOR i := 0 TO 7
	3323	/// j := i*32
	3324	/// k := __b[j+2:j] * 32
	3325	/// result[j+31:j] := __a[k+31:k]
	3326	/// ENDFOR
	3327	/// \endcode
	3328	///
	3329	/// \headerfile <immintrin.h>
	3330	///
	3331	/// This intrinsic corresponds to the \c VPERMPS instruction.
	3332	///
	3333	/// \param __a
	3334	/// A 256-bit vector of [8 x float] containing the source values.
	3335	/// \param __b
	3336	/// A 256-bit vector of [8 x i32] containing indexes of values to use from
	3337	/// \a __a.
	3338	/// \returns A 256-bit vector of [8 x float] containing the result.
811	3339	static __inline__ __m256 __DEFAULT_FN_ATTRS256
812	3340	_mm256_permutevar8x32_ps(__m256 __a, __m256i __b)
813	3341	{
814	3342	return (__m256)__builtin_ia32_permvarsf256((__v8sf)__a, (__v8si)__b);
815	3343	}
816	3344
	3345	/// Sets the result's 256-bit vector of [4 x i64] result to copies of elements
	3346	/// of the 256-bit vector of [4 x i64] in \a V as specified by the
	3347	/// immediate value \a M.
	3348	///
	3349	/// \code{.operation}
	3350	/// FOR i := 0 TO 3
	3351	/// j := i*64
	3352	/// k := (M >> i2)[1:0] 64
	3353	/// result[j+63:j] := V[k+63:k]
	3354	/// ENDFOR
	3355	/// \endcode
	3356	///
	3357	/// \headerfile <immintrin.h>
	3358	///
	3359	/// \code
	3360	/// __m256i _mm256_permute4x64_epi64(__m256i V, const int M);
	3361	/// \endcode
	3362	///
	3363	/// This intrinsic corresponds to the \c VPERMQ instruction.
	3364	///
	3365	/// \param V
	3366	/// A 256-bit vector of [4 x i64] containing the source values.
	3367	/// \param M
	3368	/// An immediate 8-bit value specifying which elements to copy from \a V.
	3369	/// \a M[1:0] specifies the index in \a a for element 0 of the result,
	3370	/// \a M[3:2] specifies the index for element 1, and so forth.
	3371	/// \returns A 256-bit vector of [4 x i64] containing the result.
817	3372	#define _mm256_permute4x64_epi64(V, M) \
818	3373	((__m256i)__builtin_ia32_permdi256((__v4di)(__m256i)(V), (int)(M)))
819	3374
	3375	/// Sets each half of the 256-bit result either to zero or to one of the
	3376	/// four possible 128-bit halves of the 256-bit vectors \a V1 and \a V2,
	3377	/// as specified by the immediate value \a M.
	3378	///
	3379	/// \code{.operation}
	3380	/// FOR i := 0 TO 1
	3381	/// j := i*128
	3382	/// k := M >> (i*4)
	3383	/// IF k[3] == 0
	3384	/// CASE (k[1:0]) OF
	3385	/// 0: result[127+j:j] := V1[127:0]
	3386	/// 1: result[127+j:j] := V1[255:128]
	3387	/// 2: result[127+j:j] := V2[127:0]
	3388	/// 3: result[127+j:j] := V2[255:128]
	3389	/// ESAC
	3390	/// ELSE
	3391	/// result[127+j:j] := 0
	3392	/// FI
	3393	/// ENDFOR
	3394	/// \endcode
	3395	///
	3396	/// \headerfile <immintrin.h>
	3397	///
	3398	/// \code
	3399	/// __m256i _mm256_permute2x128_si256(__m256i V1, __m256i V2, const int M);
	3400	/// \endcode
	3401	///
	3402	/// This intrinsic corresponds to the \c VPERM2I128 instruction.
	3403	///
	3404	/// \param V1
	3405	/// A 256-bit integer vector containing source values.
	3406	/// \param V2
	3407	/// A 256-bit integer vector containing source values.
	3408	/// \param M
	3409	/// An immediate value specifying how to form the result. Bits [3:0]
	3410	/// control the lower half of the result, bits [7:4] control the upper half.
	3411	/// Within each 4-bit control value, if bit 3 is 1, the result is zero,
	3412	/// otherwise bits [1:0] determine the source as follows. \n
	3413	/// 0: the lower half of \a V1 \n
	3414	/// 1: the upper half of \a V1 \n
	3415	/// 2: the lower half of \a V2 \n
	3416	/// 3: the upper half of \a V2
	3417	/// \returns A 256-bit integer vector containing the result.
820	3418	#define _mm256_permute2x128_si256(V1, V2, M) \
821	3419	((__m256i)__builtin_ia32_permti256((__m256i)(V1), (__m256i)(V2), (int)(M)))
822	3420
	3421	/// Extracts half of the 256-bit vector \a V to the 128-bit result. If bit 0
	3422	/// of the immediate \a M is zero, extracts the lower half of the result;
	3423	/// otherwise, extracts the upper half.
	3424	///
	3425	/// \headerfile <immintrin.h>
	3426	///
	3427	/// \code
	3428	/// __m128i _mm256_extracti128_si256(__m256i V, const int M);
	3429	/// \endcode
	3430	///
	3431	/// This intrinsic corresponds to the \c VEXTRACTI128 instruction.
	3432	///
	3433	/// \param V
	3434	/// A 256-bit integer vector containing the source values.
	3435	/// \param M
	3436	/// An immediate value specifying which half of \a V to extract.
	3437	/// \returns A 128-bit integer vector containing the result.
823	3438	#define _mm256_extracti128_si256(V, M) \
824	3439	((__m128i)__builtin_ia32_extract128i256((__v4di)(__m256i)(V), (int)(M)))
825	3440
	3441	/// Copies the 256-bit vector \a V1 to the result, then overwrites half of the
	3442	/// result with the 128-bit vector \a V2. If bit 0 of the immediate \a M
	3443	/// is zero, overwrites the lower half of the result; otherwise,
	3444	/// overwrites the upper half.
	3445	///
	3446	/// \headerfile <immintrin.h>
	3447	///
	3448	/// \code
	3449	/// __m256i _mm256_inserti128_si256(__m256i V1, __m128i V2, const int M);
	3450	/// \endcode
	3451	///
	3452	/// This intrinsic corresponds to the \c VINSERTI128 instruction.
	3453	///
	3454	/// \param V1
	3455	/// A 256-bit integer vector containing a source value.
	3456	/// \param V2
	3457	/// A 128-bit integer vector containing a source value.
	3458	/// \param M
	3459	/// An immediate value specifying where to put \a V2 in the result.
	3460	/// \returns A 256-bit integer vector containing the result.
826	3461	#define _mm256_inserti128_si256(V1, V2, M) \
827	3462	((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(V1), \
828	3463	(__v2di)(__m128i)(V2), (int)(M)))
829	3464
	3465	/// Conditionally loads eight 32-bit integer elements from memory \a __X, if
	3466	/// the most significant bit of the corresponding element in the mask
	3467	/// \a __M is set; otherwise, sets that element of the result to zero.
	3468	/// Returns the 256-bit [8 x i32] result.
	3469	///
	3470	/// \code{.operation}
	3471	/// FOR i := 0 TO 7
	3472	/// j := i*32
	3473	/// IF __M[j+31] == 1
	3474	/// result[j+31:j] := Load32(__X+(i*4))
	3475	/// ELSE
	3476	/// result[j+31:j] := 0
	3477	/// FI
	3478	/// ENDFOR
	3479	/// \endcode
	3480	///
	3481	/// \headerfile <immintrin.h>
	3482	///
	3483	/// This intrinsic corresponds to the \c VPMASKMOVD instruction.
	3484	///
	3485	/// \param __X
	3486	/// A pointer to the memory used for loading values.
	3487	/// \param __M
	3488	/// A 256-bit vector of [8 x i32] containing the mask bits.
	3489	/// \returns A 256-bit vector of [8 x i32] containing the loaded or zeroed
	3490	/// elements.
830	3491	static __inline__ __m256i __DEFAULT_FN_ATTRS256
831	3492	_mm256_maskload_epi32(int const *__X, __m256i __M)
832	3493	{
833	3494	return (__m256i)__builtin_ia32_maskloadd256((const __v8si *)__X, (__v8si)__M);
834	3495	}
835	3496
	3497	/// Conditionally loads four 64-bit integer elements from memory \a __X, if
	3498	/// the most significant bit of the corresponding element in the mask
	3499	/// \a __M is set; otherwise, sets that element of the result to zero.
	3500	/// Returns the 256-bit [4 x i64] result.
	3501	///
	3502	/// \code{.operation}
	3503	/// FOR i := 0 TO 3
	3504	/// j := i*64
	3505	/// IF __M[j+63] == 1
	3506	/// result[j+63:j] := Load64(__X+(i*8))
	3507	/// ELSE
	3508	/// result[j+63:j] := 0
	3509	/// FI
	3510	/// ENDFOR
	3511	/// \endcode
	3512	///
	3513	/// \headerfile <immintrin.h>
	3514	///
	3515	/// This intrinsic corresponds to the \c VPMASKMOVQ instruction.
	3516	///
	3517	/// \param __X
	3518	/// A pointer to the memory used for loading values.
	3519	/// \param __M
	3520	/// A 256-bit vector of [4 x i64] containing the mask bits.
	3521	/// \returns A 256-bit vector of [4 x i64] containing the loaded or zeroed
	3522	/// elements.
836	3523	static __inline__ __m256i __DEFAULT_FN_ATTRS256
837	3524	_mm256_maskload_epi64(long long const *__X, __m256i __M)
838	3525	{
839	3526	return (__m256i)__builtin_ia32_maskloadq256((const __v4di *)__X, (__v4di)__M);
840	3527	}
841	3528
	3529	/// Conditionally loads four 32-bit integer elements from memory \a __X, if
	3530	/// the most significant bit of the corresponding element in the mask
	3531	/// \a __M is set; otherwise, sets that element of the result to zero.
	3532	/// Returns the 128-bit [4 x i32] result.
	3533	///
	3534	/// \code{.operation}
	3535	/// FOR i := 0 TO 3
	3536	/// j := i*32
	3537	/// IF __M[j+31] == 1
	3538	/// result[j+31:j] := Load32(__X+(i*4))
	3539	/// ELSE
	3540	/// result[j+31:j] := 0
	3541	/// FI
	3542	/// ENDFOR
	3543	/// \endcode
	3544	///
	3545	/// \headerfile <immintrin.h>
	3546	///
	3547	/// This intrinsic corresponds to the \c VPMASKMOVD instruction.
	3548	///
	3549	/// \param __X
	3550	/// A pointer to the memory used for loading values.
	3551	/// \param __M
	3552	/// A 128-bit vector of [4 x i32] containing the mask bits.
	3553	/// \returns A 128-bit vector of [4 x i32] containing the loaded or zeroed
	3554	/// elements.
842	3555	static __inline__ __m128i __DEFAULT_FN_ATTRS128
843	3556	_mm_maskload_epi32(int const *__X, __m128i __M)
844	3557	{
845	3558	return (__m128i)__builtin_ia32_maskloadd((const __v4si *)__X, (__v4si)__M);
846	3559	}
847	3560
	3561	/// Conditionally loads two 64-bit integer elements from memory \a __X, if
	3562	/// the most significant bit of the corresponding element in the mask
	3563	/// \a __M is set; otherwise, sets that element of the result to zero.
	3564	/// Returns the 128-bit [2 x i64] result.
	3565	///
	3566	/// \code{.operation}
	3567	/// FOR i := 0 TO 1
	3568	/// j := i*64
	3569	/// IF __M[j+63] == 1
	3570	/// result[j+63:j] := Load64(__X+(i*8))
	3571	/// ELSE
	3572	/// result[j+63:j] := 0
	3573	/// FI
	3574	/// ENDFOR
	3575	/// \endcode
	3576	///
	3577	/// \headerfile <immintrin.h>
	3578	///
	3579	/// This intrinsic corresponds to the \c VPMASKMOVQ instruction.
	3580	///
	3581	/// \param __X
	3582	/// A pointer to the memory used for loading values.
	3583	/// \param __M
	3584	/// A 128-bit vector of [2 x i64] containing the mask bits.
	3585	/// \returns A 128-bit vector of [2 x i64] containing the loaded or zeroed
	3586	/// elements.
848	3587	static __inline__ __m128i __DEFAULT_FN_ATTRS128
849	3588	_mm_maskload_epi64(long long const *__X, __m128i __M)
850	3589	{
851	3590	return (__m128i)__builtin_ia32_maskloadq((const __v2di *)__X, (__v2di)__M);
852	3591	}
853	3592
	3593	/// Conditionally stores eight 32-bit integer elements from the 256-bit vector
	3594	/// of [8 x i32] in \a __Y to memory \a __X, if the most significant bit of
	3595	/// the corresponding element in the mask \a __M is set; otherwise, the
	3596	/// memory element is unchanged.
	3597	///
	3598	/// \code{.operation}
	3599	/// FOR i := 0 TO 7
	3600	/// j := i*32
	3601	/// IF __M[j+31] == 1
	3602	/// Store32(__X+(i*4), __Y[j+31:j])
	3603	/// FI
	3604	/// ENDFOR
	3605	/// \endcode
	3606	///
	3607	/// \headerfile <immintrin.h>
	3608	///
	3609	/// This intrinsic corresponds to the \c VPMASKMOVD instruction.
	3610	///
	3611	/// \param __X
	3612	/// A pointer to the memory used for storing values.
	3613	/// \param __M
	3614	/// A 256-bit vector of [8 x i32] containing the mask bits.
	3615	/// \param __Y
	3616	/// A 256-bit vector of [8 x i32] containing the values to store.
854	3617	static __inline__ void __DEFAULT_FN_ATTRS256
855	3618	_mm256_maskstore_epi32(int *__X, __m256i __M, __m256i __Y)
856	3619	{
857	3620	__builtin_ia32_maskstored256((__v8si *)__X, (__v8si)__M, (__v8si)__Y);
858	3621	}
859	3622
	3623	/// Conditionally stores four 64-bit integer elements from the 256-bit vector
	3624	/// of [4 x i64] in \a __Y to memory \a __X, if the most significant bit of
	3625	/// the corresponding element in the mask \a __M is set; otherwise, the
	3626	/// memory element is unchanged.
	3627	///
	3628	/// \code{.operation}
	3629	/// FOR i := 0 TO 3
	3630	/// j := i*64
	3631	/// IF __M[j+63] == 1
	3632	/// Store64(__X+(i*8), __Y[j+63:j])
	3633	/// FI
	3634	/// ENDFOR
	3635	/// \endcode
	3636	///
	3637	/// \headerfile <immintrin.h>
	3638	///
	3639	/// This intrinsic corresponds to the \c VPMASKMOVQ instruction.
	3640	///
	3641	/// \param __X
	3642	/// A pointer to the memory used for storing values.
	3643	/// \param __M
	3644	/// A 256-bit vector of [4 x i64] containing the mask bits.
	3645	/// \param __Y
	3646	/// A 256-bit vector of [4 x i64] containing the values to store.
860	3647	static __inline__ void __DEFAULT_FN_ATTRS256
861	3648	_mm256_maskstore_epi64(long long *__X, __m256i __M, __m256i __Y)
862	3649	{
863	3650	__builtin_ia32_maskstoreq256((__v4di *)__X, (__v4di)__M, (__v4di)__Y);
864	3651	}
865	3652
	3653	/// Conditionally stores four 32-bit integer elements from the 128-bit vector
	3654	/// of [4 x i32] in \a __Y to memory \a __X, if the most significant bit of
	3655	/// the corresponding element in the mask \a __M is set; otherwise, the
	3656	/// memory element is unchanged.
	3657	///
	3658	/// \code{.operation}
	3659	/// FOR i := 0 TO 3
	3660	/// j := i*32
	3661	/// IF __M[j+31] == 1
	3662	/// Store32(__X+(i*4), __Y[j+31:j])
	3663	/// FI
	3664	/// ENDFOR
	3665	/// \endcode
	3666	///
	3667	/// \headerfile <immintrin.h>
	3668	///
	3669	/// This intrinsic corresponds to the \c VPMASKMOVD instruction.
	3670	///
	3671	/// \param __X
	3672	/// A pointer to the memory used for storing values.
	3673	/// \param __M
	3674	/// A 128-bit vector of [4 x i32] containing the mask bits.
	3675	/// \param __Y
	3676	/// A 128-bit vector of [4 x i32] containing the values to store.
866	3677	static __inline__ void __DEFAULT_FN_ATTRS128
867	3678	_mm_maskstore_epi32(int *__X, __m128i __M, __m128i __Y)
868	3679	{
869	3680	__builtin_ia32_maskstored((__v4si *)__X, (__v4si)__M, (__v4si)__Y);
870	3681	}
871	3682
	3683	/// Conditionally stores two 64-bit integer elements from the 128-bit vector
	3684	/// of [2 x i64] in \a __Y to memory \a __X, if the most significant bit of
	3685	/// the corresponding element in the mask \a __M is set; otherwise, the
	3686	/// memory element is unchanged.
	3687	///
	3688	/// \code{.operation}
	3689	/// FOR i := 0 TO 1
	3690	/// j := i*64
	3691	/// IF __M[j+63] == 1
	3692	/// Store64(__X+(i*8), __Y[j+63:j])
	3693	/// FI
	3694	/// ENDFOR
	3695	/// \endcode
	3696	///
	3697	/// \headerfile <immintrin.h>
	3698	///
	3699	/// This intrinsic corresponds to the \c VPMASKMOVQ instruction.
	3700	///
	3701	/// \param __X
	3702	/// A pointer to the memory used for storing values.
	3703	/// \param __M
	3704	/// A 128-bit vector of [2 x i64] containing the mask bits.
	3705	/// \param __Y
	3706	/// A 128-bit vector of [2 x i64] containing the values to store.
872	3707	static __inline__ void __DEFAULT_FN_ATTRS128
873	3708	_mm_maskstore_epi64(long long *__X, __m128i __M, __m128i __Y)
874	3709	{
875	3710	__builtin_ia32_maskstoreq(( __v2di *)__X, (__v2di)__M, (__v2di)__Y);
876	3711	}
877	3712
	3713	/// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __X
	3714	/// left by the number of bits given in the corresponding element of the
	3715	/// 256-bit vector of [8 x i32] in \a __Y, shifting in zero bits, and
	3716	/// returns the result. If the shift count for any element is greater than
	3717	/// 31, the result for that element is zero.
	3718	///
	3719	/// \headerfile <immintrin.h>
	3720	///
	3721	/// This intrinsic corresponds to the \c VPSLLVD instruction.
	3722	///
	3723	/// \param __X
	3724	/// A 256-bit vector of [8 x i32] to be shifted.
	3725	/// \param __Y
	3726	/// A 256-bit vector of [8 x i32] containing the unsigned shift counts (in
	3727	/// bits).
	3728	/// \returns A 256-bit vector of [8 x i32] containing the result.
878	3729	static __inline__ __m256i __DEFAULT_FN_ATTRS256
879	3730	_mm256_sllv_epi32(__m256i __X, __m256i __Y)
880	3731	{
881	3732	return (__m256i)__builtin_ia32_psllv8si((__v8si)__X, (__v8si)__Y);
882	3733	}
883	3734
	3735	/// Shifts each 32-bit element of the 128-bit vector of [4 x i32] in \a __X
	3736	/// left by the number of bits given in the corresponding element of the
	3737	/// 128-bit vector of [4 x i32] in \a __Y, shifting in zero bits, and
	3738	/// returns the result. If the shift count for any element is greater than
	3739	/// 31, the result for that element is zero.
	3740	///
	3741	/// \headerfile <immintrin.h>
	3742	///
	3743	/// This intrinsic corresponds to the \c VPSLLVD instruction.
	3744	///
	3745	/// \param __X
	3746	/// A 128-bit vector of [4 x i32] to be shifted.
	3747	/// \param __Y
	3748	/// A 128-bit vector of [4 x i32] containing the unsigned shift counts (in
	3749	/// bits).
	3750	/// \returns A 128-bit vector of [4 x i32] containing the result.
884	3751	static __inline__ __m128i __DEFAULT_FN_ATTRS128
885	3752	_mm_sllv_epi32(__m128i __X, __m128i __Y)
886	3753	{
887	3754	return (__m128i)__builtin_ia32_psllv4si((__v4si)__X, (__v4si)__Y);
888	3755	}
889	3756
	3757	/// Shifts each 64-bit element of the 256-bit vector of [4 x i64] in \a __X
	3758	/// left by the number of bits given in the corresponding element of the
	3759	/// 128-bit vector of [4 x i64] in \a __Y, shifting in zero bits, and
	3760	/// returns the result. If the shift count for any element is greater than
	3761	/// 63, the result for that element is zero.
	3762	///
	3763	/// \headerfile <immintrin.h>
	3764	///
	3765	/// This intrinsic corresponds to the \c VPSLLVQ instruction.
	3766	///
	3767	/// \param __X
	3768	/// A 256-bit vector of [4 x i64] to be shifted.
	3769	/// \param __Y
	3770	/// A 256-bit vector of [4 x i64] containing the unsigned shift counts (in
	3771	/// bits).
	3772	/// \returns A 256-bit vector of [4 x i64] containing the result.
890	3773	static __inline__ __m256i __DEFAULT_FN_ATTRS256
891	3774	_mm256_sllv_epi64(__m256i __X, __m256i __Y)
892	3775	{
893	3776	return (__m256i)__builtin_ia32_psllv4di((__v4di)__X, (__v4di)__Y);
894	3777	}
895	3778
	3779	/// Shifts each 64-bit element of the 128-bit vector of [2 x i64] in \a __X
	3780	/// left by the number of bits given in the corresponding element of the
	3781	/// 128-bit vector of [2 x i64] in \a __Y, shifting in zero bits, and
	3782	/// returns the result. If the shift count for any element is greater than
	3783	/// 63, the result for that element is zero.
	3784	///
	3785	/// \headerfile <immintrin.h>
	3786	///
	3787	/// This intrinsic corresponds to the \c VPSLLVQ instruction.
	3788	///
	3789	/// \param __X
	3790	/// A 128-bit vector of [2 x i64] to be shifted.
	3791	/// \param __Y
	3792	/// A 128-bit vector of [2 x i64] containing the unsigned shift counts (in
	3793	/// bits).
	3794	/// \returns A 128-bit vector of [2 x i64] containing the result.
896	3795	static __inline__ __m128i __DEFAULT_FN_ATTRS128
897	3796	_mm_sllv_epi64(__m128i __X, __m128i __Y)
898	3797	{
899	3798	return (__m128i)__builtin_ia32_psllv2di((__v2di)__X, (__v2di)__Y);
900	3799	}
901	3800
	3801	/// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __X
	3802	/// right by the number of bits given in the corresponding element of the
	3803	/// 256-bit vector of [8 x i32] in \a __Y, shifting in sign bits, and
	3804	/// returns the result. If the shift count for any element is greater than
	3805	/// 31, the result for that element is 0 or -1 according to the sign bit
	3806	/// for that element.
	3807	///
	3808	/// \headerfile <immintrin.h>
	3809	///
	3810	/// This intrinsic corresponds to the \c VPSRAVD instruction.
	3811	///
	3812	/// \param __X
	3813	/// A 256-bit vector of [8 x i32] to be shifted.
	3814	/// \param __Y
	3815	/// A 256-bit vector of [8 x i32] containing the unsigned shift counts (in
	3816	/// bits).
	3817	/// \returns A 256-bit vector of [8 x i32] containing the result.
902	3818	static __inline__ __m256i __DEFAULT_FN_ATTRS256
903	3819	_mm256_srav_epi32(__m256i __X, __m256i __Y)
904	3820	{
905	3821	return (__m256i)__builtin_ia32_psrav8si((__v8si)__X, (__v8si)__Y);
906	3822	}
907	3823
	3824	/// Shifts each 32-bit element of the 128-bit vector of [4 x i32] in \a __X
	3825	/// right by the number of bits given in the corresponding element of the
	3826	/// 128-bit vector of [4 x i32] in \a __Y, shifting in sign bits, and
	3827	/// returns the result. If the shift count for any element is greater than
	3828	/// 31, the result for that element is 0 or -1 according to the sign bit
	3829	/// for that element.
	3830	///
	3831	/// \headerfile <immintrin.h>
	3832	///
	3833	/// This intrinsic corresponds to the \c VPSRAVD instruction.
	3834	///
	3835	/// \param __X
	3836	/// A 128-bit vector of [4 x i32] to be shifted.
	3837	/// \param __Y
	3838	/// A 128-bit vector of [4 x i32] containing the unsigned shift counts (in
	3839	/// bits).
	3840	/// \returns A 128-bit vector of [4 x i32] containing the result.
908	3841	static __inline__ __m128i __DEFAULT_FN_ATTRS128
909	3842	_mm_srav_epi32(__m128i __X, __m128i __Y)
910	3843	{
911	3844	return (__m128i)__builtin_ia32_psrav4si((__v4si)__X, (__v4si)__Y);
912	3845	}
913	3846
	3847	/// Shifts each 32-bit element of the 256-bit vector of [8 x i32] in \a __X
	3848	/// right by the number of bits given in the corresponding element of the
	3849	/// 256-bit vector of [8 x i32] in \a __Y, shifting in zero bits, and
	3850	/// returns the result. If the shift count for any element is greater than
	3851	/// 31, the result for that element is zero.
	3852	///
	3853	/// \headerfile <immintrin.h>
	3854	///
	3855	/// This intrinsic corresponds to the \c VPSRLVD instruction.
	3856	///
	3857	/// \param __X
	3858	/// A 256-bit vector of [8 x i32] to be shifted.
	3859	/// \param __Y
	3860	/// A 256-bit vector of [8 x i32] containing the unsigned shift counts (in
	3861	/// bits).
	3862	/// \returns A 256-bit vector of [8 x i32] containing the result.
914	3863	static __inline__ __m256i __DEFAULT_FN_ATTRS256
915	3864	_mm256_srlv_epi32(__m256i __X, __m256i __Y)
916	3865	{
917	3866	return (__m256i)__builtin_ia32_psrlv8si((__v8si)__X, (__v8si)__Y);
918	3867	}
919	3868
	3869	/// Shifts each 32-bit element of the 128-bit vector of [4 x i32] in \a __X
	3870	/// right by the number of bits given in the corresponding element of the
	3871	/// 128-bit vector of [4 x i32] in \a __Y, shifting in zero bits, and
	3872	/// returns the result. If the shift count for any element is greater than
	3873	/// 31, the result for that element is zero.
	3874	///
	3875	/// \headerfile <immintrin.h>
	3876	///
	3877	/// This intrinsic corresponds to the \c VPSRLVD instruction.
	3878	///
	3879	/// \param __X
	3880	/// A 128-bit vector of [4 x i32] to be shifted.
	3881	/// \param __Y
	3882	/// A 128-bit vector of [4 x i32] containing the unsigned shift counts (in
	3883	/// bits).
	3884	/// \returns A 128-bit vector of [4 x i32] containing the result.
920	3885	static __inline__ __m128i __DEFAULT_FN_ATTRS128
921	3886	_mm_srlv_epi32(__m128i __X, __m128i __Y)
922	3887	{
923	3888	return (__m128i)__builtin_ia32_psrlv4si((__v4si)__X, (__v4si)__Y);
924	3889	}
925	3890
	3891	/// Shifts each 64-bit element of the 256-bit vector of [4 x i64] in \a __X
	3892	/// right by the number of bits given in the corresponding element of the
	3893	/// 128-bit vector of [4 x i64] in \a __Y, shifting in zero bits, and
	3894	/// returns the result. If the shift count for any element is greater than
	3895	/// 63, the result for that element is zero.
	3896	///
	3897	/// \headerfile <immintrin.h>
	3898	///
	3899	/// This intrinsic corresponds to the \c VPSRLVQ instruction.
	3900	///
	3901	/// \param __X
	3902	/// A 256-bit vector of [4 x i64] to be shifted.
	3903	/// \param __Y
	3904	/// A 256-bit vector of [4 x i64] containing the unsigned shift counts (in
	3905	/// bits).
	3906	/// \returns A 256-bit vector of [4 x i64] containing the result.
926	3907	static __inline__ __m256i __DEFAULT_FN_ATTRS256
927	3908	_mm256_srlv_epi64(__m256i __X, __m256i __Y)
928	3909	{
929	3910	return (__m256i)__builtin_ia32_psrlv4di((__v4di)__X, (__v4di)__Y);
930	3911	}
931	3912
	3913	/// Shifts each 64-bit element of the 128-bit vector of [2 x i64] in \a __X
	3914	/// right by the number of bits given in the corresponding element of the
	3915	/// 128-bit vector of [2 x i64] in \a __Y, shifting in zero bits, and
	3916	/// returns the result. If the shift count for any element is greater than
	3917	/// 63, the result for that element is zero.
	3918	///
	3919	/// \headerfile <immintrin.h>
	3920	///
	3921	/// This intrinsic corresponds to the \c VPSRLVQ instruction.
	3922	///
	3923	/// \param __X
	3924	/// A 128-bit vector of [2 x i64] to be shifted.
	3925	/// \param __Y
	3926	/// A 128-bit vector of [2 x i64] containing the unsigned shift counts (in
	3927	/// bits).
	3928	/// \returns A 128-bit vector of [2 x i64] containing the result.
932	3929	static __inline__ __m128i __DEFAULT_FN_ATTRS128
933	3930	_mm_srlv_epi64(__m128i __X, __m128i __Y)
934	3931	{
935	3932	return (__m128i)__builtin_ia32_psrlv2di((__v2di)__X, (__v2di)__Y);
936	3933	}
937	3934
	3935	/// Conditionally gathers two 64-bit floating-point values, either from the
	3936	/// 128-bit vector of [2 x double] in \a a, or from memory \a m using scaled
	3937	/// indexes from the 128-bit vector of [4 x i32] in \a i. The 128-bit vector
	3938	/// of [2 x double] in \a mask determines the source for each element.
	3939	///
	3940	/// \code{.operation}
	3941	/// FOR element := 0 to 1
	3942	/// j := element*64
	3943	/// k := element*32
	3944	/// IF mask[j+63] == 0
	3945	/// result[j+63:j] := a[j+63:j]
	3946	/// ELSE
	3947	/// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
	3948	/// FI
	3949	/// ENDFOR
	3950	/// \endcode
	3951	///
	3952	/// \headerfile <immintrin.h>
	3953	///
	3954	/// \code
	3955	/// __m128d _mm_mask_i32gather_pd(__m128d a, const double *m, __m128i i,
	3956	/// __m128d mask, const int s);
	3957	/// \endcode
	3958	///
	3959	/// This intrinsic corresponds to the \c VGATHERDPD instruction.
	3960	///
	3961	/// \param a
	3962	/// A 128-bit vector of [2 x double] used as the source when a mask bit is
	3963	/// zero.
	3964	/// \param m
	3965	/// A pointer to the memory used for loading values.
	3966	/// \param i
	3967	/// A 128-bit vector of [4 x i32] containing signed indexes into \a m. Only
	3968	/// the first two elements are used.
	3969	/// \param mask
	3970	/// A 128-bit vector of [2 x double] containing the mask. The most
	3971	/// significant bit of each element in the mask vector represents the mask
	3972	/// bits. If a mask bit is zero, the corresponding value from vector \a a
	3973	/// is gathered; otherwise the value is loaded from memory.
	3974	/// \param s
	3975	/// A literal constant scale factor for the indexes in \a i. Must be
	3976	/// 1, 2, 4, or 8.
	3977	/// \returns A 128-bit vector of [2 x double] containing the gathered values.
938	3978	#define _mm_mask_i32gather_pd(a, m, i, mask, s) \
939	3979	((__m128d)__builtin_ia32_gatherd_pd((__v2df)(__m128i)(a), \
940	3980	(double const *)(m), \
941	3981	(__v4si)(__m128i)(i), \
942	3982	(__v2df)(__m128d)(mask), (s)))
943	3983
	3984	/// Conditionally gathers four 64-bit floating-point values, either from the
	3985	/// 256-bit vector of [4 x double] in \a a, or from memory \a m using scaled
	3986	/// indexes from the 128-bit vector of [4 x i32] in \a i. The 256-bit vector
	3987	/// of [4 x double] in \a mask determines the source for each element.
	3988	///
	3989	/// \code{.operation}
	3990	/// FOR element := 0 to 3
	3991	/// j := element*64
	3992	/// k := element*32
	3993	/// IF mask[j+63] == 0
	3994	/// result[j+63:j] := a[j+63:j]
	3995	/// ELSE
	3996	/// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
	3997	/// FI
	3998	/// ENDFOR
	3999	/// \endcode
	4000	///
	4001	/// \headerfile <immintrin.h>
	4002	///
	4003	/// \code
	4004	/// __m256d _mm256_mask_i32gather_pd(__m256d a, const double *m, __m128i i,
	4005	/// __m256d mask, const int s);
	4006	/// \endcode
	4007	///
	4008	/// This intrinsic corresponds to the \c VGATHERDPD instruction.
	4009	///
	4010	/// \param a
	4011	/// A 256-bit vector of [4 x double] used as the source when a mask bit is
	4012	/// zero.
	4013	/// \param m
	4014	/// A pointer to the memory used for loading values.
	4015	/// \param i
	4016	/// A 128-bit vector of [4 x i32] containing signed indexes into \a m.
	4017	/// \param mask
	4018	/// A 256-bit vector of [4 x double] containing the mask. The most
	4019	/// significant bit of each element in the mask vector represents the mask
	4020	/// bits. If a mask bit is zero, the corresponding value from vector \a a
	4021	/// is gathered; otherwise the value is loaded from memory.
	4022	/// \param s
	4023	/// A literal constant scale factor for the indexes in \a i. Must be
	4024	/// 1, 2, 4, or 8.
	4025	/// \returns A 256-bit vector of [4 x double] containing the gathered values.
944	4026	#define _mm256_mask_i32gather_pd(a, m, i, mask, s) \
945	4027	((__m256d)__builtin_ia32_gatherd_pd256((__v4df)(__m256d)(a), \
946	4028	(double const *)(m), \
947	4029	(__v4si)(__m128i)(i), \
948	4030	(__v4df)(__m256d)(mask), (s)))
949	4031
	4032	/// Conditionally gathers two 64-bit floating-point values, either from the
	4033	/// 128-bit vector of [2 x double] in \a a, or from memory \a m using scaled
	4034	/// indexes from the 128-bit vector of [2 x i64] in \a i. The 128-bit vector
	4035	/// of [2 x double] in \a mask determines the source for each element.
	4036	///
	4037	/// \code{.operation}
	4038	/// FOR element := 0 to 1
	4039	/// j := element*64
	4040	/// k := element*64
	4041	/// IF mask[j+63] == 0
	4042	/// result[j+63:j] := a[j+63:j]
	4043	/// ELSE
	4044	/// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
	4045	/// FI
	4046	/// ENDFOR
	4047	/// \endcode
	4048	///
	4049	/// \headerfile <immintrin.h>
	4050	///
	4051	/// \code
	4052	/// __m128d _mm_mask_i64gather_pd(__m128d a, const double *m, __m128i i,
	4053	/// __m128d mask, const int s);
	4054	/// \endcode
	4055	///
	4056	/// This intrinsic corresponds to the \c VGATHERQPD instruction.
	4057	///
	4058	/// \param a
	4059	/// A 128-bit vector of [2 x double] used as the source when a mask bit is
	4060	/// zero.
	4061	/// \param m
	4062	/// A pointer to the memory used for loading values.
	4063	/// \param i
	4064	/// A 128-bit vector of [2 x i64] containing signed indexes into \a m.
	4065	/// \param mask
	4066	/// A 128-bit vector of [2 x double] containing the mask. The most
	4067	/// significant bit of each element in the mask vector represents the mask
	4068	/// bits. If a mask bit is zero, the corresponding value from vector \a a
	4069	/// is gathered; otherwise the value is loaded from memory.
	4070	/// \param s
	4071	/// A literal constant scale factor for the indexes in \a i. Must be
	4072	/// 1, 2, 4, or 8.
	4073	/// \returns A 128-bit vector of [2 x double] containing the gathered values.
950	4074	#define _mm_mask_i64gather_pd(a, m, i, mask, s) \
951	4075	((__m128d)__builtin_ia32_gatherq_pd((__v2df)(__m128d)(a), \
952	4076	(double const *)(m), \
953	4077	(__v2di)(__m128i)(i), \
954	4078	(__v2df)(__m128d)(mask), (s)))
955	4079
	4080	/// Conditionally gathers four 64-bit floating-point values, either from the
	4081	/// 256-bit vector of [4 x double] in \a a, or from memory \a m using scaled
	4082	/// indexes from the 256-bit vector of [4 x i64] in \a i. The 256-bit vector
	4083	/// of [4 x double] in \a mask determines the source for each element.
	4084	///
	4085	/// \code{.operation}
	4086	/// FOR element := 0 to 3
	4087	/// j := element*64
	4088	/// k := element*64
	4089	/// IF mask[j+63] == 0
	4090	/// result[j+63:j] := a[j+63:j]
	4091	/// ELSE
	4092	/// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
	4093	/// FI
	4094	/// ENDFOR
	4095	/// \endcode
	4096	///
	4097	/// \headerfile <immintrin.h>
	4098	///
	4099	/// \code
	4100	/// __m256d _mm256_mask_i64gather_pd(__m256d a, const double *m, __m256i i,
	4101	/// __m256d mask, const int s);
	4102	/// \endcode
	4103	///
	4104	/// This intrinsic corresponds to the \c VGATHERQPD instruction.
	4105	///
	4106	/// \param a
	4107	/// A 256-bit vector of [4 x double] used as the source when a mask bit is
	4108	/// zero.
	4109	/// \param m
	4110	/// A pointer to the memory used for loading values.
	4111	/// \param i
	4112	/// A 256-bit vector of [4 x i64] containing signed indexes into \a m.
	4113	/// \param mask
	4114	/// A 256-bit vector of [4 x double] containing the mask. The most
	4115	/// significant bit of each element in the mask vector represents the mask
	4116	/// bits. If a mask bit is zero, the corresponding value from vector \a a
	4117	/// is gathered; otherwise the value is loaded from memory.
	4118	/// \param s
	4119	/// A literal constant scale factor for the indexes in \a i. Must be
	4120	/// 1, 2, 4, or 8.
	4121	/// \returns A 256-bit vector of [4 x double] containing the gathered values.
956	4122	#define _mm256_mask_i64gather_pd(a, m, i, mask, s) \
957	4123	((__m256d)__builtin_ia32_gatherq_pd256((__v4df)(__m256d)(a), \
958	4124	(double const *)(m), \
959	4125	(__v4di)(__m256i)(i), \
960	4126	(__v4df)(__m256d)(mask), (s)))
961	4127
	4128	/// Conditionally gathers four 32-bit floating-point values, either from the
	4129	/// 128-bit vector of [4 x float] in \a a, or from memory \a m using scaled
	4130	/// indexes from the 128-bit vector of [4 x i32] in \a i. The 128-bit vector
	4131	/// of [4 x float] in \a mask determines the source for each element.
	4132	///
	4133	/// \code{.operation}
	4134	/// FOR element := 0 to 3
	4135	/// j := element*32
	4136	/// k := element*32
	4137	/// IF mask[j+31] == 0
	4138	/// result[j+31:j] := a[j+31:j]
	4139	/// ELSE
	4140	/// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
	4141	/// FI
	4142	/// ENDFOR
	4143	/// \endcode
	4144	///
	4145	/// \headerfile <immintrin.h>
	4146	///
	4147	/// \code
	4148	/// __m128 _mm_mask_i32gather_ps(__m128 a, const float *m, __m128i i,
	4149	/// __m128 mask, const int s);
	4150	/// \endcode
	4151	///
	4152	/// This intrinsic corresponds to the \c VGATHERDPS instruction.
	4153	///
	4154	/// \param a
	4155	/// A 128-bit vector of [4 x float] used as the source when a mask bit is
	4156	/// zero.
	4157	/// \param m
	4158	/// A pointer to the memory used for loading values.
	4159	/// \param i
	4160	/// A 128-bit vector of [4 x i32] containing signed indexes into \a m.
	4161	/// \param mask
	4162	/// A 128-bit vector of [4 x float] containing the mask. The most
	4163	/// significant bit of each element in the mask vector represents the mask
	4164	/// bits. If a mask bit is zero, the corresponding value from vector \a a
	4165	/// is gathered; otherwise the value is loaded from memory.
	4166	/// \param s
	4167	/// A literal constant scale factor for the indexes in \a i. Must be
	4168	/// 1, 2, 4, or 8.
	4169	/// \returns A 128-bit vector of [4 x float] containing the gathered values.
962	4170	#define _mm_mask_i32gather_ps(a, m, i, mask, s) \
963	4171	((__m128)__builtin_ia32_gatherd_ps((__v4sf)(__m128)(a), \
964	4172	(float const *)(m), \
965	4173	(__v4si)(__m128i)(i), \
966	4174	(__v4sf)(__m128)(mask), (s)))
967	4175
	4176	/// Conditionally gathers eight 32-bit floating-point values, either from the
	4177	/// 256-bit vector of [8 x float] in \a a, or from memory \a m using scaled
	4178	/// indexes from the 256-bit vector of [8 x i32] in \a i. The 256-bit vector
	4179	/// of [8 x float] in \a mask determines the source for each element.
	4180	///
	4181	/// \code{.operation}
	4182	/// FOR element := 0 to 7
	4183	/// j := element*32
	4184	/// k := element*32
	4185	/// IF mask[j+31] == 0
	4186	/// result[j+31:j] := a[j+31:j]
	4187	/// ELSE
	4188	/// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
	4189	/// FI
	4190	/// ENDFOR
	4191	/// \endcode
	4192	///
	4193	/// \headerfile <immintrin.h>
	4194	///
	4195	/// \code
	4196	/// __m256 _mm256_mask_i32gather_ps(__m256 a, const float *m, __m256i i,
	4197	/// __m256 mask, const int s);
	4198	/// \endcode
	4199	///
	4200	/// This intrinsic corresponds to the \c VGATHERDPS instruction.
	4201	///
	4202	/// \param a
	4203	/// A 256-bit vector of [8 x float] used as the source when a mask bit is
	4204	/// zero.
	4205	/// \param m
	4206	/// A pointer to the memory used for loading values.
	4207	/// \param i
	4208	/// A 256-bit vector of [8 x i32] containing signed indexes into \a m.
	4209	/// \param mask
	4210	/// A 256-bit vector of [8 x float] containing the mask. The most
	4211	/// significant bit of each element in the mask vector represents the mask
	4212	/// bits. If a mask bit is zero, the corresponding value from vector \a a
	4213	/// is gathered; otherwise the value is loaded from memory.
	4214	/// \param s
	4215	/// A literal constant scale factor for the indexes in \a i. Must be
	4216	/// 1, 2, 4, or 8.
	4217	/// \returns A 256-bit vector of [8 x float] containing the gathered values.
968	4218	#define _mm256_mask_i32gather_ps(a, m, i, mask, s) \
969	4219	((__m256)__builtin_ia32_gatherd_ps256((__v8sf)(__m256)(a), \
970	4220	(float const *)(m), \
971	4221	(__v8si)(__m256i)(i), \
972	4222	(__v8sf)(__m256)(mask), (s)))
973	4223
	4224	/// Conditionally gathers two 32-bit floating-point values, either from the
	4225	/// 128-bit vector of [4 x float] in \a a, or from memory \a m using scaled
	4226	/// indexes from the 128-bit vector of [2 x i64] in \a i. The 128-bit vector
	4227	/// of [4 x float] in \a mask determines the source for the lower two
	4228	/// elements. The upper two elements of the result are zeroed.
	4229	///
	4230	/// \code{.operation}
	4231	/// FOR element := 0 to 1
	4232	/// j := element*32
	4233	/// k := element*64
	4234	/// IF mask[j+31] == 0
	4235	/// result[j+31:j] := a[j+31:j]
	4236	/// ELSE
	4237	/// result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s)
	4238	/// FI
	4239	/// ENDFOR
	4240	/// result[127:64] := 0
	4241	/// \endcode
	4242	///
	4243	/// \headerfile <immintrin.h>
	4244	///
	4245	/// \code
	4246	/// __m128 _mm_mask_i64gather_ps(__m128 a, const float *m, __m128i i,
	4247	/// __m128 mask, const int s);
	4248	/// \endcode
	4249	///
	4250	/// This intrinsic corresponds to the \c VGATHERQPS instruction.
	4251	///
	4252	/// \param a
	4253	/// A 128-bit vector of [4 x float] used as the source when a mask bit is
	4254	/// zero. Only the first two elements are used.
	4255	/// \param m
	4256	/// A pointer to the memory used for loading values.
	4257	/// \param i
	4258	/// A 128-bit vector of [2 x i64] containing signed indexes into \a m.
	4259	/// \param mask
	4260	/// A 128-bit vector of [4 x float] containing the mask. The most
	4261	/// significant bit of each element in the mask vector represents the mask
	4262	/// bits. If a mask bit is zero, the corresponding value from vector \a a
	4263	/// is gathered; otherwise the value is loaded from memory. Only the first
	4264	/// two elements are used.
	4265	/// \param s
	4266	/// A literal constant scale factor for the indexes in \a i. Must be
	4267	/// 1, 2, 4, or 8.
	4268	/// \returns A 128-bit vector of [4 x float] containing the gathered values.
974	4269	#define _mm_mask_i64gather_ps(a, m, i, mask, s) \
975	4270	((__m128)__builtin_ia32_gatherq_ps((__v4sf)(__m128)(a), \
976	4271	(float const *)(m), \
977	4272	(__v2di)(__m128i)(i), \
978	4273	(__v4sf)(__m128)(mask), (s)))
979	4274
	4275	/// Conditionally gathers four 32-bit floating-point values, either from the
	4276	/// 128-bit vector of [4 x float] in \a a, or from memory \a m using scaled
	4277	/// indexes from the 256-bit vector of [4 x i64] in \a i. The 128-bit vector
	4278	/// of [4 x float] in \a mask determines the source for each element.
	4279	///
	4280	/// \code{.operation}
	4281	/// FOR element := 0 to 3
	4282	/// j := element*32
	4283	/// k := element*64
	4284	/// IF mask[j+31] == 0
	4285	/// result[j+31:j] := a[j+31:j]
	4286	/// ELSE
	4287	/// result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s)
	4288	/// FI
	4289	/// ENDFOR
	4290	/// \endcode
	4291	///
	4292	/// \headerfile <immintrin.h>
	4293	///
	4294	/// \code
	4295	/// __m128 _mm256_mask_i64gather_ps(__m128 a, const float *m, __m256i i,
	4296	/// __m128 mask, const int s);
	4297	/// \endcode
	4298	///
	4299	/// This intrinsic corresponds to the \c VGATHERQPS instruction.
	4300	///
	4301	/// \param a
	4302	/// A 128-bit vector of [4 x float] used as the source when a mask bit is
	4303	/// zero.
	4304	/// \param m
	4305	/// A pointer to the memory used for loading values.
	4306	/// \param i
	4307	/// A 256-bit vector of [4 x i64] containing signed indexes into \a m.
	4308	/// \param mask
	4309	/// A 128-bit vector of [4 x float] containing the mask. The most
	4310	/// significant bit of each element in the mask vector represents the mask
	4311	/// bits. If a mask bit is zero, the corresponding value from vector \a a
	4312	/// is gathered; otherwise the value is loaded from memory.
	4313	/// \param s
	4314	/// A literal constant scale factor for the indexes in \a i. Must be
	4315	/// 1, 2, 4, or 8.
	4316	/// \returns A 128-bit vector of [4 x float] containing the gathered values.
980	4317	#define _mm256_mask_i64gather_ps(a, m, i, mask, s) \
981	4318	((__m128)__builtin_ia32_gatherq_ps256((__v4sf)(__m128)(a), \
982	4319	(float const *)(m), \
983	4320	(__v4di)(__m256i)(i), \
984	4321	(__v4sf)(__m128)(mask), (s)))
985	4322
	4323	/// Conditionally gathers four 32-bit integer values, either from the
	4324	/// 128-bit vector of [4 x i32] in \a a, or from memory \a m using scaled
	4325	/// indexes from the 128-bit vector of [4 x i32] in \a i. The 128-bit vector
	4326	/// of [4 x i32] in \a mask determines the source for each element.
	4327	///
	4328	/// \code{.operation}
	4329	/// FOR element := 0 to 3
	4330	/// j := element*32
	4331	/// k := element*32
	4332	/// IF mask[j+31] == 0
	4333	/// result[j+31:j] := a[j+31:j]
	4334	/// ELSE
	4335	/// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
	4336	/// FI
	4337	/// ENDFOR
	4338	/// \endcode
	4339	///
	4340	/// \headerfile <immintrin.h>
	4341	///
	4342	/// \code
	4343	/// __m128i _mm_mask_i32gather_epi32(__m128i a, const int *m, __m128i i,
	4344	/// __m128i mask, const int s);
	4345	/// \endcode
	4346	///
	4347	/// This intrinsic corresponds to the \c VPGATHERDD instruction.
	4348	///
	4349	/// \param a
	4350	/// A 128-bit vector of [4 x i32] used as the source when a mask bit is
	4351	/// zero.
	4352	/// \param m
	4353	/// A pointer to the memory used for loading values.
	4354	/// \param i
	4355	/// A 128-bit vector of [4 x i32] containing signed indexes into \a m.
	4356	/// \param mask
	4357	/// A 128-bit vector of [4 x i32] containing the mask. The most significant
	4358	/// bit of each element in the mask vector represents the mask bits. If a
	4359	/// mask bit is zero, the corresponding value from vector \a a is gathered;
	4360	/// otherwise the value is loaded from memory.
	4361	/// \param s
	4362	/// A literal constant scale factor for the indexes in \a i. Must be
	4363	/// 1, 2, 4, or 8.
	4364	/// \returns A 128-bit vector of [4 x i32] containing the gathered values.
986	4365	#define _mm_mask_i32gather_epi32(a, m, i, mask, s) \
987	4366	((__m128i)__builtin_ia32_gatherd_d((__v4si)(__m128i)(a), \
988	4367	(int const *)(m), \
989	4368	(__v4si)(__m128i)(i), \
990	4369	(__v4si)(__m128i)(mask), (s)))
991	4370
	4371	/// Conditionally gathers eight 32-bit integer values, either from the
	4372	/// 256-bit vector of [8 x i32] in \a a, or from memory \a m using scaled
	4373	/// indexes from the 256-bit vector of [8 x i32] in \a i. The 256-bit vector
	4374	/// of [8 x i32] in \a mask determines the source for each element.
	4375	///
	4376	/// \code{.operation}
	4377	/// FOR element := 0 to 7
	4378	/// j := element*32
	4379	/// k := element*32
	4380	/// IF mask[j+31] == 0
	4381	/// result[j+31:j] := a[j+31:j]
	4382	/// ELSE
	4383	/// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
	4384	/// FI
	4385	/// ENDFOR
	4386	/// \endcode
	4387	///
	4388	/// \headerfile <immintrin.h>
	4389	///
	4390	/// \code
	4391	/// __m256i _mm256_mask_i32gather_epi32(__m256i a, const int *m, __m256i i,
	4392	/// __m256i mask, const int s);
	4393	/// \endcode
	4394	///
	4395	/// This intrinsic corresponds to the \c VPGATHERDD instruction.
	4396	///
	4397	/// \param a
	4398	/// A 256-bit vector of [8 x i32] used as the source when a mask bit is
	4399	/// zero.
	4400	/// \param m
	4401	/// A pointer to the memory used for loading values.
	4402	/// \param i
	4403	/// A 256-bit vector of [8 x i32] containing signed indexes into \a m.
	4404	/// \param mask
	4405	/// A 256-bit vector of [8 x i32] containing the mask. The most significant
	4406	/// bit of each element in the mask vector represents the mask bits. If a
	4407	/// mask bit is zero, the corresponding value from vector \a a is gathered;
	4408	/// otherwise the value is loaded from memory.
	4409	/// \param s
	4410	/// A literal constant scale factor for the indexes in \a i. Must be
	4411	/// 1, 2, 4, or 8.
	4412	/// \returns A 256-bit vector of [8 x i32] containing the gathered values.
992	4413	#define _mm256_mask_i32gather_epi32(a, m, i, mask, s) \
993	4414	((__m256i)__builtin_ia32_gatherd_d256((__v8si)(__m256i)(a), \
994	4415	(int const *)(m), \
995	4416	(__v8si)(__m256i)(i), \
996	4417	(__v8si)(__m256i)(mask), (s)))
997	4418
	4419	/// Conditionally gathers two 32-bit integer values, either from the
	4420	/// 128-bit vector of [4 x i32] in \a a, or from memory \a m using scaled
	4421	/// indexes from the 128-bit vector of [2 x i64] in \a i. The 128-bit vector
	4422	/// of [4 x i32] in \a mask determines the source for the lower two
	4423	/// elements. The upper two elements of the result are zeroed.
	4424	///
	4425	/// \code{.operation}
	4426	/// FOR element := 0 to 1
	4427	/// j := element*32
	4428	/// k := element*64
	4429	/// IF mask[j+31] == 0
	4430	/// result[j+31:j] := a[j+31:j]
	4431	/// ELSE
	4432	/// result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s)
	4433	/// FI
	4434	/// ENDFOR
	4435	/// result[127:64] := 0
	4436	/// \endcode
	4437	///
	4438	/// \headerfile <immintrin.h>
	4439	///
	4440	/// \code
	4441	/// __m128i _mm_mask_i64gather_epi32(__m128i a, const int *m, __m128i i,
	4442	/// __m128i mask, const int s);
	4443	/// \endcode
	4444	///
	4445	/// This intrinsic corresponds to the \c VPGATHERQD instruction.
	4446	///
	4447	/// \param a
	4448	/// A 128-bit vector of [4 x i32] used as the source when a mask bit is
	4449	/// zero. Only the first two elements are used.
	4450	/// \param m
	4451	/// A pointer to the memory used for loading values.
	4452	/// \param i
	4453	/// A 128-bit vector of [2 x i64] containing indexes into \a m.
	4454	/// \param mask
	4455	/// A 128-bit vector of [4 x i32] containing the mask. The most significant
	4456	/// bit of each element in the mask vector represents the mask bits. If a
	4457	/// mask bit is zero, the corresponding value from vector \a a is gathered;
	4458	/// otherwise the value is loaded from memory. Only the first two elements
	4459	/// are used.
	4460	/// \param s
	4461	/// A literal constant scale factor for the indexes in \a i. Must be
	4462	/// 1, 2, 4, or 8.
	4463	/// \returns A 128-bit vector of [4 x i32] containing the gathered values.
998	4464	#define _mm_mask_i64gather_epi32(a, m, i, mask, s) \
999	4465	((__m128i)__builtin_ia32_gatherq_d((__v4si)(__m128i)(a), \
1000	4466	(int const *)(m), \
1001	4467	(__v2di)(__m128i)(i), \
1002	4468	(__v4si)(__m128i)(mask), (s)))
1003	4469
	4470	/// Conditionally gathers four 32-bit integer values, either from the
	4471	/// 128-bit vector of [4 x i32] in \a a, or from memory \a m using scaled
	4472	/// indexes from the 256-bit vector of [4 x i64] in \a i. The 128-bit vector
	4473	/// of [4 x i32] in \a mask determines the source for each element.
	4474	///
	4475	/// \code{.operation}
	4476	/// FOR element := 0 to 3
	4477	/// j := element*32
	4478	/// k := element*64
	4479	/// IF mask[j+31] == 0
	4480	/// result[j+31:j] := a[j+31:j]
	4481	/// ELSE
	4482	/// result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s)
	4483	/// FI
	4484	/// ENDFOR
	4485	/// \endcode
	4486	///
	4487	/// \headerfile <immintrin.h>
	4488	///
	4489	/// \code
	4490	/// __m128i _mm256_mask_i64gather_epi32(__m128i a, const int *m, __m256i i,
	4491	/// __m128i mask, const int s);
	4492	/// \endcode
	4493	///
	4494	/// This intrinsic corresponds to the \c VPGATHERQD instruction.
	4495	///
	4496	/// \param a
	4497	/// A 128-bit vector of [4 x i32] used as the source when a mask bit is
	4498	/// zero.
	4499	/// \param m
	4500	/// A pointer to the memory used for loading values.
	4501	/// \param i
	4502	/// A 256-bit vector of [4 x i64] containing signed indexes into \a m.
	4503	/// \param mask
	4504	/// A 128-bit vector of [4 x i32] containing the mask. The most significant
	4505	/// bit of each element in the mask vector represents the mask bits. If a
	4506	/// mask bit is zero, the corresponding value from vector \a a is gathered;
	4507	/// otherwise the value is loaded from memory.
	4508	/// \param s
	4509	/// A literal constant scale factor for the indexes in \a i. Must be
	4510	/// 1, 2, 4, or 8.
	4511	/// \returns A 128-bit vector of [4 x i32] containing the gathered values.
1004	4512	#define _mm256_mask_i64gather_epi32(a, m, i, mask, s) \
1005	4513	((__m128i)__builtin_ia32_gatherq_d256((__v4si)(__m128i)(a), \
1006	4514	(int const *)(m), \
1007	4515	(__v4di)(__m256i)(i), \
1008	4516	(__v4si)(__m128i)(mask), (s)))
1009	4517
	4518	/// Conditionally gathers two 64-bit integer values, either from the
	4519	/// 128-bit vector of [2 x i64] in \a a, or from memory \a m using scaled
	4520	/// indexes from the 128-bit vector of [4 x i32] in \a i. The 128-bit vector
	4521	/// of [2 x i64] in \a mask determines the source for each element.
	4522	///
	4523	/// \code{.operation}
	4524	/// FOR element := 0 to 1
	4525	/// j := element*64
	4526	/// k := element*32
	4527	/// IF mask[j+63] == 0
	4528	/// result[j+63:j] := a[j+63:j]
	4529	/// ELSE
	4530	/// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
	4531	/// FI
	4532	/// ENDFOR
	4533	/// \endcode
	4534	///
	4535	/// \headerfile <immintrin.h>
	4536	///
	4537	/// \code
	4538	/// __m128i _mm_mask_i32gather_epi64(__m128i a, const long long *m, __m128i i,
	4539	/// __m128i mask, const int s);
	4540	/// \endcode
	4541	///
	4542	/// This intrinsic corresponds to the \c VPGATHERDQ instruction.
	4543	///
	4544	/// \param a
	4545	/// A 128-bit vector of [2 x i64] used as the source when a mask bit is
	4546	/// zero.
	4547	/// \param m
	4548	/// A pointer to the memory used for loading values.
	4549	/// \param i
	4550	/// A 128-bit vector of [4 x i32] containing signed indexes into \a m. Only
	4551	/// the first two elements are used.
	4552	/// \param mask
	4553	/// A 128-bit vector of [2 x i64] containing the mask. The most significant
	4554	/// bit of each element in the mask vector represents the mask bits. If a
	4555	/// mask bit is zero, the corresponding value from vector \a a is gathered;
	4556	/// otherwise the value is loaded from memory.
	4557	/// \param s
	4558	/// A literal constant scale factor for the indexes in \a i. Must be
	4559	/// 1, 2, 4, or 8.
	4560	/// \returns A 128-bit vector of [2 x i64] containing the gathered values.
1010	4561	#define _mm_mask_i32gather_epi64(a, m, i, mask, s) \
1011	4562	((__m128i)__builtin_ia32_gatherd_q((__v2di)(__m128i)(a), \
1012	4563	(long long const *)(m), \
1013	4564	(__v4si)(__m128i)(i), \
1014	4565	(__v2di)(__m128i)(mask), (s)))
1015	4566
	4567	/// Conditionally gathers four 64-bit integer values, either from the
	4568	/// 256-bit vector of [4 x i64] in \a a, or from memory \a m using scaled
	4569	/// indexes from the 128-bit vector of [4 x i32] in \a i. The 256-bit vector
	4570	/// of [4 x i64] in \a mask determines the source for each element.
	4571	///
	4572	/// \code{.operation}
	4573	/// FOR element := 0 to 3
	4574	/// j := element*64
	4575	/// k := element*32
	4576	/// IF mask[j+63] == 0
	4577	/// result[j+63:j] := a[j+63:j]
	4578	/// ELSE
	4579	/// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
	4580	/// FI
	4581	/// ENDFOR
	4582	/// \endcode
	4583	///
	4584	/// \headerfile <immintrin.h>
	4585	///
	4586	/// \code
	4587	/// __m256i _mm256_mask_i32gather_epi64(__m256i a, const long long *m,
	4588	/// __m128i i, __m256i mask, const int s);
	4589	/// \endcode
	4590	///
	4591	/// This intrinsic corresponds to the \c VPGATHERDQ instruction.
	4592	///
	4593	/// \param a
	4594	/// A 256-bit vector of [4 x i64] used as the source when a mask bit is
	4595	/// zero.
	4596	/// \param m
	4597	/// A pointer to the memory used for loading values.
	4598	/// \param i
	4599	/// A 128-bit vector of [4 x i32] containing signed indexes into \a m.
	4600	/// \param mask
	4601	/// A 256-bit vector of [4 x i64] containing the mask. The most significant
	4602	/// bit of each element in the mask vector represents the mask bits. If a
	4603	/// mask bit is zero, the corresponding value from vector \a a is gathered;
	4604	/// otherwise the value is loaded from memory.
	4605	/// \param s
	4606	/// A literal constant scale factor for the indexes in \a i. Must be
	4607	/// 1, 2, 4, or 8.
	4608	/// \returns A 256-bit vector of [4 x i64] containing the gathered values.
1016	4609	#define _mm256_mask_i32gather_epi64(a, m, i, mask, s) \
1017	4610	((__m256i)__builtin_ia32_gatherd_q256((__v4di)(__m256i)(a), \
1018	4611	(long long const *)(m), \
1019	4612	(__v4si)(__m128i)(i), \
1020	4613	(__v4di)(__m256i)(mask), (s)))
1021	4614
	4615	/// Conditionally gathers two 64-bit integer values, either from the
	4616	/// 128-bit vector of [2 x i64] in \a a, or from memory \a m using scaled
	4617	/// indexes from the 128-bit vector of [2 x i64] in \a i. The 128-bit vector
	4618	/// of [2 x i64] in \a mask determines the source for each element.
	4619	///
	4620	/// \code{.operation}
	4621	/// FOR element := 0 to 1
	4622	/// j := element*64
	4623	/// k := element*64
	4624	/// IF mask[j+63] == 0
	4625	/// result[j+63:j] := a[j+63:j]
	4626	/// ELSE
	4627	/// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
	4628	/// FI
	4629	/// ENDFOR
	4630	/// \endcode
	4631	///
	4632	/// \headerfile <immintrin.h>
	4633	///
	4634	/// \code
	4635	/// __m128i _mm_mask_i64gather_epi64(__m128i a, const long long *m, __m128i i,
	4636	/// __m128i mask, const int s);
	4637	/// \endcode
	4638	///
	4639	/// This intrinsic corresponds to the \c VPGATHERQQ instruction.
	4640	///
	4641	/// \param a
	4642	/// A 128-bit vector of [2 x i64] used as the source when a mask bit is
	4643	/// zero.
	4644	/// \param m
	4645	/// A pointer to the memory used for loading values.
	4646	/// \param i
	4647	/// A 128-bit vector of [2 x i64] containing signed indexes into \a m.
	4648	/// \param mask
	4649	/// A 128-bit vector of [2 x i64] containing the mask. The most significant
	4650	/// bit of each element in the mask vector represents the mask bits. If a
	4651	/// mask bit is zero, the corresponding value from vector \a a is gathered;
	4652	/// otherwise the value is loaded from memory.
	4653	/// \param s
	4654	/// A literal constant scale factor for the indexes in \a i. Must be
	4655	/// 1, 2, 4, or 8.
	4656	/// \returns A 128-bit vector of [2 x i64] containing the gathered values.
1022	4657	#define _mm_mask_i64gather_epi64(a, m, i, mask, s) \
1023	4658	((__m128i)__builtin_ia32_gatherq_q((__v2di)(__m128i)(a), \
1024	4659	(long long const *)(m), \
1025	4660	(__v2di)(__m128i)(i), \
1026	4661	(__v2di)(__m128i)(mask), (s)))
1027	4662
	4663	/// Conditionally gathers four 64-bit integer values, either from the
	4664	/// 256-bit vector of [4 x i64] in \a a, or from memory \a m using scaled
	4665	/// indexes from the 256-bit vector of [4 x i64] in \a i. The 256-bit vector
	4666	/// of [4 x i64] in \a mask determines the source for each element.
	4667	///
	4668	/// \code{.operation}
	4669	/// FOR element := 0 to 3
	4670	/// j := element*64
	4671	/// k := element*64
	4672	/// IF mask[j+63] == 0
	4673	/// result[j+63:j] := a[j+63:j]
	4674	/// ELSE
	4675	/// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
	4676	/// FI
	4677	/// ENDFOR
	4678	/// \endcode
	4679	///
	4680	/// \headerfile <immintrin.h>
	4681	///
	4682	/// \code
	4683	/// __m256i _mm256_mask_i64gather_epi64(__m256i a, const long long *m,
	4684	/// __m256i i, __m256i mask, const int s);
	4685	/// \endcode
	4686	///
	4687	/// This intrinsic corresponds to the \c VPGATHERQQ instruction.
	4688	///
	4689	/// \param a
	4690	/// A 256-bit vector of [4 x i64] used as the source when a mask bit is
	4691	/// zero.
	4692	/// \param m
	4693	/// A pointer to the memory used for loading values.
	4694	/// \param i
	4695	/// A 256-bit vector of [4 x i64] containing signed indexes into \a m.
	4696	/// \param mask
	4697	/// A 256-bit vector of [4 x i64] containing the mask. The most significant
	4698	/// bit of each element in the mask vector represents the mask bits. If a
	4699	/// mask bit is zero, the corresponding value from vector \a a is gathered;
	4700	/// otherwise the value is loaded from memory.
	4701	/// \param s
	4702	/// A literal constant scale factor for the indexes in \a i. Must be
	4703	/// 1, 2, 4, or 8.
	4704	/// \returns A 256-bit vector of [4 x i64] containing the gathered values.
1028	4705	#define _mm256_mask_i64gather_epi64(a, m, i, mask, s) \
1029	4706	((__m256i)__builtin_ia32_gatherq_q256((__v4di)(__m256i)(a), \
1030	4707	(long long const *)(m), \
1031	4708	(__v4di)(__m256i)(i), \
1032	4709	(__v4di)(__m256i)(mask), (s)))
1033	4710
	4711	/// Gathers two 64-bit floating-point values from memory \a m using scaled
	4712	/// indexes from the 128-bit vector of [4 x i32] in \a i.
	4713	///
	4714	/// \code{.operation}
	4715	/// FOR element := 0 to 1
	4716	/// j := element*64
	4717	/// k := element*32
	4718	/// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
	4719	/// ENDFOR
	4720	/// \endcode
	4721	///
	4722	/// \headerfile <immintrin.h>
	4723	///
	4724	/// \code
	4725	/// __m128d _mm_i32gather_pd(const double *m, __m128i i, const int s);
	4726	/// \endcode
	4727	///
	4728	/// This intrinsic corresponds to the \c VGATHERDPD instruction.
	4729	///
	4730	/// \param m
	4731	/// A pointer to the memory used for loading values.
	4732	/// \param i
	4733	/// A 128-bit vector of [4 x i32] containing signed indexes into \a m. Only
	4734	/// the first two elements are used.
	4735	/// \param s
	4736	/// A literal constant scale factor for the indexes in \a i. Must be
	4737	/// 1, 2, 4, or 8.
	4738	/// \returns A 128-bit vector of [2 x double] containing the gathered values.
1034	4739	#define _mm_i32gather_pd(m, i, s) \
1035	4740	((__m128d)__builtin_ia32_gatherd_pd((__v2df)_mm_undefined_pd(), \
1036	4741	(double const *)(m), \
...	...	@@ -1039,6 +4744,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
1039	4744	_mm_setzero_pd()), \
1040	4745	(s)))
1041	4746
	4747	/// Gathers four 64-bit floating-point values from memory \a m using scaled
	4748	/// indexes from the 128-bit vector of [4 x i32] in \a i.
	4749	///
	4750	/// \code{.operation}
	4751	/// FOR element := 0 to 3
	4752	/// j := element*64
	4753	/// k := element*32
	4754	/// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
	4755	/// ENDFOR
	4756	/// \endcode
	4757	///
	4758	/// \headerfile <immintrin.h>
	4759	///
	4760	/// \code
	4761	/// __m256d _mm256_i32gather_pd(const double *m, __m128i i, const int s);
	4762	/// \endcode
	4763	///
	4764	/// This intrinsic corresponds to the \c VGATHERDPD instruction.
	4765	///
	4766	/// \param m
	4767	/// A pointer to the memory used for loading values.
	4768	/// \param i
	4769	/// A 128-bit vector of [4 x i32] containing signed indexes into \a m.
	4770	/// \param s
	4771	/// A literal constant scale factor for the indexes in \a i. Must be
	4772	/// 1, 2, 4, or 8.
	4773	/// \returns A 256-bit vector of [4 x double] containing the gathered values.
1042	4774	#define _mm256_i32gather_pd(m, i, s) \
1043	4775	((__m256d)__builtin_ia32_gatherd_pd256((__v4df)_mm256_undefined_pd(), \
1044	4776	(double const *)(m), \
...	...	@@ -1048,6 +4780,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
1048	4780	_CMP_EQ_OQ), \
1049	4781	(s)))
1050	4782
	4783	/// Gathers two 64-bit floating-point values from memory \a m using scaled
	4784	/// indexes from the 128-bit vector of [2 x i64] in \a i.
	4785	///
	4786	/// \code{.operation}
	4787	/// FOR element := 0 to 1
	4788	/// j := element*64
	4789	/// k := element*64
	4790	/// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
	4791	/// ENDFOR
	4792	/// \endcode
	4793	///
	4794	/// \headerfile <immintrin.h>
	4795	///
	4796	/// \code
	4797	/// __m128d _mm_i64gather_pd(const double *m, __m128i i, const int s);
	4798	/// \endcode
	4799	///
	4800	/// This intrinsic corresponds to the \c VGATHERQPD instruction.
	4801	///
	4802	/// \param m
	4803	/// A pointer to the memory used for loading values.
	4804	/// \param i
	4805	/// A 128-bit vector of [2 x i64] containing signed indexes into \a m.
	4806	/// \param s
	4807	/// A literal constant scale factor for the indexes in \a i. Must be
	4808	/// 1, 2, 4, or 8.
	4809	/// \returns A 128-bit vector of [2 x double] containing the gathered values.
1051	4810	#define _mm_i64gather_pd(m, i, s) \
1052	4811	((__m128d)__builtin_ia32_gatherq_pd((__v2df)_mm_undefined_pd(), \
1053	4812	(double const *)(m), \
...	...	@@ -1056,6 +4815,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
1056	4815	_mm_setzero_pd()), \
1057	4816	(s)))
1058	4817
	4818	/// Gathers four 64-bit floating-point values from memory \a m using scaled
	4819	/// indexes from the 256-bit vector of [4 x i64] in \a i.
	4820	///
	4821	/// \code{.operation}
	4822	/// FOR element := 0 to 3
	4823	/// j := element*64
	4824	/// k := element*64
	4825	/// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
	4826	/// ENDFOR
	4827	/// \endcode
	4828	///
	4829	/// \headerfile <immintrin.h>
	4830	///
	4831	/// \code
	4832	/// __m256d _mm256_i64gather_pd(const double *m, __m256i i, const int s);
	4833	/// \endcode
	4834	///
	4835	/// This intrinsic corresponds to the \c VGATHERQPD instruction.
	4836	///
	4837	/// \param m
	4838	/// A pointer to the memory used for loading values.
	4839	/// \param i
	4840	/// A 256-bit vector of [4 x i64] containing signed indexes into \a m.
	4841	/// \param s
	4842	/// A literal constant scale factor for the indexes in \a i. Must be
	4843	/// 1, 2, 4, or 8.
	4844	/// \returns A 256-bit vector of [4 x double] containing the gathered values.
1059	4845	#define _mm256_i64gather_pd(m, i, s) \
1060	4846	((__m256d)__builtin_ia32_gatherq_pd256((__v4df)_mm256_undefined_pd(), \
1061	4847	(double const *)(m), \
...	...	@@ -1065,6 +4851,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
1065	4851	_CMP_EQ_OQ), \
1066	4852	(s)))
1067	4853
	4854	/// Gathers four 32-bit floating-point values from memory \a m using scaled
	4855	/// indexes from the 128-bit vector of [4 x i32] in \a i.
	4856	///
	4857	/// \code{.operation}
	4858	/// FOR element := 0 to 3
	4859	/// j := element*32
	4860	/// k := element*32
	4861	/// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
	4862	/// ENDFOR
	4863	/// \endcode
	4864	///
	4865	/// \headerfile <immintrin.h>
	4866	///
	4867	/// \code
	4868	/// __m128 _mm_i32gather_ps(const float *m, __m128i i, const int s);
	4869	/// \endcode
	4870	///
	4871	/// This intrinsic corresponds to the \c VGATHERDPS instruction.
	4872	///
	4873	/// \param m
	4874	/// A pointer to the memory used for loading values.
	4875	/// \param i
	4876	/// A 128-bit vector of [4 x i32] containing signed indexes into \a m.
	4877	/// \param s
	4878	/// A literal constant scale factor for the indexes in \a i. Must be
	4879	/// 1, 2, 4, or 8.
	4880	/// \returns A 128-bit vector of [4 x float] containing the gathered values.
1068	4881	#define _mm_i32gather_ps(m, i, s) \
1069	4882	((__m128)__builtin_ia32_gatherd_ps((__v4sf)_mm_undefined_ps(), \
1070	4883	(float const *)(m), \
...	...	@@ -1073,6 +4886,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
1073	4886	_mm_setzero_ps()), \
1074	4887	(s)))
1075	4888
	4889	/// Gathers eight 32-bit floating-point values from memory \a m using scaled
	4890	/// indexes from the 256-bit vector of [8 x i32] in \a i.
	4891	///
	4892	/// \code{.operation}
	4893	/// FOR element := 0 to 7
	4894	/// j := element*32
	4895	/// k := element*32
	4896	/// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
	4897	/// ENDFOR
	4898	/// \endcode
	4899	///
	4900	/// \headerfile <immintrin.h>
	4901	///
	4902	/// \code
	4903	/// __m256 _mm256_i32gather_ps(const float *m, __m256i i, const int s);
	4904	/// \endcode
	4905	///
	4906	/// This intrinsic corresponds to the \c VGATHERDPS instruction.
	4907	///
	4908	/// \param m
	4909	/// A pointer to the memory used for loading values.
	4910	/// \param i
	4911	/// A 256-bit vector of [8 x i32] containing signed indexes into \a m.
	4912	/// \param s
	4913	/// A literal constant scale factor for the indexes in \a i. Must be
	4914	/// 1, 2, 4, or 8.
	4915	/// \returns A 256-bit vector of [8 x float] containing the gathered values.
1076	4916	#define _mm256_i32gather_ps(m, i, s) \
1077	4917	((__m256)__builtin_ia32_gatherd_ps256((__v8sf)_mm256_undefined_ps(), \
1078	4918	(float const *)(m), \
...	...	@@ -1082,6 +4922,35 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
1082	4922	_CMP_EQ_OQ), \
1083	4923	(s)))
1084	4924
	4925	/// Gathers two 32-bit floating-point values from memory \a m using scaled
	4926	/// indexes from the 128-bit vector of [2 x i64] in \a i. The upper two
	4927	/// elements of the result are zeroed.
	4928	///
	4929	/// \code{.operation}
	4930	/// FOR element := 0 to 1
	4931	/// j := element*32
	4932	/// k := element*64
	4933	/// result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s)
	4934	/// ENDFOR
	4935	/// result[127:64] := 0
	4936	/// \endcode
	4937	///
	4938	/// \headerfile <immintrin.h>
	4939	///
	4940	/// \code
	4941	/// __m128 _mm_i64gather_ps(const float *m, __m128i i, const int s);
	4942	/// \endcode
	4943	///
	4944	/// This intrinsic corresponds to the \c VGATHERQPS instruction.
	4945	///
	4946	/// \param m
	4947	/// A pointer to the memory used for loading values.
	4948	/// \param i
	4949	/// A 128-bit vector of [2 x i64] containing signed indexes into \a m.
	4950	/// \param s
	4951	/// A literal constant scale factor for the indexes in \a i. Must be
	4952	/// 1, 2, 4, or 8.
	4953	/// \returns A 128-bit vector of [4 x float] containing the gathered values.
1085	4954	#define _mm_i64gather_ps(m, i, s) \
1086	4955	((__m128)__builtin_ia32_gatherq_ps((__v4sf)_mm_undefined_ps(), \
1087	4956	(float const *)(m), \
...	...	@@ -1090,6 +4959,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
1090	4959	_mm_setzero_ps()), \
1091	4960	(s)))
1092	4961
	4962	/// Gathers four 32-bit floating-point values from memory \a m using scaled
	4963	/// indexes from the 256-bit vector of [4 x i64] in \a i.
	4964	///
	4965	/// \code{.operation}
	4966	/// FOR element := 0 to 3
	4967	/// j := element*32
	4968	/// k := element*64
	4969	/// result[j+31:j] := Load32(m + SignExtend(i[k+64:k])*s)
	4970	/// ENDFOR
	4971	/// \endcode
	4972	///
	4973	/// \headerfile <immintrin.h>
	4974	///
	4975	/// \code
	4976	/// __m128 _mm256_i64gather_ps(const float *m, __m256i i, const int s);
	4977	/// \endcode
	4978	///
	4979	/// This intrinsic corresponds to the \c VGATHERQPS instruction.
	4980	///
	4981	/// \param m
	4982	/// A pointer to the memory used for loading values.
	4983	/// \param i
	4984	/// A 256-bit vector of [4 x i64] containing signed indexes into \a m.
	4985	/// \param s
	4986	/// A literal constant scale factor for the indexes in \a i. Must be
	4987	/// 1, 2, 4, or 8.
	4988	/// \returns A 128-bit vector of [4 x float] containing the gathered values.
1093	4989	#define _mm256_i64gather_ps(m, i, s) \
1094	4990	((__m128)__builtin_ia32_gatherq_ps256((__v4sf)_mm_undefined_ps(), \
1095	4991	(float const *)(m), \
...	...	@@ -1098,44 +4994,263 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
1098	4994	_mm_setzero_ps()), \
1099	4995	(s)))
1100	4996
	4997	/// Gathers four 32-bit floating-point values from memory \a m using scaled
	4998	/// indexes from the 128-bit vector of [4 x i32] in \a i.
	4999	///
	5000	/// \code{.operation}
	5001	/// FOR element := 0 to 3
	5002	/// j := element*32
	5003	/// k := element*32
	5004	/// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
	5005	/// ENDFOR
	5006	/// \endcode
	5007	///
	5008	/// \headerfile <immintrin.h>
	5009	///
	5010	/// \code
	5011	/// __m128i _mm_i32gather_epi32(const int *m, __m128i i, const int s);
	5012	/// \endcode
	5013	///
	5014	/// This intrinsic corresponds to the \c VPGATHERDD instruction.
	5015	///
	5016	/// \param m
	5017	/// A pointer to the memory used for loading values.
	5018	/// \param i
	5019	/// A 128-bit vector of [4 x i32] containing signed indexes into \a m.
	5020	/// \param s
	5021	/// A literal constant scale factor for the indexes in \a i. Must be
	5022	/// 1, 2, 4, or 8.
	5023	/// \returns A 128-bit vector of [4 x i32] containing the gathered values.
1101	5024	#define _mm_i32gather_epi32(m, i, s) \
1102	5025	((__m128i)__builtin_ia32_gatherd_d((__v4si)_mm_undefined_si128(), \
1103	5026	(int const *)(m), (__v4si)(__m128i)(i), \
1104	5027	(__v4si)_mm_set1_epi32(-1), (s)))
1105	5028
	5029	/// Gathers eight 32-bit floating-point values from memory \a m using scaled
	5030	/// indexes from the 256-bit vector of [8 x i32] in \a i.
	5031	///
	5032	/// \code{.operation}
	5033	/// FOR element := 0 to 7
	5034	/// j := element*32
	5035	/// k := element*32
	5036	/// result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
	5037	/// ENDFOR
	5038	/// \endcode
	5039	///
	5040	/// \headerfile <immintrin.h>
	5041	///
	5042	/// \code
	5043	/// __m256i _mm256_i32gather_epi32(const int *m, __m256i i, const int s);
	5044	/// \endcode
	5045	///
	5046	/// This intrinsic corresponds to the \c VPGATHERDD instruction.
	5047	///
	5048	/// \param m
	5049	/// A pointer to the memory used for loading values.
	5050	/// \param i
	5051	/// A 256-bit vector of [8 x i32] containing signed indexes into \a m.
	5052	/// \param s
	5053	/// A literal constant scale factor for the indexes in \a i. Must be
	5054	/// 1, 2, 4, or 8.
	5055	/// \returns A 256-bit vector of [8 x i32] containing the gathered values.
1106	5056	#define _mm256_i32gather_epi32(m, i, s) \
1107	5057	((__m256i)__builtin_ia32_gatherd_d256((__v8si)_mm256_undefined_si256(), \
1108	5058	(int const *)(m), (__v8si)(__m256i)(i), \
1109	5059	(__v8si)_mm256_set1_epi32(-1), (s)))
1110	5060
	5061	/// Gathers two 32-bit integer values from memory \a m using scaled indexes
	5062	/// from the 128-bit vector of [2 x i64] in \a i. The upper two elements
	5063	/// of the result are zeroed.
	5064	///
	5065	/// \code{.operation}
	5066	/// FOR element := 0 to 1
	5067	/// j := element*32
	5068	/// k := element*64
	5069	/// result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s)
	5070	/// ENDFOR
	5071	/// result[127:64] := 0
	5072	/// \endcode
	5073	///
	5074	/// \headerfile <immintrin.h>
	5075	///
	5076	/// \code
	5077	/// __m128i _mm_i64gather_epi32(const int *m, __m128i i, const int s);
	5078	/// \endcode
	5079	///
	5080	/// This intrinsic corresponds to the \c VPGATHERQD instruction.
	5081	///
	5082	/// \param m
	5083	/// A pointer to the memory used for loading values.
	5084	/// \param i
	5085	/// A 128-bit vector of [2 x i64] containing signed indexes into \a m.
	5086	/// \param s
	5087	/// A literal constant scale factor for the indexes in \a i. Must be
	5088	/// 1, 2, 4, or 8.
	5089	/// \returns A 128-bit vector of [4 x i32] containing the gathered values.
1111	5090	#define _mm_i64gather_epi32(m, i, s) \
1112	5091	((__m128i)__builtin_ia32_gatherq_d((__v4si)_mm_undefined_si128(), \
1113	5092	(int const *)(m), (__v2di)(__m128i)(i), \
1114	5093	(__v4si)_mm_set1_epi32(-1), (s)))
1115	5094
	5095	/// Gathers four 32-bit integer values from memory \a m using scaled indexes
	5096	/// from the 256-bit vector of [4 x i64] in \a i.
	5097	///
	5098	/// \code{.operation}
	5099	/// FOR element := 0 to 3
	5100	/// j := element*32
	5101	/// k := element*64
	5102	/// result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s)
	5103	/// ENDFOR
	5104	/// \endcode
	5105	///
	5106	/// \headerfile <immintrin.h>
	5107	///
	5108	/// \code
	5109	/// __m128i _mm256_i64gather_epi32(const int *m, __m256i i, const int s);
	5110	/// \endcode
	5111	///
	5112	/// This intrinsic corresponds to the \c VPGATHERQD instruction.
	5113	///
	5114	/// \param m
	5115	/// A pointer to the memory used for loading values.
	5116	/// \param i
	5117	/// A 256-bit vector of [4 x i64] containing signed indexes into \a m.
	5118	/// \param s
	5119	/// A literal constant scale factor for the indexes in \a i. Must be
	5120	/// 1, 2, 4, or 8.
	5121	/// \returns A 128-bit vector of [4 x i32] containing the gathered values.
1116	5122	#define _mm256_i64gather_epi32(m, i, s) \
1117	5123	((__m128i)__builtin_ia32_gatherq_d256((__v4si)_mm_undefined_si128(), \
1118	5124	(int const *)(m), (__v4di)(__m256i)(i), \
1119	5125	(__v4si)_mm_set1_epi32(-1), (s)))
1120	5126
	5127	/// Gathers two 64-bit integer values from memory \a m using scaled indexes
	5128	/// from the 128-bit vector of [4 x i32] in \a i.
	5129	///
	5130	/// \code{.operation}
	5131	/// FOR element := 0 to 1
	5132	/// j := element*64
	5133	/// k := element*32
	5134	/// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
	5135	/// ENDFOR
	5136	/// \endcode
	5137	///
	5138	/// \headerfile <immintrin.h>
	5139	///
	5140	/// \code
	5141	/// __m128i _mm_i32gather_epi64(const long long *m, __m128i i, const int s);
	5142	/// \endcode
	5143	///
	5144	/// This intrinsic corresponds to the \c VPGATHERDQ instruction.
	5145	///
	5146	/// \param m
	5147	/// A pointer to the memory used for loading values.
	5148	/// \param i
	5149	/// A 128-bit vector of [4 x i32] containing signed indexes into \a m. Only
	5150	/// the first two elements are used.
	5151	/// \param s
	5152	/// A literal constant scale factor for the indexes in \a i. Must be
	5153	/// 1, 2, 4, or 8.
	5154	/// \returns A 128-bit vector of [2 x i64] containing the gathered values.
1121	5155	#define _mm_i32gather_epi64(m, i, s) \
1122	5156	((__m128i)__builtin_ia32_gatherd_q((__v2di)_mm_undefined_si128(), \
1123	5157	(long long const *)(m), \
1124	5158	(__v4si)(__m128i)(i), \
1125	5159	(__v2di)_mm_set1_epi64x(-1), (s)))
1126	5160
	5161	/// Gathers four 64-bit integer values from memory \a m using scaled indexes
	5162	/// from the 128-bit vector of [4 x i32] in \a i.
	5163	///
	5164	/// \code{.operation}
	5165	/// FOR element := 0 to 3
	5166	/// j := element*64
	5167	/// k := element*32
	5168	/// result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
	5169	/// ENDFOR
	5170	/// \endcode
	5171	///
	5172	/// \headerfile <immintrin.h>
	5173	///
	5174	/// \code
	5175	/// __m256i _mm256_i32gather_epi64(const long long *m, __m128i i, const int s);
	5176	/// \endcode
	5177	///
	5178	/// This intrinsic corresponds to the \c VPGATHERDQ instruction.
	5179	///
	5180	/// \param m
	5181	/// A pointer to the memory used for loading values.
	5182	/// \param i
	5183	/// A 128-bit vector of [4 x i32] containing signed indexes into \a m.
	5184	/// \param s
	5185	/// A literal constant scale factor for the indexes in \a i. Must be
	5186	/// 1, 2, 4, or 8.
	5187	/// \returns A 256-bit vector of [4 x i64] containing the gathered values.
1127	5188	#define _mm256_i32gather_epi64(m, i, s) \
1128	5189	((__m256i)__builtin_ia32_gatherd_q256((__v4di)_mm256_undefined_si256(), \
1129	5190	(long long const *)(m), \
1130	5191	(__v4si)(__m128i)(i), \
1131	5192	(__v4di)_mm256_set1_epi64x(-1), (s)))
1132	5193
	5194	/// Gathers two 64-bit integer values from memory \a m using scaled indexes
	5195	/// from the 128-bit vector of [2 x i64] in \a i.
	5196	///
	5197	/// \code{.operation}
	5198	/// FOR element := 0 to 1
	5199	/// j := element*64
	5200	/// k := element*64
	5201	/// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
	5202	/// ENDFOR
	5203	/// \endcode
	5204	///
	5205	/// \headerfile <immintrin.h>
	5206	///
	5207	/// \code
	5208	/// __m128i _mm_i64gather_epi64(const long long *m, __m128i i, const int s);
	5209	/// \endcode
	5210	///
	5211	/// This intrinsic corresponds to the \c VPGATHERQQ instruction.
	5212	///
	5213	/// \param m
	5214	/// A pointer to the memory used for loading values.
	5215	/// \param i
	5216	/// A 128-bit vector of [2 x i64] containing signed indexes into \a m.
	5217	/// \param s
	5218	/// A literal constant scale factor for the indexes in \a i. Must be
	5219	/// 1, 2, 4, or 8.
	5220	/// \returns A 128-bit vector of [2 x i64] containing the gathered values.
1133	5221	#define _mm_i64gather_epi64(m, i, s) \
1134	5222	((__m128i)__builtin_ia32_gatherq_q((__v2di)_mm_undefined_si128(), \
1135	5223	(long long const *)(m), \
1136	5224	(__v2di)(__m128i)(i), \
1137	5225	(__v2di)_mm_set1_epi64x(-1), (s)))
1138	5226
	5227	/// Gathers four 64-bit integer values from memory \a m using scaled indexes
	5228	/// from the 256-bit vector of [4 x i64] in \a i.
	5229	///
	5230	/// \code{.operation}
	5231	/// FOR element := 0 to 3
	5232	/// j := element*64
	5233	/// k := element*64
	5234	/// result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
	5235	/// ENDFOR
	5236	/// \endcode
	5237	///
	5238	/// \headerfile <immintrin.h>
	5239	///
	5240	/// \code
	5241	/// __m256i _mm256_i64gather_epi64(const long long *m, __m256i i, const int s);
	5242	/// \endcode
	5243	///
	5244	/// This intrinsic corresponds to the \c VPGATHERQQ instruction.
	5245	///
	5246	/// \param m
	5247	/// A pointer to the memory used for loading values.
	5248	/// \param i
	5249	/// A 256-bit vector of [4 x i64] containing signed indexes into \a m.
	5250	/// \param s
	5251	/// A literal constant scale factor for the indexes in \a i. Must be
	5252	/// 1, 2, 4, or 8.
	5253	/// \returns A 256-bit vector of [4 x i64] containing the gathered values.
1139	5254	#define _mm256_i64gather_epi64(m, i, s) \
1140	5255	((__m256i)__builtin_ia32_gatherq_q256((__v4di)_mm256_undefined_si256(), \
1141	5256	(long long const *)(m), \

lib/include/avx512fintrin.h+17-7

...	...	@@ -397,14 +397,15 @@ _mm512_broadcastsd_pd(__m128d __A)
397	397	static __inline __m512d __DEFAULT_FN_ATTRS512
398	398	_mm512_castpd256_pd512(__m256d __a)
399	399	{
400		return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, -1, -1, -1, -1);
	400	return __builtin_shufflevector(__a, __builtin_nondeterministic_value(__a), 0,
	401	1, 2, 3, 4, 5, 6, 7);
401	402	}
402	403
403	404	static __inline __m512 __DEFAULT_FN_ATTRS512
404	405	_mm512_castps256_ps512(__m256 __a)
405	406	{
406		return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7,
407		-1, -1, -1, -1, -1, -1, -1, -1);
	407	return __builtin_shufflevector(__a, __builtin_nondeterministic_value(__a), 0,
	408	1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
408	409	}
409	410
410	411	static __inline __m128d __DEFAULT_FN_ATTRS512
...	...	@@ -446,7 +447,10 @@ _mm512_castpd_si512 (__m512d __A)
446	447	static __inline__ __m512d __DEFAULT_FN_ATTRS512
447	448	_mm512_castpd128_pd512 (__m128d __A)
448	449	{
449		return __builtin_shufflevector( __A, __A, 0, 1, -1, -1, -1, -1, -1, -1);
	450	__m256d __B = __builtin_nondeterministic_value(__B);
	451	return __builtin_shufflevector(
	452	__builtin_shufflevector(__A, __builtin_nondeterministic_value(__A), 0, 1, 2, 3),
	453	__B, 0, 1, 2, 3, 4, 5, 6, 7);
450	454	}
451	455
452	456	static __inline __m512d __DEFAULT_FN_ATTRS512
...	...	@@ -464,19 +468,25 @@ _mm512_castps_si512 (__m512 __A)
464	468	static __inline__ __m512 __DEFAULT_FN_ATTRS512
465	469	_mm512_castps128_ps512 (__m128 __A)
466	470	{
467		return __builtin_shufflevector( __A, __A, 0, 1, 2, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
	471	__m256 __B = __builtin_nondeterministic_value(__B);
	472	return __builtin_shufflevector(
	473	__builtin_shufflevector(__A, __builtin_nondeterministic_value(__A), 0, 1, 2, 3, 4, 5, 6, 7),
	474	__B, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
468	475	}
469	476
470	477	static __inline__ __m512i __DEFAULT_FN_ATTRS512
471	478	_mm512_castsi128_si512 (__m128i __A)
472	479	{
473		return __builtin_shufflevector( __A, __A, 0, 1, -1, -1, -1, -1, -1, -1);
	480	__m256i __B = __builtin_nondeterministic_value(__B);
	481	return __builtin_shufflevector(
	482	__builtin_shufflevector(__A, __builtin_nondeterministic_value(__A), 0, 1, 2, 3),
	483	__B, 0, 1, 2, 3, 4, 5, 6, 7);
474	484	}
475	485
476	486	static __inline__ __m512i __DEFAULT_FN_ATTRS512
477	487	_mm512_castsi256_si512 (__m256i __A)
478	488	{
479		return __builtin_shufflevector( __A, __A, 0, 1, 2, 3, -1, -1, -1, -1);
	489	return __builtin_shufflevector( __A, __builtin_nondeterministic_value(__A), 0, 1, 2, 3, 4, 5, 6, 7);
480	490	}
481	491
482	492	static __inline __m512 __DEFAULT_FN_ATTRS512

lib/include/avx512fp16intrin.h+12-8

...	...	@@ -192,22 +192,26 @@ _mm512_castph512_ph256(__m512h __a) {
192	192
193	193	static __inline__ __m256h __DEFAULT_FN_ATTRS256
194	194	_mm256_castph128_ph256(__m128h __a) {
195		return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, -1, -1, -1,
196		-1, -1, -1, -1, -1);
	195	return __builtin_shufflevector(__a, __builtin_nondeterministic_value(__a),
	196	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
197	197	}
198	198
199	199	static __inline__ __m512h __DEFAULT_FN_ATTRS512
200	200	_mm512_castph128_ph512(__m128h __a) {
201		return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, -1, -1, -1,
202		-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
203		-1, -1, -1, -1, -1, -1, -1, -1, -1);
	201	__m256h __b = __builtin_nondeterministic_value(__b);
	202	return __builtin_shufflevector(
	203	__builtin_shufflevector(__a, __builtin_nondeterministic_value(__a),
	204	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15),
	205	__b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
	206	20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31);
204	207	}
205	208
206	209	static __inline__ __m512h __DEFAULT_FN_ATTRS512
207	210	_mm512_castph256_ph512(__m256h __a) {
208		return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
209		12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1,
210		-1, -1, -1, -1, -1, -1, -1, -1);
	211	return __builtin_shufflevector(__a, __builtin_nondeterministic_value(__a), 0,
	212	1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
	213	15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
	214	27, 28, 29, 30, 31);
211	215	}
212	216
213	217	/// Constructs a 256-bit floating-point vector of [16 x half] from a

lib/include/avxintrin.h+20-7

...	...	@@ -3017,8 +3017,11 @@ _mm256_zeroupper(void)
3017	3017	static __inline __m128 __DEFAULT_FN_ATTRS128
3018	3018	_mm_broadcast_ss(float const *__a)
3019	3019	{
3020		float __f = *__a;
3021		return __extension__ (__m128)(__v4sf){ __f, __f, __f, __f };
	3020	struct __mm_broadcast_ss_struct {
	3021	float __f;
	3022	} __attribute__((__packed__, __may_alias__));
	3023	float __f = ((const struct __mm_broadcast_ss_struct*)__a)->__f;
	3024	return __extension__ (__m128){ __f, __f, __f, __f };
3022	3025	}
3023	3026
3024	3027	/// Loads a scalar double-precision floating point value from the
...	...	@@ -3036,7 +3039,10 @@ _mm_broadcast_ss(float const *__a)
3036	3039	static __inline __m256d __DEFAULT_FN_ATTRS
3037	3040	_mm256_broadcast_sd(double const *__a)
3038	3041	{
3039		double __d = *__a;
	3042	struct __mm256_broadcast_sd_struct {
	3043	double __d;
	3044	} __attribute__((__packed__, __may_alias__));
	3045	double __d = ((const struct __mm256_broadcast_sd_struct*)__a)->__d;
3040	3046	return __extension__ (__m256d)(__v4df){ __d, __d, __d, __d };
3041	3047	}
3042	3048
...	...	@@ -3055,7 +3061,10 @@ _mm256_broadcast_sd(double const *__a)
3055	3061	static __inline __m256 __DEFAULT_FN_ATTRS
3056	3062	_mm256_broadcast_ss(float const *__a)
3057	3063	{
3058		float __f = *__a;
	3064	struct __mm256_broadcast_ss_struct {
	3065	float __f;
	3066	} __attribute__((__packed__, __may_alias__));
	3067	float __f = ((const struct __mm256_broadcast_ss_struct*)__a)->__f;
3059	3068	return __extension__ (__m256)(__v8sf){ __f, __f, __f, __f, __f, __f, __f, __f };
3060	3069	}
3061	3070
...	...	@@ -4499,7 +4508,8 @@ _mm256_castsi256_si128(__m256i __a)
4499	4508	static __inline __m256d __DEFAULT_FN_ATTRS
4500	4509	_mm256_castpd128_pd256(__m128d __a)
4501	4510	{
4502		return __builtin_shufflevector((__v2df)__a, (__v2df)__a, 0, 1, -1, -1);
	4511	return __builtin_shufflevector(
	4512	(__v2df)__a, (__v2df)__builtin_nondeterministic_value(__a), 0, 1, 2, 3);
4503	4513	}
4504	4514
4505	4515	/// Constructs a 256-bit floating-point vector of [8 x float] from a
...	...	@@ -4520,7 +4530,9 @@ _mm256_castpd128_pd256(__m128d __a)
4520	4530	static __inline __m256 __DEFAULT_FN_ATTRS
4521	4531	_mm256_castps128_ps256(__m128 __a)
4522	4532	{
4523		return __builtin_shufflevector((__v4sf)__a, (__v4sf)__a, 0, 1, 2, 3, -1, -1, -1, -1);
	4533	return __builtin_shufflevector((__v4sf)__a,
	4534	(__v4sf)__builtin_nondeterministic_value(__a),
	4535	0, 1, 2, 3, 4, 5, 6, 7);
4524	4536	}
4525	4537
4526	4538	/// Constructs a 256-bit integer vector from a 128-bit integer vector.
...	...	@@ -4539,7 +4551,8 @@ _mm256_castps128_ps256(__m128 __a)
4539	4551	static __inline __m256i __DEFAULT_FN_ATTRS
4540	4552	_mm256_castsi128_si256(__m128i __a)
4541	4553	{
4542		return __builtin_shufflevector((__v2di)__a, (__v2di)__a, 0, 1, -1, -1);
	4554	return __builtin_shufflevector(
	4555	(__v2di)__a, (__v2di)__builtin_nondeterministic_value(__a), 0, 1, 2, 3);
4543	4556	}
4544	4557
4545	4558	/// Constructs a 256-bit floating-point vector of [4 x double] from a

lib/include/avxvnniint16intrin.h created+473

...	...	@@ -0,0 +1,473 @@
	1	/*===----------- avxvnniint16intrin.h - AVXVNNIINT16 intrinsics-------------===
	2	*
	3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	* See https://llvm.org/LICENSE.txt for license information.
	5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	*
	7	*===-----------------------------------------------------------------------===
	8	*/
	9
	10	#ifndef __IMMINTRIN_H
	11	#error \
	12	"Never use <avxvnniint16intrin.h> directly; include <immintrin.h> instead."
	13	#endif // __IMMINTRIN_H
	14
	15	#ifndef __AVXVNNIINT16INTRIN_H
	16	#define __AVXVNNIINT16INTRIN_H
	17
	18	/* Define the default attributes for the functions in this file. */
	19	#define __DEFAULT_FN_ATTRS128 \
	20	__attribute__((__always_inline__, __nodebug__, __target__("avxvnniint16"), \
	21	__min_vector_width__(128)))
	22	#define __DEFAULT_FN_ATTRS256 \
	23	__attribute__((__always_inline__, __nodebug__, __target__("avxvnniint16"), \
	24	__min_vector_width__(256)))
	25
	26	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
	27	/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
	28	/// signed 16-bit results. Sum these 2 results with the corresponding
	29	/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
	30	///
	31	/// \headerfile <immintrin.h>
	32	///
	33	/// \code
	34	/// __m128i _mm_dpwsud_epi32(__m128i __W, __m128i __A, __m128i __B)
	35	/// \endcode
	36	///
	37	/// This intrinsic corresponds to the \c VPDPWSUD instruction.
	38	///
	39	/// \param __W
	40	/// A 128-bit vector of [4 x int].
	41	/// \param __A
	42	/// A 128-bit vector of [8 x short].
	43	/// \param __B
	44	/// A 128-bit vector of [8 x unsigned short].
	45	/// \returns
	46	/// A 128-bit vector of [4 x int].
	47	///
	48	/// \code{.operation}
	49	/// FOR j := 0 to 3
	50	/// tmp1.dword := SignExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
	51	/// tmp2.dword := SignExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	52	/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2
	53	/// ENDFOR
	54	/// dst[MAX:128] := 0
	55	/// \endcode
	56	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsud_epi32(__m128i __W,
	57	__m128i __A,
	58	__m128i __B) {
	59	return (__m128i)__builtin_ia32_vpdpwsud128((__v4si)__W, (__v4si)__A,
	60	(__v4si)__B);
	61	}
	62
	63	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
	64	/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
	65	/// signed 16-bit results. Sum these 2 results with the corresponding
	66	/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
	67	///
	68	/// \headerfile <immintrin.h>
	69	///
	70	/// \code
	71	/// __m256i _mm256_dpwsud_epi32(__m256i __W, __m256i __A, __m256i __B)
	72	/// \endcode
	73	///
	74	/// This intrinsic corresponds to the \c VPDPWSUD instruction.
	75	///
	76	/// \param __W
	77	/// A 256-bit vector of [8 x int].
	78	/// \param __A
	79	/// A 256-bit vector of [16 x short].
	80	/// \param __B
	81	/// A 256-bit vector of [16 x unsigned short].
	82	/// \returns
	83	/// A 256-bit vector of [8 x int].
	84	///
	85	/// \code{.operation}
	86	/// FOR j := 0 to 7
	87	/// tmp1.dword := SignExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
	88	/// tmp2.dword := SignExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	89	/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2
	90	/// ENDFOR
	91	/// dst[MAX:256] := 0
	92	/// \endcode
	93	static __inline__ __m256i __DEFAULT_FN_ATTRS256
	94	_mm256_dpwsud_epi32(__m256i __W, __m256i __A, __m256i __B) {
	95	return (__m256i)__builtin_ia32_vpdpwsud256((__v8si)__W, (__v8si)__A,
	96	(__v8si)__B);
	97	}
	98
	99	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
	100	/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
	101	/// signed 16-bit results. Sum these 2 results with the corresponding
	102	/// 32-bit integer in \a __W with signed saturation, and store the packed
	103	/// 32-bit results in \a dst.
	104	///
	105	/// \headerfile <immintrin.h>
	106	///
	107	/// \code
	108	/// __m128i _mm_dpwsuds_epi32(__m128i __W, __m128i __A, __m128i __B)
	109	/// \endcode
	110	///
	111	/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
	112	///
	113	/// \param __W
	114	/// A 128-bit vector of [4 x int].
	115	/// \param __A
	116	/// A 128-bit vector of [8 x short].
	117	/// \param __B
	118	/// A 128-bit vector of [8 x unsigned short].
	119	/// \returns
	120	/// A 128-bit vector of [4 x int].
	121	///
	122	/// \code{.operation}
	123	/// FOR j := 0 to 3
	124	/// tmp1.dword := SignExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
	125	/// tmp2.dword := SignExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	126	/// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
	127	/// ENDFOR
	128	/// dst[MAX:128] := 0
	129	/// \endcode
	130	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsuds_epi32(__m128i __W,
	131	__m128i __A,
	132	__m128i __B) {
	133	return (__m128i)__builtin_ia32_vpdpwsuds128((__v4si)__W, (__v4si)__A,
	134	(__v4si)__B);
	135	}
	136
	137	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
	138	/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
	139	/// signed 16-bit results. Sum these 2 results with the corresponding
	140	/// 32-bit integer in \a __W with signed saturation, and store the packed
	141	/// 32-bit results in \a dst.
	142	///
	143	/// \headerfile <immintrin.h>
	144	///
	145	/// \code
	146	/// __m256i _mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B)
	147	/// \endcode
	148	///
	149	/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
	150	///
	151	/// \param __W
	152	/// A 256-bit vector of [8 x int].
	153	/// \param __A
	154	/// A 256-bit vector of [16 x short].
	155	/// \param __B
	156	/// A 256-bit vector of [16 x unsigned short].
	157	/// \returns
	158	/// A 256-bit vector of [8 x int].
	159	///
	160	/// \code{.operation}
	161	/// FOR j := 0 to 7
	162	/// tmp1.dword := SignExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
	163	/// tmp2.dword := SignExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	164	/// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
	165	/// ENDFOR
	166	/// dst[MAX:256] := 0
	167	/// \endcode
	168	static __inline__ __m256i __DEFAULT_FN_ATTRS256
	169	_mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B) {
	170	return (__m256i)__builtin_ia32_vpdpwsuds256((__v8si)__W, (__v8si)__A,
	171	(__v8si)__B);
	172	}
	173
	174	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
	175	/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
	176	/// signed 16-bit results. Sum these 2 results with the corresponding
	177	/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
	178	///
	179	/// \headerfile <immintrin.h>
	180	///
	181	/// \code
	182	/// __m128i _mm_dpbusd_epi32(__m128i __W, __m128i __A, __m128i __B)
	183	/// \endcode
	184	///
	185	/// This intrinsic corresponds to the \c VPDPWUSD instruction.
	186	///
	187	/// \param __W
	188	/// A 128-bit vector of [4 x int].
	189	/// \param __A
	190	/// A 128-bit vector of [8 x unsigned short].
	191	/// \param __B
	192	/// A 128-bit vector of [8 x short].
	193	/// \returns
	194	/// A 128-bit vector of [4 x int].
	195	///
	196	/// \code{.operation}
	197	/// FOR j := 0 to 3
	198	/// tmp1.dword := ZeroExtend32(__A.word[2j]) SignExtend32(__B.word[2*j])
	199	/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
	200	/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2
	201	/// ENDFOR
	202	/// dst[MAX:128] := 0
	203	/// \endcode
	204	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusd_epi32(__m128i __W,
	205	__m128i __A,
	206	__m128i __B) {
	207	return (__m128i)__builtin_ia32_vpdpwusd128((__v4si)__W, (__v4si)__A,
	208	(__v4si)__B);
	209	}
	210
	211	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
	212	/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
	213	/// signed 16-bit results. Sum these 2 results with the corresponding
	214	/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
	215	///
	216	/// \headerfile <immintrin.h>
	217	///
	218	/// \code
	219	/// __m256i _mm256_dpwusd_epi32(__m256i __W, __m256i __A, __m256i __B)
	220	/// \endcode
	221	///
	222	/// This intrinsic corresponds to the \c VPDPWUSD instruction.
	223	///
	224	/// \param __W
	225	/// A 256-bit vector of [8 x int].
	226	/// \param __A
	227	/// A 256-bit vector of [16 x unsigned short].
	228	/// \param __B
	229	/// A 256-bit vector of [16 x short].
	230	/// \returns
	231	/// A 256-bit vector of [8 x int].
	232	///
	233	/// \code{.operation}
	234	/// FOR j := 0 to 7
	235	/// tmp1.dword := ZeroExtend32(__A.word[2j]) SignExtend32(__B.word[2*j])
	236	/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
	237	/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2
	238	/// ENDFOR
	239	/// dst[MAX:256] := 0
	240	/// \endcode
	241	static __inline__ __m256i __DEFAULT_FN_ATTRS256
	242	_mm256_dpwusd_epi32(__m256i __W, __m256i __A, __m256i __B) {
	243	return (__m256i)__builtin_ia32_vpdpwusd256((__v8si)__W, (__v8si)__A,
	244	(__v8si)__B);
	245	}
	246
	247	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
	248	/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
	249	/// signed 16-bit results. Sum these 2 results with the corresponding
	250	/// 32-bit integer in \a __W with signed saturation, and store the packed
	251	/// 32-bit results in \a dst.
	252	///
	253	/// \headerfile <immintrin.h>
	254	///
	255	/// \code
	256	/// __m128i _mm_dpwusds_epi32(__m128i __W, __m128i __A, __m128i __B)
	257	/// \endcode
	258	///
	259	/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
	260	///
	261	/// \param __W
	262	/// A 128-bit vector of [4 x int].
	263	/// \param __A
	264	/// A 128-bit vector of [8 x unsigned short].
	265	/// \param __B
	266	/// A 128-bit vector of [8 x short].
	267	/// \returns
	268	/// A 128-bit vector of [4 x int].
	269	///
	270	/// \code{.operation}
	271	/// FOR j := 0 to 3
	272	/// tmp1.dword := ZeroExtend32(__A.word[2j]) SignExtend32(__B.word[2*j])
	273	/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
	274	/// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
	275	/// ENDFOR
	276	/// dst[MAX:128] := 0
	277	/// \endcode
	278	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusds_epi32(__m128i __W,
	279	__m128i __A,
	280	__m128i __B) {
	281	return (__m128i)__builtin_ia32_vpdpwusds128((__v4si)__W, (__v4si)__A,
	282	(__v4si)__B);
	283	}
	284
	285	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
	286	/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
	287	/// signed 16-bit results. Sum these 2 results with the corresponding
	288	/// 32-bit integer in \a __W with signed saturation, and store the packed
	289	/// 32-bit results in \a dst.
	290	///
	291	/// \headerfile <immintrin.h>
	292	///
	293	/// \code
	294	/// __m256i _mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B)
	295	/// \endcode
	296	///
	297	/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
	298	///
	299	/// \param __W
	300	/// A 256-bit vector of [8 x int].
	301	/// \param __A
	302	/// A 256-bit vector of [16 x unsigned short].
	303	/// \param __B
	304	/// A 256-bit vector of [16 x short].
	305	/// \returns
	306	/// A 256-bit vector of [8 x int].
	307	///
	308	/// \code{.operation}
	309	/// FOR j := 0 to 7
	310	/// tmp1.dword := ZeroExtend32(__A.word[2j]) SignExtend32(__B.word[2*j])
	311	/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
	312	/// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
	313	/// ENDFOR
	314	/// dst[MAX:256] := 0
	315	/// \endcode
	316	static __inline__ __m256i __DEFAULT_FN_ATTRS256
	317	_mm256_dpwusds_epi32(__m256i __W, __m256i __A, __m256i __B) {
	318	return (__m256i)__builtin_ia32_vpdpwusds256((__v8si)__W, (__v8si)__A,
	319	(__v8si)__B);
	320	}
	321
	322	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
	323	/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
	324	/// signed 16-bit results. Sum these 2 results with the corresponding
	325	/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
	326	///
	327	/// \headerfile <immintrin.h>
	328	///
	329	/// \code
	330	/// __m128i _mm_dpwuud_epi32(__m128i __W, __m128i __A, __m128i __B)
	331	/// \endcode
	332	///
	333	/// This intrinsic corresponds to the \c VPDPWUUD instruction.
	334	///
	335	/// \param __W
	336	/// A 128-bit vector of [4 x unsigned int].
	337	/// \param __A
	338	/// A 128-bit vector of [8 x unsigned short].
	339	/// \param __B
	340	/// A 128-bit vector of [8 x unsigned short].
	341	/// \returns
	342	/// A 128-bit vector of [4 x unsigned int].
	343	///
	344	/// \code{.operation}
	345	/// FOR j := 0 to 3
	346	/// tmp1.dword := ZeroExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
	347	/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	348	/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2
	349	/// ENDFOR
	350	/// dst[MAX:128] := 0
	351	/// \endcode
	352	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuud_epi32(__m128i __W,
	353	__m128i __A,
	354	__m128i __B) {
	355	return (__m128i)__builtin_ia32_vpdpwuud128((__v4si)__W, (__v4si)__A,
	356	(__v4si)__B);
	357	}
	358
	359	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
	360	/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
	361	/// signed 16-bit results. Sum these 2 results with the corresponding
	362	/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
	363	///
	364	/// \headerfile <immintrin.h>
	365	///
	366	/// \code
	367	/// __m256i _mm256_dpwuud_epi32(__m256i __W, __m256i __A, __m256i __B)
	368	/// \endcode
	369	///
	370	/// This intrinsic corresponds to the \c VPDPWUUD instruction.
	371	///
	372	/// \param __W
	373	/// A 256-bit vector of [8 x unsigned int].
	374	/// \param __A
	375	/// A 256-bit vector of [16 x unsigned short].
	376	/// \param __B
	377	/// A 256-bit vector of [16 x unsigned short].
	378	/// \returns
	379	/// A 256-bit vector of [8 x unsigned int].
	380	///
	381	/// \code{.operation}
	382	/// FOR j := 0 to 7
	383	/// tmp1.dword := ZeroExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
	384	/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	385	/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2
	386	/// ENDFOR
	387	/// dst[MAX:256] := 0
	388	/// \endcode
	389	static __inline__ __m256i __DEFAULT_FN_ATTRS256
	390	_mm256_dpwuud_epi32(__m256i __W, __m256i __A, __m256i __B) {
	391	return (__m256i)__builtin_ia32_vpdpwuud256((__v8si)__W, (__v8si)__A,
	392	(__v8si)__B);
	393	}
	394
	395	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
	396	/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
	397	/// signed 16-bit results. Sum these 2 results with the corresponding
	398	/// 32-bit integer in \a __W with signed saturation, and store the packed
	399	/// 32-bit results in \a dst.
	400	///
	401	/// \headerfile <immintrin.h>
	402	///
	403	/// \code
	404	/// __m128i _mm_dpwsuds_epi32(__m128i __W, __m128i __A, __m128i __B)
	405	/// \endcode
	406	///
	407	/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
	408	///
	409	/// \param __W
	410	/// A 128-bit vector of [4 x unsigned int].
	411	/// \param __A
	412	/// A 128-bit vector of [8 x unsigned short].
	413	/// \param __B
	414	/// A 128-bit vector of [8 x unsigned short].
	415	/// \returns
	416	/// A 128-bit vector of [4 x unsigned int].
	417	///
	418	/// \code{.operation}
	419	/// FOR j := 0 to 3
	420	/// tmp1.dword := ZeroExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
	421	/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	422	/// dst.dword[j] := UNSIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
	423	/// ENDFOR
	424	/// dst[MAX:128] := 0
	425	/// \endcode
	426	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuuds_epi32(__m128i __W,
	427	__m128i __A,
	428	__m128i __B) {
	429	return (__m128i)__builtin_ia32_vpdpwuuds128((__v4si)__W, (__v4si)__A,
	430	(__v4si)__B);
	431	}
	432
	433	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
	434	/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
	435	/// signed 16-bit results. Sum these 2 results with the corresponding
	436	/// 32-bit integer in \a __W with signed saturation, and store the packed
	437	/// 32-bit results in \a dst.
	438	///
	439	/// \headerfile <immintrin.h>
	440	///
	441	/// \code
	442	/// __m256i _mm256_dpwuuds_epi32(__m256i __W, __m256i __A, __m256i __B)
	443	/// \endcode
	444	///
	445	/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
	446	///
	447	/// \param __W
	448	/// A 256-bit vector of [8 x unsigned int].
	449	/// \param __A
	450	/// A 256-bit vector of [16 x unsigned short].
	451	/// \param __B
	452	/// A 256-bit vector of [16 x unsigned short].
	453	/// \returns
	454	/// A 256-bit vector of [8 x unsigned int].
	455	///
	456	/// \code{.operation}
	457	/// FOR j := 0 to 7
	458	/// tmp1.dword := ZeroExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
	459	/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	460	/// dst.dword[j] := UNSIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
	461	/// ENDFOR
	462	/// dst[MAX:256] := 0
	463	/// \endcode
	464	static __inline__ __m256i __DEFAULT_FN_ATTRS256
	465	_mm256_dpwuuds_epi32(__m256i __W, __m256i __A, __m256i __B) {
	466	return (__m256i)__builtin_ia32_vpdpwuuds256((__v8si)__W, (__v8si)__A,
	467	(__v8si)__B);
	468	}
	469
	470	#undef __DEFAULT_FN_ATTRS128
	471	#undef __DEFAULT_FN_ATTRS256
	472
	473	#endif // __AVXVNNIINT16INTRIN_H

lib/include/bmi2intrin.h+187-13

...	...	@@ -7,8 +7,8 @@
7	7	*===-----------------------------------------------------------------------===
8	8	*/
9	9
10		#if !defined __X86INTRIN_H && !defined __IMMINTRIN_H
11		#error "Never use <bmi2intrin.h> directly; include <x86intrin.h> instead."
	10	#ifndef __IMMINTRIN_H
	11	#error "Never use <bmi2intrin.h> directly; include <immintrin.h> instead."
12	12	#endif
13	13
14	14	#ifndef __BMI2INTRIN_H
...	...	@@ -17,44 +17,228 @@
17	17	/* Define the default attributes for the functions in this file. */
18	18	#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("bmi2")))
19	19
	20	/// Copies the unsigned 32-bit integer \a __X and zeroes the upper bits
	21	/// starting at bit number \a __Y.
	22	///
	23	/// \code{.operation}
	24	/// i := __Y[7:0]
	25	/// result := __X
	26	/// IF i < 32
	27	/// result[31:i] := 0
	28	/// FI
	29	/// \endcode
	30	///
	31	/// \headerfile <immintrin.h>
	32	///
	33	/// This intrinsic corresponds to the \c BZHI instruction.
	34	///
	35	/// \param __X
	36	/// The 32-bit source value to copy.
	37	/// \param __Y
	38	/// The lower 8 bits specify the bit number of the lowest bit to zero.
	39	/// \returns The partially zeroed 32-bit value.
20	40	static __inline__ unsigned int __DEFAULT_FN_ATTRS
21	41	_bzhi_u32(unsigned int __X, unsigned int __Y)
22	42	{
23	43	return __builtin_ia32_bzhi_si(__X, __Y);
24	44	}
25	45
	46	/// Deposit (scatter) low-order bits from the unsigned 32-bit integer \a __X
	47	/// into the 32-bit result, according to the mask in the unsigned 32-bit
	48	/// integer \a __Y. All other bits of the result are zero.
	49	///
	50	/// \code{.operation}
	51	/// i := 0
	52	/// result := 0
	53	/// FOR m := 0 TO 31
	54	/// IF __Y[m] == 1
	55	/// result[m] := __X[i]
	56	/// i := i + 1
	57	/// ENDIF
	58	/// ENDFOR
	59	/// \endcode
	60	///
	61	/// \headerfile <immintrin.h>
	62	///
	63	/// This intrinsic corresponds to the \c PDEP instruction.
	64	///
	65	/// \param __X
	66	/// The 32-bit source value to copy.
	67	/// \param __Y
	68	/// The 32-bit mask specifying where to deposit source bits.
	69	/// \returns The 32-bit result.
26	70	static __inline__ unsigned int __DEFAULT_FN_ATTRS
27	71	_pdep_u32(unsigned int __X, unsigned int __Y)
28	72	{
29	73	return __builtin_ia32_pdep_si(__X, __Y);
30	74	}
31	75
	76	/// Extract (gather) bits from the unsigned 32-bit integer \a __X into the
	77	/// low-order bits of the 32-bit result, according to the mask in the
	78	/// unsigned 32-bit integer \a __Y. All other bits of the result are zero.
	79	///
	80	/// \code{.operation}
	81	/// i := 0
	82	/// result := 0
	83	/// FOR m := 0 TO 31
	84	/// IF __Y[m] == 1
	85	/// result[i] := __X[m]
	86	/// i := i + 1
	87	/// ENDIF
	88	/// ENDFOR
	89	/// \endcode
	90	///
	91	/// \headerfile <immintrin.h>
	92	///
	93	/// This intrinsic corresponds to the \c PEXT instruction.
	94	///
	95	/// \param __X
	96	/// The 32-bit source value to copy.
	97	/// \param __Y
	98	/// The 32-bit mask specifying which source bits to extract.
	99	/// \returns The 32-bit result.
32	100	static __inline__ unsigned int __DEFAULT_FN_ATTRS
33	101	_pext_u32(unsigned int __X, unsigned int __Y)
34	102	{
35	103	return __builtin_ia32_pext_si(__X, __Y);
36	104	}
37	105
	106	/// Multiplies the unsigned 32-bit integers \a __X and \a __Y to form a
	107	/// 64-bit product. Stores the upper 32 bits of the product in the
	108	/// memory at \a __P and returns the lower 32 bits.
	109	///
	110	/// \code{.operation}
	111	/// Store32(__P, (__X * __Y)[63:32])
	112	/// result := (__X * __Y)[31:0]
	113	/// \endcode
	114	///
	115	/// \headerfile <immintrin.h>
	116	///
	117	/// This intrinsic corresponds to the \c MULX instruction.
	118	///
	119	/// \param __X
	120	/// An unsigned 32-bit multiplicand.
	121	/// \param __Y
	122	/// An unsigned 32-bit multiplicand.
	123	/// \param __P
	124	/// A pointer to memory for storing the upper half of the product.
	125	/// \returns The lower half of the product.
	126	static __inline__ unsigned int __DEFAULT_FN_ATTRS
	127	_mulx_u32(unsigned int __X, unsigned int __Y, unsigned int *__P)
	128	{
	129	unsigned long long __res = (unsigned long long) __X * __Y;
	130	*__P = (unsigned int)(__res >> 32);
	131	return (unsigned int)__res;
	132	}
	133
38	134	#ifdef __x86_64__
39	135
	136	/// Copies the unsigned 64-bit integer \a __X and zeroes the upper bits
	137	/// starting at bit number \a __Y.
	138	///
	139	/// \code{.operation}
	140	/// i := __Y[7:0]
	141	/// result := __X
	142	/// IF i < 64
	143	/// result[63:i] := 0
	144	/// FI
	145	/// \endcode
	146	///
	147	/// \headerfile <immintrin.h>
	148	///
	149	/// This intrinsic corresponds to the \c BZHI instruction.
	150	///
	151	/// \param __X
	152	/// The 64-bit source value to copy.
	153	/// \param __Y
	154	/// The lower 8 bits specify the bit number of the lowest bit to zero.
	155	/// \returns The partially zeroed 64-bit value.
40	156	static __inline__ unsigned long long __DEFAULT_FN_ATTRS
41	157	_bzhi_u64(unsigned long long __X, unsigned long long __Y)
42	158	{
43	159	return __builtin_ia32_bzhi_di(__X, __Y);
44	160	}
45	161
	162	/// Deposit (scatter) low-order bits from the unsigned 64-bit integer \a __X
	163	/// into the 64-bit result, according to the mask in the unsigned 64-bit
	164	/// integer \a __Y. All other bits of the result are zero.
	165	///
	166	/// \code{.operation}
	167	/// i := 0
	168	/// result := 0
	169	/// FOR m := 0 TO 63
	170	/// IF __Y[m] == 1
	171	/// result[m] := __X[i]
	172	/// i := i + 1
	173	/// ENDIF
	174	/// ENDFOR
	175	/// \endcode
	176	///
	177	/// \headerfile <immintrin.h>
	178	///
	179	/// This intrinsic corresponds to the \c PDEP instruction.
	180	///
	181	/// \param __X
	182	/// The 64-bit source value to copy.
	183	/// \param __Y
	184	/// The 64-bit mask specifying where to deposit source bits.
	185	/// \returns The 64-bit result.
46	186	static __inline__ unsigned long long __DEFAULT_FN_ATTRS
47	187	_pdep_u64(unsigned long long __X, unsigned long long __Y)
48	188	{
49	189	return __builtin_ia32_pdep_di(__X, __Y);
50	190	}
51	191
	192	/// Extract (gather) bits from the unsigned 64-bit integer \a __X into the
	193	/// low-order bits of the 64-bit result, according to the mask in the
	194	/// unsigned 64-bit integer \a __Y. All other bits of the result are zero.
	195	///
	196	/// \code{.operation}
	197	/// i := 0
	198	/// result := 0
	199	/// FOR m := 0 TO 63
	200	/// IF __Y[m] == 1
	201	/// result[i] := __X[m]
	202	/// i := i + 1
	203	/// ENDIF
	204	/// ENDFOR
	205	/// \endcode
	206	///
	207	/// \headerfile <immintrin.h>
	208	///
	209	/// This intrinsic corresponds to the \c PEXT instruction.
	210	///
	211	/// \param __X
	212	/// The 64-bit source value to copy.
	213	/// \param __Y
	214	/// The 64-bit mask specifying which source bits to extract.
	215	/// \returns The 64-bit result.
52	216	static __inline__ unsigned long long __DEFAULT_FN_ATTRS
53	217	_pext_u64(unsigned long long __X, unsigned long long __Y)
54	218	{
55	219	return __builtin_ia32_pext_di(__X, __Y);
56	220	}
57	221
	222	/// Multiplies the unsigned 64-bit integers \a __X and \a __Y to form a
	223	/// 128-bit product. Stores the upper 64 bits of the product to the
	224	/// memory addressed by \a __P and returns the lower 64 bits.
	225	///
	226	/// \code{.operation}
	227	/// Store64(__P, (__X * __Y)[127:64])
	228	/// result := (__X * __Y)[63:0]
	229	/// \endcode
	230	///
	231	/// \headerfile <immintrin.h>
	232	///
	233	/// This intrinsic corresponds to the \c MULX instruction.
	234	///
	235	/// \param __X
	236	/// An unsigned 64-bit multiplicand.
	237	/// \param __Y
	238	/// An unsigned 64-bit multiplicand.
	239	/// \param __P
	240	/// A pointer to memory for storing the upper half of the product.
	241	/// \returns The lower half of the product.
58	242	static __inline__ unsigned long long __DEFAULT_FN_ATTRS
59	243	_mulx_u64 (unsigned long long __X, unsigned long long __Y,
60	244	unsigned long long *__P)
...	...	@@ -64,17 +248,7 @@ _mulx_u64 (unsigned long long __X, unsigned long long __Y,
64	248	return (unsigned long long) __res;
65	249	}
66	250
67		#else /* !__x86_64__ */
68
69		static __inline__ unsigned int __DEFAULT_FN_ATTRS
70		_mulx_u32 (unsigned int __X, unsigned int __Y, unsigned int *__P)
71		{
72		unsigned long long __res = (unsigned long long) __X * __Y;
73		*__P = (unsigned int) (__res >> 32);
74		return (unsigned int) __res;
75		}
76
77		#endif /* !__x86_64__ */
	251	#endif /* __x86_64__ */
78	252
79	253	#undef __DEFAULT_FN_ATTRS
80	254

lib/include/clflushoptintrin.h+9

...	...	@@ -17,6 +17,15 @@
17	17	/* Define the default attributes for the functions in this file. */
18	18	#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("clflushopt")))
19	19
	20	/// Invalidates all levels of the cache hierarchy and flushes modified data to
	21	/// memory for the cache line specified by the address \a __m.
	22	///
	23	/// \headerfile <immintrin.h>
	24	///
	25	/// This intrinsic corresponds to the \c CLFLUSHOPT instruction.
	26	///
	27	/// \param __m
	28	/// An address within the cache line to flush and invalidate.
20	29	static __inline__ void __DEFAULT_FN_ATTRS
21	30	_mm_clflushopt(void const * __m) {
22	31	__builtin_ia32_clflushopt(__m);

lib/include/clzerointrin.h+7-5

...	...	@@ -6,7 +6,7 @@
6	6	*
7	7	*===-----------------------------------------------------------------------===
8	8	*/
9		#if !defined __X86INTRIN_H && !defined __IMMINTRIN_H
	9	#ifndef __X86INTRIN_H
10	10	#error "Never use <clzerointrin.h> directly; include <x86intrin.h> instead."
11	11	#endif
12	12
...	...	@@ -17,14 +17,16 @@
17	17	#define __DEFAULT_FN_ATTRS \
18	18	__attribute__((__always_inline__, __nodebug__, __target__("clzero")))
19	19
20		/// Loads the cache line address and zero's out the cacheline
	20	/// Zeroes out the cache line for the address \a __line. This uses a
	21	/// non-temporal store. Calling \c _mm_sfence() afterward might be needed
	22	/// to enforce ordering.
21	23	///
22		/// \headerfile <clzerointrin.h>
	24	/// \headerfile <x86intrin.h>
23	25	///
24		/// This intrinsic corresponds to the <c> CLZERO </c> instruction.
	26	/// This intrinsic corresponds to the \c CLZERO instruction.
25	27	///
26	28	/// \param __line
27		/// A pointer to a cacheline which needs to be zeroed out.
	29	/// An address within the cache line to zero out.
28	30	static __inline__ void __DEFAULT_FN_ATTRS
29	31	_mm_clzero (void * __line)
30	32	{

lib/include/cuda_wrappers/bits/shared_ptr_base.h created+9

...	...	@@ -0,0 +1,9 @@
	1	// CUDA headers define __noinline__ which interferes with libstdc++'s use of
	2	// `__attribute((__noinline__))`. In order to avoid compilation error,
	3	// temporarily unset __noinline__ when we include affected libstdc++ header.
	4
	5	#pragma push_macro("__noinline__")
	6	#undef __noinline__
	7	#include_next "bits/shared_ptr_base.h"
	8
	9	#pragma pop_macro("__noinline__")

lib/include/fmaintrin.h+564

...	...	@@ -18,192 +18,756 @@
18	18	#define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("fma"), __min_vector_width__(128)))
19	19	#define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("fma"), __min_vector_width__(256)))
20	20
	21	/// Computes a multiply-add of 128-bit vectors of [4 x float].
	22	/// For each element, computes <c> (__A * __B) + __C </c>.
	23	///
	24	/// \headerfile <immintrin.h>
	25	///
	26	/// This intrinsic corresponds to the \c VFMADD213PS instruction.
	27	///
	28	/// \param __A
	29	/// A 128-bit vector of [4 x float] containing the multiplicand.
	30	/// \param __B
	31	/// A 128-bit vector of [4 x float] containing the multiplier.
	32	/// \param __C
	33	/// A 128-bit vector of [4 x float] containing the addend.
	34	/// \returns A 128-bit vector of [4 x float] containing the result.
21	35	static __inline__ __m128 __DEFAULT_FN_ATTRS128
22	36	_mm_fmadd_ps(__m128 __A, __m128 __B, __m128 __C)
23	37	{
24	38	return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
25	39	}
26	40
	41	/// Computes a multiply-add of 128-bit vectors of [2 x double].
	42	/// For each element, computes <c> (__A * __B) + __C </c>.
	43	///
	44	/// \headerfile <immintrin.h>
	45	///
	46	/// This intrinsic corresponds to the \c VFMADD213PD instruction.
	47	///
	48	/// \param __A
	49	/// A 128-bit vector of [2 x double] containing the multiplicand.
	50	/// \param __B
	51	/// A 128-bit vector of [2 x double] containing the multiplier.
	52	/// \param __C
	53	/// A 128-bit vector of [2 x double] containing the addend.
	54	/// \returns A 128-bit [2 x double] vector containing the result.
27	55	static __inline__ __m128d __DEFAULT_FN_ATTRS128
28	56	_mm_fmadd_pd(__m128d __A, __m128d __B, __m128d __C)
29	57	{
30	58	return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, (__v2df)__C);
31	59	}
32	60
	61	/// Computes a scalar multiply-add of the single-precision values in the
	62	/// low 32 bits of 128-bit vectors of [4 x float].
	63	/// \code
	64	/// result[31:0] = (__A[31:0] * __B[31:0]) + __C[31:0]
	65	/// result[127:32] = __A[127:32]
	66	/// \endcode
	67	///
	68	/// \headerfile <immintrin.h>
	69	///
	70	/// This intrinsic corresponds to the \c VFMADD213SS instruction.
	71	///
	72	/// \param __A
	73	/// A 128-bit vector of [4 x float] containing the multiplicand in the low
	74	/// 32 bits.
	75	/// \param __B
	76	/// A 128-bit vector of [4 x float] containing the multiplier in the low
	77	/// 32 bits.
	78	/// \param __C
	79	/// A 128-bit vector of [4 x float] containing the addend in the low
	80	/// 32 bits.
	81	/// \returns A 128-bit vector of [4 x float] containing the result in the low
	82	/// 32 bits and a copy of \a __A[127:32] in the upper 96 bits.
33	83	static __inline__ __m128 __DEFAULT_FN_ATTRS128
34	84	_mm_fmadd_ss(__m128 __A, __m128 __B, __m128 __C)
35	85	{
36	86	return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
37	87	}
38	88
	89	/// Computes a scalar multiply-add of the double-precision values in the
	90	/// low 64 bits of 128-bit vectors of [2 x double].
	91	/// \code
	92	/// result[63:0] = (__A[63:0] * __B[63:0]) + __C[63:0]
	93	/// result[127:64] = __A[127:64]
	94	/// \endcode
	95	///
	96	/// \headerfile <immintrin.h>
	97	///
	98	/// This intrinsic corresponds to the \c VFMADD213SD instruction.
	99	///
	100	/// \param __A
	101	/// A 128-bit vector of [2 x double] containing the multiplicand in the low
	102	/// 64 bits.
	103	/// \param __B
	104	/// A 128-bit vector of [2 x double] containing the multiplier in the low
	105	/// 64 bits.
	106	/// \param __C
	107	/// A 128-bit vector of [2 x double] containing the addend in the low
	108	/// 64 bits.
	109	/// \returns A 128-bit vector of [2 x double] containing the result in the low
	110	/// 64 bits and a copy of \a __A[127:64] in the upper 64 bits.
39	111	static __inline__ __m128d __DEFAULT_FN_ATTRS128
40	112	_mm_fmadd_sd(__m128d __A, __m128d __B, __m128d __C)
41	113	{
42	114	return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, (__v2df)__B, (__v2df)__C);
43	115	}
44	116
	117	/// Computes a multiply-subtract of 128-bit vectors of [4 x float].
	118	/// For each element, computes <c> (__A * __B) - __C </c>.
	119	///
	120	/// \headerfile <immintrin.h>
	121	///
	122	/// This intrinsic corresponds to the \c VFMSUB213PS instruction.
	123	///
	124	/// \param __A
	125	/// A 128-bit vector of [4 x float] containing the multiplicand.
	126	/// \param __B
	127	/// A 128-bit vector of [4 x float] containing the multiplier.
	128	/// \param __C
	129	/// A 128-bit vector of [4 x float] containing the subtrahend.
	130	/// \returns A 128-bit vector of [4 x float] containing the result.
45	131	static __inline__ __m128 __DEFAULT_FN_ATTRS128
46	132	_mm_fmsub_ps(__m128 __A, __m128 __B, __m128 __C)
47	133	{
48	134	return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
49	135	}
50	136
	137	/// Computes a multiply-subtract of 128-bit vectors of [2 x double].
	138	/// For each element, computes <c> (__A * __B) - __C </c>.
	139	///
	140	/// \headerfile <immintrin.h>
	141	///
	142	/// This intrinsic corresponds to the \c VFMSUB213PD instruction.
	143	///
	144	/// \param __A
	145	/// A 128-bit vector of [2 x double] containing the multiplicand.
	146	/// \param __B
	147	/// A 128-bit vector of [2 x double] containing the multiplier.
	148	/// \param __C
	149	/// A 128-bit vector of [2 x double] containing the addend.
	150	/// \returns A 128-bit vector of [2 x double] containing the result.
51	151	static __inline__ __m128d __DEFAULT_FN_ATTRS128
52	152	_mm_fmsub_pd(__m128d __A, __m128d __B, __m128d __C)
53	153	{
54	154	return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, -(__v2df)__C);
55	155	}
56	156
	157	/// Computes a scalar multiply-subtract of the single-precision values in
	158	/// the low 32 bits of 128-bit vectors of [4 x float].
	159	/// \code
	160	/// result[31:0] = (__A[31:0] * __B[31:0]) - __C[31:0]
	161	/// result[127:32] = __A[127:32]
	162	/// \endcode
	163	///
	164	/// \headerfile <immintrin.h>
	165	///
	166	/// This intrinsic corresponds to the \c VFMSUB213SS instruction.
	167	///
	168	/// \param __A
	169	/// A 128-bit vector of [4 x float] containing the multiplicand in the low
	170	/// 32 bits.
	171	/// \param __B
	172	/// A 128-bit vector of [4 x float] containing the multiplier in the low
	173	/// 32 bits.
	174	/// \param __C
	175	/// A 128-bit vector of [4 x float] containing the subtrahend in the low
	176	/// 32 bits.
	177	/// \returns A 128-bit vector of [4 x float] containing the result in the low
	178	/// 32 bits, and a copy of \a __A[127:32] in the upper 96 bits.
57	179	static __inline__ __m128 __DEFAULT_FN_ATTRS128
58	180	_mm_fmsub_ss(__m128 __A, __m128 __B, __m128 __C)
59	181	{
60	182	return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
61	183	}
62	184
	185	/// Computes a scalar multiply-subtract of the double-precision values in
	186	/// the low 64 bits of 128-bit vectors of [2 x double].
	187	/// \code
	188	/// result[63:0] = (__A[63:0] * __B[63:0]) - __C[63:0]
	189	/// result[127:64] = __A[127:64]
	190	/// \endcode
	191	///
	192	/// \headerfile <immintrin.h>
	193	///
	194	/// This intrinsic corresponds to the \c VFMSUB213SD instruction.
	195	///
	196	/// \param __A
	197	/// A 128-bit vector of [2 x double] containing the multiplicand in the low
	198	/// 64 bits.
	199	/// \param __B
	200	/// A 128-bit vector of [2 x double] containing the multiplier in the low
	201	/// 64 bits.
	202	/// \param __C
	203	/// A 128-bit vector of [2 x double] containing the subtrahend in the low
	204	/// 64 bits.
	205	/// \returns A 128-bit vector of [2 x double] containing the result in the low
	206	/// 64 bits, and a copy of \a __A[127:64] in the upper 64 bits.
63	207	static __inline__ __m128d __DEFAULT_FN_ATTRS128
64	208	_mm_fmsub_sd(__m128d __A, __m128d __B, __m128d __C)
65	209	{
66	210	return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, (__v2df)__B, -(__v2df)__C);
67	211	}
68	212
	213	/// Computes a negated multiply-add of 128-bit vectors of [4 x float].
	214	/// For each element, computes <c> -(__A * __B) + __C </c>.
	215	///
	216	/// \headerfile <immintrin.h>
	217	///
	218	/// This intrinsic corresponds to the \c VFNMADD213DPS instruction.
	219	///
	220	/// \param __A
	221	/// A 128-bit vector of [4 x float] containing the multiplicand.
	222	/// \param __B
	223	/// A 128-bit vector of [4 x float] containing the multiplier.
	224	/// \param __C
	225	/// A 128-bit vector of [4 x float] containing the addend.
	226	/// \returns A 128-bit [4 x float] vector containing the result.
69	227	static __inline__ __m128 __DEFAULT_FN_ATTRS128
70	228	_mm_fnmadd_ps(__m128 __A, __m128 __B, __m128 __C)
71	229	{
72	230	return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
73	231	}
74	232
	233	/// Computes a negated multiply-add of 128-bit vectors of [2 x double].
	234	/// For each element, computes <c> -(__A * __B) + __C </c>.
	235	///
	236	/// \headerfile <immintrin.h>
	237	///
	238	/// This intrinsic corresponds to the \c VFNMADD213PD instruction.
	239	///
	240	/// \param __A
	241	/// A 128-bit vector of [2 x double] containing the multiplicand.
	242	/// \param __B
	243	/// A 128-bit vector of [2 x double] containing the multiplier.
	244	/// \param __C
	245	/// A 128-bit vector of [2 x double] containing the addend.
	246	/// \returns A 128-bit vector of [2 x double] containing the result.
75	247	static __inline__ __m128d __DEFAULT_FN_ATTRS128
76	248	_mm_fnmadd_pd(__m128d __A, __m128d __B, __m128d __C)
77	249	{
78	250	return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, (__v2df)__C);
79	251	}
80	252
	253	/// Computes a scalar negated multiply-add of the single-precision values in
	254	/// the low 32 bits of 128-bit vectors of [4 x float].
	255	/// \code
	256	/// result[31:0] = -(__A[31:0] * __B[31:0]) + __C[31:0]
	257	/// result[127:32] = __A[127:32]
	258	/// \endcode
	259	///
	260	/// \headerfile <immintrin.h>
	261	///
	262	/// This intrinsic corresponds to the \c VFNMADD213SS instruction.
	263	///
	264	/// \param __A
	265	/// A 128-bit vector of [4 x float] containing the multiplicand in the low
	266	/// 32 bits.
	267	/// \param __B
	268	/// A 128-bit vector of [4 x float] containing the multiplier in the low
	269	/// 32 bits.
	270	/// \param __C
	271	/// A 128-bit vector of [4 x float] containing the addend in the low
	272	/// 32 bits.
	273	/// \returns A 128-bit vector of [4 x float] containing the result in the low
	274	/// 32 bits, and a copy of \a __A[127:32] in the upper 96 bits.
81	275	static __inline__ __m128 __DEFAULT_FN_ATTRS128
82	276	_mm_fnmadd_ss(__m128 __A, __m128 __B, __m128 __C)
83	277	{
84	278	return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, -(__v4sf)__B, (__v4sf)__C);
85	279	}
86	280
	281	/// Computes a scalar negated multiply-add of the double-precision values
	282	/// in the low 64 bits of 128-bit vectors of [2 x double].
	283	/// \code
	284	/// result[63:0] = -(__A[63:0] * __B[63:0]) + __C[63:0]
	285	/// result[127:64] = __A[127:64]
	286	/// \endcode
	287	///
	288	/// \headerfile <immintrin.h>
	289	///
	290	/// This intrinsic corresponds to the \c VFNMADD213SD instruction.
	291	///
	292	/// \param __A
	293	/// A 128-bit vector of [2 x double] containing the multiplicand in the low
	294	/// 64 bits.
	295	/// \param __B
	296	/// A 128-bit vector of [2 x double] containing the multiplier in the low
	297	/// 64 bits.
	298	/// \param __C
	299	/// A 128-bit vector of [2 x double] containing the addend in the low
	300	/// 64 bits.
	301	/// \returns A 128-bit vector of [2 x double] containing the result in the low
	302	/// 64 bits, and a copy of \a __A[127:64] in the upper 64 bits.
87	303	static __inline__ __m128d __DEFAULT_FN_ATTRS128
88	304	_mm_fnmadd_sd(__m128d __A, __m128d __B, __m128d __C)
89	305	{
90	306	return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, -(__v2df)__B, (__v2df)__C);
91	307	}
92	308
	309	/// Computes a negated multiply-subtract of 128-bit vectors of [4 x float].
	310	/// For each element, computes <c> -(__A * __B) - __C </c>.
	311	///
	312	/// \headerfile <immintrin.h>
	313	///
	314	/// This intrinsic corresponds to the \c VFNMSUB213PS instruction.
	315	///
	316	/// \param __A
	317	/// A 128-bit vector of [4 x float] containing the multiplicand.
	318	/// \param __B
	319	/// A 128-bit vector of [4 x float] containing the multiplier.
	320	/// \param __C
	321	/// A 128-bit vector of [4 x float] containing the subtrahend.
	322	/// \returns A 128-bit vector of [4 x float] containing the result.
93	323	static __inline__ __m128 __DEFAULT_FN_ATTRS128
94	324	_mm_fnmsub_ps(__m128 __A, __m128 __B, __m128 __C)
95	325	{
96	326	return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
97	327	}
98	328
	329	/// Computes a negated multiply-subtract of 128-bit vectors of [2 x double].
	330	/// For each element, computes <c> -(__A * __B) - __C </c>.
	331	///
	332	/// \headerfile <immintrin.h>
	333	///
	334	/// This intrinsic corresponds to the \c VFNMSUB213PD instruction.
	335	///
	336	/// \param __A
	337	/// A 128-bit vector of [2 x double] containing the multiplicand.
	338	/// \param __B
	339	/// A 128-bit vector of [2 x double] containing the multiplier.
	340	/// \param __C
	341	/// A 128-bit vector of [2 x double] containing the subtrahend.
	342	/// \returns A 128-bit vector of [2 x double] containing the result.
99	343	static __inline__ __m128d __DEFAULT_FN_ATTRS128
100	344	_mm_fnmsub_pd(__m128d __A, __m128d __B, __m128d __C)
101	345	{
102	346	return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, -(__v2df)__C);
103	347	}
104	348
	349	/// Computes a scalar negated multiply-subtract of the single-precision
	350	/// values in the low 32 bits of 128-bit vectors of [4 x float].
	351	/// \code
	352	/// result[31:0] = -(__A[31:0] * __B[31:0]) - __C[31:0]
	353	/// result[127:32] = __A[127:32]
	354	/// \endcode
	355	///
	356	/// \headerfile <immintrin.h>
	357	///
	358	/// This intrinsic corresponds to the \c VFNMSUB213SS instruction.
	359	///
	360	/// \param __A
	361	/// A 128-bit vector of [4 x float] containing the multiplicand in the low
	362	/// 32 bits.
	363	/// \param __B
	364	/// A 128-bit vector of [4 x float] containing the multiplier in the low
	365	/// 32 bits.
	366	/// \param __C
	367	/// A 128-bit vector of [4 x float] containing the subtrahend in the low
	368	/// 32 bits.
	369	/// \returns A 128-bit vector of [4 x float] containing the result in the low
	370	/// 32 bits, and a copy of \a __A[127:32] in the upper 96 bits.
105	371	static __inline__ __m128 __DEFAULT_FN_ATTRS128
106	372	_mm_fnmsub_ss(__m128 __A, __m128 __B, __m128 __C)
107	373	{
108	374	return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, -(__v4sf)__B, -(__v4sf)__C);
109	375	}
110	376
	377	/// Computes a scalar negated multiply-subtract of the double-precision
	378	/// values in the low 64 bits of 128-bit vectors of [2 x double].
	379	/// \code
	380	/// result[63:0] = -(__A[63:0] * __B[63:0]) - __C[63:0]
	381	/// result[127:64] = __A[127:64]
	382	/// \endcode
	383	///
	384	/// \headerfile <immintrin.h>
	385	///
	386	/// This intrinsic corresponds to the \c VFNMSUB213SD instruction.
	387	///
	388	/// \param __A
	389	/// A 128-bit vector of [2 x double] containing the multiplicand in the low
	390	/// 64 bits.
	391	/// \param __B
	392	/// A 128-bit vector of [2 x double] containing the multiplier in the low
	393	/// 64 bits.
	394	/// \param __C
	395	/// A 128-bit vector of [2 x double] containing the subtrahend in the low
	396	/// 64 bits.
	397	/// \returns A 128-bit vector of [2 x double] containing the result in the low
	398	/// 64 bits, and a copy of \a __A[127:64] in the upper 64 bits.
111	399	static __inline__ __m128d __DEFAULT_FN_ATTRS128
112	400	_mm_fnmsub_sd(__m128d __A, __m128d __B, __m128d __C)
113	401	{
114	402	return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, -(__v2df)__B, -(__v2df)__C);
115	403	}
116	404
	405	/// Computes a multiply with alternating add/subtract of 128-bit vectors of
	406	/// [4 x float].
	407	/// \code
	408	/// result[31:0] = (__A[31:0] * __B[31:0]) - __C[31:0]
	409	/// result[63:32] = (__A[63:32] * __B[63:32]) + __C[63:32]
	410	/// result[95:64] = (__A[95:64] * __B[95:64]) - __C[95:64]
	411	/// result[127:96] = (__A[127:96] * __B[127:96]) + __C[127:96]
	412	/// \endcode
	413	///
	414	/// \headerfile <immintrin.h>
	415	///
	416	/// This intrinsic corresponds to the \c VFMADDSUB213PS instruction.
	417	///
	418	/// \param __A
	419	/// A 128-bit vector of [4 x float] containing the multiplicand.
	420	/// \param __B
	421	/// A 128-bit vector of [4 x float] containing the multiplier.
	422	/// \param __C
	423	/// A 128-bit vector of [4 x float] containing the addend/subtrahend.
	424	/// \returns A 128-bit vector of [4 x float] containing the result.
117	425	static __inline__ __m128 __DEFAULT_FN_ATTRS128
118	426	_mm_fmaddsub_ps(__m128 __A, __m128 __B, __m128 __C)
119	427	{
120	428	return (__m128)__builtin_ia32_vfmaddsubps((__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
121	429	}
122	430
	431	/// Computes a multiply with alternating add/subtract of 128-bit vectors of
	432	/// [2 x double].
	433	/// \code
	434	/// result[63:0] = (__A[63:0] * __B[63:0]) - __C[63:0]
	435	/// result[127:64] = (__A[127:64] * __B[127:64]) + __C[127:64]
	436	/// \endcode
	437	///
	438	/// \headerfile <immintrin.h>
	439	///
	440	/// This intrinsic corresponds to the \c VFMADDSUB213PD instruction.
	441	///
	442	/// \param __A
	443	/// A 128-bit vector of [2 x double] containing the multiplicand.
	444	/// \param __B
	445	/// A 128-bit vector of [2 x double] containing the multiplier.
	446	/// \param __C
	447	/// A 128-bit vector of [2 x double] containing the addend/subtrahend.
	448	/// \returns A 128-bit vector of [2 x double] containing the result.
123	449	static __inline__ __m128d __DEFAULT_FN_ATTRS128
124	450	_mm_fmaddsub_pd(__m128d __A, __m128d __B, __m128d __C)
125	451	{
126	452	return (__m128d)__builtin_ia32_vfmaddsubpd((__v2df)__A, (__v2df)__B, (__v2df)__C);
127	453	}
128	454
	455	/// Computes a multiply with alternating add/subtract of 128-bit vectors of
	456	/// [4 x float].
	457	/// \code
	458	/// result[31:0] = (__A[31:0] * __B[31:0]) + __C[31:0]
	459	/// result[63:32] = (__A[63:32] * __B[63:32]) - __C[63:32]
	460	/// result[95:64] = (__A[95:64] * __B[95:64]) + __C[95:64]
	461	/// result[127:96 = (__A[127:96] * __B[127:96]) - __C[127:96]
	462	/// \endcode
	463	///
	464	/// \headerfile <immintrin.h>
	465	///
	466	/// This intrinsic corresponds to the \c VFMSUBADD213PS instruction.
	467	///
	468	/// \param __A
	469	/// A 128-bit vector of [4 x float] containing the multiplicand.
	470	/// \param __B
	471	/// A 128-bit vector of [4 x float] containing the multiplier.
	472	/// \param __C
	473	/// A 128-bit vector of [4 x float] containing the addend/subtrahend.
	474	/// \returns A 128-bit vector of [4 x float] containing the result.
129	475	static __inline__ __m128 __DEFAULT_FN_ATTRS128
130	476	_mm_fmsubadd_ps(__m128 __A, __m128 __B, __m128 __C)
131	477	{
132	478	return (__m128)__builtin_ia32_vfmaddsubps((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
133	479	}
134	480
	481	/// Computes a multiply with alternating add/subtract of 128-bit vectors of
	482	/// [2 x double].
	483	/// \code
	484	/// result[63:0] = (__A[63:0] * __B[63:0]) + __C[63:0]
	485	/// result[127:64] = (__A[127:64] * __B[127:64]) - __C[127:64]
	486	/// \endcode
	487	///
	488	/// \headerfile <immintrin.h>
	489	///
	490	/// This intrinsic corresponds to the \c VFMADDSUB213PD instruction.
	491	///
	492	/// \param __A
	493	/// A 128-bit vector of [2 x double] containing the multiplicand.
	494	/// \param __B
	495	/// A 128-bit vector of [2 x double] containing the multiplier.
	496	/// \param __C
	497	/// A 128-bit vector of [2 x double] containing the addend/subtrahend.
	498	/// \returns A 128-bit vector of [2 x double] containing the result.
135	499	static __inline__ __m128d __DEFAULT_FN_ATTRS128
136	500	_mm_fmsubadd_pd(__m128d __A, __m128d __B, __m128d __C)
137	501	{
138	502	return (__m128d)__builtin_ia32_vfmaddsubpd((__v2df)__A, (__v2df)__B, -(__v2df)__C);
139	503	}
140	504
	505	/// Computes a multiply-add of 256-bit vectors of [8 x float].
	506	/// For each element, computes <c> (__A * __B) + __C </c>.
	507	///
	508	/// \headerfile <immintrin.h>
	509	///
	510	/// This intrinsic corresponds to the \c VFMADD213PS instruction.
	511	///
	512	/// \param __A
	513	/// A 256-bit vector of [8 x float] containing the multiplicand.
	514	/// \param __B
	515	/// A 256-bit vector of [8 x float] containing the multiplier.
	516	/// \param __C
	517	/// A 256-bit vector of [8 x float] containing the addend.
	518	/// \returns A 256-bit vector of [8 x float] containing the result.
141	519	static __inline__ __m256 __DEFAULT_FN_ATTRS256
142	520	_mm256_fmadd_ps(__m256 __A, __m256 __B, __m256 __C)
143	521	{
144	522	return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, (__v8sf)__C);
145	523	}
146	524
	525	/// Computes a multiply-add of 256-bit vectors of [4 x double].
	526	/// For each element, computes <c> (__A * __B) + __C </c>.
	527	///
	528	/// \headerfile <immintrin.h>
	529	///
	530	/// This intrinsic corresponds to the \c VFMADD213PD instruction.
	531	///
	532	/// \param __A
	533	/// A 256-bit vector of [4 x double] containing the multiplicand.
	534	/// \param __B
	535	/// A 256-bit vector of [4 x double] containing the multiplier.
	536	/// \param __C
	537	/// A 256-bit vector of [4 x double] containing the addend.
	538	/// \returns A 256-bit vector of [4 x double] containing the result.
147	539	static __inline__ __m256d __DEFAULT_FN_ATTRS256
148	540	_mm256_fmadd_pd(__m256d __A, __m256d __B, __m256d __C)
149	541	{
150	542	return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, (__v4df)__C);
151	543	}
152	544
	545	/// Computes a multiply-subtract of 256-bit vectors of [8 x float].
	546	/// For each element, computes <c> (__A * __B) - __C </c>.
	547	///
	548	/// \headerfile <immintrin.h>
	549	///
	550	/// This intrinsic corresponds to the \c VFMSUB213PS instruction.
	551	///
	552	/// \param __A
	553	/// A 256-bit vector of [8 x float] containing the multiplicand.
	554	/// \param __B
	555	/// A 256-bit vector of [8 x float] containing the multiplier.
	556	/// \param __C
	557	/// A 256-bit vector of [8 x float] containing the subtrahend.
	558	/// \returns A 256-bit vector of [8 x float] containing the result.
153	559	static __inline__ __m256 __DEFAULT_FN_ATTRS256
154	560	_mm256_fmsub_ps(__m256 __A, __m256 __B, __m256 __C)
155	561	{
156	562	return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C);
157	563	}
158	564
	565	/// Computes a multiply-subtract of 256-bit vectors of [4 x double].
	566	/// For each element, computes <c> (__A * __B) - __C </c>.
	567	///
	568	/// \headerfile <immintrin.h>
	569	///
	570	/// This intrinsic corresponds to the \c VFMSUB213PD instruction.
	571	///
	572	/// \param __A
	573	/// A 256-bit vector of [4 x double] containing the multiplicand.
	574	/// \param __B
	575	/// A 256-bit vector of [4 x double] containing the multiplier.
	576	/// \param __C
	577	/// A 256-bit vector of [4 x double] containing the subtrahend.
	578	/// \returns A 256-bit vector of [4 x double] containing the result.
159	579	static __inline__ __m256d __DEFAULT_FN_ATTRS256
160	580	_mm256_fmsub_pd(__m256d __A, __m256d __B, __m256d __C)
161	581	{
162	582	return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, -(__v4df)__C);
163	583	}
164	584
	585	/// Computes a negated multiply-add of 256-bit vectors of [8 x float].
	586	/// For each element, computes <c> -(__A * __B) + __C </c>.
	587	///
	588	/// \headerfile <immintrin.h>
	589	///
	590	/// This intrinsic corresponds to the \c VFNMADD213PS instruction.
	591	///
	592	/// \param __A
	593	/// A 256-bit vector of [8 x float] containing the multiplicand.
	594	/// \param __B
	595	/// A 256-bit vector of [8 x float] containing the multiplier.
	596	/// \param __C
	597	/// A 256-bit vector of [8 x float] containing the addend.
	598	/// \returns A 256-bit vector of [8 x float] containing the result.
165	599	static __inline__ __m256 __DEFAULT_FN_ATTRS256
166	600	_mm256_fnmadd_ps(__m256 __A, __m256 __B, __m256 __C)
167	601	{
168	602	return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, (__v8sf)__C);
169	603	}
170	604
	605	/// Computes a negated multiply-add of 256-bit vectors of [4 x double].
	606	/// For each element, computes <c> -(__A * __B) + __C </c>.
	607	///
	608	/// \headerfile <immintrin.h>
	609	///
	610	/// This intrinsic corresponds to the \c VFNMADD213PD instruction.
	611	///
	612	/// \param __A
	613	/// A 256-bit vector of [4 x double] containing the multiplicand.
	614	/// \param __B
	615	/// A 256-bit vector of [4 x double] containing the multiplier.
	616	/// \param __C
	617	/// A 256-bit vector of [4 x double] containing the addend.
	618	/// \returns A 256-bit vector of [4 x double] containing the result.
171	619	static __inline__ __m256d __DEFAULT_FN_ATTRS256
172	620	_mm256_fnmadd_pd(__m256d __A, __m256d __B, __m256d __C)
173	621	{
174	622	return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, (__v4df)__C);
175	623	}
176	624
	625	/// Computes a negated multiply-subtract of 256-bit vectors of [8 x float].
	626	/// For each element, computes <c> -(__A * __B) - __C </c>.
	627	///
	628	/// \headerfile <immintrin.h>
	629	///
	630	/// This intrinsic corresponds to the \c VFNMSUB213PS instruction.
	631	///
	632	/// \param __A
	633	/// A 256-bit vector of [8 x float] containing the multiplicand.
	634	/// \param __B
	635	/// A 256-bit vector of [8 x float] containing the multiplier.
	636	/// \param __C
	637	/// A 256-bit vector of [8 x float] containing the subtrahend.
	638	/// \returns A 256-bit vector of [8 x float] containing the result.
177	639	static __inline__ __m256 __DEFAULT_FN_ATTRS256
178	640	_mm256_fnmsub_ps(__m256 __A, __m256 __B, __m256 __C)
179	641	{
180	642	return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, -(__v8sf)__C);
181	643	}
182	644
	645	/// Computes a negated multiply-subtract of 256-bit vectors of [4 x double].
	646	/// For each element, computes <c> -(__A * __B) - __C </c>.
	647	///
	648	/// \headerfile <immintrin.h>
	649	///
	650	/// This intrinsic corresponds to the \c VFNMSUB213PD instruction.
	651	///
	652	/// \param __A
	653	/// A 256-bit vector of [4 x double] containing the multiplicand.
	654	/// \param __B
	655	/// A 256-bit vector of [4 x double] containing the multiplier.
	656	/// \param __C
	657	/// A 256-bit vector of [4 x double] containing the subtrahend.
	658	/// \returns A 256-bit vector of [4 x double] containing the result.
183	659	static __inline__ __m256d __DEFAULT_FN_ATTRS256
184	660	_mm256_fnmsub_pd(__m256d __A, __m256d __B, __m256d __C)
185	661	{
186	662	return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, -(__v4df)__C);
187	663	}
188	664
	665	/// Computes a multiply with alternating add/subtract of 256-bit vectors of
	666	/// [8 x float].
	667	/// \code
	668	/// result[31:0] = (__A[31:0] * __B[31:0]) - __C[31:0]
	669	/// result[63:32] = (__A[63:32] * __B[63:32]) + __C[63:32]
	670	/// result[95:64] = (__A[95:64] * __B[95:64]) - __C[95:64]
	671	/// result[127:96] = (__A[127:96] * __B[127:96]) + __C[127:96]
	672	/// result[159:128] = (__A[159:128] * __B[159:128]) - __C[159:128]
	673	/// result[191:160] = (__A[191:160] * __B[191:160]) + __C[191:160]
	674	/// result[223:192] = (__A[223:192] * __B[223:192]) - __C[223:192]
	675	/// result[255:224] = (__A[255:224] * __B[255:224]) + __C[255:224]
	676	/// \endcode
	677	///
	678	/// \headerfile <immintrin.h>
	679	///
	680	/// This intrinsic corresponds to the \c VFMADDSUB213PS instruction.
	681	///
	682	/// \param __A
	683	/// A 256-bit vector of [8 x float] containing the multiplicand.
	684	/// \param __B
	685	/// A 256-bit vector of [8 x float] containing the multiplier.
	686	/// \param __C
	687	/// A 256-bit vector of [8 x float] containing the addend/subtrahend.
	688	/// \returns A 256-bit vector of [8 x float] containing the result.
189	689	static __inline__ __m256 __DEFAULT_FN_ATTRS256
190	690	_mm256_fmaddsub_ps(__m256 __A, __m256 __B, __m256 __C)
191	691	{
192	692	return (__m256)__builtin_ia32_vfmaddsubps256((__v8sf)__A, (__v8sf)__B, (__v8sf)__C);
193	693	}
194	694
	695	/// Computes a multiply with alternating add/subtract of 256-bit vectors of
	696	/// [4 x double].
	697	/// \code
	698	/// result[63:0] = (__A[63:0] * __B[63:0]) - __C[63:0]
	699	/// result[127:64] = (__A[127:64] * __B[127:64]) + __C[127:64]
	700	/// result[191:128] = (__A[191:128] * __B[191:128]) - __C[191:128]
	701	/// result[255:192] = (__A[255:192] * __B[255:192]) + __C[255:192]
	702	/// \endcode
	703	///
	704	/// \headerfile <immintrin.h>
	705	///
	706	/// This intrinsic corresponds to the \c VFMADDSUB213PD instruction.
	707	///
	708	/// \param __A
	709	/// A 256-bit vector of [4 x double] containing the multiplicand.
	710	/// \param __B
	711	/// A 256-bit vector of [4 x double] containing the multiplier.
	712	/// \param __C
	713	/// A 256-bit vector of [4 x double] containing the addend/subtrahend.
	714	/// \returns A 256-bit vector of [4 x double] containing the result.
195	715	static __inline__ __m256d __DEFAULT_FN_ATTRS256
196	716	_mm256_fmaddsub_pd(__m256d __A, __m256d __B, __m256d __C)
197	717	{
198	718	return (__m256d)__builtin_ia32_vfmaddsubpd256((__v4df)__A, (__v4df)__B, (__v4df)__C);
199	719	}
200	720
	721	/// Computes a vector multiply with alternating add/subtract of 256-bit
	722	/// vectors of [8 x float].
	723	/// \code
	724	/// result[31:0] = (__A[31:0] * __B[31:0]) + __C[31:0]
	725	/// result[63:32] = (__A[63:32] * __B[63:32]) - __C[63:32]
	726	/// result[95:64] = (__A[95:64] * __B[95:64]) + __C[95:64]
	727	/// result[127:96] = (__A[127:96] * __B[127:96]) - __C[127:96]
	728	/// result[159:128] = (__A[159:128] * __B[159:128]) + __C[159:128]
	729	/// result[191:160] = (__A[191:160] * __B[191:160]) - __C[191:160]
	730	/// result[223:192] = (__A[223:192] * __B[223:192]) + __C[223:192]
	731	/// result[255:224] = (__A[255:224] * __B[255:224]) - __C[255:224]
	732	/// \endcode
	733	///
	734	/// \headerfile <immintrin.h>
	735	///
	736	/// This intrinsic corresponds to the \c VFMSUBADD213PS instruction.
	737	///
	738	/// \param __A
	739	/// A 256-bit vector of [8 x float] containing the multiplicand.
	740	/// \param __B
	741	/// A 256-bit vector of [8 x float] containing the multiplier.
	742	/// \param __C
	743	/// A 256-bit vector of [8 x float] containing the addend/subtrahend.
	744	/// \returns A 256-bit vector of [8 x float] containing the result.
201	745	static __inline__ __m256 __DEFAULT_FN_ATTRS256
202	746	_mm256_fmsubadd_ps(__m256 __A, __m256 __B, __m256 __C)
203	747	{
204	748	return (__m256)__builtin_ia32_vfmaddsubps256((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C);
205	749	}
206	750
	751	/// Computes a vector multiply with alternating add/subtract of 256-bit
	752	/// vectors of [4 x double].
	753	/// \code
	754	/// result[63:0] = (__A[63:0] * __B[63:0]) + __C[63:0]
	755	/// result[127:64] = (__A[127:64] * __B[127:64]) - __C[127:64]
	756	/// result[191:128] = (__A[191:128] * __B[191:128]) + __C[191:128]
	757	/// result[255:192] = (__A[255:192] * __B[255:192]) - __C[255:192]
	758	/// \endcode
	759	///
	760	/// \headerfile <immintrin.h>
	761	///
	762	/// This intrinsic corresponds to the \c VFMSUBADD213PD instruction.
	763	///
	764	/// \param __A
	765	/// A 256-bit vector of [4 x double] containing the multiplicand.
	766	/// \param __B
	767	/// A 256-bit vector of [4 x double] containing the multiplier.
	768	/// \param __C
	769	/// A 256-bit vector of [4 x double] containing the addend/subtrahend.
	770	/// \returns A 256-bit vector of [4 x double] containing the result.
207	771	static __inline__ __m256d __DEFAULT_FN_ATTRS256
208	772	_mm256_fmsubadd_pd(__m256d __A, __m256d __B, __m256d __C)
209	773	{

lib/include/immintrin.h+116-8

...	...	@@ -269,6 +269,26 @@
269	269	#include <avxneconvertintrin.h>
270	270	#endif
271	271
	272	#if !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \|\| \
	273	defined(__SHA512__)
	274	#include <sha512intrin.h>
	275	#endif
	276
	277	#if !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \|\| \
	278	defined(__SM3__)
	279	#include <sm3intrin.h>
	280	#endif
	281
	282	#if !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \|\| \
	283	defined(__SM4__)
	284	#include <sm4intrin.h>
	285	#endif
	286
	287	#if !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \|\| \
	288	defined(__AVXVNNIINT16__)
	289	#include <avxvnniint16intrin.h>
	290	#endif
	291
272	292	#if !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \|\| \
273	293	defined(__RDPID__)
274	294	/// Returns the value of the IA32_TSC_AUX MSR (0xc0000103).
...	...	@@ -284,30 +304,53 @@ _rdpid_u32(void) {
284	304
285	305	#if !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \|\| \
286	306	defined(__RDRND__)
	307	/// Returns a 16-bit hardware-generated random value.
	308	///
	309	/// \headerfile <immintrin.h>
	310	///
	311	/// This intrinsic corresponds to the <c> RDRAND </c> instruction.
	312	///
	313	/// \param __p
	314	/// A pointer to a 16-bit memory location to place the random value.
	315	/// \returns 1 if the value was successfully generated, 0 otherwise.
287	316	static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
288	317	_rdrand16_step(unsigned short *__p)
289	318	{
290	319	return (int)__builtin_ia32_rdrand16_step(__p);
291	320	}
292	321
	322	/// Returns a 32-bit hardware-generated random value.
	323	///
	324	/// \headerfile <immintrin.h>
	325	///
	326	/// This intrinsic corresponds to the <c> RDRAND </c> instruction.
	327	///
	328	/// \param __p
	329	/// A pointer to a 32-bit memory location to place the random value.
	330	/// \returns 1 if the value was successfully generated, 0 otherwise.
293	331	static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
294	332	_rdrand32_step(unsigned int *__p)
295	333	{
296	334	return (int)__builtin_ia32_rdrand32_step(__p);
297	335	}
298	336
299		#ifdef __x86_64__
	337	/// Returns a 64-bit hardware-generated random value.
	338	///
	339	/// \headerfile <immintrin.h>
	340	///
	341	/// This intrinsic corresponds to the <c> RDRAND </c> instruction.
	342	///
	343	/// \param __p
	344	/// A pointer to a 64-bit memory location to place the random value.
	345	/// \returns 1 if the value was successfully generated, 0 otherwise.
300	346	static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
301	347	_rdrand64_step(unsigned long long *__p)
302	348	{
	349	#ifdef __x86_64__
303	350	return (int)__builtin_ia32_rdrand64_step(__p);
304		}
305	351	#else
306		// We need to emulate the functionality of 64-bit rdrand with 2 32-bit
307		// rdrand instructions.
308		static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
309		_rdrand64_step(unsigned long long *__p)
310		{
	352	// We need to emulate the functionality of 64-bit rdrand with 2 32-bit
	353	// rdrand instructions.
311	354	unsigned int __lo, __hi;
312	355	unsigned int __res_lo = __builtin_ia32_rdrand32_step(&__lo);
313	356	unsigned int __res_hi = __builtin_ia32_rdrand32_step(&__hi);
...	...	@@ -318,55 +361,115 @@ _rdrand64_step(unsigned long long *__p)
318	361	*__p = 0;
319	362	return 0;
320	363	}
321		}
322	364	#endif
	365	}
323	366	#endif /* __RDRND__ */
324	367
325	368	#if !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \|\| \
326	369	defined(__FSGSBASE__)
327	370	#ifdef __x86_64__
	371	/// Reads the FS base register.
	372	///
	373	/// \headerfile <immintrin.h>
	374	///
	375	/// This intrinsic corresponds to the <c> RDFSBASE </c> instruction.
	376	///
	377	/// \returns The lower 32 bits of the FS base register.
328	378	static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
329	379	_readfsbase_u32(void)
330	380	{
331	381	return __builtin_ia32_rdfsbase32();
332	382	}
333	383
	384	/// Reads the FS base register.
	385	///
	386	/// \headerfile <immintrin.h>
	387	///
	388	/// This intrinsic corresponds to the <c> RDFSBASE </c> instruction.
	389	///
	390	/// \returns The contents of the FS base register.
334	391	static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
335	392	_readfsbase_u64(void)
336	393	{
337	394	return __builtin_ia32_rdfsbase64();
338	395	}
339	396
	397	/// Reads the GS base register.
	398	///
	399	/// \headerfile <immintrin.h>
	400	///
	401	/// This intrinsic corresponds to the <c> RDGSBASE </c> instruction.
	402	///
	403	/// \returns The lower 32 bits of the GS base register.
340	404	static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
341	405	_readgsbase_u32(void)
342	406	{
343	407	return __builtin_ia32_rdgsbase32();
344	408	}
345	409
	410	/// Reads the GS base register.
	411	///
	412	/// \headerfile <immintrin.h>
	413	///
	414	/// This intrinsic corresponds to the <c> RDGSBASE </c> instruction.
	415	///
	416	/// \returns The contents of the GS base register.
346	417	static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
347	418	_readgsbase_u64(void)
348	419	{
349	420	return __builtin_ia32_rdgsbase64();
350	421	}
351	422
	423	/// Modifies the FS base register.
	424	///
	425	/// \headerfile <immintrin.h>
	426	///
	427	/// This intrinsic corresponds to the <c> WRFSBASE </c> instruction.
	428	///
	429	/// \param __V
	430	/// Value to use for the lower 32 bits of the FS base register.
352	431	static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
353	432	_writefsbase_u32(unsigned int __V)
354	433	{
355	434	__builtin_ia32_wrfsbase32(__V);
356	435	}
357	436
	437	/// Modifies the FS base register.
	438	///
	439	/// \headerfile <immintrin.h>
	440	///
	441	/// This intrinsic corresponds to the <c> WRFSBASE </c> instruction.
	442	///
	443	/// \param __V
	444	/// Value to use for the FS base register.
358	445	static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
359	446	_writefsbase_u64(unsigned long long __V)
360	447	{
361	448	__builtin_ia32_wrfsbase64(__V);
362	449	}
363	450
	451	/// Modifies the GS base register.
	452	///
	453	/// \headerfile <immintrin.h>
	454	///
	455	/// This intrinsic corresponds to the <c> WRGSBASE </c> instruction.
	456	///
	457	/// \param __V
	458	/// Value to use for the lower 32 bits of the GS base register.
364	459	static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
365	460	_writegsbase_u32(unsigned int __V)
366	461	{
367	462	__builtin_ia32_wrgsbase32(__V);
368	463	}
369	464
	465	/// Modifies the GS base register.
	466	///
	467	/// \headerfile <immintrin.h>
	468	///
	469	/// This intrinsic corresponds to the <c> WRFSBASE </c> instruction.
	470	///
	471	/// \param __V
	472	/// Value to use for GS base register.
370	473	static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
371	474	_writegsbase_u64(unsigned long long __V)
372	475	{
...	...	@@ -538,6 +641,11 @@ _storebe_i64(void * __P, long long __D) {
538	641	#include <amxintrin.h>
539	642	#endif
540	643
	644	#if !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \|\| \
	645	defined(__AMX_COMPLEX__)
	646	#include <amxcomplexintrin.h>
	647	#endif
	648
541	649	#if !(defined(_MSC_VER) \|\| defined(__SCE__)) \|\| __has_feature(modules) \|\| \
542	650	defined(__AVX512VP2INTERSECT__)
543	651	#include <avx512vp2intersectintrin.h>

lib/include/limits.h+5-1

...	...	@@ -52,7 +52,11 @@
52	52	#define LONG_MIN (-__LONG_MAX__ -1L)
53	53
54	54	#define UCHAR_MAX (__SCHAR_MAX__*2 +1)
55		#define USHRT_MAX (__SHRT_MAX__ *2 +1)
	55	#if __SHRT_WIDTH__ < __INT_WIDTH__
	56	#define USHRT_MAX (__SHRT_MAX__ * 2 + 1)
	57	#else
	58	#define USHRT_MAX (__SHRT_MAX__ * 2U + 1U)
	59	#endif
56	60	#define UINT_MAX (__INT_MAX__ *2U +1U)
57	61	#define ULONG_MAX (__LONG_MAX__ *2UL+1UL)
58	62

lib/include/llvm_libc_wrappers/ctype.h created+85

...	...	@@ -0,0 +1,85 @@
	1	//===-- Wrapper for C standard ctype.h declarations on the GPU ------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef __CLANG_LLVM_LIBC_WRAPPERS_CTYPE_H__
	10	#define __CLANG_LLVM_LIBC_WRAPPERS_CTYPE_H__
	11
	12	#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__)
	13	#error "This file is for GPU offloading compilation only"
	14	#endif
	15
	16	#include_next <ctype.h>
	17
	18	#if __has_include(<llvm-libc-decls/ctype.h>)
	19
	20	#if defined(__HIP__) \|\| defined(__CUDA__)
	21	#define __LIBC_ATTRS __attribute__((device))
	22	#endif
	23
	24	// The GNU headers like to provide these as macros, we need to undefine them so
	25	// they do not conflict with the following definitions for the GPU.
	26
	27	#pragma push_macro("isalnum")
	28	#pragma push_macro("isalpha")
	29	#pragma push_macro("isblank")
	30	#pragma push_macro("iscntrl")
	31	#pragma push_macro("isdigit")
	32	#pragma push_macro("isgraph")
	33	#pragma push_macro("islower")
	34	#pragma push_macro("isprint")
	35	#pragma push_macro("ispunct")
	36	#pragma push_macro("isspace")
	37	#pragma push_macro("isupper")
	38	#pragma push_macro("isxdigit")
	39	#pragma push_macro("tolower")
	40	#pragma push_macro("toupper")
	41
	42	#undef isalnum
	43	#undef isalpha
	44	#undef iscntrl
	45	#undef isdigit
	46	#undef islower
	47	#undef isgraph
	48	#undef isprint
	49	#undef ispunct
	50	#undef isspace
	51	#undef isupper
	52	#undef isblank
	53	#undef isxdigit
	54	#undef tolower
	55	#undef toupper
	56
	57	#pragma omp begin declare target
	58
	59	#include <llvm-libc-decls/ctype.h>
	60
	61	#pragma omp end declare target
	62
	63	// Restore the original macros when compiling on the host.
	64	#if !defined(__NVPTX__) && !defined(__AMDGPU__)
	65	#pragma pop_macro("isalnum")
	66	#pragma pop_macro("isalpha")
	67	#pragma pop_macro("isblank")
	68	#pragma pop_macro("iscntrl")
	69	#pragma pop_macro("isdigit")
	70	#pragma pop_macro("isgraph")
	71	#pragma pop_macro("islower")
	72	#pragma pop_macro("isprint")
	73	#pragma pop_macro("ispunct")
	74	#pragma pop_macro("isspace")
	75	#pragma pop_macro("isupper")
	76	#pragma pop_macro("isxdigit")
	77	#pragma pop_macro("tolower")
	78	#pragma pop_macro("toupper")
	79	#endif
	80
	81	#undef __LIBC_ATTRS
	82
	83	#endif
	84
	85	#endif // __CLANG_LLVM_LIBC_WRAPPERS_CTYPE_H__

lib/include/llvm_libc_wrappers/inttypes.h created+34

...	...	@@ -0,0 +1,34 @@
	1	//===-- Wrapper for C standard inttypes.h declarations on the GPU ---------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef __CLANG_LLVM_LIBC_WRAPPERS_INTTYPES_H__
	10	#define __CLANG_LLVM_LIBC_WRAPPERS_INTTYPES_H__
	11
	12	#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__)
	13	#error "This file is for GPU offloading compilation only"
	14	#endif
	15
	16	#include_next <inttypes.h>
	17
	18	#if __has_include(<llvm-libc-decls/inttypes.h>)
	19
	20	#if defined(__HIP__) \|\| defined(__CUDA__)
	21	#define __LIBC_ATTRS __attribute__((device))
	22	#endif
	23
	24	#pragma omp begin declare target
	25
	26	#include <llvm-libc-decls/inttypes.h>
	27
	28	#pragma omp end declare target
	29
	30	#undef __LIBC_ATTRS
	31
	32	#endif
	33
	34	#endif // __CLANG_LLVM_LIBC_WRAPPERS_INTTYPES_H__

lib/include/llvm_libc_wrappers/stdio.h created+34

...	...	@@ -0,0 +1,34 @@
	1	//===-- Wrapper for C standard stdio.h declarations on the GPU ------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__
	10	#define __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__
	11
	12	#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__)
	13	#error "This file is for GPU offloading compilation only"
	14	#endif
	15
	16	#include_next <stdio.h>
	17
	18	#if __has_include(<llvm-libc-decls/stdio.h>)
	19
	20	#if defined(__HIP__) \|\| defined(__CUDA__)
	21	#define __LIBC_ATTRS __attribute__((device))
	22	#endif
	23
	24	#pragma omp begin declare target
	25
	26	#include <llvm-libc-decls/stdio.h>
	27
	28	#pragma omp end declare target
	29
	30	#undef __LIBC_ATTRS
	31
	32	#endif
	33
	34	#endif // __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__

lib/include/llvm_libc_wrappers/stdlib.h created+42

...	...	@@ -0,0 +1,42 @@
	1	//===-- Wrapper for C standard stdlib.h declarations on the GPU -----------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef __CLANG_LLVM_LIBC_WRAPPERS_STDLIB_H__
	10	#define __CLANG_LLVM_LIBC_WRAPPERS_STDLIB_H__
	11
	12	#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__)
	13	#error "This file is for GPU offloading compilation only"
	14	#endif
	15
	16	#include_next <stdlib.h>
	17
	18	#if __has_include(<llvm-libc-decls/stdlib.h>)
	19
	20	#if defined(__HIP__) \|\| defined(__CUDA__)
	21	#define __LIBC_ATTRS __attribute__((device))
	22	#endif
	23
	24	#pragma omp begin declare target
	25
	26	// The LLVM C library uses this type so we forward declare it.
	27	typedef void (*__atexithandler_t)(void);
	28
	29	// Enforce ABI compatibility with the structs used by the LLVM C library.
	30	_Static_assert(__builtin_offsetof(div_t, quot) == 0, "ABI mismatch!");
	31	_Static_assert(__builtin_offsetof(ldiv_t, quot) == 0, "ABI mismatch!");
	32	_Static_assert(__builtin_offsetof(lldiv_t, quot) == 0, "ABI mismatch!");
	33
	34	#include <llvm-libc-decls/stdlib.h>
	35
	36	#pragma omp end declare target
	37
	38	#undef __LIBC_ATTRS
	39
	40	#endif
	41
	42	#endif // __CLANG_LLVM_LIBC_WRAPPERS_STDLIB_H__

lib/include/llvm_libc_wrappers/string.h created+37

...	...	@@ -0,0 +1,37 @@
	1	//===-- Wrapper for C standard string.h declarations on the GPU -----------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef __CLANG_LLVM_LIBC_WRAPPERS_STRING_H__
	10	#define __CLANG_LLVM_LIBC_WRAPPERS_STRING_H__
	11
	12	#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__)
	13	#error "This file is for GPU offloading compilation only"
	14	#endif
	15
	16	// FIXME: The GNU headers provide C++ standard compliant headers when in C++
	17	// mode and the LLVM libc does not. We cannot enable memchr, strchr, strchrnul,
	18	// strpbrk, strrchr, strstr, or strcasestr until this is addressed.
	19	#include_next <string.h>
	20
	21	#if __has_include(<llvm-libc-decls/string.h>)
	22
	23	#if defined(__HIP__) \|\| defined(__CUDA__)
	24	#define __LIBC_ATTRS __attribute__((device))
	25	#endif
	26
	27	#pragma omp begin declare target
	28
	29	#include <llvm-libc-decls/string.h>
	30
	31	#pragma omp end declare target
	32
	33	#undef __LIBC_ATTRS
	34
	35	#endif
	36
	37	#endif // __CLANG_LLVM_LIBC_WRAPPERS_STRING_H__

lib/include/mwaitxintrin.h+29

...	...	@@ -16,12 +16,41 @@
16	16
17	17	/* Define the default attributes for the functions in this file. */
18	18	#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("mwaitx")))
	19
	20	/// Establishes a linear address memory range to be monitored and puts
	21	/// the processor in the monitor event pending state. Data stored in the
	22	/// monitored address range causes the processor to exit the pending state.
	23	///
	24	/// \headerfile <x86intrin.h>
	25	///
	26	/// This intrinsic corresponds to the \c MONITORX instruction.
	27	///
	28	/// \param __p
	29	/// The memory range to be monitored. The size of the range is determined by
	30	/// CPUID function 0000_0005h.
	31	/// \param __extensions
	32	/// Optional extensions for the monitoring state.
	33	/// \param __hints
	34	/// Optional hints for the monitoring state.
19	35	static __inline__ void __DEFAULT_FN_ATTRS
20	36	_mm_monitorx(void * __p, unsigned __extensions, unsigned __hints)
21	37	{
22	38	__builtin_ia32_monitorx(__p, __extensions, __hints);
23	39	}
24	40
	41	/// Used with the \c MONITORX instruction to wait while the processor is in
	42	/// the monitor event pending state. Data stored in the monitored address
	43	/// range, or an interrupt, causes the processor to exit the pending state.
	44	///
	45	/// \headerfile <x86intrin.h>
	46	///
	47	/// This intrinsic corresponds to the \c MWAITX instruction.
	48	///
	49	/// \param __extensions
	50	/// Optional extensions for the monitoring state, which can vary by
	51	/// processor.
	52	/// \param __hints
	53	/// Optional hints for the monitoring state, which can vary by processor.
25	54	static __inline__ void __DEFAULT_FN_ATTRS
26	55	_mm_mwaitx(unsigned __extensions, unsigned __hints, unsigned __clock)
27	56	{

lib/include/opencl-c-base.h+3

...	...	@@ -474,6 +474,9 @@ typedef enum memory_order
474	474	#define CLK_HALF_FLOAT 0x10DD
475	475	#define CLK_FLOAT 0x10DE
476	476	#define CLK_UNORM_INT24 0x10DF
	477	#if __OPENCL_C_VERSION__ >= CL_VERSION_3_0
	478	#define CLK_UNORM_INT_101010_2 0x10E0
	479	#endif // __OPENCL_C_VERSION__ >= CL_VERSION_3_0
477	480
478	481	// Channel order, numbering must be aligned with cl_channel_order in cl.h
479	482	//

lib/include/openmp_wrappers/__clang_openmp_device_functions.h-1

...	...	@@ -40,7 +40,6 @@ extern "C" {
40	40
41	41	// Import types which will be used by __clang_hip_libdevice_declares.h
42	42	#ifndef __cplusplus
43		#include <stdbool.h>
44	43	#include <stdint.h>
45	44	#endif
46	45

lib/include/openmp_wrappers/new+1-1

...	...	@@ -13,7 +13,7 @@
13	13	// which do not use nothrow_t are provided without the <new> header.
14	14	#include_next <new>
15	15
16		#if defined(__NVPTX__) && defined(_OPENMP)
	16	#if (defined(__NVPTX__) \|\| defined(__AMDGPU__)) && defined(_OPENMP)
17	17
18	18	#include <cstdlib>
19	19

lib/include/pmmintrin.h+12-6

...	...	@@ -253,9 +253,12 @@ _mm_movedup_pd(__m128d __a)
253	253	/// the processor in the monitor event pending state. Data stored in the
254	254	/// monitored address range causes the processor to exit the pending state.
255	255	///
	256	/// The \c MONITOR instruction can be used in kernel mode, and in other modes
	257	/// if MSR <c> C001_0015h[MonMwaitUserEn] </c> is set.
	258	///
256	259	/// \headerfile <x86intrin.h>
257	260	///
258		/// This intrinsic corresponds to the <c> MONITOR </c> instruction.
	261	/// This intrinsic corresponds to the \c MONITOR instruction.
259	262	///
260	263	/// \param __p
261	264	/// The memory range to be monitored. The size of the range is determined by
...	...	@@ -270,19 +273,22 @@ _mm_monitor(void const *__p, unsigned __extensions, unsigned __hints)
270	273	__builtin_ia32_monitor(__p, __extensions, __hints);
271	274	}
272	275
273		/// Used with the MONITOR instruction to wait while the processor is in
	276	/// Used with the \c MONITOR instruction to wait while the processor is in
274	277	/// the monitor event pending state. Data stored in the monitored address
275		/// range causes the processor to exit the pending state.
	278	/// range, or an interrupt, causes the processor to exit the pending state.
	279	///
	280	/// The \c MWAIT instruction can be used in kernel mode, and in other modes if
	281	/// MSR <c> C001_0015h[MonMwaitUserEn] </c> is set.
276	282	///
277	283	/// \headerfile <x86intrin.h>
278	284	///
279		/// This intrinsic corresponds to the <c> MWAIT </c> instruction.
	285	/// This intrinsic corresponds to the \c MWAIT instruction.
280	286	///
281	287	/// \param __extensions
282		/// Optional extensions for the monitoring state, which may vary by
	288	/// Optional extensions for the monitoring state, which can vary by
283	289	/// processor.
284	290	/// \param __hints
285		/// Optional hints for the monitoring state, which may vary by processor.
	291	/// Optional hints for the monitoring state, which can vary by processor.
286	292	static __inline__ void __DEFAULT_FN_ATTRS
287	293	_mm_mwait(unsigned __extensions, unsigned __hints)
288	294	{

lib/include/ppc_wrappers/emmintrin.h+2-1

...	...	@@ -46,6 +46,7 @@
46	46
47	47	/* SSE2 */
48	48	typedef __vector double __v2df;
	49	typedef __vector float __v4f;
49	50	typedef __vector long long __v2di;
50	51	typedef __vector unsigned long long __v2du;
51	52	typedef __vector int __v4si;
...	...	@@ -951,7 +952,7 @@ extern __inline __m128d
951	952	_mm_cvtpi32_pd(__m64 __A) {
952	953	__v4si __temp;
953	954	__v2di __tmp2;
954		__v2df __result;
	955	__v4f __result;
955	956
956	957	__temp = (__v4si)vec_splats(__A);
957	958	__tmp2 = (__v2di)vec_unpackl(__temp);

lib/include/ppc_wrappers/smmintrin.h+2-2

...	...	@@ -305,9 +305,9 @@ extern __inline int
305	305	extern __inline __m128i
306	306	__attribute__((__gnu_inline__, __always_inline__, __artificial__))
307	307	_mm_blend_epi16(__m128i __A, __m128i __B, const int __imm8) {
308		__v16qi __charmask = vec_splats((signed char)__imm8);
	308	__v16qu __charmask = vec_splats((unsigned char)__imm8);
309	309	__charmask = vec_gb(__charmask);
310		__v8hu __shortmask = (__v8hu)vec_unpackh(__charmask);
	310	__v8hu __shortmask = (__v8hu)vec_unpackh((__v16qi)__charmask);
311	311	#ifdef __BIG_ENDIAN__
312	312	__shortmask = vec_reve(__shortmask);
313	313	#endif

lib/include/rdseedintrin.h+65-2

...	...	@@ -7,8 +7,8 @@
7	7	*===-----------------------------------------------------------------------===
8	8	*/
9	9
10		#if !defined __X86INTRIN_H && !defined __IMMINTRIN_H
11		#error "Never use <rdseedintrin.h> directly; include <x86intrin.h> instead."
	10	#ifndef __IMMINTRIN_H
	11	#error "Never use <rdseedintrin.h> directly; include <immintrin.h> instead."
12	12	#endif
13	13
14	14	#ifndef __RDSEEDINTRIN_H
...	...	@@ -17,12 +17,54 @@
17	17	/* Define the default attributes for the functions in this file. */
18	18	#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("rdseed")))
19	19
	20	/// Stores a hardware-generated 16-bit random value in the memory at \a __p.
	21	///
	22	/// The random number generator complies with NIST SP800-90B and SP800-90C.
	23	///
	24	/// \code{.operation}
	25	/// IF HW_NRND_GEN.ready == 1
	26	/// Store16(__p, HW_NRND_GEN.data)
	27	/// result := 1
	28	/// ELSE
	29	/// Store16(__p, 0)
	30	/// result := 0
	31	/// END
	32	/// \endcode
	33	///
	34	/// \headerfile <immintrin.h>
	35	///
	36	/// This intrinsic corresponds to the \c RDSEED instruction.
	37	///
	38	/// \param __p
	39	/// Pointer to memory for storing the 16-bit random number.
	40	/// \returns 1 if a random number was generated, 0 if not.
20	41	static __inline__ int __DEFAULT_FN_ATTRS
21	42	_rdseed16_step(unsigned short *__p)
22	43	{
23	44	return (int) __builtin_ia32_rdseed16_step(__p);
24	45	}
25	46
	47	/// Stores a hardware-generated 32-bit random value in the memory at \a __p.
	48	///
	49	/// The random number generator complies with NIST SP800-90B and SP800-90C.
	50	///
	51	/// \code{.operation}
	52	/// IF HW_NRND_GEN.ready == 1
	53	/// Store32(__p, HW_NRND_GEN.data)
	54	/// result := 1
	55	/// ELSE
	56	/// Store32(__p, 0)
	57	/// result := 0
	58	/// END
	59	/// \endcode
	60	///
	61	/// \headerfile <immintrin.h>
	62	///
	63	/// This intrinsic corresponds to the \c RDSEED instruction.
	64	///
	65	/// \param __p
	66	/// Pointer to memory for storing the 32-bit random number.
	67	/// \returns 1 if a random number was generated, 0 if not.
26	68	static __inline__ int __DEFAULT_FN_ATTRS
27	69	_rdseed32_step(unsigned int *__p)
28	70	{
...	...	@@ -30,6 +72,27 @@ _rdseed32_step(unsigned int *__p)
30	72	}
31	73
32	74	#ifdef __x86_64__
	75	/// Stores a hardware-generated 64-bit random value in the memory at \a __p.
	76	///
	77	/// The random number generator complies with NIST SP800-90B and SP800-90C.
	78	///
	79	/// \code{.operation}
	80	/// IF HW_NRND_GEN.ready == 1
	81	/// Store64(__p, HW_NRND_GEN.data)
	82	/// result := 1
	83	/// ELSE
	84	/// Store64(__p, 0)
	85	/// result := 0
	86	/// END
	87	/// \endcode
	88	///
	89	/// \headerfile <immintrin.h>
	90	///
	91	/// This intrinsic corresponds to the \c RDSEED instruction.
	92	///
	93	/// \param __p
	94	/// Pointer to memory for storing the 64-bit random number.
	95	/// \returns 1 if a random number was generated, 0 if not.
33	96	static __inline__ int __DEFAULT_FN_ATTRS
34	97	_rdseed64_step(unsigned long long *__p)
35	98	{

lib/include/riscv_ntlh.h created+28

...	...	@@ -0,0 +1,28 @@
	1	/*===---- riscv_ntlh.h - RISC-V NTLH intrinsics ----------------------------===
	2	*
	3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	* See https://llvm.org/LICENSE.txt for license information.
	5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	*
	7	*===-----------------------------------------------------------------------===
	8	*/
	9
	10	#ifndef __RISCV_NTLH_H
	11	#define __RISCV_NTLH_H
	12
	13	#ifndef __riscv_zihintntl
	14	#error "NTLH intrinsics require the NTLH extension."
	15	#endif
	16
	17	enum {
	18	__RISCV_NTLH_INNERMOST_PRIVATE = 2,
	19	__RISCV_NTLH_ALL_PRIVATE,
	20	__RISCV_NTLH_INNERMOST_SHARED,
	21	__RISCV_NTLH_ALL
	22	};
	23
	24	#define __riscv_ntl_load(PTR, DOMAIN) __builtin_riscv_ntl_load((PTR), (DOMAIN))
	25	#define __riscv_ntl_store(PTR, VAL, DOMAIN) \
	26	__builtin_riscv_ntl_store((PTR), (VAL), (DOMAIN))
	27
	28	#endif
		\ No newline at end of file

lib/include/riscv_vector.h+240-51

...	...	@@ -25,52 +25,15 @@ extern "C" {
25	25	#pragma clang riscv intrinsic vector
26	26
27	27
28		#define __riscv_vlenb() __builtin_rvv_vlenb()
29
30		enum RVV_CSR {
31		RVV_VSTART = 0,
32		RVV_VXSAT,
33		RVV_VXRM,
34		RVV_VCSR,
	28	enum __RISCV_FRM {
	29	__RISCV_FRM_RNE = 0,
	30	__RISCV_FRM_RTZ = 1,
	31	__RISCV_FRM_RDN = 2,
	32	__RISCV_FRM_RUP = 3,
	33	__RISCV_FRM_RMM = 4,
35	34	};
36	35
37		static __inline__ __attribute__((__always_inline__, __nodebug__))
38		unsigned long __riscv_vread_csr(enum RVV_CSR __csr) {
39		unsigned long __rv = 0;
40		switch (__csr) {
41		case RVV_VSTART:
42		__asm__ __volatile__ ("csrr\t%0, vstart" : "=r"(__rv) : : "memory");
43		break;
44		case RVV_VXSAT:
45		__asm__ __volatile__ ("csrr\t%0, vxsat" : "=r"(__rv) : : "memory");
46		break;
47		case RVV_VXRM:
48		__asm__ __volatile__ ("csrr\t%0, vxrm" : "=r"(__rv) : : "memory");
49		break;
50		case RVV_VCSR:
51		__asm__ __volatile__ ("csrr\t%0, vcsr" : "=r"(__rv) : : "memory");
52		break;
53		}
54		return __rv;
55		}
56
57		static __inline__ __attribute__((__always_inline__, __nodebug__))
58		void __riscv_vwrite_csr(enum RVV_CSR __csr, unsigned long __value) {
59		switch (__csr) {
60		case RVV_VSTART:
61		__asm__ __volatile__ ("csrw\tvstart, %z0" : : "rJ"(__value) : "memory");
62		break;
63		case RVV_VXSAT:
64		__asm__ __volatile__ ("csrw\tvxsat, %z0" : : "rJ"(__value) : "memory");
65		break;
66		case RVV_VXRM:
67		__asm__ __volatile__ ("csrw\tvxrm, %z0" : : "rJ"(__value) : "memory");
68		break;
69		case RVV_VCSR:
70		__asm__ __volatile__ ("csrw\tvcsr, %z0" : : "rJ"(__value) : "memory");
71		break;
72		}
73		}
	36	#define __riscv_vlenb() __builtin_rvv_vlenb()
74	37
75	38	#define __riscv_vsetvl_e8mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 6)
76	39	#define __riscv_vsetvl_e8mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 7)
...	...	@@ -130,6 +93,13 @@ void __riscv_vwrite_csr(enum RVV_CSR __csr, unsigned long __value) {
130	93	#define __riscv_vsetvlmax_e64m8() __builtin_rvv_vsetvlimax(3, 3)
131	94	#endif
132	95
	96
	97	enum __RISCV_VXRM {
	98	__RISCV_VXRM_RNU = 0,
	99	__RISCV_VXRM_RNE = 1,
	100	__RISCV_VXRM_RDN = 2,
	101	__RISCV_VXRM_ROD = 3,
	102	};
133	103	typedef __rvv_bool64_t vbool64_t;
134	104	typedef __rvv_bool32_t vbool32_t;
135	105	typedef __rvv_bool16_t vbool16_t;
...	...	@@ -139,70 +109,289 @@ typedef __rvv_bool2_t vbool2_t;
139	109	typedef __rvv_bool1_t vbool1_t;
140	110	typedef __rvv_int8mf8_t vint8mf8_t;
141	111	typedef __rvv_uint8mf8_t vuint8mf8_t;
	112	typedef __rvv_int8mf8x2_t vint8mf8x2_t;
	113	typedef __rvv_uint8mf8x2_t vuint8mf8x2_t;
	114	typedef __rvv_int8mf8x3_t vint8mf8x3_t;
	115	typedef __rvv_uint8mf8x3_t vuint8mf8x3_t;
	116	typedef __rvv_int8mf8x4_t vint8mf8x4_t;
	117	typedef __rvv_uint8mf8x4_t vuint8mf8x4_t;
	118	typedef __rvv_int8mf8x5_t vint8mf8x5_t;
	119	typedef __rvv_uint8mf8x5_t vuint8mf8x5_t;
	120	typedef __rvv_int8mf8x6_t vint8mf8x6_t;
	121	typedef __rvv_uint8mf8x6_t vuint8mf8x6_t;
	122	typedef __rvv_int8mf8x7_t vint8mf8x7_t;
	123	typedef __rvv_uint8mf8x7_t vuint8mf8x7_t;
	124	typedef __rvv_int8mf8x8_t vint8mf8x8_t;
	125	typedef __rvv_uint8mf8x8_t vuint8mf8x8_t;
142	126	typedef __rvv_int8mf4_t vint8mf4_t;
143	127	typedef __rvv_uint8mf4_t vuint8mf4_t;
	128	typedef __rvv_int8mf4x2_t vint8mf4x2_t;
	129	typedef __rvv_uint8mf4x2_t vuint8mf4x2_t;
	130	typedef __rvv_int8mf4x3_t vint8mf4x3_t;
	131	typedef __rvv_uint8mf4x3_t vuint8mf4x3_t;
	132	typedef __rvv_int8mf4x4_t vint8mf4x4_t;
	133	typedef __rvv_uint8mf4x4_t vuint8mf4x4_t;
	134	typedef __rvv_int8mf4x5_t vint8mf4x5_t;
	135	typedef __rvv_uint8mf4x5_t vuint8mf4x5_t;
	136	typedef __rvv_int8mf4x6_t vint8mf4x6_t;
	137	typedef __rvv_uint8mf4x6_t vuint8mf4x6_t;
	138	typedef __rvv_int8mf4x7_t vint8mf4x7_t;
	139	typedef __rvv_uint8mf4x7_t vuint8mf4x7_t;
	140	typedef __rvv_int8mf4x8_t vint8mf4x8_t;
	141	typedef __rvv_uint8mf4x8_t vuint8mf4x8_t;
144	142	typedef __rvv_int8mf2_t vint8mf2_t;
145	143	typedef __rvv_uint8mf2_t vuint8mf2_t;
	144	typedef __rvv_int8mf2x2_t vint8mf2x2_t;
	145	typedef __rvv_uint8mf2x2_t vuint8mf2x2_t;
	146	typedef __rvv_int8mf2x3_t vint8mf2x3_t;
	147	typedef __rvv_uint8mf2x3_t vuint8mf2x3_t;
	148	typedef __rvv_int8mf2x4_t vint8mf2x4_t;
	149	typedef __rvv_uint8mf2x4_t vuint8mf2x4_t;
	150	typedef __rvv_int8mf2x5_t vint8mf2x5_t;
	151	typedef __rvv_uint8mf2x5_t vuint8mf2x5_t;
	152	typedef __rvv_int8mf2x6_t vint8mf2x6_t;
	153	typedef __rvv_uint8mf2x6_t vuint8mf2x6_t;
	154	typedef __rvv_int8mf2x7_t vint8mf2x7_t;
	155	typedef __rvv_uint8mf2x7_t vuint8mf2x7_t;
	156	typedef __rvv_int8mf2x8_t vint8mf2x8_t;
	157	typedef __rvv_uint8mf2x8_t vuint8mf2x8_t;
146	158	typedef __rvv_int8m1_t vint8m1_t;
147	159	typedef __rvv_uint8m1_t vuint8m1_t;
	160	typedef __rvv_int8m1x2_t vint8m1x2_t;
	161	typedef __rvv_uint8m1x2_t vuint8m1x2_t;
	162	typedef __rvv_int8m1x3_t vint8m1x3_t;
	163	typedef __rvv_uint8m1x3_t vuint8m1x3_t;
	164	typedef __rvv_int8m1x4_t vint8m1x4_t;
	165	typedef __rvv_uint8m1x4_t vuint8m1x4_t;
	166	typedef __rvv_int8m1x5_t vint8m1x5_t;
	167	typedef __rvv_uint8m1x5_t vuint8m1x5_t;
	168	typedef __rvv_int8m1x6_t vint8m1x6_t;
	169	typedef __rvv_uint8m1x6_t vuint8m1x6_t;
	170	typedef __rvv_int8m1x7_t vint8m1x7_t;
	171	typedef __rvv_uint8m1x7_t vuint8m1x7_t;
	172	typedef __rvv_int8m1x8_t vint8m1x8_t;
	173	typedef __rvv_uint8m1x8_t vuint8m1x8_t;
148	174	typedef __rvv_int8m2_t vint8m2_t;
149	175	typedef __rvv_uint8m2_t vuint8m2_t;
	176	typedef __rvv_int8m2x2_t vint8m2x2_t;
	177	typedef __rvv_uint8m2x2_t vuint8m2x2_t;
	178	typedef __rvv_int8m2x3_t vint8m2x3_t;
	179	typedef __rvv_uint8m2x3_t vuint8m2x3_t;
	180	typedef __rvv_int8m2x4_t vint8m2x4_t;
	181	typedef __rvv_uint8m2x4_t vuint8m2x4_t;
150	182	typedef __rvv_int8m4_t vint8m4_t;
151	183	typedef __rvv_uint8m4_t vuint8m4_t;
	184	typedef __rvv_int8m4x2_t vint8m4x2_t;
	185	typedef __rvv_uint8m4x2_t vuint8m4x2_t;
152	186	typedef __rvv_int8m8_t vint8m8_t;
153	187	typedef __rvv_uint8m8_t vuint8m8_t;
154	188	typedef __rvv_int16mf4_t vint16mf4_t;
155	189	typedef __rvv_uint16mf4_t vuint16mf4_t;
	190	typedef __rvv_int16mf4x2_t vint16mf4x2_t;
	191	typedef __rvv_uint16mf4x2_t vuint16mf4x2_t;
	192	typedef __rvv_int16mf4x3_t vint16mf4x3_t;
	193	typedef __rvv_uint16mf4x3_t vuint16mf4x3_t;
	194	typedef __rvv_int16mf4x4_t vint16mf4x4_t;
	195	typedef __rvv_uint16mf4x4_t vuint16mf4x4_t;
	196	typedef __rvv_int16mf4x5_t vint16mf4x5_t;
	197	typedef __rvv_uint16mf4x5_t vuint16mf4x5_t;
	198	typedef __rvv_int16mf4x6_t vint16mf4x6_t;
	199	typedef __rvv_uint16mf4x6_t vuint16mf4x6_t;
	200	typedef __rvv_int16mf4x7_t vint16mf4x7_t;
	201	typedef __rvv_uint16mf4x7_t vuint16mf4x7_t;
	202	typedef __rvv_int16mf4x8_t vint16mf4x8_t;
	203	typedef __rvv_uint16mf4x8_t vuint16mf4x8_t;
156	204	typedef __rvv_int16mf2_t vint16mf2_t;
157	205	typedef __rvv_uint16mf2_t vuint16mf2_t;
	206	typedef __rvv_int16mf2x2_t vint16mf2x2_t;
	207	typedef __rvv_uint16mf2x2_t vuint16mf2x2_t;
	208	typedef __rvv_int16mf2x3_t vint16mf2x3_t;
	209	typedef __rvv_uint16mf2x3_t vuint16mf2x3_t;
	210	typedef __rvv_int16mf2x4_t vint16mf2x4_t;
	211	typedef __rvv_uint16mf2x4_t vuint16mf2x4_t;
	212	typedef __rvv_int16mf2x5_t vint16mf2x5_t;
	213	typedef __rvv_uint16mf2x5_t vuint16mf2x5_t;
	214	typedef __rvv_int16mf2x6_t vint16mf2x6_t;
	215	typedef __rvv_uint16mf2x6_t vuint16mf2x6_t;
	216	typedef __rvv_int16mf2x7_t vint16mf2x7_t;
	217	typedef __rvv_uint16mf2x7_t vuint16mf2x7_t;
	218	typedef __rvv_int16mf2x8_t vint16mf2x8_t;
	219	typedef __rvv_uint16mf2x8_t vuint16mf2x8_t;
158	220	typedef __rvv_int16m1_t vint16m1_t;
159	221	typedef __rvv_uint16m1_t vuint16m1_t;
	222	typedef __rvv_int16m1x2_t vint16m1x2_t;
	223	typedef __rvv_uint16m1x2_t vuint16m1x2_t;
	224	typedef __rvv_int16m1x3_t vint16m1x3_t;
	225	typedef __rvv_uint16m1x3_t vuint16m1x3_t;
	226	typedef __rvv_int16m1x4_t vint16m1x4_t;
	227	typedef __rvv_uint16m1x4_t vuint16m1x4_t;
	228	typedef __rvv_int16m1x5_t vint16m1x5_t;
	229	typedef __rvv_uint16m1x5_t vuint16m1x5_t;
	230	typedef __rvv_int16m1x6_t vint16m1x6_t;
	231	typedef __rvv_uint16m1x6_t vuint16m1x6_t;
	232	typedef __rvv_int16m1x7_t vint16m1x7_t;
	233	typedef __rvv_uint16m1x7_t vuint16m1x7_t;
	234	typedef __rvv_int16m1x8_t vint16m1x8_t;
	235	typedef __rvv_uint16m1x8_t vuint16m1x8_t;
160	236	typedef __rvv_int16m2_t vint16m2_t;
161	237	typedef __rvv_uint16m2_t vuint16m2_t;
	238	typedef __rvv_int16m2x2_t vint16m2x2_t;
	239	typedef __rvv_uint16m2x2_t vuint16m2x2_t;
	240	typedef __rvv_int16m2x3_t vint16m2x3_t;
	241	typedef __rvv_uint16m2x3_t vuint16m2x3_t;
	242	typedef __rvv_int16m2x4_t vint16m2x4_t;
	243	typedef __rvv_uint16m2x4_t vuint16m2x4_t;
162	244	typedef __rvv_int16m4_t vint16m4_t;
163	245	typedef __rvv_uint16m4_t vuint16m4_t;
	246	typedef __rvv_int16m4x2_t vint16m4x2_t;
	247	typedef __rvv_uint16m4x2_t vuint16m4x2_t;
164	248	typedef __rvv_int16m8_t vint16m8_t;
165	249	typedef __rvv_uint16m8_t vuint16m8_t;
166	250	typedef __rvv_int32mf2_t vint32mf2_t;
167	251	typedef __rvv_uint32mf2_t vuint32mf2_t;
	252	typedef __rvv_int32mf2x2_t vint32mf2x2_t;
	253	typedef __rvv_uint32mf2x2_t vuint32mf2x2_t;
	254	typedef __rvv_int32mf2x3_t vint32mf2x3_t;
	255	typedef __rvv_uint32mf2x3_t vuint32mf2x3_t;
	256	typedef __rvv_int32mf2x4_t vint32mf2x4_t;
	257	typedef __rvv_uint32mf2x4_t vuint32mf2x4_t;
	258	typedef __rvv_int32mf2x5_t vint32mf2x5_t;
	259	typedef __rvv_uint32mf2x5_t vuint32mf2x5_t;
	260	typedef __rvv_int32mf2x6_t vint32mf2x6_t;
	261	typedef __rvv_uint32mf2x6_t vuint32mf2x6_t;
	262	typedef __rvv_int32mf2x7_t vint32mf2x7_t;
	263	typedef __rvv_uint32mf2x7_t vuint32mf2x7_t;
	264	typedef __rvv_int32mf2x8_t vint32mf2x8_t;
	265	typedef __rvv_uint32mf2x8_t vuint32mf2x8_t;
168	266	typedef __rvv_int32m1_t vint32m1_t;
169	267	typedef __rvv_uint32m1_t vuint32m1_t;
	268	typedef __rvv_int32m1x2_t vint32m1x2_t;
	269	typedef __rvv_uint32m1x2_t vuint32m1x2_t;
	270	typedef __rvv_int32m1x3_t vint32m1x3_t;
	271	typedef __rvv_uint32m1x3_t vuint32m1x3_t;
	272	typedef __rvv_int32m1x4_t vint32m1x4_t;
	273	typedef __rvv_uint32m1x4_t vuint32m1x4_t;
	274	typedef __rvv_int32m1x5_t vint32m1x5_t;
	275	typedef __rvv_uint32m1x5_t vuint32m1x5_t;
	276	typedef __rvv_int32m1x6_t vint32m1x6_t;
	277	typedef __rvv_uint32m1x6_t vuint32m1x6_t;
	278	typedef __rvv_int32m1x7_t vint32m1x7_t;
	279	typedef __rvv_uint32m1x7_t vuint32m1x7_t;
	280	typedef __rvv_int32m1x8_t vint32m1x8_t;
	281	typedef __rvv_uint32m1x8_t vuint32m1x8_t;
170	282	typedef __rvv_int32m2_t vint32m2_t;
171	283	typedef __rvv_uint32m2_t vuint32m2_t;
	284	typedef __rvv_int32m2x2_t vint32m2x2_t;
	285	typedef __rvv_uint32m2x2_t vuint32m2x2_t;
	286	typedef __rvv_int32m2x3_t vint32m2x3_t;
	287	typedef __rvv_uint32m2x3_t vuint32m2x3_t;
	288	typedef __rvv_int32m2x4_t vint32m2x4_t;
	289	typedef __rvv_uint32m2x4_t vuint32m2x4_t;
172	290	typedef __rvv_int32m4_t vint32m4_t;
173	291	typedef __rvv_uint32m4_t vuint32m4_t;
	292	typedef __rvv_int32m4x2_t vint32m4x2_t;
	293	typedef __rvv_uint32m4x2_t vuint32m4x2_t;
174	294	typedef __rvv_int32m8_t vint32m8_t;
175	295	typedef __rvv_uint32m8_t vuint32m8_t;
176	296	typedef __rvv_int64m1_t vint64m1_t;
177	297	typedef __rvv_uint64m1_t vuint64m1_t;
	298	typedef __rvv_int64m1x2_t vint64m1x2_t;
	299	typedef __rvv_uint64m1x2_t vuint64m1x2_t;
	300	typedef __rvv_int64m1x3_t vint64m1x3_t;
	301	typedef __rvv_uint64m1x3_t vuint64m1x3_t;
	302	typedef __rvv_int64m1x4_t vint64m1x4_t;
	303	typedef __rvv_uint64m1x4_t vuint64m1x4_t;
	304	typedef __rvv_int64m1x5_t vint64m1x5_t;
	305	typedef __rvv_uint64m1x5_t vuint64m1x5_t;
	306	typedef __rvv_int64m1x6_t vint64m1x6_t;
	307	typedef __rvv_uint64m1x6_t vuint64m1x6_t;
	308	typedef __rvv_int64m1x7_t vint64m1x7_t;
	309	typedef __rvv_uint64m1x7_t vuint64m1x7_t;
	310	typedef __rvv_int64m1x8_t vint64m1x8_t;
	311	typedef __rvv_uint64m1x8_t vuint64m1x8_t;
178	312	typedef __rvv_int64m2_t vint64m2_t;
179	313	typedef __rvv_uint64m2_t vuint64m2_t;
	314	typedef __rvv_int64m2x2_t vint64m2x2_t;
	315	typedef __rvv_uint64m2x2_t vuint64m2x2_t;
	316	typedef __rvv_int64m2x3_t vint64m2x3_t;
	317	typedef __rvv_uint64m2x3_t vuint64m2x3_t;
	318	typedef __rvv_int64m2x4_t vint64m2x4_t;
	319	typedef __rvv_uint64m2x4_t vuint64m2x4_t;
180	320	typedef __rvv_int64m4_t vint64m4_t;
181	321	typedef __rvv_uint64m4_t vuint64m4_t;
	322	typedef __rvv_int64m4x2_t vint64m4x2_t;
	323	typedef __rvv_uint64m4x2_t vuint64m4x2_t;
182	324	typedef __rvv_int64m8_t vint64m8_t;
183	325	typedef __rvv_uint64m8_t vuint64m8_t;
184		#if defined(__riscv_zvfh)
185	326	typedef __rvv_float16mf4_t vfloat16mf4_t;
	327	typedef __rvv_float16mf4x2_t vfloat16mf4x2_t;
	328	typedef __rvv_float16mf4x3_t vfloat16mf4x3_t;
	329	typedef __rvv_float16mf4x4_t vfloat16mf4x4_t;
	330	typedef __rvv_float16mf4x5_t vfloat16mf4x5_t;
	331	typedef __rvv_float16mf4x6_t vfloat16mf4x6_t;
	332	typedef __rvv_float16mf4x7_t vfloat16mf4x7_t;
	333	typedef __rvv_float16mf4x8_t vfloat16mf4x8_t;
186	334	typedef __rvv_float16mf2_t vfloat16mf2_t;
	335	typedef __rvv_float16mf2x2_t vfloat16mf2x2_t;
	336	typedef __rvv_float16mf2x3_t vfloat16mf2x3_t;
	337	typedef __rvv_float16mf2x4_t vfloat16mf2x4_t;
	338	typedef __rvv_float16mf2x5_t vfloat16mf2x5_t;
	339	typedef __rvv_float16mf2x6_t vfloat16mf2x6_t;
	340	typedef __rvv_float16mf2x7_t vfloat16mf2x7_t;
	341	typedef __rvv_float16mf2x8_t vfloat16mf2x8_t;
187	342	typedef __rvv_float16m1_t vfloat16m1_t;
	343	typedef __rvv_float16m1x2_t vfloat16m1x2_t;
	344	typedef __rvv_float16m1x3_t vfloat16m1x3_t;
	345	typedef __rvv_float16m1x4_t vfloat16m1x4_t;
	346	typedef __rvv_float16m1x5_t vfloat16m1x5_t;
	347	typedef __rvv_float16m1x6_t vfloat16m1x6_t;
	348	typedef __rvv_float16m1x7_t vfloat16m1x7_t;
	349	typedef __rvv_float16m1x8_t vfloat16m1x8_t;
188	350	typedef __rvv_float16m2_t vfloat16m2_t;
	351	typedef __rvv_float16m2x2_t vfloat16m2x2_t;
	352	typedef __rvv_float16m2x3_t vfloat16m2x3_t;
	353	typedef __rvv_float16m2x4_t vfloat16m2x4_t;
189	354	typedef __rvv_float16m4_t vfloat16m4_t;
	355	typedef __rvv_float16m4x2_t vfloat16m4x2_t;
190	356	typedef __rvv_float16m8_t vfloat16m8_t;
191		#endif
192		#if (__riscv_v_elen_fp >= 32)
193	357	typedef __rvv_float32mf2_t vfloat32mf2_t;
	358	typedef __rvv_float32mf2x2_t vfloat32mf2x2_t;
	359	typedef __rvv_float32mf2x3_t vfloat32mf2x3_t;
	360	typedef __rvv_float32mf2x4_t vfloat32mf2x4_t;
	361	typedef __rvv_float32mf2x5_t vfloat32mf2x5_t;
	362	typedef __rvv_float32mf2x6_t vfloat32mf2x6_t;
	363	typedef __rvv_float32mf2x7_t vfloat32mf2x7_t;
	364	typedef __rvv_float32mf2x8_t vfloat32mf2x8_t;
194	365	typedef __rvv_float32m1_t vfloat32m1_t;
	366	typedef __rvv_float32m1x2_t vfloat32m1x2_t;
	367	typedef __rvv_float32m1x3_t vfloat32m1x3_t;
	368	typedef __rvv_float32m1x4_t vfloat32m1x4_t;
	369	typedef __rvv_float32m1x5_t vfloat32m1x5_t;
	370	typedef __rvv_float32m1x6_t vfloat32m1x6_t;
	371	typedef __rvv_float32m1x7_t vfloat32m1x7_t;
	372	typedef __rvv_float32m1x8_t vfloat32m1x8_t;
195	373	typedef __rvv_float32m2_t vfloat32m2_t;
	374	typedef __rvv_float32m2x2_t vfloat32m2x2_t;
	375	typedef __rvv_float32m2x3_t vfloat32m2x3_t;
	376	typedef __rvv_float32m2x4_t vfloat32m2x4_t;
196	377	typedef __rvv_float32m4_t vfloat32m4_t;
	378	typedef __rvv_float32m4x2_t vfloat32m4x2_t;
197	379	typedef __rvv_float32m8_t vfloat32m8_t;
198		#endif
199		#if (__riscv_v_elen_fp >= 64)
200	380	typedef __rvv_float64m1_t vfloat64m1_t;
	381	typedef __rvv_float64m1x2_t vfloat64m1x2_t;
	382	typedef __rvv_float64m1x3_t vfloat64m1x3_t;
	383	typedef __rvv_float64m1x4_t vfloat64m1x4_t;
	384	typedef __rvv_float64m1x5_t vfloat64m1x5_t;
	385	typedef __rvv_float64m1x6_t vfloat64m1x6_t;
	386	typedef __rvv_float64m1x7_t vfloat64m1x7_t;
	387	typedef __rvv_float64m1x8_t vfloat64m1x8_t;
201	388	typedef __rvv_float64m2_t vfloat64m2_t;
	389	typedef __rvv_float64m2x2_t vfloat64m2x2_t;
	390	typedef __rvv_float64m2x3_t vfloat64m2x3_t;
	391	typedef __rvv_float64m2x4_t vfloat64m2x4_t;
202	392	typedef __rvv_float64m4_t vfloat64m4_t;
	393	typedef __rvv_float64m4x2_t vfloat64m4x2_t;
203	394	typedef __rvv_float64m8_t vfloat64m8_t;
204		#endif
205
206	395	#define __riscv_v_intrinsic_overloading 1
207	396
208	397	#ifdef __cplusplus

lib/include/sha512intrin.h created+200

...	...	@@ -0,0 +1,200 @@
	1	/*===--------------- sha512intrin.h - SHA512 intrinsics -----------------===
	2	*
	3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	* See https://llvm.org/LICENSE.txt for license information.
	5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	*
	7	*===-----------------------------------------------------------------------===
	8	*/
	9
	10	#ifndef __IMMINTRIN_H
	11	#error "Never use <sha512intrin.h> directly; include <immintrin.h> instead."
	12	#endif // __IMMINTRIN_H
	13
	14	#ifndef __SHA512INTRIN_H
	15	#define __SHA512INTRIN_H
	16
	17	#define __DEFAULT_FN_ATTRS256 \
	18	__attribute__((__always_inline__, __nodebug__, __target__("sha512"), \
	19	__min_vector_width__(256)))
	20
	21	/// This intrinisc is one of the two SHA512 message scheduling instructions.
	22	/// The intrinsic performs an intermediate calculation for the next four
	23	/// SHA512 message qwords. The calculated results are stored in \a dst.
	24	///
	25	/// \headerfile <immintrin.h>
	26	///
	27	/// \code
	28	/// __m256i _mm256_sha512msg1_epi64(__m256i __A, __m128i __B)
	29	/// \endcode
	30	///
	31	/// This intrinsic corresponds to the \c VSHA512MSG1 instruction.
	32	///
	33	/// \param __A
	34	/// A 256-bit vector of [4 x long long].
	35	/// \param __B
	36	/// A 128-bit vector of [2 x long long].
	37	/// \returns
	38	/// A 256-bit vector of [4 x long long].
	39	///
	40	/// \code{.operation}
	41	/// DEFINE ROR64(qword, n) {
	42	/// count := n % 64
	43	/// dest := (qword >> count) \| (qword << (64 - count))
	44	/// RETURN dest
	45	/// }
	46	/// DEFINE SHR64(qword, n) {
	47	/// RETURN qword >> n
	48	/// }
	49	/// DEFINE s0(qword):
	50	/// RETURN ROR64(qword,1) ^ ROR64(qword, 8) ^ SHR64(qword, 7)
	51	/// }
	52	/// W[4] := __B.qword[0]
	53	/// W[3] := __A.qword[3]
	54	/// W[2] := __A.qword[2]
	55	/// W[1] := __A.qword[1]
	56	/// W[0] := __A.qword[0]
	57	/// dst.qword[3] := W[3] + s0(W[4])
	58	/// dst.qword[2] := W[2] + s0(W[3])
	59	/// dst.qword[1] := W[1] + s0(W[2])
	60	/// dst.qword[0] := W[0] + s0(W[1])
	61	/// dst[MAX:256] := 0
	62	/// \endcode
	63	static __inline__ __m256i __DEFAULT_FN_ATTRS256
	64	_mm256_sha512msg1_epi64(__m256i __A, __m128i __B) {
	65	return (__m256i)__builtin_ia32_vsha512msg1((__v4du)__A, (__v2du)__B);
	66	}
	67
	68	/// This intrinisc is one of the two SHA512 message scheduling instructions.
	69	/// The intrinsic performs the final calculation for the next four SHA512
	70	/// message qwords. The calculated results are stored in \a dst.
	71	///
	72	/// \headerfile <immintrin.h>
	73	///
	74	/// \code
	75	/// __m256i _mm256_sha512msg2_epi64(__m256i __A, __m256i __B)
	76	/// \endcode
	77	///
	78	/// This intrinsic corresponds to the \c VSHA512MSG2 instruction.
	79	///
	80	/// \param __A
	81	/// A 256-bit vector of [4 x long long].
	82	/// \param __B
	83	/// A 256-bit vector of [4 x long long].
	84	/// \returns
	85	/// A 256-bit vector of [4 x long long].
	86	///
	87	/// \code{.operation}
	88	/// DEFINE ROR64(qword, n) {
	89	/// count := n % 64
	90	/// dest := (qword >> count) \| (qword << (64 - count))
	91	/// RETURN dest
	92	/// }
	93	/// DEFINE SHR64(qword, n) {
	94	/// RETURN qword >> n
	95	/// }
	96	/// DEFINE s1(qword) {
	97	/// RETURN ROR64(qword,19) ^ ROR64(qword, 61) ^ SHR64(qword, 6)
	98	/// }
	99	/// W[14] := __B.qword[2]
	100	/// W[15] := __B.qword[3]
	101	/// W[16] := __A.qword[0] + s1(W[14])
	102	/// W[17] := __A.qword[1] + s1(W[15])
	103	/// W[18] := __A.qword[2] + s1(W[16])
	104	/// W[19] := __A.qword[3] + s1(W[17])
	105	/// dst.qword[3] := W[19]
	106	/// dst.qword[2] := W[18]
	107	/// dst.qword[1] := W[17]
	108	/// dst.qword[0] := W[16]
	109	/// dst[MAX:256] := 0
	110	/// \endcode
	111	static __inline__ __m256i __DEFAULT_FN_ATTRS256
	112	_mm256_sha512msg2_epi64(__m256i __A, __m256i __B) {
	113	return (__m256i)__builtin_ia32_vsha512msg2((__v4du)__A, (__v4du)__B);
	114	}
	115
	116	/// This intrinisc performs two rounds of SHA512 operation using initial SHA512
	117	/// state (C,D,G,H) from \a __A, an initial SHA512 state (A,B,E,F) from
	118	/// \a __A, and a pre-computed sum of the next two round message qwords and
	119	/// the corresponding round constants from \a __C (only the two lower qwords
	120	/// of the third operand). The updated SHA512 state (A,B,E,F) is written to
	121	/// \a __A, and \a __A can be used as the updated state (C,D,G,H) in later
	122	/// rounds.
	123	///
	124	/// \headerfile <immintrin.h>
	125	///
	126	/// \code
	127	/// __m256i _mm256_sha512rnds2_epi64(__m256i __A, __m256i __B, __m128i __C)
	128	/// \endcode
	129	///
	130	/// This intrinsic corresponds to the \c VSHA512RNDS2 instruction.
	131	///
	132	/// \param __A
	133	/// A 256-bit vector of [4 x long long].
	134	/// \param __B
	135	/// A 256-bit vector of [4 x long long].
	136	/// \param __C
	137	/// A 128-bit vector of [2 x long long].
	138	/// \returns
	139	/// A 256-bit vector of [4 x long long].
	140	///
	141	/// \code{.operation}
	142	/// DEFINE ROR64(qword, n) {
	143	/// count := n % 64
	144	/// dest := (qword >> count) \| (qword << (64 - count))
	145	/// RETURN dest
	146	/// }
	147	/// DEFINE SHR64(qword, n) {
	148	/// RETURN qword >> n
	149	/// }
	150	/// DEFINE cap_sigma0(qword) {
	151	/// RETURN ROR64(qword,28) ^ ROR64(qword, 34) ^ ROR64(qword, 39)
	152	/// }
	153	/// DEFINE cap_sigma1(qword) {
	154	/// RETURN ROR64(qword,14) ^ ROR64(qword, 18) ^ ROR64(qword, 41)
	155	/// }
	156	/// DEFINE MAJ(a,b,c) {
	157	/// RETURN (a & b) ^ (a & c) ^ (b & c)
	158	/// }
	159	/// DEFINE CH(e,f,g) {
	160	/// RETURN (e & f) ^ (g & ~e)
	161	/// }
	162	/// A[0] := __B.qword[3]
	163	/// B[0] := __B.qword[2]
	164	/// C[0] := __C.qword[3]
	165	/// D[0] := __C.qword[2]
	166	/// E[0] := __B.qword[1]
	167	/// F[0] := __B.qword[0]
	168	/// G[0] := __C.qword[1]
	169	/// H[0] := __C.qword[0]
	170	/// WK[0]:= __A.qword[0]
	171	/// WK[1]:= __A.qword[1]
	172	/// FOR i := 0 to 1:
	173	/// A[i+1] := CH(E[i], F[i], G[i]) +
	174	/// cap_sigma1(E[i]) + WK[i] + H[i] +
	175	/// MAJ(A[i], B[i], C[i]) +
	176	/// cap_sigma0(A[i])
	177	/// B[i+1] := A[i]
	178	/// C[i+1] := B[i]
	179	/// D[i+1] := C[i]
	180	/// E[i+1] := CH(E[i], F[i], G[i]) +
	181	/// cap_sigma1(E[i]) + WK[i] + H[i] + D[i]
	182	/// F[i+1] := E[i]
	183	/// G[i+1] := F[i]
	184	/// H[i+1] := G[i]
	185	/// ENDFOR
	186	/// dst.qword[3] := A[2]
	187	/// dst.qword[2] := B[2]
	188	/// dst.qword[1] := E[2]
	189	/// dst.qword[0] := F[2]
	190	/// dst[MAX:256] := 0
	191	/// \endcode
	192	static __inline__ __m256i __DEFAULT_FN_ATTRS256
	193	_mm256_sha512rnds2_epi64(__m256i __A, __m256i __B, __m128i __C) {
	194	return (__m256i)__builtin_ia32_vsha512rnds2((__v4du)__A, (__v4du)__B,
	195	(__v2du)__C);
	196	}
	197
	198	#undef __DEFAULT_FN_ATTRS256
	199
	200	#endif // __SHA512INTRIN_H

lib/include/shaintrin.h+128

...	...	@@ -17,39 +17,167 @@
17	17	/* Define the default attributes for the functions in this file. */
18	18	#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("sha"), __min_vector_width__(128)))
19	19
	20	/// Performs four iterations of the inner loop of the SHA-1 message digest
	21	/// algorithm using the starting SHA-1 state (A, B, C, D) from the 128-bit
	22	/// vector of [4 x i32] in \a V1 and the next four 32-bit elements of the
	23	/// message from the 128-bit vector of [4 x i32] in \a V2. Note that the
	24	/// SHA-1 state variable E must have already been added to \a V2
	25	/// (\c _mm_sha1nexte_epu32() can perform this step). Returns the updated
	26	/// SHA-1 state (A, B, C, D) as a 128-bit vector of [4 x i32].
	27	///
	28	/// The SHA-1 algorithm has an inner loop of 80 iterations, twenty each
	29	/// with a different combining function and rounding constant. This
	30	/// intrinsic performs four iterations using a combining function and
	31	/// rounding constant selected by \a M[1:0].
	32	///
	33	/// \headerfile <immintrin.h>
	34	///
	35	/// \code
	36	/// __m128i _mm_sha1rnds4_epu32(__m128i V1, __m128i V2, const int M);
	37	/// \endcode
	38	///
	39	/// This intrinsic corresponds to the \c SHA1RNDS4 instruction.
	40	///
	41	/// \param V1
	42	/// A 128-bit vector of [4 x i32] containing the initial SHA-1 state.
	43	/// \param V2
	44	/// A 128-bit vector of [4 x i32] containing the next four elements of
	45	/// the message, plus SHA-1 state variable E.
	46	/// \param M
	47	/// An immediate value where bits [1:0] select among four possible
	48	/// combining functions and rounding constants (not specified here).
	49	/// \returns A 128-bit vector of [4 x i32] containing the updated SHA-1 state.
20	50	#define _mm_sha1rnds4_epu32(V1, V2, M) \
21	51	__builtin_ia32_sha1rnds4((__v4si)(__m128i)(V1), (__v4si)(__m128i)(V2), (M))
22	52
	53	/// Calculates the SHA-1 state variable E from the SHA-1 state variables in
	54	/// the 128-bit vector of [4 x i32] in \a __X, adds that to the next set of
	55	/// four message elements in the 128-bit vector of [4 x i32] in \a __Y, and
	56	/// returns the result.
	57	///
	58	/// \headerfile <immintrin.h>
	59	///
	60	/// This intrinsic corresponds to the \c SHA1NEXTE instruction.
	61	///
	62	/// \param __X
	63	/// A 128-bit vector of [4 x i32] containing the current SHA-1 state.
	64	/// \param __Y
	65	/// A 128-bit vector of [4 x i32] containing the next four elements of the
	66	/// message.
	67	/// \returns A 128-bit vector of [4 x i32] containing the updated SHA-1
	68	/// values.
23	69	static __inline__ __m128i __DEFAULT_FN_ATTRS
24	70	_mm_sha1nexte_epu32(__m128i __X, __m128i __Y)
25	71	{
26	72	return (__m128i)__builtin_ia32_sha1nexte((__v4si)__X, (__v4si)__Y);
27	73	}
28	74
	75	/// Performs an intermediate calculation for deriving the next four SHA-1
	76	/// message elements using previous message elements from the 128-bit
	77	/// vectors of [4 x i32] in \a __X and \a __Y, and returns the result.
	78	///
	79	/// \headerfile <immintrin.h>
	80	///
	81	/// This intrinsic corresponds to the \c SHA1MSG1 instruction.
	82	///
	83	/// \param __X
	84	/// A 128-bit vector of [4 x i32] containing previous message elements.
	85	/// \param __Y
	86	/// A 128-bit vector of [4 x i32] containing previous message elements.
	87	/// \returns A 128-bit vector of [4 x i32] containing the derived SHA-1
	88	/// elements.
29	89	static __inline__ __m128i __DEFAULT_FN_ATTRS
30	90	_mm_sha1msg1_epu32(__m128i __X, __m128i __Y)
31	91	{
32	92	return (__m128i)__builtin_ia32_sha1msg1((__v4si)__X, (__v4si)__Y);
33	93	}
34	94
	95	/// Performs the final calculation for deriving the next four SHA-1 message
	96	/// elements using previous message elements from the 128-bit vectors of
	97	/// [4 x i32] in \a __X and \a __Y, and returns the result.
	98	///
	99	/// \headerfile <immintrin.h>
	100	///
	101	/// This intrinsic corresponds to the \c SHA1MSG2 instruction.
	102	///
	103	/// \param __X
	104	/// A 128-bit vector of [4 x i32] containing an intermediate result.
	105	/// \param __Y
	106	/// A 128-bit vector of [4 x i32] containing previous message values.
	107	/// \returns A 128-bit vector of [4 x i32] containing the updated SHA-1
	108	/// values.
35	109	static __inline__ __m128i __DEFAULT_FN_ATTRS
36	110	_mm_sha1msg2_epu32(__m128i __X, __m128i __Y)
37	111	{
38	112	return (__m128i)__builtin_ia32_sha1msg2((__v4si)__X, (__v4si)__Y);
39	113	}
40	114
	115	/// Performs two rounds of SHA-256 operation using the following inputs: a
	116	/// starting SHA-256 state (C, D, G, H) from the 128-bit vector of
	117	/// [4 x i32] in \a __X; a starting SHA-256 state (A, B, E, F) from the
	118	/// 128-bit vector of [4 x i32] in \a __Y; and a pre-computed sum of the
	119	/// next two message elements (unsigned 32-bit integers) and corresponding
	120	/// rounding constants from the 128-bit vector of [4 x i32] in \a __Z.
	121	/// Returns the updated SHA-256 state (A, B, E, F) as a 128-bit vector of
	122	/// [4 x i32].
	123	///
	124	/// The SHA-256 algorithm has a core loop of 64 iterations. This intrinsic
	125	/// performs two of those iterations.
	126	///
	127	/// \headerfile <immintrin.h>
	128	///
	129	/// This intrinsic corresponds to the \c SHA256RNDS2 instruction.
	130	///
	131	/// \param __X
	132	/// A 128-bit vector of [4 x i32] containing part of the initial SHA-256
	133	/// state.
	134	/// \param __Y
	135	/// A 128-bit vector of [4 x i32] containing part of the initial SHA-256
	136	/// state.
	137	/// \param __Z
	138	/// A 128-bit vector of [4 x i32] containing additional input to the
	139	/// SHA-256 operation.
	140	/// \returns A 128-bit vector of [4 x i32] containing the updated SHA-1 state.
41	141	static __inline__ __m128i __DEFAULT_FN_ATTRS
42	142	_mm_sha256rnds2_epu32(__m128i __X, __m128i __Y, __m128i __Z)
43	143	{
44	144	return (__m128i)__builtin_ia32_sha256rnds2((__v4si)__X, (__v4si)__Y, (__v4si)__Z);
45	145	}
46	146
	147	/// Performs an intermediate calculation for deriving the next four SHA-256
	148	/// message elements using previous message elements from the 128-bit
	149	/// vectors of [4 x i32] in \a __X and \a __Y, and returns the result.
	150	///
	151	/// \headerfile <immintrin.h>
	152	///
	153	/// This intrinsic corresponds to the \c SHA256MSG1 instruction.
	154	///
	155	/// \param __X
	156	/// A 128-bit vector of [4 x i32] containing previous message elements.
	157	/// \param __Y
	158	/// A 128-bit vector of [4 x i32] containing previous message elements.
	159	/// \returns A 128-bit vector of [4 x i32] containing the updated SHA-256
	160	/// values.
47	161	static __inline__ __m128i __DEFAULT_FN_ATTRS
48	162	_mm_sha256msg1_epu32(__m128i __X, __m128i __Y)
49	163	{
50	164	return (__m128i)__builtin_ia32_sha256msg1((__v4si)__X, (__v4si)__Y);
51	165	}
52	166
	167	/// Performs the final calculation for deriving the next four SHA-256 message
	168	/// elements using previous message elements from the 128-bit vectors of
	169	/// [4 x i32] in \a __X and \a __Y, and returns the result.
	170	///
	171	/// \headerfile <immintrin.h>
	172	///
	173	/// This intrinsic corresponds to the \c SHA256MSG2 instruction.
	174	///
	175	/// \param __X
	176	/// A 128-bit vector of [4 x i32] containing an intermediate result.
	177	/// \param __Y
	178	/// A 128-bit vector of [4 x i32] containing previous message values.
	179	/// \returns A 128-bit vector of [4 x i32] containing the updated SHA-256
	180	/// values.
53	181	static __inline__ __m128i __DEFAULT_FN_ATTRS
54	182	_mm_sha256msg2_epu32(__m128i __X, __m128i __Y)
55	183	{

lib/include/sifive_vector.h created+16

...	...	@@ -0,0 +1,16 @@
	1	//===----- sifive_vector.h - SiFive Vector definitions --------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _SIFIVE_VECTOR_H_
	10	#define _SIFIVE_VECTOR_H_
	11
	12	#include "riscv_vector.h"
	13
	14	#pragma clang riscv intrinsic sifive_vector
	15
	16	#endif //_SIFIVE_VECTOR_H_

lib/include/sm3intrin.h created+238

...	...	@@ -0,0 +1,238 @@
	1	/*===-------------------- sm3intrin.h - SM3 intrinsics ---------------------===
	2	*
	3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	* See https://llvm.org/LICENSE.txt for license information.
	5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	*
	7	*===-----------------------------------------------------------------------===
	8	*/
	9
	10	#ifndef __IMMINTRIN_H
	11	#error "Never use <sm3intrin.h> directly; include <immintrin.h> instead."
	12	#endif // __IMMINTRIN_H
	13
	14	#ifndef __SM3INTRIN_H
	15	#define __SM3INTRIN_H
	16
	17	#define __DEFAULT_FN_ATTRS128 \
	18	__attribute__((__always_inline__, __nodebug__, __target__("sm3"), \
	19	__min_vector_width__(128)))
	20
	21	/// This intrinisc is one of the two SM3 message scheduling intrinsics. The
	22	/// intrinsic performs an initial calculation for the next four SM3 message
	23	/// words. The calculated results are stored in \a dst.
	24	///
	25	/// \headerfile <immintrin.h>
	26	///
	27	/// \code
	28	/// __m128i _mm_sm3msg1_epi32(__m128i __A, __m128i __B, __m128i __C)
	29	/// \endcode
	30	///
	31	/// This intrinsic corresponds to the \c VSM3MSG1 instruction.
	32	///
	33	/// \param __A
	34	/// A 128-bit vector of [4 x int].
	35	/// \param __B
	36	/// A 128-bit vector of [4 x int].
	37	/// \param __C
	38	/// A 128-bit vector of [4 x int].
	39	/// \returns
	40	/// A 128-bit vector of [4 x int].
	41	///
	42	/// \code{.operation}
	43	/// DEFINE ROL32(dword, n) {
	44	/// count := n % 32
	45	/// dest := (dword << count) \| (dword >> (32 - count))
	46	/// RETURN dest
	47	/// }
	48	/// DEFINE P1(x) {
	49	/// RETURN x ^ ROL32(x, 15) ^ ROL32(x, 23)
	50	/// }
	51	/// W[0] := __C.dword[0]
	52	/// W[1] := __C.dword[1]
	53	/// W[2] := __C.dword[2]
	54	/// W[3] := __C.dword[3]
	55	/// W[7] := __A.dword[0]
	56	/// W[8] := __A.dword[1]
	57	/// W[9] := __A.dword[2]
	58	/// W[10] := __A.dword[3]
	59	/// W[13] := __B.dword[0]
	60	/// W[14] := __B.dword[1]
	61	/// W[15] := __B.dword[2]
	62	/// TMP0 := W[7] ^ W[0] ^ ROL32(W[13], 15)
	63	/// TMP1 := W[8] ^ W[1] ^ ROL32(W[14], 15)
	64	/// TMP2 := W[9] ^ W[2] ^ ROL32(W[15], 15)
	65	/// TMP3 := W[10] ^ W[3]
	66	/// dst.dword[0] := P1(TMP0)
	67	/// dst.dword[1] := P1(TMP1)
	68	/// dst.dword[2] := P1(TMP2)
	69	/// dst.dword[3] := P1(TMP3)
	70	/// dst[MAX:128] := 0
	71	/// \endcode
	72	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_sm3msg1_epi32(__m128i __A,
	73	__m128i __B,
	74	__m128i __C) {
	75	return (__m128i)__builtin_ia32_vsm3msg1((__v4su)__A, (__v4su)__B,
	76	(__v4su)__C);
	77	}
	78
	79	/// This intrinisc is one of the two SM3 message scheduling intrinsics. The
	80	/// intrinsic performs the final calculation for the next four SM3 message
	81	/// words. The calculated results are stored in \a dst.
	82	///
	83	/// \headerfile <immintrin.h>
	84	///
	85	/// \code
	86	/// __m128i _mm_sm3msg2_epi32(__m128i __A, __m128i __B, __m128i __C)
	87	/// \endcode
	88	///
	89	/// This intrinsic corresponds to the \c VSM3MSG2 instruction.
	90	///
	91	/// \param __A
	92	/// A 128-bit vector of [4 x int].
	93	/// \param __B
	94	/// A 128-bit vector of [4 x int].
	95	/// \param __C
	96	/// A 128-bit vector of [4 x int].
	97	/// \returns
	98	/// A 128-bit vector of [4 x int].
	99	///
	100	/// \code{.operation}
	101	/// DEFINE ROL32(dword, n) {
	102	/// count := n % 32
	103	/// dest := (dword << count) \| (dword >> (32-count))
	104	/// RETURN dest
	105	/// }
	106	/// WTMP[0] := __A.dword[0]
	107	/// WTMP[1] := __A.dword[1]
	108	/// WTMP[2] := __A.dword[2]
	109	/// WTMP[3] := __A.dword[3]
	110	/// W[3] := __B.dword[0]
	111	/// W[4] := __B.dword[1]
	112	/// W[5] := __B.dword[2]
	113	/// W[6] := __B.dword[3]
	114	/// W[10] := __C.dword[0]
	115	/// W[11] := __C.dword[1]
	116	/// W[12] := __C.dword[2]
	117	/// W[13] := __C.dword[3]
	118	/// W[16] := ROL32(W[3], 7) ^ W[10] ^ WTMP[0]
	119	/// W[17] := ROL32(W[4], 7) ^ W[11] ^ WTMP[1]
	120	/// W[18] := ROL32(W[5], 7) ^ W[12] ^ WTMP[2]
	121	/// W[19] := ROL32(W[6], 7) ^ W[13] ^ WTMP[3]
	122	/// W[19] := W[19] ^ ROL32(W[16], 6) ^ ROL32(W[16], 15) ^ ROL32(W[16], 30)
	123	/// dst.dword[0] := W[16]
	124	/// dst.dword[1] := W[17]
	125	/// dst.dword[2] := W[18]
	126	/// dst.dword[3] := W[19]
	127	/// dst[MAX:128] := 0
	128	/// \endcode
	129	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_sm3msg2_epi32(__m128i __A,
	130	__m128i __B,
	131	__m128i __C) {
	132	return (__m128i)__builtin_ia32_vsm3msg2((__v4su)__A, (__v4su)__B,
	133	(__v4su)__C);
	134	}
	135
	136	/// This intrinsic performs two rounds of SM3 operation using initial SM3 state
	137	/// (C, D, G, H) from \a __A, an initial SM3 states (A, B, E, F)
	138	/// from \a __B and a pre-computed words from the \a __C. \a __A with
	139	/// initial SM3 state of (C, D, G, H) assumes input of non-rotated left
	140	/// variables from previous state. The updated SM3 state (A, B, E, F) is
	141	/// written to \a __A. The \a imm8 should contain the even round number
	142	/// for the first of the two rounds computed by this instruction. The
	143	/// computation masks the \a imm8 value by AND’ing it with 0x3E so that only
	144	/// even round numbers from 0 through 62 are used for this operation. The
	145	/// calculated results are stored in \a dst.
	146	///
	147	/// \headerfile <immintrin.h>
	148	///
	149	/// \code
	150	/// __m128i _mm_sm3rnds2_epi32(__m128i __A, __m128i __B, __m128i __C, const int
	151	/// imm8) \endcode
	152	///
	153	/// This intrinsic corresponds to the \c VSM3RNDS2 instruction.
	154	///
	155	/// \param __A
	156	/// A 128-bit vector of [4 x int].
	157	/// \param __B
	158	/// A 128-bit vector of [4 x int].
	159	/// \param __C
	160	/// A 128-bit vector of [4 x int].
	161	/// \param imm8
	162	/// A 8-bit constant integer.
	163	/// \returns
	164	/// A 128-bit vector of [4 x int].
	165	///
	166	/// \code{.operation}
	167	/// DEFINE ROL32(dword, n) {
	168	/// count := n % 32
	169	/// dest := (dword << count) \| (dword >> (32-count))
	170	/// RETURN dest
	171	/// }
	172	/// DEFINE P0(dword) {
	173	/// RETURN dword ^ ROL32(dword, 9) ^ ROL32(dword, 17)
	174	/// }
	175	/// DEFINE FF(x,y,z, round){
	176	/// IF round < 16
	177	/// RETURN (x ^ y ^ z)
	178	/// ELSE
	179	/// RETURN (x & y) \| (x & z) \| (y & z)
	180	/// FI
	181	/// }
	182	/// DEFINE GG(x, y, z, round){
	183	/// IF round < 16
	184	/// RETURN (x ^ y ^ z)
	185	/// ELSE
	186	/// RETURN (x & y) \| (~x & z)
	187	/// FI
	188	/// }
	189	/// A[0] := __B.dword[3]
	190	/// B[0] := __B.dword[2]
	191	/// C[0] := __A.dword[3]
	192	/// D[0] := __A.dword[2]
	193	/// E[0] := __B.dword[1]
	194	/// F[0] := __B.dword[0]
	195	/// G[0] := __A.dword[1]
	196	/// H[0] := __A.dword[0]
	197	/// W[0] := __C.dword[0]
	198	/// W[1] := __C.dword[1]
	199	/// W[4] := __C.dword[2]
	200	/// W[5] := __C.dword[3]
	201	/// C[0] := ROL32(C[0], 9)
	202	/// D[0] := ROL32(D[0], 9)
	203	/// G[0] := ROL32(G[0], 19)
	204	/// H[0] := ROL32(H[0], 19)
	205	/// ROUND := __D & 0x3E
	206	/// IF ROUND < 16
	207	/// CONST := 0x79CC4519
	208	/// ELSE
	209	/// CONST := 0x7A879D8A
	210	/// FI
	211	/// CONST := ROL32(CONST,ROUND)
	212	/// FOR i:= 0 to 1
	213	/// S1 := ROL32((ROL32(A[i], 12) + E[i] + CONST), 7)
	214	/// S2 := S1 ^ ROL32(A[i], 12)
	215	/// T1 := FF(A[i], B[i], C[i], ROUND) + D[i] + S2 + (W[i] ^ W[i+4])
	216	/// T2 := GG(E[i], F[i], G[i], ROUND) + H[i] + S1 + W[i]
	217	/// D[i+1] := C[i]
	218	/// C[i+1] := ROL32(B[i],9)
	219	/// B[i+1] := A[i]
	220	/// A[i+1] := T1
	221	/// H[i+1] := G[i]
	222	/// G[i+1] := ROL32(F[i], 19)
	223	/// F[i+1] := E[i]
	224	/// E[i+1] := P0(T2)
	225	/// CONST := ROL32(CONST, 1)
	226	/// ENDFOR
	227	/// dst.dword[3] := A[2]
	228	/// dst.dword[2] := B[2]
	229	/// dst.dword[1] := E[2]
	230	/// dst.dword[0] := F[2]
	231	/// dst[MAX:128] := 0
	232	/// \endcode
	233	#define _mm_sm3rnds2_epi32(A, B, C, D) \
	234	(__m128i) __builtin_ia32_vsm3rnds2((__v4su)A, (__v4su)B, (__v4su)C, (int)D)
	235
	236	#undef __DEFAULT_FN_ATTRS128
	237
	238	#endif // __SM3INTRIN_H

lib/include/sm4intrin.h created+269

...	...	@@ -0,0 +1,269 @@
	1	/*===--------------- sm4intrin.h - SM4 intrinsics -----------------===
	2	*
	3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	* See https://llvm.org/LICENSE.txt for license information.
	5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	*
	7	*===-----------------------------------------------------------------------===
	8	*/
	9
	10	#ifndef __IMMINTRIN_H
	11	#error "Never use <sm4intrin.h> directly; include <immintrin.h> instead."
	12	#endif // __IMMINTRIN_H
	13
	14	#ifndef __SM4INTRIN_H
	15	#define __SM4INTRIN_H
	16
	17	/// This intrinsic performs four rounds of SM4 key expansion. The intrinsic
	18	/// operates on independent 128-bit lanes. The calculated results are
	19	/// stored in \a dst.
	20	/// \headerfile <immintrin.h>
	21	///
	22	/// \code
	23	/// __m128i _mm_sm4key4_epi32(__m128i __A, __m128i __B)
	24	/// \endcode
	25	///
	26	/// This intrinsic corresponds to the \c VSM4KEY4 instruction.
	27	///
	28	/// \param __A
	29	/// A 128-bit vector of [4 x int].
	30	/// \param __B
	31	/// A 128-bit vector of [4 x int].
	32	/// \returns
	33	/// A 128-bit vector of [4 x int].
	34	///
	35	/// \code{.operation}
	36	/// DEFINE ROL32(dword, n) {
	37	/// count := n % 32
	38	/// dest := (dword << count) \| (dword >> (32-count))
	39	/// RETURN dest
	40	/// }
	41	/// DEFINE SBOX_BYTE(dword, i) {
	42	/// RETURN sbox[dword.byte[i]]
	43	/// }
	44	/// DEFINE lower_t(dword) {
	45	/// tmp.byte[0] := SBOX_BYTE(dword, 0)
	46	/// tmp.byte[1] := SBOX_BYTE(dword, 1)
	47	/// tmp.byte[2] := SBOX_BYTE(dword, 2)
	48	/// tmp.byte[3] := SBOX_BYTE(dword, 3)
	49	/// RETURN tmp
	50	/// }
	51	/// DEFINE L_KEY(dword) {
	52	/// RETURN dword ^ ROL32(dword, 13) ^ ROL32(dword, 23)
	53	/// }
	54	/// DEFINE T_KEY(dword) {
	55	/// RETURN L_KEY(lower_t(dword))
	56	/// }
	57	/// DEFINE F_KEY(X0, X1, X2, X3, round_key) {
	58	/// RETURN X0 ^ T_KEY(X1 ^ X2 ^ X3 ^ round_key)
	59	/// }
	60	/// FOR i:= 0 to 0
	61	/// P[0] := __B.xmm[i].dword[0]
	62	/// P[1] := __B.xmm[i].dword[1]
	63	/// P[2] := __B.xmm[i].dword[2]
	64	/// P[3] := __B.xmm[i].dword[3]
	65	/// C[0] := F_KEY(P[0], P[1], P[2], P[3], __A.xmm[i].dword[0])
	66	/// C[1] := F_KEY(P[1], P[2], P[3], C[0], __A.xmm[i].dword[1])
	67	/// C[2] := F_KEY(P[2], P[3], C[0], C[1], __A.xmm[i].dword[2])
	68	/// C[3] := F_KEY(P[3], C[0], C[1], C[2], __A.xmm[i].dword[3])
	69	/// DEST.xmm[i].dword[0] := C[0]
	70	/// DEST.xmm[i].dword[1] := C[1]
	71	/// DEST.xmm[i].dword[2] := C[2]
	72	/// DEST.xmm[i].dword[3] := C[3]
	73	/// ENDFOR
	74	/// DEST[MAX:128] := 0
	75	/// \endcode
	76	#define _mm_sm4key4_epi32(A, B) \
	77	(__m128i) __builtin_ia32_vsm4key4128((__v4su)A, (__v4su)B)
	78
	79	/// This intrinsic performs four rounds of SM4 key expansion. The intrinsic
	80	/// operates on independent 128-bit lanes. The calculated results are
	81	/// stored in \a dst.
	82	/// \headerfile <immintrin.h>
	83	///
	84	/// \code
	85	/// __m256i _mm256_sm4key4_epi32(__m256i __A, __m256i __B)
	86	/// \endcode
	87	///
	88	/// This intrinsic corresponds to the \c VSM4KEY4 instruction.
	89	///
	90	/// \param __A
	91	/// A 256-bit vector of [8 x int].
	92	/// \param __B
	93	/// A 256-bit vector of [8 x int].
	94	/// \returns
	95	/// A 256-bit vector of [8 x int].
	96	///
	97	/// \code{.operation}
	98	/// DEFINE ROL32(dword, n) {
	99	/// count := n % 32
	100	/// dest := (dword << count) \| (dword >> (32-count))
	101	/// RETURN dest
	102	/// }
	103	/// DEFINE SBOX_BYTE(dword, i) {
	104	/// RETURN sbox[dword.byte[i]]
	105	/// }
	106	/// DEFINE lower_t(dword) {
	107	/// tmp.byte[0] := SBOX_BYTE(dword, 0)
	108	/// tmp.byte[1] := SBOX_BYTE(dword, 1)
	109	/// tmp.byte[2] := SBOX_BYTE(dword, 2)
	110	/// tmp.byte[3] := SBOX_BYTE(dword, 3)
	111	/// RETURN tmp
	112	/// }
	113	/// DEFINE L_KEY(dword) {
	114	/// RETURN dword ^ ROL32(dword, 13) ^ ROL32(dword, 23)
	115	/// }
	116	/// DEFINE T_KEY(dword) {
	117	/// RETURN L_KEY(lower_t(dword))
	118	/// }
	119	/// DEFINE F_KEY(X0, X1, X2, X3, round_key) {
	120	/// RETURN X0 ^ T_KEY(X1 ^ X2 ^ X3 ^ round_key)
	121	/// }
	122	/// FOR i:= 0 to 1
	123	/// P[0] := __B.xmm[i].dword[0]
	124	/// P[1] := __B.xmm[i].dword[1]
	125	/// P[2] := __B.xmm[i].dword[2]
	126	/// P[3] := __B.xmm[i].dword[3]
	127	/// C[0] := F_KEY(P[0], P[1], P[2], P[3], __A.xmm[i].dword[0])
	128	/// C[1] := F_KEY(P[1], P[2], P[3], C[0], __A.xmm[i].dword[1])
	129	/// C[2] := F_KEY(P[2], P[3], C[0], C[1], __A.xmm[i].dword[2])
	130	/// C[3] := F_KEY(P[3], C[0], C[1], C[2], __A.xmm[i].dword[3])
	131	/// DEST.xmm[i].dword[0] := C[0]
	132	/// DEST.xmm[i].dword[1] := C[1]
	133	/// DEST.xmm[i].dword[2] := C[2]
	134	/// DEST.xmm[i].dword[3] := C[3]
	135	/// ENDFOR
	136	/// DEST[MAX:256] := 0
	137	/// \endcode
	138	#define _mm256_sm4key4_epi32(A, B) \
	139	(__m256i) __builtin_ia32_vsm4key4256((__v8su)A, (__v8su)B)
	140
	141	/// This intrinisc performs four rounds of SM4 encryption. The intrinisc
	142	/// operates on independent 128-bit lanes. The calculated results are
	143	/// stored in \a dst.
	144	/// \headerfile <immintrin.h>
	145	///
	146	/// \code
	147	/// __m128i _mm_sm4rnds4_epi32(__m128i __A, __m128i __B)
	148	/// \endcode
	149	///
	150	/// This intrinsic corresponds to the \c VSM4RNDS4 instruction.
	151	///
	152	/// \param __A
	153	/// A 128-bit vector of [4 x int].
	154	/// \param __B
	155	/// A 128-bit vector of [4 x int].
	156	/// \returns
	157	/// A 128-bit vector of [4 x int].
	158	///
	159	/// \code{.operation}
	160	/// DEFINE ROL32(dword, n) {
	161	/// count := n % 32
	162	/// dest := (dword << count) \| (dword >> (32-count))
	163	/// RETURN dest
	164	/// }
	165	/// DEFINE lower_t(dword) {
	166	/// tmp.byte[0] := SBOX_BYTE(dword, 0)
	167	/// tmp.byte[1] := SBOX_BYTE(dword, 1)
	168	/// tmp.byte[2] := SBOX_BYTE(dword, 2)
	169	/// tmp.byte[3] := SBOX_BYTE(dword, 3)
	170	/// RETURN tmp
	171	/// }
	172	/// DEFINE L_RND(dword) {
	173	/// tmp := dword
	174	/// tmp := tmp ^ ROL32(dword, 2)
	175	/// tmp := tmp ^ ROL32(dword, 10)
	176	/// tmp := tmp ^ ROL32(dword, 18)
	177	/// tmp := tmp ^ ROL32(dword, 24)
	178	/// RETURN tmp
	179	/// }
	180	/// DEFINE T_RND(dword) {
	181	/// RETURN L_RND(lower_t(dword))
	182	/// }
	183	/// DEFINE F_RND(X0, X1, X2, X3, round_key) {
	184	/// RETURN X0 ^ T_RND(X1 ^ X2 ^ X3 ^ round_key)
	185	/// }
	186	/// FOR i:= 0 to 0
	187	/// P[0] := __B.xmm[i].dword[0]
	188	/// P[1] := __B.xmm[i].dword[1]
	189	/// P[2] := __B.xmm[i].dword[2]
	190	/// P[3] := __B.xmm[i].dword[3]
	191	/// C[0] := F_RND(P[0], P[1], P[2], P[3], __A.xmm[i].dword[0])
	192	/// C[1] := F_RND(P[1], P[2], P[3], C[0], __A.xmm[i].dword[1])
	193	/// C[2] := F_RND(P[2], P[3], C[0], C[1], __A.xmm[i].dword[2])
	194	/// C[3] := F_RND(P[3], C[0], C[1], C[2], __A.xmm[i].dword[3])
	195	/// DEST.xmm[i].dword[0] := C[0]
	196	/// DEST.xmm[i].dword[1] := C[1]
	197	/// DEST.xmm[i].dword[2] := C[2]
	198	/// DEST.xmm[i].dword[3] := C[3]
	199	/// ENDFOR
	200	/// DEST[MAX:128] := 0
	201	/// \endcode
	202	#define _mm_sm4rnds4_epi32(A, B) \
	203	(__m128i) __builtin_ia32_vsm4rnds4128((__v4su)A, (__v4su)B)
	204
	205	/// This intrinisc performs four rounds of SM4 encryption. The intrinisc
	206	/// operates on independent 128-bit lanes. The calculated results are
	207	/// stored in \a dst.
	208	/// \headerfile <immintrin.h>
	209	///
	210	/// \code
	211	/// __m256i _mm256_sm4rnds4_epi32(__m256i __A, __m256i __B)
	212	/// \endcode
	213	///
	214	/// This intrinsic corresponds to the \c VSM4RNDS4 instruction.
	215	///
	216	/// \param __A
	217	/// A 256-bit vector of [8 x int].
	218	/// \param __B
	219	/// A 256-bit vector of [8 x int].
	220	/// \returns
	221	/// A 256-bit vector of [8 x int].
	222	///
	223	/// \code{.operation}
	224	/// DEFINE ROL32(dword, n) {
	225	/// count := n % 32
	226	/// dest := (dword << count) \| (dword >> (32-count))
	227	/// RETURN dest
	228	/// }
	229	/// DEFINE lower_t(dword) {
	230	/// tmp.byte[0] := SBOX_BYTE(dword, 0)
	231	/// tmp.byte[1] := SBOX_BYTE(dword, 1)
	232	/// tmp.byte[2] := SBOX_BYTE(dword, 2)
	233	/// tmp.byte[3] := SBOX_BYTE(dword, 3)
	234	/// RETURN tmp
	235	/// }
	236	/// DEFINE L_RND(dword) {
	237	/// tmp := dword
	238	/// tmp := tmp ^ ROL32(dword, 2)
	239	/// tmp := tmp ^ ROL32(dword, 10)
	240	/// tmp := tmp ^ ROL32(dword, 18)
	241	/// tmp := tmp ^ ROL32(dword, 24)
	242	/// RETURN tmp
	243	/// }
	244	/// DEFINE T_RND(dword) {
	245	/// RETURN L_RND(lower_t(dword))
	246	/// }
	247	/// DEFINE F_RND(X0, X1, X2, X3, round_key) {
	248	/// RETURN X0 ^ T_RND(X1 ^ X2 ^ X3 ^ round_key)
	249	/// }
	250	/// FOR i:= 0 to 0
	251	/// P[0] := __B.xmm[i].dword[0]
	252	/// P[1] := __B.xmm[i].dword[1]
	253	/// P[2] := __B.xmm[i].dword[2]
	254	/// P[3] := __B.xmm[i].dword[3]
	255	/// C[0] := F_RND(P[0], P[1], P[2], P[3], __A.xmm[i].dword[0])
	256	/// C[1] := F_RND(P[1], P[2], P[3], C[0], __A.xmm[i].dword[1])
	257	/// C[2] := F_RND(P[2], P[3], C[0], C[1], __A.xmm[i].dword[2])
	258	/// C[3] := F_RND(P[3], C[0], C[1], C[2], __A.xmm[i].dword[3])
	259	/// DEST.xmm[i].dword[0] := C[0]
	260	/// DEST.xmm[i].dword[1] := C[1]
	261	/// DEST.xmm[i].dword[2] := C[2]
	262	/// DEST.xmm[i].dword[3] := C[3]
	263	/// ENDFOR
	264	/// DEST[MAX:256] := 0
	265	/// \endcode
	266	#define _mm256_sm4rnds4_epi32(A, B) \
	267	(__m256i) __builtin_ia32_vsm4rnds4256((__v8su)A, (__v8su)B)
	268
	269	#endif // __SM4INTRIN_H

lib/include/stdalign.h+5

...	...	@@ -10,6 +10,10 @@
10	10	#ifndef __STDALIGN_H
11	11	#define __STDALIGN_H
12	12
	13	/* FIXME: This is using the placeholder dates Clang produces for these macros
	14	in C2x mode; switch to the correct values once they've been published. */
	15	#if defined(__cplusplus) \|\| \
	16	(defined(__STDC_VERSION__) && __STDC_VERSION__ < 202000L)
13	17	#ifndef __cplusplus
14	18	#define alignas _Alignas
15	19	#define alignof _Alignof
...	...	@@ -17,5 +21,6 @@
17	21
18	22	#define __alignas_is_defined 1
19	23	#define __alignof_is_defined 1
	24	#endif /* __STDC_VERSION__ */
20	25
21	26	#endif /* __STDALIGN_H */

lib/include/stdatomic.h+9-2

...	...	@@ -45,9 +45,16 @@ extern "C" {
45	45	#define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE
46	46
47	47	/* 7.17.2 Initialization */
48
	48	/* FIXME: This is using the placeholder dates Clang produces for these macros
	49	in C2x mode; switch to the correct values once they've been published. */
	50	#if (defined(__STDC_VERSION__) && __STDC_VERSION__ < 202000L) \|\| \
	51	defined(__cplusplus)
	52	/* ATOMIC_VAR_INIT was removed in C2x, but still remains in C++23. */
49	53	#define ATOMIC_VAR_INIT(value) (value)
50		#if ((defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201710L) \|\| \
	54	#endif
	55
	56	#if ((defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201710L && \
	57	__STDC_VERSION__ < 202000L) \|\| \
51	58	(defined(__cplusplus) && __cplusplus >= 202002L)) && \
52	59	!defined(_CLANG_DISABLE_CRT_DEPRECATION_WARNINGS)
53	60	/* ATOMIC_VAR_INIT was deprecated in C17 and C++20. */

lib/include/stddef.h+5

...	...	@@ -103,6 +103,11 @@ using ::std::nullptr_t;
103	103	typedef typeof(nullptr) nullptr_t;
104	104	#endif /* defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202000L */
105	105
	106	#if defined(__need_STDDEF_H_misc) && defined(__STDC_VERSION__) && \
	107	__STDC_VERSION__ >= 202000L
	108	#define unreachable() __builtin_unreachable()
	109	#endif /* defined(__need_STDDEF_H_misc) && >= C23 */
	110
106	111	#if defined(__need_STDDEF_H_misc)
107	112	#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \|\| \
108	113	(defined(__cplusplus) && __cplusplus >= 201103L)

lib/include/wasm_simd128.h+132-12

...	...	@@ -961,17 +961,17 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_popcnt(v128_t __a) {
961	961
962	962	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shl(v128_t __a,
963	963	uint32_t __b) {
964		return (v128_t)((__i8x16)__a << __b);
	964	return (v128_t)((__i8x16)__a << (__b & 0x7));
965	965	}
966	966
967	967	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shr(v128_t __a,
968	968	uint32_t __b) {
969		return (v128_t)((__i8x16)__a >> __b);
	969	return (v128_t)((__i8x16)__a >> (__b & 0x7));
970	970	}
971	971
972	972	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_shr(v128_t __a,
973	973	uint32_t __b) {
974		return (v128_t)((__u8x16)__a >> __b);
	974	return (v128_t)((__u8x16)__a >> (__b & 0x7));
975	975	}
976	976
977	977	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_add(v128_t __a,
...	...	@@ -1047,17 +1047,17 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i16x8_bitmask(v128_t __a) {
1047	1047
1048	1048	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_shl(v128_t __a,
1049	1049	uint32_t __b) {
1050		return (v128_t)((__i16x8)__a << __b);
	1050	return (v128_t)((__i16x8)__a << (__b & 0xF));
1051	1051	}
1052	1052
1053	1053	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_shr(v128_t __a,
1054	1054	uint32_t __b) {
1055		return (v128_t)((__i16x8)__a >> __b);
	1055	return (v128_t)((__i16x8)__a >> (__b & 0xF));
1056	1056	}
1057	1057
1058	1058	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_shr(v128_t __a,
1059	1059	uint32_t __b) {
1060		return (v128_t)((__u16x8)__a >> __b);
	1060	return (v128_t)((__u16x8)__a >> (__b & 0xF));
1061	1061	}
1062	1062
1063	1063	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_add(v128_t __a,
...	...	@@ -1138,17 +1138,17 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i32x4_bitmask(v128_t __a) {
1138	1138
1139	1139	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_shl(v128_t __a,
1140	1140	uint32_t __b) {
1141		return (v128_t)((__i32x4)__a << __b);
	1141	return (v128_t)((__i32x4)__a << (__b & 0x1F));
1142	1142	}
1143	1143
1144	1144	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_shr(v128_t __a,
1145	1145	uint32_t __b) {
1146		return (v128_t)((__i32x4)__a >> __b);
	1146	return (v128_t)((__i32x4)__a >> (__b & 0x1F));
1147	1147	}
1148	1148
1149	1149	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_shr(v128_t __a,
1150	1150	uint32_t __b) {
1151		return (v128_t)((__u32x4)__a >> __b);
	1151	return (v128_t)((__u32x4)__a >> (__b & 0x1F));
1152	1152	}
1153	1153
1154	1154	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_add(v128_t __a,
...	...	@@ -1209,17 +1209,17 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i64x2_bitmask(v128_t __a) {
1209	1209
1210	1210	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_shl(v128_t __a,
1211	1211	uint32_t __b) {
1212		return (v128_t)((__i64x2)__a << (int64_t)__b);
	1212	return (v128_t)((__i64x2)__a << ((int64_t)__b & 0x3F));
1213	1213	}
1214	1214
1215	1215	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_shr(v128_t __a,
1216	1216	uint32_t __b) {
1217		return (v128_t)((__i64x2)__a >> (int64_t)__b);
	1217	return (v128_t)((__i64x2)__a >> ((int64_t)__b & 0x3F));
1218	1218	}
1219	1219
1220	1220	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u64x2_shr(v128_t __a,
1221	1221	uint32_t __b) {
1222		return (v128_t)((__u64x2)__a >> (int64_t)__b);
	1222	return (v128_t)((__u64x2)__a >> ((int64_t)__b & 0x3F));
1223	1223	}
1224	1224
1225	1225	static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_add(v128_t __a,
...	...	@@ -1760,6 +1760,126 @@ wasm_u64x2_load_32x2(const void *__mem) {
1760	1760	__DEPRECATED_WASM_MACRO("wasm_v64x2_shuffle", "wasm_i64x2_shuffle") \
1761	1761	wasm_i64x2_shuffle(__a, __b, __c0, __c1)
1762	1762
	1763	// Relaxed SIMD intrinsics
	1764
	1765	#define __RELAXED_FN_ATTRS \
	1766	__attribute__((__always_inline__, __nodebug__, __target__("relaxed-simd"), \
	1767	__min_vector_width__(128)))
	1768
	1769	static __inline__ v128_t __RELAXED_FN_ATTRS
	1770	wasm_f32x4_relaxed_madd(v128_t __a, v128_t __b, v128_t __c) {
	1771	return (v128_t)__builtin_wasm_relaxed_madd_f32x4((__f32x4)__a, (__f32x4)__b,
	1772	(__f32x4)__c);
	1773	}
	1774
	1775	static __inline__ v128_t __RELAXED_FN_ATTRS
	1776	wasm_f32x4_relaxed_nmadd(v128_t __a, v128_t __b, v128_t __c) {
	1777	return (v128_t)__builtin_wasm_relaxed_nmadd_f32x4((__f32x4)__a, (__f32x4)__b,
	1778	(__f32x4)__c);
	1779	}
	1780
	1781	static __inline__ v128_t __RELAXED_FN_ATTRS
	1782	wasm_f64x2_relaxed_madd(v128_t __a, v128_t __b, v128_t __c) {
	1783	return (v128_t)__builtin_wasm_relaxed_madd_f64x2((__f64x2)__a, (__f64x2)__b,
	1784	(__f64x2)__c);
	1785	}
	1786
	1787	static __inline__ v128_t __RELAXED_FN_ATTRS
	1788	wasm_f64x2_relaxed_nmadd(v128_t __a, v128_t __b, v128_t __c) {
	1789	return (v128_t)__builtin_wasm_relaxed_nmadd_f64x2((__f64x2)__a, (__f64x2)__b,
	1790	(__f64x2)__c);
	1791	}
	1792
	1793	static __inline__ v128_t __RELAXED_FN_ATTRS
	1794	wasm_i8x16_relaxed_laneselect(v128_t __a, v128_t __b, v128_t __m) {
	1795	return (v128_t)__builtin_wasm_relaxed_laneselect_i8x16(
	1796	(__i8x16)__a, (__i8x16)__b, (__i8x16)__m);
	1797	}
	1798
	1799	static __inline__ v128_t __RELAXED_FN_ATTRS
	1800	wasm_i16x8_relaxed_laneselect(v128_t __a, v128_t __b, v128_t __m) {
	1801	return (v128_t)__builtin_wasm_relaxed_laneselect_i16x8(
	1802	(__i16x8)__a, (__i16x8)__b, (__i16x8)__m);
	1803	}
	1804
	1805	static __inline__ v128_t __RELAXED_FN_ATTRS
	1806	wasm_i32x4_relaxed_laneselect(v128_t __a, v128_t __b, v128_t __m) {
	1807	return (v128_t)__builtin_wasm_relaxed_laneselect_i32x4(
	1808	(__i32x4)__a, (__i32x4)__b, (__i32x4)__m);
	1809	}
	1810
	1811	static __inline__ v128_t __RELAXED_FN_ATTRS
	1812	wasm_i64x2_relaxed_laneselect(v128_t __a, v128_t __b, v128_t __m) {
	1813	return (v128_t)__builtin_wasm_relaxed_laneselect_i64x2(
	1814	(__i64x2)__a, (__i64x2)__b, (__i64x2)__m);
	1815	}
	1816
	1817	static __inline__ v128_t __RELAXED_FN_ATTRS
	1818	wasm_i8x16_relaxed_swizzle(v128_t __a, v128_t __s) {
	1819	return (v128_t)__builtin_wasm_relaxed_swizzle_i8x16((__i8x16)__a,
	1820	(__i8x16)__s);
	1821	}
	1822
	1823	static __inline__ v128_t __RELAXED_FN_ATTRS wasm_f32x4_relaxed_min(v128_t __a,
	1824	v128_t __b) {
	1825	return (v128_t)__builtin_wasm_relaxed_min_f32x4((__f32x4)__a, (__f32x4)__b);
	1826	}
	1827
	1828	static __inline__ v128_t __RELAXED_FN_ATTRS wasm_f32x4_relaxed_max(v128_t __a,
	1829	v128_t __b) {
	1830	return (v128_t)__builtin_wasm_relaxed_max_f32x4((__f32x4)__a, (__f32x4)__b);
	1831	}
	1832
	1833	static __inline__ v128_t __RELAXED_FN_ATTRS wasm_f64x2_relaxed_min(v128_t __a,
	1834	v128_t __b) {
	1835	return (v128_t)__builtin_wasm_relaxed_min_f64x2((__f64x2)__a, (__f64x2)__b);
	1836	}
	1837
	1838	static __inline__ v128_t __RELAXED_FN_ATTRS wasm_f64x2_relaxed_max(v128_t __a,
	1839	v128_t __b) {
	1840	return (v128_t)__builtin_wasm_relaxed_max_f64x2((__f64x2)__a, (__f64x2)__b);
	1841	}
	1842
	1843	static __inline__ v128_t __RELAXED_FN_ATTRS
	1844	wasm_i32x4_relaxed_trunc_f32x4(v128_t __a) {
	1845	return (v128_t)__builtin_wasm_relaxed_trunc_s_i32x4_f32x4((__f32x4)__a);
	1846	}
	1847
	1848	static __inline__ v128_t __RELAXED_FN_ATTRS
	1849	wasm_u32x4_relaxed_trunc_f32x4(v128_t __a) {
	1850	return (v128_t)__builtin_wasm_relaxed_trunc_u_i32x4_f32x4((__f32x4)__a);
	1851	}
	1852
	1853	static __inline__ v128_t __RELAXED_FN_ATTRS
	1854	wasm_i32x4_relaxed_trunc_f64x2_zero(v128_t __a) {
	1855	return (v128_t)__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2((__f64x2)__a);
	1856	}
	1857
	1858	static __inline__ v128_t __RELAXED_FN_ATTRS
	1859	wasm_u32x4_relaxed_trunc_f64x2_zero(v128_t __a) {
	1860	return (v128_t)__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2((__f64x2)__a);
	1861	}
	1862
	1863	static __inline__ v128_t __RELAXED_FN_ATTRS
	1864	wasm_i16x8_relaxed_q15mulr(v128_t __a, v128_t __b) {
	1865	return (v128_t)__builtin_wasm_relaxed_q15mulr_s_i16x8((__i16x8)__a,
	1866	(__i16x8)__b);
	1867	}
	1868
	1869	static __inline__ v128_t __RELAXED_FN_ATTRS
	1870	wasm_i16x8_relaxed_dot_i8x16_i7x16(v128_t __a, v128_t __b) {
	1871	return (v128_t)__builtin_wasm_relaxed_dot_i8x16_i7x16_s_i16x8((__i8x16)__a,
	1872	(__i8x16)__b);
	1873	}
	1874
	1875	static __inline__ v128_t __RELAXED_FN_ATTRS
	1876	wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v128_t __a, v128_t __b, v128_t __c) {
	1877	return (v128_t)__builtin_wasm_relaxed_dot_i8x16_i7x16_add_s_i32x4(
	1878	(__i8x16)__a, (__i8x16)__b, (__i32x4)__c);
	1879	}
	1880
	1881	// Deprecated intrinsics
	1882
1763	1883	static __inline__ v128_t __DEPRECATED_FN_ATTRS("wasm_i8x16_swizzle")
1764	1884	wasm_v8x16_swizzle(v128_t __a, v128_t __b) {
1765	1885	return wasm_i8x16_swizzle(__a, __b);

lib/include/xsavecintrin.h+50

...	...	@@ -17,12 +17,62 @@
17	17	/* Define the default attributes for the functions in this file. */
18	18	#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("xsavec")))
19	19
	20	/// Performs a full or partial save of processor state to the memory at
	21	/// \a __p. The exact state saved depends on the 64-bit mask \a __m and
	22	/// processor control register \c XCR0.
	23	///
	24	/// \code{.operation}
	25	/// mask[62:0] := __m[62:0] AND XCR0[62:0]
	26	/// FOR i := 0 TO 62
	27	/// IF mask[i] == 1
	28	/// CASE (i) OF
	29	/// 0: save X87 FPU state
	30	/// 1: save SSE state
	31	/// DEFAULT: __p.Ext_Save_Area[i] := ProcessorState[i]
	32	/// FI
	33	/// ENDFOR
	34	/// __p.Header.XSTATE_BV[62:0] := INIT_FUNCTION(mask[62:0])
	35	/// \endcode
	36	///
	37	/// \headerfile <immintrin.h>
	38	///
	39	/// This intrinsic corresponds to the \c XSAVEC instruction.
	40	///
	41	/// \param __p
	42	/// Pointer to the save area; must be 64-byte aligned.
	43	/// \param __m
	44	/// A 64-bit mask indicating what state should be saved.
20	45	static __inline__ void __DEFAULT_FN_ATTRS
21	46	_xsavec(void *__p, unsigned long long __m) {
22	47	__builtin_ia32_xsavec(__p, __m);
23	48	}
24	49
25	50	#ifdef __x86_64__
	51	/// Performs a full or partial save of processor state to the memory at
	52	/// \a __p. The exact state saved depends on the 64-bit mask \a __m and
	53	/// processor control register \c XCR0.
	54	///
	55	/// \code{.operation}
	56	/// mask[62:0] := __m[62:0] AND XCR0[62:0]
	57	/// FOR i := 0 TO 62
	58	/// IF mask[i] == 1
	59	/// CASE (i) OF
	60	/// 0: save X87 FPU state
	61	/// 1: save SSE state
	62	/// DEFAULT: __p.Ext_Save_Area[i] := ProcessorState[i]
	63	/// FI
	64	/// ENDFOR
	65	/// __p.Header.XSTATE_BV[62:0] := INIT_FUNCTION(mask[62:0])
	66	/// \endcode
	67	///
	68	/// \headerfile <immintrin.h>
	69	///
	70	/// This intrinsic corresponds to the \c XSAVEC64 instruction.
	71	///
	72	/// \param __p
	73	/// Pointer to the save area; must be 64-byte aligned.
	74	/// \param __m
	75	/// A 64-bit mask indicating what state should be saved.
26	76	static __inline__ void __DEFAULT_FN_ATTRS
27	77	_xsavec64(void *__p, unsigned long long __m) {
28	78	__builtin_ia32_xsavec64(__p, __m);