authorgravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2025-02-05 10:23:43+01:00
committergravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2025-04-04 06:08:09+02:00
logce754724b31e141688e1203ba7bb0a2538f2b544
tree2989b6fbb7ebb1ba5370ed9a7fc1317ff935f525
parent0181cfe8adfab3e7a682aaec10f932cd3279f40a
signaturebadge-check Signed by SSH key SHA256:7B/LJ7bpR1eX8aCXSr4mtd5M45VMPKcx9zY8e95b5QM

zig cc: Update intrinsic headers to Clang 20.

80 files changed, 17990 insertions(+), 2925 deletions(-)

lib/include/adcintrin.h+5
......@@ -15,7 +15,12 @@
1515#endif
1616
1717/* Define the default attributes for the functions in this file. */
18#if defined(__cplusplus) && (__cplusplus >= 201103L)
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__)) constexpr
21#else
1822#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
23#endif
1924
2025/* Use C++ inline semantics in C++, GNU inline for C mode. */
2126#if defined(__cplusplus)
lib/include/adxintrin.h+5
......@@ -15,8 +15,13 @@
1515#define __ADXINTRIN_H
1616
1717/* Define the default attributes for the functions in this file. */
18#if defined(__cplusplus) && (__cplusplus >= 201103L)
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, __target__("adx"))) constexpr
21#else
1822#define __DEFAULT_FN_ATTRS \
1923 __attribute__((__always_inline__, __nodebug__, __target__("adx")))
24#endif
2025
2126/* Use C++ inline semantics in C++, GNU inline for C mode. */
2227#if defined(__cplusplus)
lib/include/altivec.h+8-8
......@@ -2502,37 +2502,37 @@ vec_cmplt(vector unsigned long long __a, vector unsigned long long __b) {
25022502
25032503static __inline__ vector unsigned char __ATTRS_o_ai
25042504vec_popcnt(vector signed char __a) {
2505 return (vector unsigned char)__builtin_altivec_vpopcntb(
2505 return (vector unsigned char)__builtin_elementwise_popcount(
25062506 (vector unsigned char)__a);
25072507}
25082508static __inline__ vector unsigned char __ATTRS_o_ai
25092509vec_popcnt(vector unsigned char __a) {
2510 return __builtin_altivec_vpopcntb(__a);
2510 return __builtin_elementwise_popcount(__a);
25112511}
25122512static __inline__ vector unsigned short __ATTRS_o_ai
25132513vec_popcnt(vector signed short __a) {
2514 return (vector unsigned short)__builtin_altivec_vpopcnth(
2514 return (vector unsigned short)__builtin_elementwise_popcount(
25152515 (vector unsigned short)__a);
25162516}
25172517static __inline__ vector unsigned short __ATTRS_o_ai
25182518vec_popcnt(vector unsigned short __a) {
2519 return __builtin_altivec_vpopcnth(__a);
2519 return __builtin_elementwise_popcount(__a);
25202520}
25212521static __inline__ vector unsigned int __ATTRS_o_ai
25222522vec_popcnt(vector signed int __a) {
2523 return __builtin_altivec_vpopcntw((vector unsigned int)__a);
2523 return __builtin_elementwise_popcount((vector unsigned int)__a);
25242524}
25252525static __inline__ vector unsigned int __ATTRS_o_ai
25262526vec_popcnt(vector unsigned int __a) {
2527 return __builtin_altivec_vpopcntw(__a);
2527 return __builtin_elementwise_popcount(__a);
25282528}
25292529static __inline__ vector unsigned long long __ATTRS_o_ai
25302530vec_popcnt(vector signed long long __a) {
2531 return __builtin_altivec_vpopcntd((vector unsigned long long)__a);
2531 return __builtin_elementwise_popcount((vector unsigned long long)__a);
25322532}
25332533static __inline__ vector unsigned long long __ATTRS_o_ai
25342534vec_popcnt(vector unsigned long long __a) {
2535 return __builtin_altivec_vpopcntd(__a);
2535 return __builtin_elementwise_popcount(__a);
25362536}
25372537
25382538#define vec_vclz vec_cntlz
lib/include/amxavx512intrin.h created+382
......@@ -0,0 +1,382 @@
1/*===--------------------- amxavx512intrin.h - AMXAVX512 --------------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===------------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error "Never use <amxavx512intrin.h> directly; include <immintrin.h> instead."
11#endif // __IMMINTRIN_H
12
13#ifndef __AMX_AVX512INTRIN_H
14#define __AMX_AVX512INTRIN_H
15#if defined(__x86_64__) && defined(__SSE2__)
16
17#define __DEFAULT_FN_ATTRS_AVX512 \
18 __attribute__((__always_inline__, __nodebug__, \
19 __target__("amx-avx512,avx10.2-512")))
20
21/// Moves a row from a tile register to a zmm destination register, converting
22/// the int32 source elements to fp32. The row of the tile is selected by a
23/// 32b GPR.
24///
25/// \headerfile <x86intrin.h>
26///
27/// \code
28/// __m512i _tile_cvtrowd2ps(__tile tsrc, unsigned int row);
29/// \endcode
30///
31/// \code{.operation}
32/// VL := 512
33/// VL_bytes := VL >> 3
34/// row_index := row & 0xffff
35/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes
36/// FOR i := 0 TO (VL_bytes / 4) - 1
37/// IF i + row_chunk / 4 >= tsrc.colsb / 4
38/// dst.dword[i] := 0
39/// ELSE
40/// dst.f32[i] := CONVERT_INT32_TO_FP32(tsrc.row[row_index].dword[row_chunk/4+i], RNE)
41/// FI
42/// ENDFOR
43/// dst[MAX_VL-1:VL] := 0
44/// zero_tileconfig_start()
45/// \endcode
46///
47/// This intrinsic corresponds to the \c TCVTROWD2PS instruction.
48///
49/// \param tsrc
50/// The source tile. Max size is 1024 Bytes.
51/// \param row
52/// The row of the source tile
53#define _tile_cvtrowd2ps(tsrc, row) __builtin_ia32_tcvtrowd2ps(tsrc, row)
54
55/// Moves a row from a tile register to a zmm destination register, converting
56/// the fp32 source elements to bf16. It places the resulting bf16 elements
57/// in the high 16 bits within each dword. The row of the tile is selected
58/// by a 32b GPR.
59///
60/// \headerfile <x86intrin.h>
61///
62/// \code
63/// __m512i _tile_cvtrowps2bf16h(__tile tsrc, unsigned int row);
64/// \endcode
65///
66/// \code{.operation}
67/// VL := 512
68/// VL_bytes := VL >> 3
69/// row_index := row & 0xffff
70/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes
71/// FOR i := 0 TO (VL_bytes / 4) - 1
72/// IF i + row_chunk / 4 >= tsrc.colsb / 4
73/// dst.dword[i] := 0
74/// ELSE
75/// dst.word[2*i+0] := 0
76/// dst.bf16[2*i+1] := CONVERT_FP32_TO_BF16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
77/// FI
78/// ENDFOR
79/// dst[MAX_VL-1:VL] := 0
80/// zero_tileconfig_start()
81/// \endcode
82///
83/// This intrinsic corresponds to the \c TCVTROWPS2BF16H instruction.
84///
85/// \param tsrc
86/// The source tile. Max size is 1024 Bytes.
87/// \param row
88/// The the row of the source tile.
89#define _tile_cvtrowps2bf16h(tsrc, row) \
90 __builtin_ia32_tcvtrowps2bf16h(tsrc, row)
91
92/// Moves a row from a tile register to a zmm destination register, converting
93/// the fp32 source elements to bf16. It places the resulting bf16 elements
94/// in the low 16 bits within each dword. The row of the tile is selected
95/// by a 32b GPR.
96///
97/// \headerfile <x86intrin.h>
98///
99/// \code
100/// __m512i _tile_cvtrowps2bf16l(__tile tsrc, unsigned int row);
101/// \endcode
102///
103/// \code{.operation}
104/// VL := 512
105/// VL_bytes := VL >> 3
106/// row_index := row & 0xffff
107/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes
108/// FOR i := 0 TO (VL_bytes / 4) - 1
109/// IF i + row_chunk / 4 >= tsrc.colsb / 4
110/// dst.dword[i] := 0
111/// ELSE
112/// dst.word[2*i+1] := 0
113/// dst.bf16[2*i+0] := CONVERT_FP32_TO_BF16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
114/// FI
115/// ENDFOR
116/// dst[MAX_VL-1:VL] := 0
117/// zero_tileconfig_start()
118/// \endcode
119///
120/// This intrinsic corresponds to the \c TCVTROWPS2BF16L instruction.
121///
122/// \param tsrc
123/// The source tile. Max size is 1024 Bytes.
124/// \param row
125/// The the row of the source tile.
126#define _tile_cvtrowps2bf16l(tsrc, row) \
127 __builtin_ia32_tcvtrowps2bf16l(tsrc, row)
128
129/// Moves a row from a tile register to a zmm destination register, converting
130/// the fp32 source elements to fp16. It places the resulting fp16 elements
131/// in the high 16 bits within each dword. The row of the tile is selected
132/// by a 32b GPR.
133///
134/// \headerfile <x86intrin.h>
135///
136/// \code
137/// __m512i _tile_cvtrowps2phh(__tile tsrc, unsigned int row);
138/// \endcode
139///
140/// \code{.operation}
141/// VL := 512
142/// VL_bytes := VL >> 3
143/// row_index := row & 0xffff
144/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes
145/// FOR i := 0 TO (VL_bytes / 4) - 1
146/// IF i + row_chunk / 4 >= tsrc.colsb / 4
147/// dst.dword[i] := 0
148/// ELSE
149/// dst.word[2*i+0] := 0
150/// dst.fp16[2*i+1] := CONVERT_FP32_TO_FP16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
151/// FI
152/// ENDFOR
153/// dst[MAX_VL-1:VL] := 0
154/// zero_tileconfig_start()
155/// \endcode
156///
157/// This intrinsic corresponds to the \c TCVTROWPS2PHH instruction.
158///
159/// \param tsrc
160/// The source tile. Max size is 1024 Bytes.
161/// \param row
162/// The the row of the source tile.
163#define _tile_cvtrowps2phh(tsrc, row) __builtin_ia32_tcvtrowps2phh(tsrc, row)
164
165/// Moves a row from a tile register to a zmm destination register, converting
166/// the fp32 source elements to fp16. It places the resulting fp16 elements
167/// in the low 16 bits within each dword. The row of the tile is selected
168/// by a 32b GPR.
169///
170/// \headerfile <x86intrin.h>
171///
172/// \code
173/// __m512i _tile_cvtrowps2phl(__tile tsrc, unsigned int row);
174/// \endcode
175///
176/// \code{.operation}
177/// VL := 512
178/// VL_bytes := VL >> 3
179/// row_index := row & 0xffff
180/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes
181/// FOR i := 0 TO (VL_bytes / 4) - 1
182/// IF i + row_chunk / 4 >= tsrc.colsb / 4
183/// dst.dword[i] := 0
184/// ELSE
185/// dst.word[2*i+1] := 0
186/// dst.fp16[2*i+0] := CONVERT_FP32_TO_FP16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
187/// FI
188/// ENDFOR
189/// dst[MAX_VL-1:VL] := 0
190/// zero_tileconfig_start()
191/// \endcode
192///
193/// This intrinsic corresponds to the \c TCVTROWPS2PHL instruction.
194///
195/// \param tsrc
196/// The source tile. Max size is 1024 Bytes.
197/// \param row
198/// The the row of the source tile.
199#define _tile_cvtrowps2phl(tsrc, row) __builtin_ia32_tcvtrowps2phl(tsrc, row)
200
201/// Move one row of a tile data to a v16f32 data.
202/// The row of the tile is selected by a 32b GPR.
203///
204/// \headerfile <immintrin.h>
205///
206/// \code
207/// __m512 _tile_movrow(__tile a, unsigned b);
208/// \endcode
209///
210/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction.
211///
212/// \param a
213/// The 1st source tile. Max size is 1024 Bytes.
214/// \param b
215/// The 2nd source r32. Size is 4 Bytes.
216/// \returns
217/// The destination v16f32 data. Size is 64 Bytes.
218///
219/// \code{.operation}
220/// VL := 512
221/// VL_bytes := VL>>3
222/// row_index := b&0xffff
223/// row_chunk := ((b>>16)&0xffff) * VL_bytes
224/// FOR i := 0 TO (VL_bytes-1)
225/// IF (row_chunk + i >= a.colsb)
226/// dst.byte[i] := 0
227/// ELSE
228/// dst.byte[i] := a.row[row_index].byte[row_chunk+i]
229/// ENDFOR
230/// \endcode
231#define _tile_movrow(a, b) __builtin_ia32_tilemovrow(a, b)
232
233/// This is internal intrinsic. C/C++ user should avoid calling it directly.
234
235static __inline__ __m512 __DEFAULT_FN_ATTRS_AVX512 _tile_cvtrowd2ps_internal(
236 unsigned short m, unsigned short n, _tile1024i src, unsigned u) {
237 return __builtin_ia32_tcvtrowd2ps_internal(m, n, src, u);
238}
239
240static __inline__ __m512bh __DEFAULT_FN_ATTRS_AVX512
241_tile_cvtrowps2bf16h_internal(unsigned short m, unsigned short n,
242 _tile1024i src, unsigned u) {
243 return __builtin_ia32_tcvtrowps2bf16h_internal(m, n, src, u);
244}
245
246static __inline__ __m512bh __DEFAULT_FN_ATTRS_AVX512
247_tile_cvtrowps2bf16l_internal(unsigned short m, unsigned short n,
248 _tile1024i src, unsigned u) {
249 return __builtin_ia32_tcvtrowps2bf16l_internal(m, n, src, u);
250}
251
252static __inline__ __m512h __DEFAULT_FN_ATTRS_AVX512 _tile_cvtrowps2phh_internal(
253 unsigned short m, unsigned short n, _tile1024i src, unsigned u) {
254 return __builtin_ia32_tcvtrowps2phh_internal(m, n, src, u);
255}
256
257static __inline__ __m512h __DEFAULT_FN_ATTRS_AVX512 _tile_cvtrowps2phl_internal(
258 unsigned short m, unsigned short n, _tile1024i src, unsigned u) {
259 return __builtin_ia32_tcvtrowps2phl_internal(m, n, src, u);
260}
261
262static __inline__ __m512i __DEFAULT_FN_ATTRS_AVX512 _tile_movrow_internal(
263 unsigned short m, unsigned short n, _tile1024i src, unsigned u) {
264 return (__m512i)__builtin_ia32_tilemovrow_internal(m, n, src, u);
265}
266
267/// Move a row from a tile (src0) to a v16f32 dst, converting the int32 source
268/// elements to fp32. No SIMD exceptions are generated. Rounding is done as if
269/// MXCSR.RC=RNE. Embedded rounding is not supported.
270/// The row and chunk elements of tile is fetched from 32bit src1.
271///
272/// \headerfile <immintrin.h>
273///
274/// This intrinsic corresponds to the <c> TCVTROWD2PS </c> instruction.
275///
276/// \param src0
277/// The 1st source tile. Max size is 1024 Bytes.
278/// \param src1
279/// The 2nd source r32. Size is 4 Bytes.
280/// \returns
281/// The destination v16f32 data. Size is 64 Bytes.
282__DEFAULT_FN_ATTRS_AVX512
283static __m512 __tile_cvtrowd2ps(__tile1024i src0, unsigned src1) {
284 return _tile_cvtrowd2ps_internal(src0.row, src0.col, src0.tile, src1);
285}
286
287/// Move a row from a tile (src0) to a v32bf16 dst, converting the fp32 source
288/// elements to bf16 at high 16-bits of each dword.
289/// The row and chunk elements of tile is fetched from 32bit src1.
290///
291/// \headerfile <immintrin.h>
292///
293/// This intrinsic corresponds to the <c> TCVTROWPS2BF16H </c> instruction.
294///
295/// \param src0
296/// The 1st source tile. Max size is 1024 Bytes.
297/// \param src1
298/// The 2nd source r32. Size is 4 Bytes.
299/// \returns
300/// The destination v32bf16 data. Size is 64 Bytes.
301__DEFAULT_FN_ATTRS_AVX512
302static __m512bh __tile_cvtrowps2bf16h(__tile1024i src0, unsigned src1) {
303 return _tile_cvtrowps2bf16h_internal(src0.row, src0.col, src0.tile, src1);
304}
305
306/// Move a row from a tile (src0) to a v32bf16 dst, converting the fp32 source
307/// elements to bf16 at low 16-bits of each dword.
308/// The row and chunk elements of tile is fetched from 32bit src1.
309///
310/// \headerfile <immintrin.h>
311///
312/// This intrinsic corresponds to the <c> TCVTROWPS2BF16L </c> instruction.
313///
314/// \param src0
315/// The 1st source tile. Max size is 1024 Bytes.
316/// \param src1
317/// The 2nd source r32. Size is 4 Bytes.
318/// \returns
319/// The destination v32bf16 data. Size is 64 Bytes.
320__DEFAULT_FN_ATTRS_AVX512
321static __m512bh __tile_cvtrowps2bf16l(__tile1024i src0, unsigned src1) {
322 return _tile_cvtrowps2bf16l_internal(src0.row, src0.col, src0.tile, src1);
323}
324
325/// Move a row from a tile (src0) to a v32fp16 dst, converting the fp32 source
326/// elements to fp16 at high 16-bits of each dword.
327/// The row and chunk elements of tile is fetched from 32bit src1.
328///
329/// \headerfile <immintrin.h>
330///
331/// This intrinsic corresponds to the <c> TCVTROWPS2PHH </c> instruction.
332///
333/// \param src0
334/// The 1st source tile. Max size is 1024 Bytes.
335/// \param src1
336/// The 2nd source r32. Size is 4 Bytes.
337/// \returns
338/// The destination v32fp16 data. Size is 64 Bytes.
339__DEFAULT_FN_ATTRS_AVX512
340static __m512h __tile_cvtrowps2phh(__tile1024i src0, unsigned src1) {
341 return _tile_cvtrowps2phh_internal(src0.row, src0.col, src0.tile, src1);
342}
343
344/// Move a row from a tile (src0) to a v32fp16 dst, converting the fp32 source
345/// elements to fp16 at low 16-bits of each dword.
346/// The row and chunk elements of tile is fetched from 32bit src1.
347///
348/// \headerfile <immintrin.h>
349///
350/// This intrinsic corresponds to the <c> TCVTROWPS2PHL </c> instruction.
351///
352/// \param src0
353/// The 1st source tile. Max size is 1024 Bytes.
354/// \param src1
355/// The 2nd source r32. Size is 4 Bytes.
356/// \returns
357/// The destination v32fp16 data. Size is 64 Bytes.
358__DEFAULT_FN_ATTRS_AVX512
359static __m512h __tile_cvtrowps2phl(__tile1024i src0, unsigned src1) {
360 return _tile_cvtrowps2phl_internal(src0.row, src0.col, src0.tile, src1);
361}
362
363/// Move one row of a tile data to a v16f32 data.
364/// The row of the tile is selected by a 32b GPR.
365///
366/// \headerfile <immintrin.h>
367///
368/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction.
369///
370/// \param src0
371/// The 1st source tile. Max size is 1024 Bytes.
372/// \param src1
373/// The 2nd source r32. Size is 4 Bytes.
374/// \returns
375/// The destination v16i32 data. Size is 64 Bytes.
376__DEFAULT_FN_ATTRS_AVX512
377static __m512i __tile_movrow(__tile1024i src0, unsigned src1) {
378 return (__m512i)_tile_movrow_internal(src0.row, src0.col, src0.tile, src1);
379}
380
381#endif // __x86_64__ && __SSE2__
382#endif // __AMX_AVX512INTRIN_H
lib/include/amxbf16transposeintrin.h created+94
......@@ -0,0 +1,94 @@
1/*===----- amxbf16transposeintrin.h - AMX-BF16 and AMX-TRANSPOSE ------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===------------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error \
12 "Never use <amxbf16transposeintrin.h> directly; use <immintrin.h> instead."
13#endif /* __IMMINTRIN_H */
14
15#ifndef __AMX_BF16TRANSPOSEINTRIN_H
16#define __AMX_BF16TRANSPOSEINTRIN_H
17#ifdef __x86_64__
18
19/* Define the default attributes for the functions in this file. */
20#define __DEFAULT_FN_ATTRS \
21 __attribute__((__always_inline__, __nodebug__, \
22 __target__("amx-bf16,amx-transpose")))
23
24/// Compute transpose and dot-product of BF16 (16-bit) floating-point pairs in
25/// tiles \a a and \a b, accumulating the intermediate single-precision
26/// (32-bit) floating-point elements with elements in \a dst, and store the
27/// 32-bit result back to tile \a dst.
28///
29/// \headerfile <immintrin.h>
30///
31/// \code
32/// void _tile_tdpbf16ps (__tile dst, __tile a, __tile b)
33/// \endcode
34///
35/// \code{.operation}
36/// FOR m := 0 TO dst.rows - 1
37/// tmp := dst.row[m]
38/// FOR k := 0 TO (a.colsb / 4) - 1
39/// FOR n := 0 TO (dst.colsb / 4) - 1
40/// tmp.bf32[n] += FP32(a.row[m].bf16[2*k+0]) *
41/// FP32(b.row[k].bf16[2*n+0])
42/// tmp.bf32[n] += FP32(a.row[m].bf16[2*k+1]) *
43/// FP32(b.row[k].bf16[2*n+1])
44/// ENDFOR
45/// ENDFOR
46/// write_row_and_zero(dst, m, tmp, dst.colsb)
47/// ENDFOR
48/// zero_upper_rows(dst, dst.rows)
49/// zero_tileconfig_start()
50/// \endcode
51///
52/// This intrinsic corresponds to the \c TTDPBF16PS instruction.
53///
54/// \param dst
55/// The destination tile. Max size is 1024 Bytes.
56/// \param a
57/// The 1st source tile. Max size is 1024 Bytes.
58/// \param b
59/// The 2nd source tile. Max size is 1024 Bytes.
60#define _tile_tdpbf16ps(dst, a, b) __builtin_ia32_ttdpbf16ps((dst), (a), (b))
61
62/// This is internal intrinsic. C/C++ user should avoid calling it directly.
63static __inline__ _tile1024i __DEFAULT_FN_ATTRS
64_tile_tdpbf16ps_internal(unsigned short m, unsigned short n, unsigned short k,
65 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
66 return __builtin_ia32_ttdpbf16ps_internal(m, n, k, dst, src1, src2);
67}
68
69/// Compute transpose and dot-product of BF16 (16-bit) floating-point pairs in
70/// tiles src0 and src1, accumulating the intermediate single-precision
71/// (32-bit) floating-point elements with elements in "dst", and store the
72/// 32-bit result back to tile "dst".
73///
74/// \headerfile <immintrin.h>
75///
76/// This intrinsic corresponds to the <c> TTDPBF16PS </c> instruction.
77///
78/// \param dst
79/// The destination tile. Max size is 1024 Bytes.
80/// \param src0
81/// The 1st source tile. Max size is 1024 Bytes.
82/// \param src1
83/// The 2nd source tile. Max size is 1024 Bytes.
84__DEFAULT_FN_ATTRS
85static __inline__ void __tile_tdpbf16ps(__tile1024i *dst, __tile1024i src0,
86 __tile1024i src1) {
87 dst->tile = _tile_tdpbf16ps_internal(src0.row, src1.col, src0.col, dst->tile,
88 src0.tile, src1.tile);
89}
90
91#undef __DEFAULT_FN_ATTRS
92
93#endif /* __x86_64__ */
94#endif /* __AMX_BF16TRANSPOSEINTRIN_H */
lib/include/amxcomplextransposeintrin.h created+303
......@@ -0,0 +1,303 @@
1/*===----- amxcomplextransposeintrin.h - AMX-COMPLEX and AMX-TRANSPOSE ------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===------------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error \
12 "Never use <amxcomplextransposeintrin.h> directly; include <immintrin.h> instead."
13#endif // __IMMINTRIN_H
14
15#ifndef __AMX_COMPLEXTRANSPOSEINTRIN_H
16#define __AMX_COMPLEXTRANSPOSEINTRIN_H
17#ifdef __x86_64__
18
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, \
21 __target__("amx-complex,amx-transpose")))
22
23/// Perform matrix multiplication of two tiles containing complex elements and
24/// accumulate the results into a packed single precision tile. Each dword
25/// element in input tiles \a a and \a b is interpreted as a complex number
26/// with FP16 real part and FP16 imaginary part.
27/// Calculates the imaginary part of the result. For each possible combination
28/// of (transposed column of \a a, column of \a b), it performs a set of
29/// multiplication and accumulations on all corresponding complex numbers
30/// (one from \a a and one from \a b). The imaginary part of the \a a element
31/// is multiplied with the real part of the corresponding \a b element, and
32/// the real part of the \a a element is multiplied with the imaginary part
33/// of the corresponding \a b elements. The two accumulated results are
34/// added, and then accumulated into the corresponding row and column of
35/// \a dst.
36///
37/// \headerfile <x86intrin.h>
38///
39/// \code
40/// void _tile_tcmmimfp16ps(__tile dst, __tile a, __tile b);
41/// \endcode
42///
43/// \code{.operation}
44/// FOR m := 0 TO dst.rows - 1
45/// tmp := dst.row[m]
46/// FOR k := 0 TO a.rows - 1
47/// FOR n := 0 TO (dst.colsb / 4) - 1
48/// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+0]) * FP32(b.row[k].fp16[2*n+1])
49/// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+1]) * FP32(b.row[k].fp16[2*n+0])
50/// ENDFOR
51/// ENDFOR
52/// write_row_and_zero(dst, m, tmp, dst.colsb)
53/// ENDFOR
54/// zero_upper_rows(dst, dst.rows)
55/// zero_tileconfig_start()
56/// \endcode
57///
58/// This intrinsic corresponds to the \c TTCMMIMFP16PS instruction.
59///
60/// \param dst
61/// The destination tile. Max size is 1024 Bytes.
62/// \param a
63/// The 1st source tile. Max size is 1024 Bytes.
64/// \param b
65/// The 2nd source tile. Max size is 1024 Bytes.
66#define _tile_tcmmimfp16ps(dst, a, b) \
67 __builtin_ia32_ttcmmimfp16ps((dst), (a), (b))
68
69/// Perform matrix multiplication of two tiles containing complex elements and
70/// accumulate the results into a packed single precision tile. Each dword
71/// element in input tiles \a a and \a b is interpreted as a complex number
72/// with FP16 real part and FP16 imaginary part.
73/// Calculates the real part of the result. For each possible combination
74/// of (rtransposed colum of \a a, column of \a b), it performs a set of
75/// multiplication and accumulations on all corresponding complex numbers
76/// (one from \a a and one from \a b). The real part of the \a a element is
77/// multiplied with the real part of the corresponding \a b element, and the
78/// negated imaginary part of the \a a element is multiplied with the
79/// imaginary part of the corresponding \a b elements. The two accumulated
80/// results are added, and then accumulated into the corresponding row and
81/// column of \a dst.
82///
83/// \headerfile <x86intrin.h>
84///
85/// \code
86/// void _tile_tcmmrlfp16ps(__tile dst, __tile a, __tile b);
87/// \endcode
88///
89/// \code{.operation}
90/// FOR m := 0 TO dst.rows - 1
91/// tmp := dst.row[m]
92/// FOR k := 0 TO a.rows - 1
93/// FOR n := 0 TO (dst.colsb / 4) - 1
94/// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+0]) * FP32(b.row[k].fp16[2*n+0])
95/// tmp.fp32[n] += FP32(-a.row[m].fp16[2*k+1]) * FP32(b.row[k].fp16[2*n+1])
96/// ENDFOR
97/// ENDFOR
98/// write_row_and_zero(dst, m, tmp, dst.colsb)
99/// ENDFOR
100/// zero_upper_rows(dst, dst.rows)
101/// zero_tileconfig_start()
102/// \endcode
103///
104/// This intrinsic corresponds to the \c TTCMMIMFP16PS instruction.
105///
106/// \param dst
107/// The destination tile. Max size is 1024 Bytes.
108/// \param a
109/// The 1st source tile. Max size is 1024 Bytes.
110/// \param b
111/// The 2nd source tile. Max size is 1024 Bytes.
112#define _tile_tcmmrlfp16ps(dst, a, b) \
113 __builtin_ia32_ttcmmrlfp16ps((dst), (a), (b))
114
115/// Perform matrix conjugate transpose and multiplication of two tiles
116/// containing complex elements and accumulate the results into a packed
117/// single precision tile. Each dword element in input tiles \a a and \a b
118/// is interpreted as a complex number with FP16 real part and FP16 imaginary
119/// part.
120/// Calculates the imaginary part of the result. For each possible combination
121/// of (transposed column of \a a, column of \a b), it performs a set of
122/// multiplication and accumulations on all corresponding complex numbers
123/// (one from \a a and one from \a b). The negated imaginary part of the \a a
124/// element is multiplied with the real part of the corresponding \a b
125/// element, and the real part of the \a a element is multiplied with the
126/// imaginary part of the corresponding \a b elements. The two accumulated
127/// results are added, and then accumulated into the corresponding row and
128/// column of \a dst.
129///
130/// \headerfile <x86intrin.h>
131///
132/// \code
133/// void _tile_conjtcmmimfp16ps(__tile dst, __tile a, __tile b);
134/// \endcode
135///
136/// \code{.operation}
137/// FOR m := 0 TO dst.rows - 1
138/// tmp := dst.row[m]
139/// FOR k := 0 TO a.rows - 1
140/// FOR n := 0 TO (dst.colsb / 4) - 1
141/// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+0]) * FP32(b.row[k].fp16[2*n+1])
142/// tmp.fp32[n] += FP32(-a.row[m].fp16[2*k+1]) * FP32(b.row[k].fp16[2*n+0])
143/// ENDFOR
144/// ENDFOR
145/// write_row_and_zero(dst, m, tmp, dst.colsb)
146/// ENDFOR
147/// zero_upper_rows(dst, dst.rows)
148/// zero_tileconfig_start()
149/// \endcode
150///
151/// This intrinsic corresponds to the \c TCONJTCMMIMFP16PS instruction.
152///
153/// \param dst
154/// The destination tile. Max size is 1024 Bytes.
155/// \param a
156/// The 1st source tile. Max size is 1024 Bytes.
157/// \param b
158/// The 2nd source tile. Max size is 1024 Bytes.
159#define _tile_conjtcmmimfp16ps(dst, a, b) \
160 __builtin_ia32_tconjtcmmimfp16ps((dst), (a), (b))
161
162/// Perform conjugate transpose of an FP16-pair of complex elements from \a a
163/// and writes the result to \a dst.
164///
165/// \headerfile <x86intrin.h>
166///
167/// \code
168/// void _tile_conjtfp16(__tile dst, __tile a);
169/// \endcode
170///
171/// \code{.operation}
172/// FOR i := 0 TO dst.rows - 1
173/// FOR j := 0 TO (dst.colsb / 4) - 1
174/// tmp.fp16[2*j+0] := a.row[j].fp16[2*i+0]
175/// tmp.fp16[2*j+1] := -a.row[j].fp16[2*i+1]
176/// ENDFOR
177/// write_row_and_zero(dst, i, tmp, dst.colsb)
178/// ENDFOR
179/// zero_upper_rows(dst, dst.rows)
180/// zero_tileconfig_start()
181/// \endcode
182///
183/// This intrinsic corresponds to the \c TCONJTFP16 instruction.
184///
185/// \param dst
186/// The destination tile. Max size is 1024 Bytes.
187/// \param a
188/// The source tile. Max size is 1024 Bytes.
189#define _tile_conjtfp16(dst, a) __builtin_ia32_tconjtfp16((dst), (a))
190
191static __inline__ _tile1024i __DEFAULT_FN_ATTRS _tile_tcmmimfp16ps_internal(
192 unsigned short m, unsigned short n, unsigned short k, _tile1024i dst,
193 _tile1024i src1, _tile1024i src2) {
194 return __builtin_ia32_ttcmmimfp16ps_internal(m, n, k, dst, src1, src2);
195}
196
197static __inline__ _tile1024i __DEFAULT_FN_ATTRS _tile_tcmmrlfp16ps_internal(
198 unsigned short m, unsigned short n, unsigned short k, _tile1024i dst,
199 _tile1024i src1, _tile1024i src2) {
200 return __builtin_ia32_ttcmmrlfp16ps_internal(m, n, k, dst, src1, src2);
201}
202
203static __inline__ _tile1024i __DEFAULT_FN_ATTRS _tile_conjtcmmimfp16ps_internal(
204 unsigned short m, unsigned short n, unsigned short k, _tile1024i dst,
205 _tile1024i src1, _tile1024i src2) {
206 return __builtin_ia32_tconjtcmmimfp16ps_internal(m, n, k, dst, src1, src2);
207}
208
209static __inline__ _tile1024i __DEFAULT_FN_ATTRS
210_tile_conjtfp16_internal(unsigned short m, unsigned short n, _tile1024i src) {
211 return __builtin_ia32_tconjtfp16_internal(m, n, src);
212}
213
214/// Perform matrix multiplication of two tiles containing complex elements and
215/// accumulate the results into a packed single precision tile. Each dword
216/// element in input tiles src0 and src1 is interpreted as a complex number
217/// with FP16 real part and FP16 imaginary part.
218/// This function calculates the imaginary part of the result.
219///
220/// \headerfile <immintrin.h>
221///
222/// This intrinsic corresponds to the <c> TTCMMIMFP16PS </c> instruction.
223///
224/// \param dst
225/// The destination tile. Max size is 1024 Bytes.
226/// \param src0
227/// The 1st source tile. Max size is 1024 Bytes.
228/// \param src1
229/// The 2nd source tile. Max size is 1024 Bytes.
230__DEFAULT_FN_ATTRS
231static void __tile_tcmmimfp16ps(__tile1024i *dst, __tile1024i src0,
232 __tile1024i src1) {
233 dst->tile = _tile_tcmmimfp16ps_internal(src0.row, src1.col, src0.col,
234 dst->tile, src0.tile, src1.tile);
235}
236
237/// Perform matrix multiplication of two tiles containing complex elements and
238/// accumulate the results into a packed single precision tile. Each dword
239/// element in input tiles src0 and src1 is interpreted as a complex number
240/// with FP16 real part and FP16 imaginary part.
241/// This function calculates the real part of the result.
242///
243/// \headerfile <immintrin.h>
244///
245/// This intrinsic corresponds to the <c> TTCMMRLFP16PS </c> instruction.
246///
247/// \param dst
248/// The destination tile. Max size is 1024 Bytes.
249/// \param src0
250/// The 1st source tile. Max size is 1024 Bytes.
251/// \param src1
252/// The 2nd source tile. Max size is 1024 Bytes.
253__DEFAULT_FN_ATTRS
254static void __tile_tcmmrlfp16ps(__tile1024i *dst, __tile1024i src0,
255 __tile1024i src1) {
256 dst->tile = _tile_tcmmrlfp16ps_internal(src0.row, src1.col, src0.col,
257 dst->tile, src0.tile, src1.tile);
258}
259
260/// Perform matrix conjugate transpose and multiplication of two tiles
261/// containing complex elements and accumulate the results into a packed
262/// single precision tile. Each dword element in input tiles src0 and src1
263/// is interpreted as a complex number with FP16 real part and FP16 imaginary
264/// part.
265/// This function calculates the imaginary part of the result.
266///
267/// \headerfile <immintrin.h>
268///
269/// This intrinsic corresponds to the <c> TCONJTCMMIMFP16PS </c> instruction.
270///
271/// \param dst
272/// The destination tile. Max size is 1024 Bytes.
273/// \param src0
274/// The 1st source tile. Max size is 1024 Bytes.
275/// \param src1
276/// The 2nd source tile. Max size is 1024 Bytes.
277__DEFAULT_FN_ATTRS
278static void __tile_conjtcmmimfp16ps(__tile1024i *dst, __tile1024i src0,
279 __tile1024i src1) {
280 dst->tile = _tile_conjtcmmimfp16ps_internal(src0.row, src1.col, src0.col,
281 dst->tile, src0.tile, src1.tile);
282}
283
284/// Perform conjugate transpose of an FP16-pair of complex elements from src and
285/// writes the result to dst.
286///
287/// \headerfile <immintrin.h>
288///
289/// This intrinsic corresponds to the <c> TCONJTFP16 </c> instruction.
290///
291/// \param dst
292/// The destination tile. Max size is 1024 Bytes.
293/// \param src
294/// The source tile. Max size is 1024 Bytes.
295__DEFAULT_FN_ATTRS
296static void __tile_conjtfp16(__tile1024i *dst, __tile1024i src) {
297 dst->tile = _tile_conjtfp16_internal(src.row, src.col, src.tile);
298}
299
300#undef __DEFAULT_FN_ATTRS
301
302#endif // __x86_64__
303#endif // __AMX_COMPLEXTRANSPOSEINTRIN_H
lib/include/amxfp16intrin.h+35
......@@ -15,6 +15,10 @@
1515#define __AMX_FP16INTRIN_H
1616#ifdef __x86_64__
1717
18/* Define the default attributes for the functions in this file. */
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, __target__("amx-fp16")))
21
1822/// Compute dot-product of FP16 (16-bit) floating-point pairs in tiles \a a
1923/// and \a b, accumulating the intermediate single-precision (32-bit)
2024/// floating-point elements with elements in \a dst, and store the 32-bit
......@@ -54,5 +58,36 @@
5458#define _tile_dpfp16ps(dst, a, b) \
5559 __builtin_ia32_tdpfp16ps(dst, a, b)
5660
61/// This is internal intrinsic. C/C++ user should avoid calling it directly.
62static __inline__ _tile1024i __DEFAULT_FN_ATTRS
63_tile_dpfp16ps_internal(unsigned short m, unsigned short n, unsigned short k,
64 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
65 return __builtin_ia32_tdpfp16ps_internal(m, n, k, dst, src1, src2);
66}
67
68/// Compute dot-product of FP16 (16-bit) floating-point pairs in tiles src0 and
69/// src1, accumulating the intermediate single-precision (32-bit) floating-point
70/// elements with elements in "dst", and store the 32-bit result back to tile
71/// "dst".
72///
73/// \headerfile <immintrin.h>
74///
75/// This intrinsic corresponds to the <c> TDPFP16PS </c> instruction.
76///
77/// \param dst
78/// The destination tile. Max size is 1024 Bytes.
79/// \param src0
80/// The 1st source tile. Max size is 1024 Bytes.
81/// \param src1
82/// The 2nd source tile. Max size is 1024 Bytes.
83__DEFAULT_FN_ATTRS
84static __inline__ void __tile_dpfp16ps(__tile1024i *dst, __tile1024i src0,
85 __tile1024i src1) {
86 dst->tile = _tile_dpfp16ps_internal(src0.row, src1.col, src0.col, dst->tile,
87 src0.tile, src1.tile);
88}
89
90#undef __DEFAULT_FN_ATTRS
91
5792#endif /* __x86_64__ */
5893#endif /* __AMX_FP16INTRIN_H */
lib/include/amxfp16transposeintrin.h created+94
......@@ -0,0 +1,94 @@
1/*===----- amxfp16transposeintrin.h - AMX-FP16 and AMX-TRANSPOSE ------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===------------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error \
12 "Never use <amxfp16transposeintrin.h> directly; use <immintrin.h> instead."
13#endif /* __IMMINTRIN_H */
14
15#ifndef __AMX_FP16TRANSPOSEINTRIN_H
16#define __AMX_FP16TRANSPOSEINTRIN_H
17#ifdef __x86_64__
18
19/* Define the default attributes for the functions in this file. */
20#define __DEFAULT_FN_ATTRS \
21 __attribute__((__always_inline__, __nodebug__, \
22 __target__("amx-fp16,amx-transpose")))
23
24/// Compute transpose and dot-product of FP16 (16-bit) floating-point pairs in
25/// tiles \a a and \a b, accumulating the intermediate single-precision
26/// (32-bit) floating-point elements with elements in \a dst, and store the
27/// 32-bit result back to tile \a dst.
28///
29/// \headerfile <immintrin.h>
30///
31/// \code
32/// void _tile_tdpfp16ps (__tile dst, __tile a, __tile b)
33/// \endcode
34///
35/// \code{.operation}
36/// FOR m := 0 TO dst.rows - 1
37/// tmp := dst.row[m]
38/// FOR k := 0 TO (a.colsb / 4) - 1
39/// FOR n := 0 TO (dst.colsb / 4) - 1
40/// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+0]) *
41/// FP32(b.row[k].fp16[2*n+0])
42/// tmp.fp32[n] += FP32(a.row[m].fp16[2*k+1]) *
43/// FP32(b.row[k].fp16[2*n+1])
44/// ENDFOR
45/// ENDFOR
46/// write_row_and_zero(dst, m, tmp, dst.colsb)
47/// ENDFOR
48/// zero_upper_rows(dst, dst.rows)
49/// zero_tileconfig_start()
50/// \endcode
51///
52/// This intrinsic corresponds to the \c TTDPFP16PS instruction.
53///
54/// \param dst
55/// The destination tile. Max size is 1024 Bytes.
56/// \param a
57/// The 1st source tile. Max size is 1024 Bytes.
58/// \param b
59/// The 2nd source tile. Max size is 1024 Bytes.
60#define _tile_tdpfp16ps(dst, a, b) __builtin_ia32_ttdpfp16ps((dst), (a), (b))
61
62/// This is internal intrinsic. C/C++ user should avoid calling it directly.
63static __inline__ _tile1024i __DEFAULT_FN_ATTRS
64_tile_tdpfp16ps_internal(unsigned short m, unsigned short n, unsigned short k,
65 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
66 return __builtin_ia32_ttdpfp16ps_internal(m, n, k, dst, src1, src2);
67}
68
69/// Compute transpose and dot-product of FP16 (16-bit) floating-point pairs in
70/// tiles src0 and src1, accumulating the intermediate single-precision
71/// (32-bit) floating-point elements with elements in "dst", and store the
72/// 32-bit result back to tile "dst".
73///
74/// \headerfile <immintrin.h>
75///
76/// This intrinsic corresponds to the <c> TTDPFP16PS </c> instruction.
77///
78/// \param dst
79/// The destination tile. Max size is 1024 Bytes.
80/// \param src0
81/// The 1st source tile. Max size is 1024 Bytes.
82/// \param src1
83/// The 2nd source tile. Max size is 1024 Bytes.
84__DEFAULT_FN_ATTRS
85static __inline__ void __tile_tdpfp16ps(__tile1024i *dst, __tile1024i src0,
86 __tile1024i src1) {
87 dst->tile = _tile_tdpfp16ps_internal(src0.row, src1.col, src0.col, dst->tile,
88 src0.tile, src1.tile);
89}
90
91#undef __DEFAULT_FN_ATTRS
92
93#endif /* __x86_64__ */
94#endif /* __AMX_FP16TRANSPOSEINTRIN_H */
lib/include/amxfp8intrin.h created+230
......@@ -0,0 +1,230 @@
1/*===------------- amxfp8intrin.h - AMX intrinsics -*- C++ -*----------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===------------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error "Never use <amxfp8intrin.h> directly; include <immintrin.h> instead."
12#endif /* __IMMINTRIN_H */
13
14#ifndef __AMXFP8INTRIN_H
15#define __AMXFP8INTRIN_H
16#ifdef __x86_64__
17
18#define __DEFAULT_FN_ATTRS_FP8 \
19 __attribute__((__always_inline__, __nodebug__, __target__("amx-fp8")))
20
21static __inline__ _tile1024i __DEFAULT_FN_ATTRS_FP8
22_tile_dpbf8ps_internal(unsigned short m, unsigned short n, unsigned short k,
23 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
24 return __builtin_ia32_tdpbf8ps_internal(m, n, k, dst, src1, src2);
25}
26
27/// Perform the dot product of a BF8 value \a src1 by a BF8 value \a src2
28/// accumulating into a Single Precision (FP32) source/dest \a dst.
29///
30/// \headerfile <immintrin.h>
31///
32/// \code
33/// void __tile_dpbf8ps (__tile1024i *dst, __tile1024i src1, __tile1024i src2)
34/// \endcode
35///
36/// \code{.operation}
37/// FOR m := 0 TO dst.rows - 1
38/// temp1[(dst.colsb / 4 - 1) : 0] = 0
39/// FOR k := 0 TO src1.colsb / 4 - 1
40/// FOR n := 0 TO dst.colsb / 4 - 1
41/// temp1[n] +=
42/// INT64(src1.row[m].float8[4*k+0]) * INT64(src2.row[k].float8[4*n+0])
43/// + INT64(src1.row[m].float8[4*k+1]) * INT64(src2.row[k].float8[4*n+1])
44/// + INT64(src1.row[m].float8[4*k+2]) * INT64(src2.row[k].float8[4*n+2])
45/// + INT64(src1.row[m].float8[4*k+3]) * INT64(src2.row[k].float8[4*n+3])
46/// ENDFOR
47/// ENDFOR
48/// FOR n := 0 TO dst.colsb / 4 - 1
49/// tmp.row[m].fp32[n] = dst.row[m].fp32[n] + FP32(temp1[n])
50/// ENDFOR
51/// write_row_and_zero(dst, m, tmp, dst.colsb)
52/// zero_upper_rows(dst, dst.rows)
53/// zero_tileconfig_start()
54/// \endcode
55///
56/// This intrinsic corresponds to the \c TDPBF8PS instruction.
57///
58/// \param dst
59/// The destination tile. Max size is 1024 Bytes.
60/// \param src1
61/// The 1st source tile. Max size is 1024 Bytes.
62/// \param src2
63/// The 2nd source tile. Max size is 1024 Bytes.
64__DEFAULT_FN_ATTRS_FP8 static void
65__tile_dpbf8ps(__tile1024i *dst, __tile1024i src1, __tile1024i src2) {
66 dst->tile = _tile_dpbf8ps_internal(src1.row, src2.col, src1.col, dst->tile,
67 src1.tile, src2.tile);
68}
69
70static __inline__ _tile1024i __DEFAULT_FN_ATTRS_FP8
71_tile_dpbhf8ps_internal(unsigned short m, unsigned short n, unsigned short k,
72 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
73 return __builtin_ia32_tdpbhf8ps_internal(m, n, k, dst, src1, src2);
74}
75
76/// Perform the dot product of a BF8 value \a src1 by an HF8 value \a src2
77/// accumulating into a Single Precision (FP32) source/dest \a dst.
78///
79/// \headerfile <immintrin.h>
80///
81/// \code
82/// void __tile_dpbhf8ps (__tile1024i dst, __tile1024i src1, __tile1024i src2)
83/// \endcode
84///
85/// \code{.operation}
86/// FOR m := 0 TO dst.rows - 1
87/// temp1[(dst.colsb / 4 - 1) : 0] = 0
88/// FOR k := 0 TO src1.colsb / 4 - 1
89/// FOR n := 0 TO dst.colsb / 4 - 1
90/// temp1[n] +=
91/// INT64(src1.row[m].float8[4*k+0]) * INT64(src2.row[k].float8[4*n+0])
92/// + INT64(src1.row[m].float8[4*k+1]) * INT64(src2.row[k].float8[4*n+1])
93/// + INT64(src1.row[m].float8[4*k+2]) * INT64(src2.row[k].float8[4*n+2])
94/// + INT64(src1.row[m].float8[4*k+3]) * INT64(src2.row[k].float8[4*n+3])
95/// ENDFOR
96/// ENDFOR
97/// FOR n := 0 TO dst.colsb / 4 - 1
98/// tmp.row[m].fp32[n] = dst.row[m].fp32[n] + FP32(temp1[n])
99/// ENDFOR
100/// write_row_and_zero(dst, m, tmp, dst.colsb)
101/// zero_upper_rows(dst, dst.rows)
102/// zero_tileconfig_start()
103/// \endcode
104///
105/// This intrinsic corresponds to the \c TDPBHF8PS instruction.
106///
107/// \param dst
108/// The destination tile. Max size is 1024 Bytes.
109/// \param src1
110/// The 1st source tile. Max size is 1024 Bytes.
111/// \param src2
112/// The 2nd source tile. Max size is 1024 Bytes.
113__DEFAULT_FN_ATTRS_FP8 static void
114__tile_dpbhf8ps(__tile1024i *dst, __tile1024i src1, __tile1024i src2) {
115 dst->tile = _tile_dpbhf8ps_internal(src1.row, src2.col, src1.col, dst->tile,
116 src1.tile, src2.tile);
117}
118
119static __inline__ _tile1024i __DEFAULT_FN_ATTRS_FP8
120_tile_dphbf8ps_internal(unsigned short m, unsigned short n, unsigned short k,
121 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
122 return __builtin_ia32_tdphbf8ps_internal(m, n, k, dst, src1, src2);
123}
124
125/// Perform the dot product of an HF8 value \a src1 by a BF8 value \a src2
126/// accumulating into a Single Precision (FP32) source/dest \a dst.
127///
128/// \headerfile <immintrin.h>
129///
130/// \code
131/// void __tile_dphbf8ps (__tile1024i dst, __tile1024i src1, __tile1024i src2)
132/// \endcode
133///
134/// \code{.operation}
135/// FOR m := 0 TO dst.rows - 1
136/// temp1[(dst.colsb / 4 - 1) : 0] = 0
137/// FOR k := 0 TO src1.colsb / 4 - 1
138/// FOR n := 0 TO dst.colsb / 4 - 1
139/// temp1[n] +=
140/// INT64(src1.row[m].float8[4*k+0]) * INT64(src2.row[k].float8[4*n+0])
141/// + INT64(src1.row[m].float8[4*k+1]) * INT64(src2.row[k].float8[4*n+1])
142/// + INT64(src1.row[m].float8[4*k+2]) * INT64(src2.row[k].float8[4*n+2])
143/// + INT64(src1.row[m].float8[4*k+3]) * INT64(src2.row[k].float8[4*n+3])
144/// ENDFOR
145/// ENDFOR
146/// FOR n := 0 TO dst.colsb / 4 - 1
147/// tmp.row[m].fp32[n] = dst.row[m].fp32[n] + FP32(temp1[n])
148/// ENDFOR
149/// write_row_and_zero(dst, m, tmp, dst.colsb)
150/// zero_upper_rows(dst, dst.rows)
151/// zero_tileconfig_start()
152/// \endcode
153///
154/// This intrinsic corresponds to the \c TDPHBF8PS instruction.
155///
156/// \param dst
157/// The destination tile. Max size is 1024 Bytes.
158/// \param src1
159/// The 1st source tile. Max size is 1024 Bytes.
160/// \param src2
161/// The 2nd source tile. Max size is 1024 Bytes.
162
163__DEFAULT_FN_ATTRS_FP8 static void
164__tile_dphbf8ps(__tile1024i *dst, __tile1024i src1, __tile1024i src2) {
165 dst->tile = _tile_dphbf8ps_internal(src1.row, src2.col, src1.col, dst->tile,
166 src1.tile, src2.tile);
167}
168
169static __inline__ _tile1024i __DEFAULT_FN_ATTRS_FP8
170_tile_dphf8ps_internal(unsigned short m, unsigned short n, unsigned short k,
171 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
172 return __builtin_ia32_tdphf8ps_internal(m, n, k, dst, src1, src2);
173}
174
175/// Perform the dot product of an HF8 value \a src1 by an HF8 value \a src2
176/// accumulating into a Single Precision (FP32) source/dest \a dst.
177///
178/// \headerfile <immintrin.h>
179///
180/// \code
181/// void __tile_dphf8ps (__tile1024i dst, __tile1024i src1, __tile1024i src2)
182/// \endcode
183///
184/// \code{.operation}
185/// FOR m := 0 TO dst.rows - 1
186/// temp1[(dst.colsb / 4 - 1) : 0] = 0
187/// FOR k := 0 TO src1.colsb / 4 - 1
188/// FOR n := 0 TO dst.colsb / 4 - 1
189/// temp1[n] +=
190/// INT64(src1.row[m].float8[4*k+0]) * INT64(src2.row[k].float8[4*n+0])
191/// + INT64(src1.row[m].float8[4*k+1]) * INT64(src2.row[k].float8[4*n+1])
192/// + INT64(src1.row[m].float8[4*k+2]) * INT64(src2.row[k].float8[4*n+2])
193/// + INT64(src1.row[m].float8[4*k+3]) * INT64(src2.row[k].float8[4*n+3])
194/// ENDFOR
195/// ENDFOR
196/// FOR n := 0 TO dst.colsb / 4 - 1
197/// tmp.row[m].fp32[n] = dst.row[m].fp32[n] + FP32(temp1[n])
198/// ENDFOR
199/// write_row_and_zero(dst, m, tmp, dst.colsb)
200/// zero_upper_rows(dst, dst.rows)
201/// zero_tileconfig_start()
202/// \endcode
203///
204/// This intrinsic corresponds to the \c TDPHF8PS instruction.
205///
206/// \param dst
207/// The destination tile. Max size is 1024 Bytes.
208/// \param src1
209/// The 1st source tile. Max size is 1024 Bytes.
210/// \param src2
211/// The 2nd source tile. Max size is 1024 Bytes.
212__DEFAULT_FN_ATTRS_FP8 static void
213__tile_dphf8ps(__tile1024i *dst, __tile1024i src1, __tile1024i src2) {
214 dst->tile = _tile_dphf8ps_internal(src1.row, src2.col, src1.col, dst->tile,
215 src1.tile, src2.tile);
216}
217
218#define _tile_dpbf8ps(dst, src1, src2) \
219 __builtin_ia32_tdpbf8ps((dst), (src1), (src2))
220#define _tile_dpbhf8ps(dst, src1, src2) \
221 __builtin_ia32_tdpbhf8ps((dst), (src1), (src2))
222#define _tile_dphbf8ps(dst, src1, src2) \
223 __builtin_ia32_tdphbf8ps((dst), (src1), (src2))
224#define _tile_dphf8ps(dst, src1, src2) \
225 __builtin_ia32_tdphf8ps((dst), (src1), (src2))
226
227#undef __DEFAULT_FN_ATTRS_FP8
228
229#endif /* __x86_64__ */
230#endif /* __AMXFP8INTRIN_H */
lib/include/amxintrin.h+5-35
......@@ -22,8 +22,6 @@
2222 __attribute__((__always_inline__, __nodebug__, __target__("amx-int8")))
2323#define __DEFAULT_FN_ATTRS_BF16 \
2424 __attribute__((__always_inline__, __nodebug__, __target__("amx-bf16")))
25#define __DEFAULT_FN_ATTRS_FP16 \
26 __attribute__((__always_inline__, __nodebug__, __target__("amx-fp16")))
2725
2826/// Load tile configuration from a 64-byte memory location specified by
2927/// "mem_addr". The tile configuration includes the tile type palette, the
......@@ -232,9 +230,11 @@ static __inline__ void __DEFAULT_FN_ATTRS_TILE _tile_release(void) {
232230/// bytes. Since there is no 2D type in llvm IR, we use vector type to
233231/// represent 2D tile and the fixed size is maximum amx tile register size.
234232typedef int _tile1024i __attribute__((__vector_size__(1024), __aligned__(64)));
233typedef int _tile1024i_1024a
234 __attribute__((__vector_size__(1024), __aligned__(1024)));
235235
236236/// This is internal intrinsic. C/C++ user should avoid calling it directly.
237static __inline__ _tile1024i __DEFAULT_FN_ATTRS_INT8
237static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TILE
238238_tile_loadd_internal(unsigned short m, unsigned short n, const void *base,
239239 __SIZE_TYPE__ stride) {
240240 return __builtin_ia32_tileloadd64_internal(m, n, base,
......@@ -242,7 +242,7 @@ _tile_loadd_internal(unsigned short m, unsigned short n, const void *base,
242242}
243243
244244/// This is internal intrinsic. C/C++ user should avoid calling it directly.
245static __inline__ _tile1024i __DEFAULT_FN_ATTRS_INT8
245static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TILE
246246_tile_loaddt1_internal(unsigned short m, unsigned short n, const void *base,
247247 __SIZE_TYPE__ stride) {
248248 return __builtin_ia32_tileloaddt164_internal(m, n, base,
......@@ -278,7 +278,7 @@ _tile_dpbuud_internal(unsigned short m, unsigned short n, unsigned short k,
278278}
279279
280280/// This is internal intrinsic. C/C++ user should avoid calling it directly.
281static __inline__ void __DEFAULT_FN_ATTRS_INT8
281static __inline__ void __DEFAULT_FN_ATTRS_TILE
282282_tile_stored_internal(unsigned short m, unsigned short n, void *base,
283283 __SIZE_TYPE__ stride, _tile1024i tile) {
284284 return __builtin_ia32_tilestored64_internal(m, n, base,
......@@ -292,13 +292,6 @@ _tile_dpbf16ps_internal(unsigned short m, unsigned short n, unsigned short k,
292292 return __builtin_ia32_tdpbf16ps_internal(m, n, k, dst, src1, src2);
293293}
294294
295/// This is internal intrinsic. C/C++ user should avoid calling it directly.
296static __inline__ _tile1024i __DEFAULT_FN_ATTRS_FP16
297_tile_dpfp16ps_internal(unsigned short m, unsigned short n, unsigned short k,
298 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
299 return __builtin_ia32_tdpfp16ps_internal(m, n, k, dst, src1, src2);
300}
301
302295/// This struct pack the shape and tile data together for user. We suggest
303296/// initializing the struct as early as possible, because compiler depends
304297/// on the shape information to do configure. The constant value is preferred
......@@ -493,32 +486,9 @@ static __inline__ void __tile_dpbf16ps(__tile1024i *dst, __tile1024i src0,
493486 src0.tile, src1.tile);
494487}
495488
496/// Compute dot-product of FP16 (16-bit) floating-point pairs in tiles src0 and
497/// src1, accumulating the intermediate single-precision (32-bit) floating-point
498/// elements with elements in "dst", and store the 32-bit result back to tile
499/// "dst".
500///
501/// \headerfile <immintrin.h>
502///
503/// This intrinsic corresponds to the <c> TDPFP16PS </c> instruction.
504///
505/// \param dst
506/// The destination tile. Max size is 1024 Bytes.
507/// \param src0
508/// The 1st source tile. Max size is 1024 Bytes.
509/// \param src1
510/// The 2nd source tile. Max size is 1024 Bytes.
511__DEFAULT_FN_ATTRS_FP16
512static __inline__ void __tile_dpfp16ps(__tile1024i *dst, __tile1024i src0,
513 __tile1024i src1) {
514 dst->tile = _tile_dpfp16ps_internal(src0.row, src1.col, src0.col, dst->tile,
515 src0.tile, src1.tile);
516}
517
518489#undef __DEFAULT_FN_ATTRS_TILE
519490#undef __DEFAULT_FN_ATTRS_INT8
520491#undef __DEFAULT_FN_ATTRS_BF16
521#undef __DEFAULT_FN_ATTRS_FP16
522492
523493#endif /* __x86_64__ */
524494#endif /* __AMXINTRIN_H */
lib/include/amxmovrsintrin.h created+48
......@@ -0,0 +1,48 @@
1/*===-------- amxmovrsintrin.h - AMX MOVRS intrinsics -*- C++ -*---------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 * ===-------------------------------------------------------------------=== */
8
9#ifndef __IMMINTRIN_H
10#error "Never use <amxmovrsintrin.h> directly; include <immintrin.h> instead."
11#endif /* __IMMINTRIN_H */
12
13#ifndef __AMXMOVRSINTRIN_H
14#define __AMXMOVRSINTRIN_H
15#ifdef __x86_64__
16
17#define __DEFAULT_FN_ATTRS_MOVRS \
18 __attribute__((__always_inline__, __nodebug__, __target__("amx-movrs")))
19
20#define _tile_loaddrs(dst, base, stride) \
21 __builtin_ia32_tileloaddrs64((dst), ((const void *)(base)), \
22 (__SIZE_TYPE__)(stride))
23#define _tile_stream_loaddrs(dst, base, stride) \
24 __builtin_ia32_tileloaddrst164((dst), ((const void *)(base)), \
25 (__SIZE_TYPE__)(stride))
26static __inline__ _tile1024i __DEFAULT_FN_ATTRS_MOVRS
27_tile_loaddrs_internal(unsigned short m, unsigned short n, const void *base,
28 __SIZE_TYPE__ stride) {
29 return __builtin_ia32_tileloaddrs64_internal(m, n, base,
30 (__SIZE_TYPE__)(stride));
31}
32static __inline__ _tile1024i __DEFAULT_FN_ATTRS_MOVRS
33_tile_loaddrst1_internal(unsigned short m, unsigned short n, const void *base,
34 __SIZE_TYPE__ stride) {
35 return __builtin_ia32_tileloaddrst164_internal(m, n, base,
36 (__SIZE_TYPE__)(stride));
37}
38static __inline__ void __DEFAULT_FN_ATTRS_MOVRS
39__tile_loaddrs(__tile1024i *dst, const void *base, __SIZE_TYPE__ stride) {
40 dst->tile = _tile_loaddrs_internal(dst->row, dst->col, base, stride);
41}
42static __inline__ void __DEFAULT_FN_ATTRS_MOVRS __tile_stream_loaddrs(
43 __tile1024i *dst, const void *base, __SIZE_TYPE__ stride) {
44 dst->tile = _tile_loaddrst1_internal(dst->row, dst->col, base, stride);
45}
46#undef __DEFAULT_FN_ATTRS_MOVRS
47#endif /* __x86_64__ */
48#endif /* __AMXMOVRSINTRIN_H */
lib/include/amxmovrstransposeintrin.h created+200
......@@ -0,0 +1,200 @@
1/* ===--- amxmovrstransposeintrin.h - AMX_MOVRS_TRANSPOSE intrinsics --------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 * ===-----------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error \
12 "Never use <amxmovrstransposeintrin.h> directly; use <immintrin.h> instead."
13#endif /* __IMMINTRIN_H */
14
15#ifndef __AMX_MOVRS_TRANSPOSEINTRIN_H
16#define __AMX_MOVRS_TRANSPOSEINTRIN_H
17#ifdef __x86_64__
18
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, \
21 __target__("amx-transpose,amx-movrs")))
22
23#define _tile_2rpntlvwz0rs(tdst, base, stride) \
24 __builtin_ia32_t2rpntlvwz0rs(tdst, base, stride)
25#define _tile_2rpntlvwz0rst1(tdst, base, stride) \
26 __builtin_ia32_t2rpntlvwz0rst1(tdst, base, stride)
27#define _tile_2rpntlvwz1rs(tdst, base, stride) \
28 __builtin_ia32_t2rpntlvwz1rs(tdst, base, stride)
29#define _tile_2rpntlvwz1rst1(tdst, base, stride) \
30 __builtin_ia32_t2rpntlvwz1rst1(tdst, base, stride)
31
32static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz0rs_internal(
33 unsigned short row, unsigned short col0, unsigned short col1,
34 _tile1024i *dst0, _tile1024i *dst1, const void *base,
35 __SIZE_TYPE__ stride) {
36 // Use __tile1024i_1024a* to escape the alignment check in
37 // clang/test/Headers/x86-intrinsics-headers-clean.cpp
38 __builtin_ia32_t2rpntlvwz0rs_internal(
39 row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
40 (__SIZE_TYPE__)(stride));
41}
42
43static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz0rst1_internal(
44 unsigned short row, unsigned short col0, unsigned short col1,
45 _tile1024i *dst0, _tile1024i *dst1, const void *base,
46 __SIZE_TYPE__ stride) {
47 __builtin_ia32_t2rpntlvwz0rst1_internal(
48 row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
49 (__SIZE_TYPE__)(stride));
50}
51
52static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz1rs_internal(
53 unsigned short row, unsigned short col0, unsigned short col1,
54 _tile1024i *dst0, _tile1024i *dst1, const void *base,
55 __SIZE_TYPE__ stride) {
56 __builtin_ia32_t2rpntlvwz1rs_internal(
57 row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
58 (__SIZE_TYPE__)(stride));
59}
60
61static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz1rst1_internal(
62 unsigned short row, unsigned short col0, unsigned short col1,
63 _tile1024i *dst0, _tile1024i *dst1, const void *base,
64 __SIZE_TYPE__ stride) {
65 __builtin_ia32_t2rpntlvwz1rst1_internal(
66 row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
67 (__SIZE_TYPE__)(stride));
68}
69
70/// Converts a pair of tiles from memory into VNNI format, and places the
71/// results in a pair of destinations specified by dst. The pair of tiles
72/// in memory is specified via a tsib; the second tile is after the first
73/// one, separated by the same stride that separates each row.
74/// The tile configuration for the destination tiles indicates the amount
75/// of data to read from memory. The instruction will load a number of rows
76/// that is equal to twice the number of rows in tmm1. The size of each row
77/// is equal to the average width of the destination tiles. If the second
78/// tile is configured with zero rows and columns, only the first tile will
79/// be written.
80/// Provides a hint to the implementation that the data will likely become
81/// read shared in the near future and the data caching can be optimized.
82///
83/// \headerfile <immintrin.h>
84///
85/// This intrinsic corresponds to the <c> T2RPNTLVWZ0RS </c> instruction.
86///
87/// \param dst0
88/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
89/// \param dst1
90/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
91/// \param base
92/// A pointer to base address.
93/// \param stride
94/// The stride between the rows' data to be loaded in memory.
95__DEFAULT_FN_ATTRS
96static void __tile_2rpntlvwz0rs(__tile1024i *dst0, __tile1024i *dst1,
97 const void *base, __SIZE_TYPE__ stride) {
98 _tile_2rpntlvwz0rs_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
99 &dst1->tile, base, stride);
100}
101
102/// Converts a pair of tiles from memory into VNNI format, and places the
103/// results in a pair of destinations specified by dst. The pair of tiles
104/// in memory is specified via a tsib; the second tile is after the first
105/// one, separated by the same stride that separates each row.
106/// The tile configuration for the destination tiles indicates the amount
107/// of data to read from memory. The instruction will load a number of rows
108/// that is equal to twice the number of rows in tmm1. The size of each row
109/// is equal to the average width of the destination tiles. If the second
110/// tile is configured with zero rows and columns, only the first tile will
111/// be written.
112///
113/// \headerfile <immintrin.h>
114///
115/// This intrinsic corresponds to the <c> T2RPNTLVWZ0T1RS </c> instruction.
116///
117/// \param dst0
118/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
119/// \param dst1
120/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
121/// \param base
122/// A pointer to base address.
123/// \param stride
124/// The stride between the rows' data to be loaded in memory.
125__DEFAULT_FN_ATTRS
126static void __tile_2rpntlvwz0rst1(__tile1024i *dst0, __tile1024i *dst1,
127 const void *base, __SIZE_TYPE__ stride) {
128 _tile_2rpntlvwz0rst1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
129 &dst1->tile, base, stride);
130}
131
132/// Converts a pair of tiles from memory into VNNI format, and places the
133/// results in a pair of destinations specified by dst. The pair of tiles
134/// in memory is specified via a tsib; the second tile is after the first
135/// one, separated by the same stride that separates each row.
136/// The tile configuration for the destination tiles indicates the amount
137/// of data to read from memory. The instruction will load a number of rows
138/// that is equal to twice the number of rows in tmm1. The size of each row
139/// is equal to the average width of the destination tiles. If the second
140/// tile is configured with zero rows and columns, only the first tile will
141/// be written. The last row will be not be read from memory but instead
142/// filled with zeros.
143/// Provides a hint to the implementation that the data will likely become
144/// read shared in the near future and the data caching can be optimized.
145///
146/// \headerfile <immintrin.h>
147///
148/// This intrinsic corresponds to the <c> T2RPNTLVWZ1 </c> instruction.
149///
150/// \param dst0
151/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
152/// \param dst1
153/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
154/// \param base
155/// A pointer to base address.
156/// \param stride
157/// The stride between the rows' data to be loaded in memory.
158__DEFAULT_FN_ATTRS
159static void __tile_2rpntlvwz1rs(__tile1024i *dst0, __tile1024i *dst1,
160 const void *base, __SIZE_TYPE__ stride) {
161 _tile_2rpntlvwz1rs_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
162 &dst1->tile, base, stride);
163}
164
165/// Converts a pair of tiles from memory into VNNI format, and places the
166/// results in a pair of destinations specified by dst. The pair of tiles
167/// in memory is specified via a tsib; the second tile is after the first
168/// one, separated by the same stride that separates each row.
169/// The tile configuration for the destination tiles indicates the amount
170/// of data to read from memory. The instruction will load a number of rows
171/// that is equal to twice the number of rows in tmm1. The size of each row
172/// is equal to the average width of the destination tiles. If the second
173/// tile is configured with zero rows and columns, only the first tile will
174/// be written. The last row will be not be read from memory but instead
175/// filled with zeros.
176/// Provides a hint to the implementation that the data will likely become
177/// read shared in the near future and the data caching can be optimized.
178///
179/// \headerfile <immintrin.h>
180///
181/// This intrinsic corresponds to the <c> T2RPNTLVWZ1T1RS </c> instruction.
182///
183/// \param dst0
184/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
185/// \param dst1
186/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
187/// \param base
188/// A pointer to base address.
189/// \param stride
190/// The stride between the rows' data to be loaded in memory.
191__DEFAULT_FN_ATTRS
192static void __tile_2rpntlvwz1rst1(__tile1024i *dst0, __tile1024i *dst1,
193 const void *base, __SIZE_TYPE__ stride) {
194 _tile_2rpntlvwz1rst1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
195 &dst1->tile, base, stride);
196}
197
198#undef __DEFAULT_FN_ATTRS
199#endif /* __x86_64__ */
200#endif /* __AMX_MOVRS_TRANSPOSEINTRIN_H */
\ No newline at end of file
lib/include/amxtf32intrin.h created+108
......@@ -0,0 +1,108 @@
1/*===------------- amxtf32intrin.h - AMX_TF32 intrinsics -*- C++ -*---------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===------------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error "Never use <amxtf32intrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifndef __AMX_TF32INTRIN_H
15#define __AMX_TF32INTRIN_H
16#ifdef __x86_64__
17
18#define __DEFAULT_FN_ATTRS_TF32 \
19 __attribute__((__always_inline__, __nodebug__, __target__("amx-tf32")))
20
21/// Do Matrix Multiplication of \a a and \a b, and then do Matrix Plus
22/// with \a srcdst.
23/// All the calculation is base on float32 but with the lower 13-bit set to 0.
24///
25/// \headerfile <immintrin.h>
26///
27/// \code
28/// void _tile_mmultf32ps(constexpr int srcdst, constexpr int a, \
29/// constexpr int b);
30/// \endcode
31///
32/// This intrinsic corresponds to the <c> TMMULTF32PS </c> instruction.
33///
34/// \param srcdst
35/// The destination tile. Max size is 1024 Bytes.
36/// \param a
37/// The 1st source tile. Max size is 1024 Bytes.
38/// \param b
39/// The 2nd source tile. Max size is 1024 Bytes.
40///
41/// \code{.operation}
42/// DEFINE zero_lower_mantissa_bits_fp32(x[31:0]) {
43/// dword[12:0] := 0
44/// dword[31:13] := x[31:13]
45/// return dword
46/// }
47///
48/// DEFINE silence_snan_fp32(x[31:0]) {
49/// IF (x.exponent == 255 and x.fraction != 0 and x.fraction[22] == 0)
50/// x.fraction[22] := 1
51/// return x
52/// }
53///
54/// elements_a := a.colsb / 4
55/// elements_dest := srcdst.colsb / 4
56///
57/// FOR m = 0 TO (srcdst.rows-1)
58/// tmp[511:0] := 0
59/// FOR k = 0 TO (elements_a-1)
60/// FOR n = 0 TO (elements_dest-1)
61/// af := silence_snan_fp32(a.row[m].fp32[k])
62/// bf := silence_snan_fp32(b.row[k].fp32[n])
63/// tmp.fp32[n] += zero_lower_mantissa_bits_fp32(af)
64/// * zero_lower_mantissa_bits_fp32(bf)
65/// ENDFOR
66/// ENDFOR
67///
68/// FOR n = 0 TO (elements_dest-1)
69/// tmp.fp32[n] += srcdst.row[m].fp32[n]
70/// ENDFOR
71/// write_row_and_zero(srcdst, m, tmp, srcdst.colsb)
72///
73/// ENDFOR
74///
75/// zero_upper_rows(srcdst, srcdst.rows)
76/// zero_tileconfig_start()
77/// \endcode
78#define _tile_mmultf32ps(srcdst, a, b) \
79 __builtin_ia32_tmmultf32ps((srcdst), (a), (b))
80
81static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TF32
82_tile_mmultf32ps_internal(unsigned short m, unsigned short n, unsigned short k,
83 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
84 return __builtin_ia32_tmmultf32ps_internal(m, n, k, dst, src1, src2);
85}
86
87/// Do Matrix Multiplication of src0 and src1, and then do Matrix Plus with dst.
88/// All the calculation is base on float32 but with the lower 13-bit set to 0.
89///
90/// \headerfile <immintrin.h>
91///
92/// This intrinsic corresponds to the <c> TMMULTF32PS </c> instruction.
93///
94/// \param dst
95/// The destination tile. Max size is 1024 Bytes.
96/// \param src0
97/// The 1st source tile. Max size is 1024 Bytes.
98/// \param src1
99/// The 2nd source tile. Max size is 1024 Bytes.
100__DEFAULT_FN_ATTRS_TF32
101static void __tile_mmultf32ps(__tile1024i *dst, __tile1024i src0,
102 __tile1024i src1) {
103 dst->tile = _tile_mmultf32ps_internal(src0.row, src1.col, src0.col, dst->tile,
104 src0.tile, src1.tile);
105}
106
107#endif // __x86_64__
108#endif // __AMX_TF32INTRIN_H
lib/include/amxtf32transposeintrin.h created+105
......@@ -0,0 +1,105 @@
1/*===--------- amxtf32transposeintrin.h - AMX-TF32 and AMX-TRANSPOSE --------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===------------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <amxtf32tranposeintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifndef __AMX_TF32TRANSPOSEINTRIN_H
15#define __AMX_TF32TRANSPOSEINTRIN_H
16#ifdef __x86_64__
17
18#define __DEFAULT_FN_ATTRS_TF32_TRANSPOSE \
19 __attribute__((__always_inline__, __nodebug__, \
20 __target__("amx-tf32,amx-transpose")))
21
22/// \code
23/// void _tile_tmmultf32ps(constexpr int srcdst, constexpr int a, \
24/// constexpr int b);
25/// \endcode
26///
27/// This intrinsic corresponds to the <c> TTMMULTF32PS </c> instruction.
28///
29/// \param srcdst
30/// The destination tile. Max size is 1024 Bytes.
31/// \param a
32/// The 1st source tile. Max size is 1024 Bytes.
33/// \param b
34/// The 2nd source tile. Max size is 1024 Bytes.
35///
36/// \code{.operation}
37/// DEFINE zero_lower_mantissa_bits_fp32(x[31:0]) {
38/// dword[12:0] := 0
39/// dword[31:13] := x[31:13]
40/// return dword
41/// }
42///
43/// DEFINE silence_snan_fp32(x[31:0]) {
44/// IF (x.exponent == 255 and x.fraction != 0 and x.fraction[22] == 0)
45/// x.fraction[22] := 1
46/// return x
47/// }
48///
49/// elements_dest:= srcdst.colsb/4
50///
51/// FOR m := 0 TO (srcdst.rows-1)
52/// tmp[511:0] := 0
53/// FOR k := 0 TO (a.rows-1)
54/// FOR n := 0 TO (elements_dest-1)
55/// a1e := silence_snan_fp32(a.row[k].fp32[m])
56/// a2e := silence_snan_fp32(b.row[k].fp32[n])
57/// s1e := zero_lower_mantissa_bits_fp32(a1e)
58/// s2e := zero_lower_mantissa_bits_fp32(a2e)
59/// tmp.fp32[n] += s1e * s2e
60/// ENDFOR
61/// ENDFOR
62///
63/// FOR n := 0 TO (elements_dest-1)
64/// tmp.fp32[n] += srcdst.row[m].fp32[n]
65/// ENDFOR
66/// write_row_and_zero(srcdst, m, tmp, srcdst.colsb)
67///
68/// ENDFOR
69///
70/// zero_upper_rows(srcdst, srcdst.rows)
71/// zero_tileconfig_start()
72/// \endcode
73#define _tile_tmmultf32ps(srcdst, a, b) \
74 __builtin_ia32_ttmmultf32ps((srcdst), (a), (b))
75
76// dst = m x n (srcdest), src1 = k x m, src2 = k x n
77static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TF32_TRANSPOSE
78_tile_tmmultf32ps_internal(unsigned short m, unsigned short n, unsigned short k,
79 _tile1024i dst, _tile1024i src1, _tile1024i src2) {
80 return __builtin_ia32_ttmmultf32ps_internal(m, n, k, dst, src1, src2);
81}
82
83/// Compute transpose and do Matrix Multiplication of src0 and src1, and then do
84/// Matrix Plus with dst. All the calculation is base on float32 but with the
85/// lower 13-bit set to 0.
86///
87/// \headerfile <immintrin.h>
88///
89/// This intrinsic corresponds to the <c> TTMMULTF32PS </c> instruction.
90///
91/// \param dst
92/// The destination tile. Max size is 1024 Bytes.
93/// \param src0
94/// The 1st source tile. Max size is 1024 Bytes.
95/// \param src1
96/// The 2nd source tile. Max size is 1024 Bytes.
97__DEFAULT_FN_ATTRS_TF32_TRANSPOSE
98static void __tile_tmmultf32ps(__tile1024i *dst, __tile1024i src0,
99 __tile1024i src1) {
100 dst->tile = _tile_tmmultf32ps_internal(src0.row, src1.col, src0.col,
101 dst->tile, src0.tile, src1.tile);
102}
103
104#endif // __x86_64__
105#endif // __AMX_TF32TRANSPOSEINTRIN_H
lib/include/amxtransposeintrin.h created+248
......@@ -0,0 +1,248 @@
1/* ===--- amxtransposeintrin.h - AMX_TRANSPOSE intrinsics -*- C++ -*---------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 * ===-----------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error "Never use <amxtransposeintrin.h> directly; use <immintrin.h> instead."
12#endif /* __IMMINTRIN_H */
13
14#ifndef __AMX_TRANSPOSEINTRIN_H
15#define __AMX_TRANSPOSEINTRIN_H
16#ifdef __x86_64__
17
18#define __DEFAULT_FN_ATTRS_TRANSPOSE \
19 __attribute__((__always_inline__, __nodebug__, __target__("amx-transpose")))
20
21#define _tile_2rpntlvwz0(tdst, base, stride) \
22 __builtin_ia32_t2rpntlvwz0(tdst, base, stride)
23#define _tile_2rpntlvwz0t1(tdst, base, stride) \
24 __builtin_ia32_t2rpntlvwz0t1(tdst, base, stride)
25#define _tile_2rpntlvwz1(tdst, base, stride) \
26 __builtin_ia32_t2rpntlvwz1(tdst, base, stride)
27#define _tile_2rpntlvwz1t1(tdst, base, stride) \
28 __builtin_ia32_t2rpntlvwz1t1(tdst, base, stride)
29
30/// Transpose 32-bit elements from \a src and write the result to \a dst.
31///
32/// \headerfile <immintrin.h>
33///
34/// \code
35/// void _tile_transposed(__tile dst, __tile src);
36/// \endcode
37///
38/// This intrinsic corresponds to the <c> TTRANSPOSED </c> instruction.
39///
40/// \param dst
41/// The destination tile. Max size is 1024 Bytes.
42/// \param src
43/// The source tile. Max size is 1024 Bytes.
44///
45/// \code{.operation}
46///
47/// FOR i := 0 TO (dst.rows-1)
48/// tmp[511:0] := 0
49/// FOR j := 0 TO (dst.colsb/4-1)
50/// tmp.dword[j] := src.row[j].dword[i]
51/// ENDFOR
52/// dst.row[i] := tmp
53/// ENDFOR
54///
55/// zero_upper_rows(dst, dst.rows)
56/// zero_tileconfig_start()
57/// \endcode
58#define _tile_transposed(dst, src) __builtin_ia32_ttransposed(dst, src)
59
60static __inline__ void __DEFAULT_FN_ATTRS_TRANSPOSE _tile_2rpntlvwz0_internal(
61 unsigned short row, unsigned short col0, unsigned short col1,
62 _tile1024i *dst0, _tile1024i *dst1, const void *base,
63 __SIZE_TYPE__ stride) {
64 // Use __tile1024i_1024a* to escape the alignment check in
65 // clang/test/Headers/x86-intrinsics-headers-clean.cpp
66 __builtin_ia32_t2rpntlvwz0_internal(row, col0, col1, (_tile1024i_1024a *)dst0,
67 (_tile1024i_1024a *)dst1, base,
68 (__SIZE_TYPE__)(stride));
69}
70
71static __inline__ void __DEFAULT_FN_ATTRS_TRANSPOSE _tile_2rpntlvwz0t1_internal(
72 unsigned short row, unsigned short col0, unsigned short col1,
73 _tile1024i *dst0, _tile1024i *dst1, const void *base,
74 __SIZE_TYPE__ stride) {
75 __builtin_ia32_t2rpntlvwz0t1_internal(
76 row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
77 (__SIZE_TYPE__)(stride));
78}
79
80static __inline__ void __DEFAULT_FN_ATTRS_TRANSPOSE _tile_2rpntlvwz1_internal(
81 unsigned short row, unsigned short col0, unsigned short col1,
82 _tile1024i *dst0, _tile1024i *dst1, const void *base,
83 __SIZE_TYPE__ stride) {
84 __builtin_ia32_t2rpntlvwz1_internal(row, col0, col1, (_tile1024i_1024a *)dst0,
85 (_tile1024i_1024a *)dst1, base,
86 (__SIZE_TYPE__)(stride));
87}
88
89static __inline__ void __DEFAULT_FN_ATTRS_TRANSPOSE _tile_2rpntlvwz1t1_internal(
90 unsigned short row, unsigned short col0, unsigned short col1,
91 _tile1024i *dst0, _tile1024i *dst1, const void *base,
92 __SIZE_TYPE__ stride) {
93 __builtin_ia32_t2rpntlvwz1t1_internal(
94 row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
95 (__SIZE_TYPE__)(stride));
96}
97
98// This is internal intrinsic. C/C++ user should avoid calling it directly.
99static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TRANSPOSE
100_tile_transposed_internal(unsigned short m, unsigned short n, _tile1024i src) {
101 return __builtin_ia32_ttransposed_internal(m, n, src);
102}
103
104/// Converts a pair of tiles from memory into VNNI format, and places the
105/// results in a pair of destinations specified by dst. The pair of tiles
106/// in memory is specified via a tsib; the second tile is after the first
107/// one, separated by the same stride that separates each row.
108/// The tile configuration for the destination tiles indicates the amount
109/// of data to read from memory. The instruction will load a number of rows
110/// that is equal to twice the number of rows in tmm1. The size of each row
111/// is equal to the average width of the destination tiles. If the second
112/// tile is configured with zero rows and columns, only the first tile will
113/// be written.
114/// Provides a hint to the implementation that the data will likely not be
115/// reused in the near future and the data caching can be optimized.
116///
117/// \headerfile <immintrin.h>
118///
119/// This intrinsic corresponds to the <c> T2RPNTLVWZ0 </c> instruction.
120///
121/// \param dst0
122/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
123/// \param dst1
124/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
125/// \param base
126/// A pointer to base address.
127/// \param stride
128/// The stride between the rows' data to be loaded in memory.
129__DEFAULT_FN_ATTRS_TRANSPOSE
130static void __tile_2rpntlvwz0(__tile1024i *dst0, __tile1024i *dst1,
131 const void *base, __SIZE_TYPE__ stride) {
132 _tile_2rpntlvwz0_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
133 &dst1->tile, base, stride);
134}
135
136/// Converts a pair of tiles from memory into VNNI format, and places the
137/// results in a pair of destinations specified by dst. The pair of tiles
138/// in memory is specified via a tsib; the second tile is after the first
139/// one, separated by the same stride that separates each row.
140/// The tile configuration for the destination tiles indicates the amount
141/// of data to read from memory. The instruction will load a number of rows
142/// that is equal to twice the number of rows in tmm1. The size of each row
143/// is equal to the average width of the destination tiles. If the second
144/// tile is configured with zero rows and columns, only the first tile will
145/// be written.
146///
147/// \headerfile <immintrin.h>
148///
149/// This intrinsic corresponds to the <c> T2RPNTLVWZ0T1 </c> instruction.
150///
151/// \param dst0
152/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
153/// \param dst1
154/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
155/// \param base
156/// A pointer to base address.
157/// \param stride
158/// The stride between the rows' data to be loaded in memory.
159__DEFAULT_FN_ATTRS_TRANSPOSE
160static void __tile_2rpntlvwz0t1(__tile1024i *dst0, __tile1024i *dst1,
161 const void *base, __SIZE_TYPE__ stride) {
162 _tile_2rpntlvwz0t1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
163 &dst1->tile, base, stride);
164}
165
166/// Converts a pair of tiles from memory into VNNI format, and places the
167/// results in a pair of destinations specified by dst. The pair of tiles
168/// in memory is specified via a tsib; the second tile is after the first
169/// one, separated by the same stride that separates each row.
170/// The tile configuration for the destination tiles indicates the amount
171/// of data to read from memory. The instruction will load a number of rows
172/// that is equal to twice the number of rows in tmm1. The size of each row
173/// is equal to the average width of the destination tiles. If the second
174/// tile is configured with zero rows and columns, only the first tile will
175/// be written. The last row will be not be read from memory but instead
176/// filled with zeros.
177/// Provides a hint to the implementation that the data will likely not be
178/// reused in the near future and the data caching can be optimized.
179///
180/// \headerfile <immintrin.h>
181///
182/// This intrinsic corresponds to the <c> T2RPNTLVWZ1 </c> instruction.
183///
184/// \param dst0
185/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
186/// \param dst1
187/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
188/// \param base
189/// A pointer to base address.
190/// \param stride
191/// The stride between the rows' data to be loaded in memory.
192__DEFAULT_FN_ATTRS_TRANSPOSE
193static void __tile_2rpntlvwz1(__tile1024i *dst0, __tile1024i *dst1,
194 const void *base, __SIZE_TYPE__ stride) {
195 _tile_2rpntlvwz1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
196 &dst1->tile, base, stride);
197}
198
199/// Converts a pair of tiles from memory into VNNI format, and places the
200/// results in a pair of destinations specified by dst. The pair of tiles
201/// in memory is specified via a tsib; the second tile is after the first
202/// one, separated by the same stride that separates each row.
203/// The tile configuration for the destination tiles indicates the amount
204/// of data to read from memory. The instruction will load a number of rows
205/// that is equal to twice the number of rows in tmm1. The size of each row
206/// is equal to the average width of the destination tiles. If the second
207/// tile is configured with zero rows and columns, only the first tile will
208/// be written. The last row will be not be read from memory but instead
209/// filled with zeros.
210/// Provides a hint to the implementation that the data will likely not be
211/// reused in the near future and the data caching can be optimized.
212///
213/// \headerfile <immintrin.h>
214///
215/// This intrinsic corresponds to the <c> T2RPNTLVWZ1T1 </c> instruction.
216///
217/// \param dst0
218/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
219/// \param dst1
220/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
221/// \param base
222/// A pointer to base address.
223/// \param stride
224/// The stride between the rows' data to be loaded in memory.
225__DEFAULT_FN_ATTRS_TRANSPOSE
226static void __tile_2rpntlvwz1t1(__tile1024i *dst0, __tile1024i *dst1,
227 const void *base, __SIZE_TYPE__ stride) {
228 _tile_2rpntlvwz1t1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
229 &dst1->tile, base, stride);
230}
231
232/// Transpose 32-bit elements from src and write the result to dst.
233///
234/// \headerfile <immintrin.h>
235///
236/// This intrinsic corresponds to the <c> TTRANSPOSED </c> instruction.
237///
238/// \param dst
239/// The destination tile. Max size is 1024 Bytes.
240/// \param src
241/// The source tile. Max size is 1024 Bytes.
242__DEFAULT_FN_ATTRS_TRANSPOSE
243static void __tile_transposed(__tile1024i *dst, __tile1024i src) {
244 dst->tile = _tile_transposed_internal(dst->row, dst->col, src.tile);
245}
246
247#endif /* __x86_64__ */
248#endif /* __AMX_TRANSPOSEINTRIN_H */
lib/include/arm_acle.h+18-21
......@@ -264,28 +264,28 @@ __rbitl(unsigned long __t) {
264264}
265265
266266/* 8.3 16-bit multiplications */
267#if defined(__ARM_FEATURE_DSP) && __ARM_FEATURE_DSP
268static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))
267#if defined(__ARM_32BIT_STATE) && __ARM_32BIT_STATE
268static __inline__ int32_t __attribute__((__always_inline__,__nodebug__, target("dsp")))
269269__smulbb(int32_t __a, int32_t __b) {
270270 return __builtin_arm_smulbb(__a, __b);
271271}
272static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))
272static __inline__ int32_t __attribute__((__always_inline__,__nodebug__, target("dsp")))
273273__smulbt(int32_t __a, int32_t __b) {
274274 return __builtin_arm_smulbt(__a, __b);
275275}
276static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))
276static __inline__ int32_t __attribute__((__always_inline__,__nodebug__, target("dsp")))
277277__smultb(int32_t __a, int32_t __b) {
278278 return __builtin_arm_smultb(__a, __b);
279279}
280static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))
280static __inline__ int32_t __attribute__((__always_inline__,__nodebug__, target("dsp")))
281281__smultt(int32_t __a, int32_t __b) {
282282 return __builtin_arm_smultt(__a, __b);
283283}
284static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))
284static __inline__ int32_t __attribute__((__always_inline__,__nodebug__, target("dsp")))
285285__smulwb(int32_t __a, int32_t __b) {
286286 return __builtin_arm_smulwb(__a, __b);
287287}
288static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))
288static __inline__ int32_t __attribute__((__always_inline__,__nodebug__, target("dsp")))
289289__smulwt(int32_t __a, int32_t __b) {
290290 return __builtin_arm_smulwt(__a, __b);
291291}
......@@ -304,46 +304,46 @@ __smulwt(int32_t __a, int32_t __b) {
304304#endif
305305
306306/* 8.4.2 Saturating addition and subtraction intrinsics */
307#if defined(__ARM_FEATURE_DSP) && __ARM_FEATURE_DSP
308static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
307#if defined(__ARM_32BIT_STATE) && __ARM_32BIT_STATE
308static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
309309__qadd(int32_t __t, int32_t __v) {
310310 return __builtin_arm_qadd(__t, __v);
311311}
312312
313static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
313static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
314314__qsub(int32_t __t, int32_t __v) {
315315 return __builtin_arm_qsub(__t, __v);
316316}
317317
318static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
318static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
319319__qdbl(int32_t __t) {
320320 return __builtin_arm_qadd(__t, __t);
321321}
322322#endif
323323
324324/* 8.4.3 Accumulating multiplications */
325#if defined(__ARM_FEATURE_DSP) && __ARM_FEATURE_DSP
326static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
325#if defined(__ARM_32BIT_STATE) && __ARM_32BIT_STATE
326static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
327327__smlabb(int32_t __a, int32_t __b, int32_t __c) {
328328 return __builtin_arm_smlabb(__a, __b, __c);
329329}
330static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
330static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
331331__smlabt(int32_t __a, int32_t __b, int32_t __c) {
332332 return __builtin_arm_smlabt(__a, __b, __c);
333333}
334static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
334static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
335335__smlatb(int32_t __a, int32_t __b, int32_t __c) {
336336 return __builtin_arm_smlatb(__a, __b, __c);
337337}
338static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
338static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
339339__smlatt(int32_t __a, int32_t __b, int32_t __c) {
340340 return __builtin_arm_smlatt(__a, __b, __c);
341341}
342static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
342static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
343343__smlawb(int32_t __a, int32_t __b, int32_t __c) {
344344 return __builtin_arm_smlawb(__a, __b, __c);
345345}
346static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
346static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("dsp")))
347347__smlawt(int32_t __a, int32_t __b, int32_t __c) {
348348 return __builtin_arm_smlawt(__a, __b, __c);
349349}
......@@ -621,8 +621,6 @@ __rintnf(float __a) {
621621#endif
622622
623623/* 8.8 CRC32 intrinsics */
624#if (defined(__ARM_FEATURE_CRC32) && __ARM_FEATURE_CRC32) || \
625 (defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE)
626624static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__, target("crc")))
627625__crc32b(uint32_t __a, uint8_t __b) {
628626 return __builtin_arm_crc32b(__a, __b);
......@@ -662,7 +660,6 @@ static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__, target
662660__crc32cd(uint32_t __a, uint64_t __b) {
663661 return __builtin_arm_crc32cd(__a, __b);
664662}
665#endif
666663
667664/* 8.6 Floating-point data-processing intrinsics */
668665/* Armv8.3-A Javascript conversion intrinsic */
lib/include/arm_neon.h+3587-1696
......@@ -359,9 +359,7 @@ __ai __attribute__((target("bf16,neon"))) bfloat16x8_t __noswap_vcombine_bf16(bf
359359})
360360__ai __attribute__((target("bf16,neon"))) float32_t vcvtah_f32_bf16(bfloat16_t __p0) {
361361 float32_t __ret;
362bfloat16_t __reint = __p0;
363int32_t __reint1 = (int32_t)(*(int16_t *) &__reint) << 16;
364 __ret = *(float32_t *) &__reint1;
362 __ret = __builtin_bit_cast(float32_t, (uint32_t)(__builtin_bit_cast(uint16_t, __p0)) << 16);
365363 return __ret;
366364}
367365__ai __attribute__((target("bf16,neon"))) bfloat16_t vcvth_bf16_f32(float32_t __p0) {
......@@ -35841,9 +35839,7 @@ __ai __attribute__((target("v8.3a,fullfp16,neon"))) float16x4_t __noswap_vcmla_f
3584135839 float16x4_t __s0_150 = __p0_150; \
3584235840 float16x4_t __s1_150 = __p1_150; \
3584335841 float16x4_t __s2_150 = __p2_150; \
35844float16x4_t __reint_150 = __s2_150; \
35845uint32x2_t __reint1_150 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_150, __p3_150), vget_lane_u32(*(uint32x2_t *) &__reint_150, __p3_150)}; \
35846 __ret_150 = vcmla_f16(__s0_150, __s1_150, *(float16x4_t *) &__reint1_150); \
35842 __ret_150 = vcmla_f16(__s0_150, __s1_150, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_150), __p3_150), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_150), __p3_150)})); \
3584735843 __ret_150; \
3584835844})
3584935845#else
......@@ -35855,9 +35851,7 @@ uint32x2_t __reint1_150 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
3585535851 float16x4_t __rev0_151; __rev0_151 = __builtin_shufflevector(__s0_151, __s0_151, 3, 2, 1, 0); \
3585635852 float16x4_t __rev1_151; __rev1_151 = __builtin_shufflevector(__s1_151, __s1_151, 3, 2, 1, 0); \
3585735853 float16x4_t __rev2_151; __rev2_151 = __builtin_shufflevector(__s2_151, __s2_151, 3, 2, 1, 0); \
35858float16x4_t __reint_151 = __rev2_151; \
35859uint32x2_t __reint1_151 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_151, __p3_151), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_151, __p3_151)}; \
35860 __ret_151 = __noswap_vcmla_f16(__rev0_151, __rev1_151, *(float16x4_t *) &__reint1_151); \
35854 __ret_151 = __noswap_vcmla_f16(__rev0_151, __rev1_151, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_151), __p3_151), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_151), __p3_151)})); \
3586135855 __ret_151 = __builtin_shufflevector(__ret_151, __ret_151, 3, 2, 1, 0); \
3586235856 __ret_151; \
3586335857})
......@@ -35869,9 +35863,7 @@ uint32x2_t __reint1_151 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
3586935863 float16x8_t __s0_152 = __p0_152; \
3587035864 float16x8_t __s1_152 = __p1_152; \
3587135865 float16x4_t __s2_152 = __p2_152; \
35872float16x4_t __reint_152 = __s2_152; \
35873uint32x4_t __reint1_152 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_152, __p3_152), vget_lane_u32(*(uint32x2_t *) &__reint_152, __p3_152), vget_lane_u32(*(uint32x2_t *) &__reint_152, __p3_152), vget_lane_u32(*(uint32x2_t *) &__reint_152, __p3_152)}; \
35874 __ret_152 = vcmlaq_f16(__s0_152, __s1_152, *(float16x8_t *) &__reint1_152); \
35866 __ret_152 = vcmlaq_f16(__s0_152, __s1_152, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_152), __p3_152), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_152), __p3_152), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_152), __p3_152), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_152), __p3_152)})); \
3587535867 __ret_152; \
3587635868})
3587735869#else
......@@ -35883,9 +35875,7 @@ uint32x4_t __reint1_152 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
3588335875 float16x8_t __rev0_153; __rev0_153 = __builtin_shufflevector(__s0_153, __s0_153, 7, 6, 5, 4, 3, 2, 1, 0); \
3588435876 float16x8_t __rev1_153; __rev1_153 = __builtin_shufflevector(__s1_153, __s1_153, 7, 6, 5, 4, 3, 2, 1, 0); \
3588535877 float16x4_t __rev2_153; __rev2_153 = __builtin_shufflevector(__s2_153, __s2_153, 3, 2, 1, 0); \
35886float16x4_t __reint_153 = __rev2_153; \
35887uint32x4_t __reint1_153 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_153, __p3_153), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_153, __p3_153), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_153, __p3_153), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_153, __p3_153)}; \
35888 __ret_153 = __noswap_vcmlaq_f16(__rev0_153, __rev1_153, *(float16x8_t *) &__reint1_153); \
35878 __ret_153 = __noswap_vcmlaq_f16(__rev0_153, __rev1_153, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_153), __p3_153), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_153), __p3_153), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_153), __p3_153), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_153), __p3_153)})); \
3588935879 __ret_153 = __builtin_shufflevector(__ret_153, __ret_153, 7, 6, 5, 4, 3, 2, 1, 0); \
3589035880 __ret_153; \
3589135881})
......@@ -35897,9 +35887,7 @@ uint32x4_t __reint1_153 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
3589735887 float16x4_t __s0_154 = __p0_154; \
3589835888 float16x4_t __s1_154 = __p1_154; \
3589935889 float16x8_t __s2_154 = __p2_154; \
35900float16x8_t __reint_154 = __s2_154; \
35901uint32x2_t __reint1_154 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_154, __p3_154), vgetq_lane_u32(*(uint32x4_t *) &__reint_154, __p3_154)}; \
35902 __ret_154 = vcmla_f16(__s0_154, __s1_154, *(float16x4_t *) &__reint1_154); \
35890 __ret_154 = vcmla_f16(__s0_154, __s1_154, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_154), __p3_154), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_154), __p3_154)})); \
3590335891 __ret_154; \
3590435892})
3590535893#else
......@@ -35911,9 +35899,7 @@ uint32x2_t __reint1_154 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
3591135899 float16x4_t __rev0_155; __rev0_155 = __builtin_shufflevector(__s0_155, __s0_155, 3, 2, 1, 0); \
3591235900 float16x4_t __rev1_155; __rev1_155 = __builtin_shufflevector(__s1_155, __s1_155, 3, 2, 1, 0); \
3591335901 float16x8_t __rev2_155; __rev2_155 = __builtin_shufflevector(__s2_155, __s2_155, 7, 6, 5, 4, 3, 2, 1, 0); \
35914float16x8_t __reint_155 = __rev2_155; \
35915uint32x2_t __reint1_155 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_155, __p3_155), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_155, __p3_155)}; \
35916 __ret_155 = __noswap_vcmla_f16(__rev0_155, __rev1_155, *(float16x4_t *) &__reint1_155); \
35902 __ret_155 = __noswap_vcmla_f16(__rev0_155, __rev1_155, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_155), __p3_155), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_155), __p3_155)})); \
3591735903 __ret_155 = __builtin_shufflevector(__ret_155, __ret_155, 3, 2, 1, 0); \
3591835904 __ret_155; \
3591935905})
......@@ -35925,9 +35911,7 @@ uint32x2_t __reint1_155 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *)
3592535911 float16x8_t __s0_156 = __p0_156; \
3592635912 float16x8_t __s1_156 = __p1_156; \
3592735913 float16x8_t __s2_156 = __p2_156; \
35928float16x8_t __reint_156 = __s2_156; \
35929uint32x4_t __reint1_156 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_156, __p3_156), vgetq_lane_u32(*(uint32x4_t *) &__reint_156, __p3_156), vgetq_lane_u32(*(uint32x4_t *) &__reint_156, __p3_156), vgetq_lane_u32(*(uint32x4_t *) &__reint_156, __p3_156)}; \
35930 __ret_156 = vcmlaq_f16(__s0_156, __s1_156, *(float16x8_t *) &__reint1_156); \
35914 __ret_156 = vcmlaq_f16(__s0_156, __s1_156, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_156), __p3_156), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_156), __p3_156), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_156), __p3_156), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_156), __p3_156)})); \
3593135915 __ret_156; \
3593235916})
3593335917#else
......@@ -35939,9 +35923,7 @@ uint32x4_t __reint1_156 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
3593935923 float16x8_t __rev0_157; __rev0_157 = __builtin_shufflevector(__s0_157, __s0_157, 7, 6, 5, 4, 3, 2, 1, 0); \
3594035924 float16x8_t __rev1_157; __rev1_157 = __builtin_shufflevector(__s1_157, __s1_157, 7, 6, 5, 4, 3, 2, 1, 0); \
3594135925 float16x8_t __rev2_157; __rev2_157 = __builtin_shufflevector(__s2_157, __s2_157, 7, 6, 5, 4, 3, 2, 1, 0); \
35942float16x8_t __reint_157 = __rev2_157; \
35943uint32x4_t __reint1_157 = (uint32x4_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_157, __p3_157), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_157, __p3_157), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_157, __p3_157), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_157, __p3_157)}; \
35944 __ret_157 = __noswap_vcmlaq_f16(__rev0_157, __rev1_157, *(float16x8_t *) &__reint1_157); \
35926 __ret_157 = __noswap_vcmlaq_f16(__rev0_157, __rev1_157, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_157), __p3_157), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_157), __p3_157), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_157), __p3_157), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_157), __p3_157)})); \
3594535927 __ret_157 = __builtin_shufflevector(__ret_157, __ret_157, 7, 6, 5, 4, 3, 2, 1, 0); \
3594635928 __ret_157; \
3594735929})
......@@ -35999,9 +35981,7 @@ __ai __attribute__((target("v8.3a,fullfp16,neon"))) float16x4_t __noswap_vcmla_r
3599935981 float16x4_t __s0_158 = __p0_158; \
3600035982 float16x4_t __s1_158 = __p1_158; \
3600135983 float16x4_t __s2_158 = __p2_158; \
36002float16x4_t __reint_158 = __s2_158; \
36003uint32x2_t __reint1_158 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_158, __p3_158), vget_lane_u32(*(uint32x2_t *) &__reint_158, __p3_158)}; \
36004 __ret_158 = vcmla_rot180_f16(__s0_158, __s1_158, *(float16x4_t *) &__reint1_158); \
35984 __ret_158 = vcmla_rot180_f16(__s0_158, __s1_158, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_158), __p3_158), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_158), __p3_158)})); \
3600535985 __ret_158; \
3600635986})
3600735987#else
......@@ -36013,9 +35993,7 @@ uint32x2_t __reint1_158 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
3601335993 float16x4_t __rev0_159; __rev0_159 = __builtin_shufflevector(__s0_159, __s0_159, 3, 2, 1, 0); \
3601435994 float16x4_t __rev1_159; __rev1_159 = __builtin_shufflevector(__s1_159, __s1_159, 3, 2, 1, 0); \
3601535995 float16x4_t __rev2_159; __rev2_159 = __builtin_shufflevector(__s2_159, __s2_159, 3, 2, 1, 0); \
36016float16x4_t __reint_159 = __rev2_159; \
36017uint32x2_t __reint1_159 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_159, __p3_159), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_159, __p3_159)}; \
36018 __ret_159 = __noswap_vcmla_rot180_f16(__rev0_159, __rev1_159, *(float16x4_t *) &__reint1_159); \
35996 __ret_159 = __noswap_vcmla_rot180_f16(__rev0_159, __rev1_159, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_159), __p3_159), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_159), __p3_159)})); \
3601935997 __ret_159 = __builtin_shufflevector(__ret_159, __ret_159, 3, 2, 1, 0); \
3602035998 __ret_159; \
3602135999})
......@@ -36027,9 +36005,7 @@ uint32x2_t __reint1_159 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
3602736005 float16x8_t __s0_160 = __p0_160; \
3602836006 float16x8_t __s1_160 = __p1_160; \
3602936007 float16x4_t __s2_160 = __p2_160; \
36030float16x4_t __reint_160 = __s2_160; \
36031uint32x4_t __reint1_160 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_160, __p3_160), vget_lane_u32(*(uint32x2_t *) &__reint_160, __p3_160), vget_lane_u32(*(uint32x2_t *) &__reint_160, __p3_160), vget_lane_u32(*(uint32x2_t *) &__reint_160, __p3_160)}; \
36032 __ret_160 = vcmlaq_rot180_f16(__s0_160, __s1_160, *(float16x8_t *) &__reint1_160); \
36008 __ret_160 = vcmlaq_rot180_f16(__s0_160, __s1_160, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_160), __p3_160), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_160), __p3_160), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_160), __p3_160), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_160), __p3_160)})); \
3603336009 __ret_160; \
3603436010})
3603536011#else
......@@ -36041,9 +36017,7 @@ uint32x4_t __reint1_160 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
3604136017 float16x8_t __rev0_161; __rev0_161 = __builtin_shufflevector(__s0_161, __s0_161, 7, 6, 5, 4, 3, 2, 1, 0); \
3604236018 float16x8_t __rev1_161; __rev1_161 = __builtin_shufflevector(__s1_161, __s1_161, 7, 6, 5, 4, 3, 2, 1, 0); \
3604336019 float16x4_t __rev2_161; __rev2_161 = __builtin_shufflevector(__s2_161, __s2_161, 3, 2, 1, 0); \
36044float16x4_t __reint_161 = __rev2_161; \
36045uint32x4_t __reint1_161 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_161, __p3_161), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_161, __p3_161), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_161, __p3_161), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_161, __p3_161)}; \
36046 __ret_161 = __noswap_vcmlaq_rot180_f16(__rev0_161, __rev1_161, *(float16x8_t *) &__reint1_161); \
36020 __ret_161 = __noswap_vcmlaq_rot180_f16(__rev0_161, __rev1_161, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_161), __p3_161), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_161), __p3_161), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_161), __p3_161), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_161), __p3_161)})); \
3604736021 __ret_161 = __builtin_shufflevector(__ret_161, __ret_161, 7, 6, 5, 4, 3, 2, 1, 0); \
3604836022 __ret_161; \
3604936023})
......@@ -36055,9 +36029,7 @@ uint32x4_t __reint1_161 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
3605536029 float16x4_t __s0_162 = __p0_162; \
3605636030 float16x4_t __s1_162 = __p1_162; \
3605736031 float16x8_t __s2_162 = __p2_162; \
36058float16x8_t __reint_162 = __s2_162; \
36059uint32x2_t __reint1_162 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_162, __p3_162), vgetq_lane_u32(*(uint32x4_t *) &__reint_162, __p3_162)}; \
36060 __ret_162 = vcmla_rot180_f16(__s0_162, __s1_162, *(float16x4_t *) &__reint1_162); \
36032 __ret_162 = vcmla_rot180_f16(__s0_162, __s1_162, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_162), __p3_162), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_162), __p3_162)})); \
3606136033 __ret_162; \
3606236034})
3606336035#else
......@@ -36069,9 +36041,7 @@ uint32x2_t __reint1_162 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
3606936041 float16x4_t __rev0_163; __rev0_163 = __builtin_shufflevector(__s0_163, __s0_163, 3, 2, 1, 0); \
3607036042 float16x4_t __rev1_163; __rev1_163 = __builtin_shufflevector(__s1_163, __s1_163, 3, 2, 1, 0); \
3607136043 float16x8_t __rev2_163; __rev2_163 = __builtin_shufflevector(__s2_163, __s2_163, 7, 6, 5, 4, 3, 2, 1, 0); \
36072float16x8_t __reint_163 = __rev2_163; \
36073uint32x2_t __reint1_163 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_163, __p3_163), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_163, __p3_163)}; \
36074 __ret_163 = __noswap_vcmla_rot180_f16(__rev0_163, __rev1_163, *(float16x4_t *) &__reint1_163); \
36044 __ret_163 = __noswap_vcmla_rot180_f16(__rev0_163, __rev1_163, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_163), __p3_163), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_163), __p3_163)})); \
3607536045 __ret_163 = __builtin_shufflevector(__ret_163, __ret_163, 3, 2, 1, 0); \
3607636046 __ret_163; \
3607736047})
......@@ -36083,9 +36053,7 @@ uint32x2_t __reint1_163 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *)
3608336053 float16x8_t __s0_164 = __p0_164; \
3608436054 float16x8_t __s1_164 = __p1_164; \
3608536055 float16x8_t __s2_164 = __p2_164; \
36086float16x8_t __reint_164 = __s2_164; \
36087uint32x4_t __reint1_164 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_164, __p3_164), vgetq_lane_u32(*(uint32x4_t *) &__reint_164, __p3_164), vgetq_lane_u32(*(uint32x4_t *) &__reint_164, __p3_164), vgetq_lane_u32(*(uint32x4_t *) &__reint_164, __p3_164)}; \
36088 __ret_164 = vcmlaq_rot180_f16(__s0_164, __s1_164, *(float16x8_t *) &__reint1_164); \
36056 __ret_164 = vcmlaq_rot180_f16(__s0_164, __s1_164, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_164), __p3_164), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_164), __p3_164), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_164), __p3_164), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_164), __p3_164)})); \
3608936057 __ret_164; \
3609036058})
3609136059#else
......@@ -36097,9 +36065,7 @@ uint32x4_t __reint1_164 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
3609736065 float16x8_t __rev0_165; __rev0_165 = __builtin_shufflevector(__s0_165, __s0_165, 7, 6, 5, 4, 3, 2, 1, 0); \
3609836066 float16x8_t __rev1_165; __rev1_165 = __builtin_shufflevector(__s1_165, __s1_165, 7, 6, 5, 4, 3, 2, 1, 0); \
3609936067 float16x8_t __rev2_165; __rev2_165 = __builtin_shufflevector(__s2_165, __s2_165, 7, 6, 5, 4, 3, 2, 1, 0); \
36100float16x8_t __reint_165 = __rev2_165; \
36101uint32x4_t __reint1_165 = (uint32x4_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_165, __p3_165), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_165, __p3_165), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_165, __p3_165), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_165, __p3_165)}; \
36102 __ret_165 = __noswap_vcmlaq_rot180_f16(__rev0_165, __rev1_165, *(float16x8_t *) &__reint1_165); \
36068 __ret_165 = __noswap_vcmlaq_rot180_f16(__rev0_165, __rev1_165, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_165), __p3_165), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_165), __p3_165), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_165), __p3_165), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_165), __p3_165)})); \
3610336069 __ret_165 = __builtin_shufflevector(__ret_165, __ret_165, 7, 6, 5, 4, 3, 2, 1, 0); \
3610436070 __ret_165; \
3610536071})
......@@ -36157,9 +36123,7 @@ __ai __attribute__((target("v8.3a,fullfp16,neon"))) float16x4_t __noswap_vcmla_r
3615736123 float16x4_t __s0_166 = __p0_166; \
3615836124 float16x4_t __s1_166 = __p1_166; \
3615936125 float16x4_t __s2_166 = __p2_166; \
36160float16x4_t __reint_166 = __s2_166; \
36161uint32x2_t __reint1_166 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_166, __p3_166), vget_lane_u32(*(uint32x2_t *) &__reint_166, __p3_166)}; \
36162 __ret_166 = vcmla_rot270_f16(__s0_166, __s1_166, *(float16x4_t *) &__reint1_166); \
36126 __ret_166 = vcmla_rot270_f16(__s0_166, __s1_166, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_166), __p3_166), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_166), __p3_166)})); \
3616336127 __ret_166; \
3616436128})
3616536129#else
......@@ -36171,9 +36135,7 @@ uint32x2_t __reint1_166 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
3617136135 float16x4_t __rev0_167; __rev0_167 = __builtin_shufflevector(__s0_167, __s0_167, 3, 2, 1, 0); \
3617236136 float16x4_t __rev1_167; __rev1_167 = __builtin_shufflevector(__s1_167, __s1_167, 3, 2, 1, 0); \
3617336137 float16x4_t __rev2_167; __rev2_167 = __builtin_shufflevector(__s2_167, __s2_167, 3, 2, 1, 0); \
36174float16x4_t __reint_167 = __rev2_167; \
36175uint32x2_t __reint1_167 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_167, __p3_167), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_167, __p3_167)}; \
36176 __ret_167 = __noswap_vcmla_rot270_f16(__rev0_167, __rev1_167, *(float16x4_t *) &__reint1_167); \
36138 __ret_167 = __noswap_vcmla_rot270_f16(__rev0_167, __rev1_167, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_167), __p3_167), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_167), __p3_167)})); \
3617736139 __ret_167 = __builtin_shufflevector(__ret_167, __ret_167, 3, 2, 1, 0); \
3617836140 __ret_167; \
3617936141})
......@@ -36185,9 +36147,7 @@ uint32x2_t __reint1_167 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
3618536147 float16x8_t __s0_168 = __p0_168; \
3618636148 float16x8_t __s1_168 = __p1_168; \
3618736149 float16x4_t __s2_168 = __p2_168; \
36188float16x4_t __reint_168 = __s2_168; \
36189uint32x4_t __reint1_168 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_168, __p3_168), vget_lane_u32(*(uint32x2_t *) &__reint_168, __p3_168), vget_lane_u32(*(uint32x2_t *) &__reint_168, __p3_168), vget_lane_u32(*(uint32x2_t *) &__reint_168, __p3_168)}; \
36190 __ret_168 = vcmlaq_rot270_f16(__s0_168, __s1_168, *(float16x8_t *) &__reint1_168); \
36150 __ret_168 = vcmlaq_rot270_f16(__s0_168, __s1_168, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_168), __p3_168), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_168), __p3_168), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_168), __p3_168), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_168), __p3_168)})); \
3619136151 __ret_168; \
3619236152})
3619336153#else
......@@ -36199,9 +36159,7 @@ uint32x4_t __reint1_168 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
3619936159 float16x8_t __rev0_169; __rev0_169 = __builtin_shufflevector(__s0_169, __s0_169, 7, 6, 5, 4, 3, 2, 1, 0); \
3620036160 float16x8_t __rev1_169; __rev1_169 = __builtin_shufflevector(__s1_169, __s1_169, 7, 6, 5, 4, 3, 2, 1, 0); \
3620136161 float16x4_t __rev2_169; __rev2_169 = __builtin_shufflevector(__s2_169, __s2_169, 3, 2, 1, 0); \
36202float16x4_t __reint_169 = __rev2_169; \
36203uint32x4_t __reint1_169 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_169, __p3_169), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_169, __p3_169), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_169, __p3_169), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_169, __p3_169)}; \
36204 __ret_169 = __noswap_vcmlaq_rot270_f16(__rev0_169, __rev1_169, *(float16x8_t *) &__reint1_169); \
36162 __ret_169 = __noswap_vcmlaq_rot270_f16(__rev0_169, __rev1_169, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_169), __p3_169), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_169), __p3_169), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_169), __p3_169), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_169), __p3_169)})); \
3620536163 __ret_169 = __builtin_shufflevector(__ret_169, __ret_169, 7, 6, 5, 4, 3, 2, 1, 0); \
3620636164 __ret_169; \
3620736165})
......@@ -36213,9 +36171,7 @@ uint32x4_t __reint1_169 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
3621336171 float16x4_t __s0_170 = __p0_170; \
3621436172 float16x4_t __s1_170 = __p1_170; \
3621536173 float16x8_t __s2_170 = __p2_170; \
36216float16x8_t __reint_170 = __s2_170; \
36217uint32x2_t __reint1_170 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_170, __p3_170), vgetq_lane_u32(*(uint32x4_t *) &__reint_170, __p3_170)}; \
36218 __ret_170 = vcmla_rot270_f16(__s0_170, __s1_170, *(float16x4_t *) &__reint1_170); \
36174 __ret_170 = vcmla_rot270_f16(__s0_170, __s1_170, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_170), __p3_170), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_170), __p3_170)})); \
3621936175 __ret_170; \
3622036176})
3622136177#else
......@@ -36227,9 +36183,7 @@ uint32x2_t __reint1_170 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
3622736183 float16x4_t __rev0_171; __rev0_171 = __builtin_shufflevector(__s0_171, __s0_171, 3, 2, 1, 0); \
3622836184 float16x4_t __rev1_171; __rev1_171 = __builtin_shufflevector(__s1_171, __s1_171, 3, 2, 1, 0); \
3622936185 float16x8_t __rev2_171; __rev2_171 = __builtin_shufflevector(__s2_171, __s2_171, 7, 6, 5, 4, 3, 2, 1, 0); \
36230float16x8_t __reint_171 = __rev2_171; \
36231uint32x2_t __reint1_171 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_171, __p3_171), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_171, __p3_171)}; \
36232 __ret_171 = __noswap_vcmla_rot270_f16(__rev0_171, __rev1_171, *(float16x4_t *) &__reint1_171); \
36186 __ret_171 = __noswap_vcmla_rot270_f16(__rev0_171, __rev1_171, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_171), __p3_171), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_171), __p3_171)})); \
3623336187 __ret_171 = __builtin_shufflevector(__ret_171, __ret_171, 3, 2, 1, 0); \
3623436188 __ret_171; \
3623536189})
......@@ -36241,9 +36195,7 @@ uint32x2_t __reint1_171 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *)
3624136195 float16x8_t __s0_172 = __p0_172; \
3624236196 float16x8_t __s1_172 = __p1_172; \
3624336197 float16x8_t __s2_172 = __p2_172; \
36244float16x8_t __reint_172 = __s2_172; \
36245uint32x4_t __reint1_172 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_172, __p3_172), vgetq_lane_u32(*(uint32x4_t *) &__reint_172, __p3_172), vgetq_lane_u32(*(uint32x4_t *) &__reint_172, __p3_172), vgetq_lane_u32(*(uint32x4_t *) &__reint_172, __p3_172)}; \
36246 __ret_172 = vcmlaq_rot270_f16(__s0_172, __s1_172, *(float16x8_t *) &__reint1_172); \
36198 __ret_172 = vcmlaq_rot270_f16(__s0_172, __s1_172, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_172), __p3_172), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_172), __p3_172), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_172), __p3_172), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_172), __p3_172)})); \
3624736199 __ret_172; \
3624836200})
3624936201#else
......@@ -36255,9 +36207,7 @@ uint32x4_t __reint1_172 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
3625536207 float16x8_t __rev0_173; __rev0_173 = __builtin_shufflevector(__s0_173, __s0_173, 7, 6, 5, 4, 3, 2, 1, 0); \
3625636208 float16x8_t __rev1_173; __rev1_173 = __builtin_shufflevector(__s1_173, __s1_173, 7, 6, 5, 4, 3, 2, 1, 0); \
3625736209 float16x8_t __rev2_173; __rev2_173 = __builtin_shufflevector(__s2_173, __s2_173, 7, 6, 5, 4, 3, 2, 1, 0); \
36258float16x8_t __reint_173 = __rev2_173; \
36259uint32x4_t __reint1_173 = (uint32x4_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_173, __p3_173), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_173, __p3_173), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_173, __p3_173), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_173, __p3_173)}; \
36260 __ret_173 = __noswap_vcmlaq_rot270_f16(__rev0_173, __rev1_173, *(float16x8_t *) &__reint1_173); \
36210 __ret_173 = __noswap_vcmlaq_rot270_f16(__rev0_173, __rev1_173, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_173), __p3_173), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_173), __p3_173), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_173), __p3_173), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_173), __p3_173)})); \
3626136211 __ret_173 = __builtin_shufflevector(__ret_173, __ret_173, 7, 6, 5, 4, 3, 2, 1, 0); \
3626236212 __ret_173; \
3626336213})
......@@ -36315,9 +36265,7 @@ __ai __attribute__((target("v8.3a,fullfp16,neon"))) float16x4_t __noswap_vcmla_r
3631536265 float16x4_t __s0_174 = __p0_174; \
3631636266 float16x4_t __s1_174 = __p1_174; \
3631736267 float16x4_t __s2_174 = __p2_174; \
36318float16x4_t __reint_174 = __s2_174; \
36319uint32x2_t __reint1_174 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_174, __p3_174), vget_lane_u32(*(uint32x2_t *) &__reint_174, __p3_174)}; \
36320 __ret_174 = vcmla_rot90_f16(__s0_174, __s1_174, *(float16x4_t *) &__reint1_174); \
36268 __ret_174 = vcmla_rot90_f16(__s0_174, __s1_174, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_174), __p3_174), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_174), __p3_174)})); \
3632136269 __ret_174; \
3632236270})
3632336271#else
......@@ -36329,9 +36277,7 @@ uint32x2_t __reint1_174 = (uint32x2_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
3632936277 float16x4_t __rev0_175; __rev0_175 = __builtin_shufflevector(__s0_175, __s0_175, 3, 2, 1, 0); \
3633036278 float16x4_t __rev1_175; __rev1_175 = __builtin_shufflevector(__s1_175, __s1_175, 3, 2, 1, 0); \
3633136279 float16x4_t __rev2_175; __rev2_175 = __builtin_shufflevector(__s2_175, __s2_175, 3, 2, 1, 0); \
36332float16x4_t __reint_175 = __rev2_175; \
36333uint32x2_t __reint1_175 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_175, __p3_175), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_175, __p3_175)}; \
36334 __ret_175 = __noswap_vcmla_rot90_f16(__rev0_175, __rev1_175, *(float16x4_t *) &__reint1_175); \
36280 __ret_175 = __noswap_vcmla_rot90_f16(__rev0_175, __rev1_175, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_175), __p3_175), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_175), __p3_175)})); \
3633536281 __ret_175 = __builtin_shufflevector(__ret_175, __ret_175, 3, 2, 1, 0); \
3633636282 __ret_175; \
3633736283})
......@@ -36343,9 +36289,7 @@ uint32x2_t __reint1_175 = (uint32x2_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
3634336289 float16x8_t __s0_176 = __p0_176; \
3634436290 float16x8_t __s1_176 = __p1_176; \
3634536291 float16x4_t __s2_176 = __p2_176; \
36346float16x4_t __reint_176 = __s2_176; \
36347uint32x4_t __reint1_176 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_176, __p3_176), vget_lane_u32(*(uint32x2_t *) &__reint_176, __p3_176), vget_lane_u32(*(uint32x2_t *) &__reint_176, __p3_176), vget_lane_u32(*(uint32x2_t *) &__reint_176, __p3_176)}; \
36348 __ret_176 = vcmlaq_rot90_f16(__s0_176, __s1_176, *(float16x8_t *) &__reint1_176); \
36292 __ret_176 = vcmlaq_rot90_f16(__s0_176, __s1_176, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_176), __p3_176), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_176), __p3_176), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_176), __p3_176), vget_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_176), __p3_176)})); \
3634936293 __ret_176; \
3635036294})
3635136295#else
......@@ -36357,9 +36301,7 @@ uint32x4_t __reint1_176 = (uint32x4_t) {vget_lane_u32(*(uint32x2_t *) &__reint_1
3635736301 float16x8_t __rev0_177; __rev0_177 = __builtin_shufflevector(__s0_177, __s0_177, 7, 6, 5, 4, 3, 2, 1, 0); \
3635836302 float16x8_t __rev1_177; __rev1_177 = __builtin_shufflevector(__s1_177, __s1_177, 7, 6, 5, 4, 3, 2, 1, 0); \
3635936303 float16x4_t __rev2_177; __rev2_177 = __builtin_shufflevector(__s2_177, __s2_177, 3, 2, 1, 0); \
36360float16x4_t __reint_177 = __rev2_177; \
36361uint32x4_t __reint1_177 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &__reint_177, __p3_177), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_177, __p3_177), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_177, __p3_177), __noswap_vget_lane_u32(*(uint32x2_t *) &__reint_177, __p3_177)}; \
36362 __ret_177 = __noswap_vcmlaq_rot90_f16(__rev0_177, __rev1_177, *(float16x8_t *) &__reint1_177); \
36304 __ret_177 = __noswap_vcmlaq_rot90_f16(__rev0_177, __rev1_177, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_177), __p3_177), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_177), __p3_177), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_177), __p3_177), __noswap_vget_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_177), __p3_177)})); \
3636336305 __ret_177 = __builtin_shufflevector(__ret_177, __ret_177, 7, 6, 5, 4, 3, 2, 1, 0); \
3636436306 __ret_177; \
3636536307})
......@@ -36371,9 +36313,7 @@ uint32x4_t __reint1_177 = (uint32x4_t) {__noswap_vget_lane_u32(*(uint32x2_t *) &
3637136313 float16x4_t __s0_178 = __p0_178; \
3637236314 float16x4_t __s1_178 = __p1_178; \
3637336315 float16x8_t __s2_178 = __p2_178; \
36374float16x8_t __reint_178 = __s2_178; \
36375uint32x2_t __reint1_178 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_178, __p3_178), vgetq_lane_u32(*(uint32x4_t *) &__reint_178, __p3_178)}; \
36376 __ret_178 = vcmla_rot90_f16(__s0_178, __s1_178, *(float16x4_t *) &__reint1_178); \
36316 __ret_178 = vcmla_rot90_f16(__s0_178, __s1_178, __builtin_bit_cast(float16x4_t, (uint32x2_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_178), __p3_178), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_178), __p3_178)})); \
3637736317 __ret_178; \
3637836318})
3637936319#else
......@@ -36385,9 +36325,7 @@ uint32x2_t __reint1_178 = (uint32x2_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
3638536325 float16x4_t __rev0_179; __rev0_179 = __builtin_shufflevector(__s0_179, __s0_179, 3, 2, 1, 0); \
3638636326 float16x4_t __rev1_179; __rev1_179 = __builtin_shufflevector(__s1_179, __s1_179, 3, 2, 1, 0); \
3638736327 float16x8_t __rev2_179; __rev2_179 = __builtin_shufflevector(__s2_179, __s2_179, 7, 6, 5, 4, 3, 2, 1, 0); \
36388float16x8_t __reint_179 = __rev2_179; \
36389uint32x2_t __reint1_179 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_179, __p3_179), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_179, __p3_179)}; \
36390 __ret_179 = __noswap_vcmla_rot90_f16(__rev0_179, __rev1_179, *(float16x4_t *) &__reint1_179); \
36328 __ret_179 = __noswap_vcmla_rot90_f16(__rev0_179, __rev1_179, __builtin_bit_cast(float16x4_t, (uint32x2_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_179), __p3_179), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_179), __p3_179)})); \
3639136329 __ret_179 = __builtin_shufflevector(__ret_179, __ret_179, 3, 2, 1, 0); \
3639236330 __ret_179; \
3639336331})
......@@ -36399,9 +36337,7 @@ uint32x2_t __reint1_179 = (uint32x2_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *)
3639936337 float16x8_t __s0_180 = __p0_180; \
3640036338 float16x8_t __s1_180 = __p1_180; \
3640136339 float16x8_t __s2_180 = __p2_180; \
36402float16x8_t __reint_180 = __s2_180; \
36403uint32x4_t __reint1_180 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_180, __p3_180), vgetq_lane_u32(*(uint32x4_t *) &__reint_180, __p3_180), vgetq_lane_u32(*(uint32x4_t *) &__reint_180, __p3_180), vgetq_lane_u32(*(uint32x4_t *) &__reint_180, __p3_180)}; \
36404 __ret_180 = vcmlaq_rot90_f16(__s0_180, __s1_180, *(float16x8_t *) &__reint1_180); \
36340 __ret_180 = vcmlaq_rot90_f16(__s0_180, __s1_180, __builtin_bit_cast(float16x8_t, (uint32x4_t) {vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_180), __p3_180), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_180), __p3_180), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_180), __p3_180), vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __s2_180), __p3_180)})); \
3640536341 __ret_180; \
3640636342})
3640736343#else
......@@ -36413,9 +36349,7 @@ uint32x4_t __reint1_180 = (uint32x4_t) {vgetq_lane_u32(*(uint32x4_t *) &__reint_
3641336349 float16x8_t __rev0_181; __rev0_181 = __builtin_shufflevector(__s0_181, __s0_181, 7, 6, 5, 4, 3, 2, 1, 0); \
3641436350 float16x8_t __rev1_181; __rev1_181 = __builtin_shufflevector(__s1_181, __s1_181, 7, 6, 5, 4, 3, 2, 1, 0); \
3641536351 float16x8_t __rev2_181; __rev2_181 = __builtin_shufflevector(__s2_181, __s2_181, 7, 6, 5, 4, 3, 2, 1, 0); \
36416float16x8_t __reint_181 = __rev2_181; \
36417uint32x4_t __reint1_181 = (uint32x4_t) {__noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_181, __p3_181), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_181, __p3_181), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_181, __p3_181), __noswap_vgetq_lane_u32(*(uint32x4_t *) &__reint_181, __p3_181)}; \
36418 __ret_181 = __noswap_vcmlaq_rot90_f16(__rev0_181, __rev1_181, *(float16x8_t *) &__reint1_181); \
36352 __ret_181 = __noswap_vcmlaq_rot90_f16(__rev0_181, __rev1_181, __builtin_bit_cast(float16x8_t, (uint32x4_t) {__noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_181), __p3_181), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_181), __p3_181), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_181), __p3_181), __noswap_vgetq_lane_u32(__builtin_bit_cast(uint32x4_t, __rev2_181), __p3_181)})); \
3641936353 __ret_181 = __builtin_shufflevector(__ret_181, __ret_181, 7, 6, 5, 4, 3, 2, 1, 0); \
3642036354 __ret_181; \
3642136355})
......@@ -36541,9 +36475,7 @@ __ai __attribute__((target("v8.3a,neon"))) float32x2_t __noswap_vcmla_f32(float3
3654136475 float32x2_t __s0_182 = __p0_182; \
3654236476 float32x2_t __s1_182 = __p1_182; \
3654336477 float32x2_t __s2_182 = __p2_182; \
36544float32x2_t __reint_182 = __s2_182; \
36545uint64x1_t __reint1_182 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_182, __p3_182)}; \
36546 __ret_182 = vcmla_f32(__s0_182, __s1_182, *(float32x2_t *) &__reint1_182); \
36478 __ret_182 = vcmla_f32(__s0_182, __s1_182, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_182), __p3_182)})); \
3654736479 __ret_182; \
3654836480})
3654936481#else
......@@ -36555,9 +36487,7 @@ uint64x1_t __reint1_182 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
3655536487 float32x2_t __rev0_183; __rev0_183 = __builtin_shufflevector(__s0_183, __s0_183, 1, 0); \
3655636488 float32x2_t __rev1_183; __rev1_183 = __builtin_shufflevector(__s1_183, __s1_183, 1, 0); \
3655736489 float32x2_t __rev2_183; __rev2_183 = __builtin_shufflevector(__s2_183, __s2_183, 1, 0); \
36558float32x2_t __reint_183 = __rev2_183; \
36559uint64x1_t __reint1_183 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_183, __p3_183)}; \
36560 __ret_183 = __noswap_vcmla_f32(__rev0_183, __rev1_183, *(float32x2_t *) &__reint1_183); \
36490 __ret_183 = __noswap_vcmla_f32(__rev0_183, __rev1_183, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_183), __p3_183)})); \
3656136491 __ret_183 = __builtin_shufflevector(__ret_183, __ret_183, 1, 0); \
3656236492 __ret_183; \
3656336493})
......@@ -36569,9 +36499,7 @@ uint64x1_t __reint1_183 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
3656936499 float32x4_t __s0_184 = __p0_184; \
3657036500 float32x4_t __s1_184 = __p1_184; \
3657136501 float32x2_t __s2_184 = __p2_184; \
36572float32x2_t __reint_184 = __s2_184; \
36573uint64x2_t __reint1_184 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_184, __p3_184), vget_lane_u64(*(uint64x1_t *) &__reint_184, __p3_184)}; \
36574 __ret_184 = vcmlaq_f32(__s0_184, __s1_184, *(float32x4_t *) &__reint1_184); \
36502 __ret_184 = vcmlaq_f32(__s0_184, __s1_184, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_184), __p3_184), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_184), __p3_184)})); \
3657536503 __ret_184; \
3657636504})
3657736505#else
......@@ -36583,9 +36511,7 @@ uint64x2_t __reint1_184 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
3658336511 float32x4_t __rev0_185; __rev0_185 = __builtin_shufflevector(__s0_185, __s0_185, 3, 2, 1, 0); \
3658436512 float32x4_t __rev1_185; __rev1_185 = __builtin_shufflevector(__s1_185, __s1_185, 3, 2, 1, 0); \
3658536513 float32x2_t __rev2_185; __rev2_185 = __builtin_shufflevector(__s2_185, __s2_185, 1, 0); \
36586float32x2_t __reint_185 = __rev2_185; \
36587uint64x2_t __reint1_185 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_185, __p3_185), vget_lane_u64(*(uint64x1_t *) &__reint_185, __p3_185)}; \
36588 __ret_185 = __noswap_vcmlaq_f32(__rev0_185, __rev1_185, *(float32x4_t *) &__reint1_185); \
36514 __ret_185 = __noswap_vcmlaq_f32(__rev0_185, __rev1_185, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_185), __p3_185), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_185), __p3_185)})); \
3658936515 __ret_185 = __builtin_shufflevector(__ret_185, __ret_185, 3, 2, 1, 0); \
3659036516 __ret_185; \
3659136517})
......@@ -36597,9 +36523,7 @@ uint64x2_t __reint1_185 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
3659736523 float32x2_t __s0_186 = __p0_186; \
3659836524 float32x2_t __s1_186 = __p1_186; \
3659936525 float32x4_t __s2_186 = __p2_186; \
36600float32x4_t __reint_186 = __s2_186; \
36601uint64x1_t __reint1_186 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_186, __p3_186)}; \
36602 __ret_186 = vcmla_f32(__s0_186, __s1_186, *(float32x2_t *) &__reint1_186); \
36526 __ret_186 = vcmla_f32(__s0_186, __s1_186, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_186), __p3_186)})); \
3660336527 __ret_186; \
3660436528})
3660536529#else
......@@ -36611,9 +36535,7 @@ uint64x1_t __reint1_186 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
3661136535 float32x2_t __rev0_187; __rev0_187 = __builtin_shufflevector(__s0_187, __s0_187, 1, 0); \
3661236536 float32x2_t __rev1_187; __rev1_187 = __builtin_shufflevector(__s1_187, __s1_187, 1, 0); \
3661336537 float32x4_t __rev2_187; __rev2_187 = __builtin_shufflevector(__s2_187, __s2_187, 3, 2, 1, 0); \
36614float32x4_t __reint_187 = __rev2_187; \
36615uint64x1_t __reint1_187 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_187, __p3_187)}; \
36616 __ret_187 = __noswap_vcmla_f32(__rev0_187, __rev1_187, *(float32x2_t *) &__reint1_187); \
36538 __ret_187 = __noswap_vcmla_f32(__rev0_187, __rev1_187, __builtin_bit_cast(float32x2_t, (uint64x1_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_187), __p3_187)})); \
3661736539 __ret_187 = __builtin_shufflevector(__ret_187, __ret_187, 1, 0); \
3661836540 __ret_187; \
3661936541})
......@@ -36625,9 +36547,7 @@ uint64x1_t __reint1_187 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *)
3662536547 float32x4_t __s0_188 = __p0_188; \
3662636548 float32x4_t __s1_188 = __p1_188; \
3662736549 float32x4_t __s2_188 = __p2_188; \
36628float32x4_t __reint_188 = __s2_188; \
36629uint64x2_t __reint1_188 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_188, __p3_188), vgetq_lane_u64(*(uint64x2_t *) &__reint_188, __p3_188)}; \
36630 __ret_188 = vcmlaq_f32(__s0_188, __s1_188, *(float32x4_t *) &__reint1_188); \
36550 __ret_188 = vcmlaq_f32(__s0_188, __s1_188, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_188), __p3_188), vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_188), __p3_188)})); \
3663136551 __ret_188; \
3663236552})
3663336553#else
......@@ -36639,9 +36559,7 @@ uint64x2_t __reint1_188 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
3663936559 float32x4_t __rev0_189; __rev0_189 = __builtin_shufflevector(__s0_189, __s0_189, 3, 2, 1, 0); \
3664036560 float32x4_t __rev1_189; __rev1_189 = __builtin_shufflevector(__s1_189, __s1_189, 3, 2, 1, 0); \
3664136561 float32x4_t __rev2_189; __rev2_189 = __builtin_shufflevector(__s2_189, __s2_189, 3, 2, 1, 0); \
36642float32x4_t __reint_189 = __rev2_189; \
36643uint64x2_t __reint1_189 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_189, __p3_189), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_189, __p3_189)}; \
36644 __ret_189 = __noswap_vcmlaq_f32(__rev0_189, __rev1_189, *(float32x4_t *) &__reint1_189); \
36562 __ret_189 = __noswap_vcmlaq_f32(__rev0_189, __rev1_189, __builtin_bit_cast(float32x4_t, (uint64x2_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_189), __p3_189), __noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_189), __p3_189)})); \
3664536563 __ret_189 = __builtin_shufflevector(__ret_189, __ret_189, 3, 2, 1, 0); \
3664636564 __ret_189; \
3664736565})
......@@ -36699,9 +36617,7 @@ __ai __attribute__((target("v8.3a,neon"))) float32x2_t __noswap_vcmla_rot180_f32
3669936617 float32x2_t __s0_190 = __p0_190; \
3670036618 float32x2_t __s1_190 = __p1_190; \
3670136619 float32x2_t __s2_190 = __p2_190; \
36702float32x2_t __reint_190 = __s2_190; \
36703uint64x1_t __reint1_190 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_190, __p3_190)}; \
36704 __ret_190 = vcmla_rot180_f32(__s0_190, __s1_190, *(float32x2_t *) &__reint1_190); \
36620 __ret_190 = vcmla_rot180_f32(__s0_190, __s1_190, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_190), __p3_190)})); \
3670536621 __ret_190; \
3670636622})
3670736623#else
......@@ -36713,9 +36629,7 @@ uint64x1_t __reint1_190 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
3671336629 float32x2_t __rev0_191; __rev0_191 = __builtin_shufflevector(__s0_191, __s0_191, 1, 0); \
3671436630 float32x2_t __rev1_191; __rev1_191 = __builtin_shufflevector(__s1_191, __s1_191, 1, 0); \
3671536631 float32x2_t __rev2_191; __rev2_191 = __builtin_shufflevector(__s2_191, __s2_191, 1, 0); \
36716float32x2_t __reint_191 = __rev2_191; \
36717uint64x1_t __reint1_191 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_191, __p3_191)}; \
36718 __ret_191 = __noswap_vcmla_rot180_f32(__rev0_191, __rev1_191, *(float32x2_t *) &__reint1_191); \
36632 __ret_191 = __noswap_vcmla_rot180_f32(__rev0_191, __rev1_191, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_191), __p3_191)})); \
3671936633 __ret_191 = __builtin_shufflevector(__ret_191, __ret_191, 1, 0); \
3672036634 __ret_191; \
3672136635})
......@@ -36727,9 +36641,7 @@ uint64x1_t __reint1_191 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
3672736641 float32x4_t __s0_192 = __p0_192; \
3672836642 float32x4_t __s1_192 = __p1_192; \
3672936643 float32x2_t __s2_192 = __p2_192; \
36730float32x2_t __reint_192 = __s2_192; \
36731uint64x2_t __reint1_192 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_192, __p3_192), vget_lane_u64(*(uint64x1_t *) &__reint_192, __p3_192)}; \
36732 __ret_192 = vcmlaq_rot180_f32(__s0_192, __s1_192, *(float32x4_t *) &__reint1_192); \
36644 __ret_192 = vcmlaq_rot180_f32(__s0_192, __s1_192, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_192), __p3_192), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_192), __p3_192)})); \
3673336645 __ret_192; \
3673436646})
3673536647#else
......@@ -36741,9 +36653,7 @@ uint64x2_t __reint1_192 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
3674136653 float32x4_t __rev0_193; __rev0_193 = __builtin_shufflevector(__s0_193, __s0_193, 3, 2, 1, 0); \
3674236654 float32x4_t __rev1_193; __rev1_193 = __builtin_shufflevector(__s1_193, __s1_193, 3, 2, 1, 0); \
3674336655 float32x2_t __rev2_193; __rev2_193 = __builtin_shufflevector(__s2_193, __s2_193, 1, 0); \
36744float32x2_t __reint_193 = __rev2_193; \
36745uint64x2_t __reint1_193 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_193, __p3_193), vget_lane_u64(*(uint64x1_t *) &__reint_193, __p3_193)}; \
36746 __ret_193 = __noswap_vcmlaq_rot180_f32(__rev0_193, __rev1_193, *(float32x4_t *) &__reint1_193); \
36656 __ret_193 = __noswap_vcmlaq_rot180_f32(__rev0_193, __rev1_193, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_193), __p3_193), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_193), __p3_193)})); \
3674736657 __ret_193 = __builtin_shufflevector(__ret_193, __ret_193, 3, 2, 1, 0); \
3674836658 __ret_193; \
3674936659})
......@@ -36755,9 +36665,7 @@ uint64x2_t __reint1_193 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
3675536665 float32x2_t __s0_194 = __p0_194; \
3675636666 float32x2_t __s1_194 = __p1_194; \
3675736667 float32x4_t __s2_194 = __p2_194; \
36758float32x4_t __reint_194 = __s2_194; \
36759uint64x1_t __reint1_194 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_194, __p3_194)}; \
36760 __ret_194 = vcmla_rot180_f32(__s0_194, __s1_194, *(float32x2_t *) &__reint1_194); \
36668 __ret_194 = vcmla_rot180_f32(__s0_194, __s1_194, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_194), __p3_194)})); \
3676136669 __ret_194; \
3676236670})
3676336671#else
......@@ -36769,9 +36677,7 @@ uint64x1_t __reint1_194 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
3676936677 float32x2_t __rev0_195; __rev0_195 = __builtin_shufflevector(__s0_195, __s0_195, 1, 0); \
3677036678 float32x2_t __rev1_195; __rev1_195 = __builtin_shufflevector(__s1_195, __s1_195, 1, 0); \
3677136679 float32x4_t __rev2_195; __rev2_195 = __builtin_shufflevector(__s2_195, __s2_195, 3, 2, 1, 0); \
36772float32x4_t __reint_195 = __rev2_195; \
36773uint64x1_t __reint1_195 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_195, __p3_195)}; \
36774 __ret_195 = __noswap_vcmla_rot180_f32(__rev0_195, __rev1_195, *(float32x2_t *) &__reint1_195); \
36680 __ret_195 = __noswap_vcmla_rot180_f32(__rev0_195, __rev1_195, __builtin_bit_cast(float32x2_t, (uint64x1_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_195), __p3_195)})); \
3677536681 __ret_195 = __builtin_shufflevector(__ret_195, __ret_195, 1, 0); \
3677636682 __ret_195; \
3677736683})
......@@ -36783,9 +36689,7 @@ uint64x1_t __reint1_195 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *)
3678336689 float32x4_t __s0_196 = __p0_196; \
3678436690 float32x4_t __s1_196 = __p1_196; \
3678536691 float32x4_t __s2_196 = __p2_196; \
36786float32x4_t __reint_196 = __s2_196; \
36787uint64x2_t __reint1_196 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_196, __p3_196), vgetq_lane_u64(*(uint64x2_t *) &__reint_196, __p3_196)}; \
36788 __ret_196 = vcmlaq_rot180_f32(__s0_196, __s1_196, *(float32x4_t *) &__reint1_196); \
36692 __ret_196 = vcmlaq_rot180_f32(__s0_196, __s1_196, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_196), __p3_196), vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_196), __p3_196)})); \
3678936693 __ret_196; \
3679036694})
3679136695#else
......@@ -36797,9 +36701,7 @@ uint64x2_t __reint1_196 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
3679736701 float32x4_t __rev0_197; __rev0_197 = __builtin_shufflevector(__s0_197, __s0_197, 3, 2, 1, 0); \
3679836702 float32x4_t __rev1_197; __rev1_197 = __builtin_shufflevector(__s1_197, __s1_197, 3, 2, 1, 0); \
3679936703 float32x4_t __rev2_197; __rev2_197 = __builtin_shufflevector(__s2_197, __s2_197, 3, 2, 1, 0); \
36800float32x4_t __reint_197 = __rev2_197; \
36801uint64x2_t __reint1_197 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_197, __p3_197), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_197, __p3_197)}; \
36802 __ret_197 = __noswap_vcmlaq_rot180_f32(__rev0_197, __rev1_197, *(float32x4_t *) &__reint1_197); \
36704 __ret_197 = __noswap_vcmlaq_rot180_f32(__rev0_197, __rev1_197, __builtin_bit_cast(float32x4_t, (uint64x2_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_197), __p3_197), __noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_197), __p3_197)})); \
3680336705 __ret_197 = __builtin_shufflevector(__ret_197, __ret_197, 3, 2, 1, 0); \
3680436706 __ret_197; \
3680536707})
......@@ -36857,9 +36759,7 @@ __ai __attribute__((target("v8.3a,neon"))) float32x2_t __noswap_vcmla_rot270_f32
3685736759 float32x2_t __s0_198 = __p0_198; \
3685836760 float32x2_t __s1_198 = __p1_198; \
3685936761 float32x2_t __s2_198 = __p2_198; \
36860float32x2_t __reint_198 = __s2_198; \
36861uint64x1_t __reint1_198 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_198, __p3_198)}; \
36862 __ret_198 = vcmla_rot270_f32(__s0_198, __s1_198, *(float32x2_t *) &__reint1_198); \
36762 __ret_198 = vcmla_rot270_f32(__s0_198, __s1_198, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_198), __p3_198)})); \
3686336763 __ret_198; \
3686436764})
3686536765#else
......@@ -36871,9 +36771,7 @@ uint64x1_t __reint1_198 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
3687136771 float32x2_t __rev0_199; __rev0_199 = __builtin_shufflevector(__s0_199, __s0_199, 1, 0); \
3687236772 float32x2_t __rev1_199; __rev1_199 = __builtin_shufflevector(__s1_199, __s1_199, 1, 0); \
3687336773 float32x2_t __rev2_199; __rev2_199 = __builtin_shufflevector(__s2_199, __s2_199, 1, 0); \
36874float32x2_t __reint_199 = __rev2_199; \
36875uint64x1_t __reint1_199 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_199, __p3_199)}; \
36876 __ret_199 = __noswap_vcmla_rot270_f32(__rev0_199, __rev1_199, *(float32x2_t *) &__reint1_199); \
36774 __ret_199 = __noswap_vcmla_rot270_f32(__rev0_199, __rev1_199, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_199), __p3_199)})); \
3687736775 __ret_199 = __builtin_shufflevector(__ret_199, __ret_199, 1, 0); \
3687836776 __ret_199; \
3687936777})
......@@ -36885,9 +36783,7 @@ uint64x1_t __reint1_199 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_1
3688536783 float32x4_t __s0_200 = __p0_200; \
3688636784 float32x4_t __s1_200 = __p1_200; \
3688736785 float32x2_t __s2_200 = __p2_200; \
36888float32x2_t __reint_200 = __s2_200; \
36889uint64x2_t __reint1_200 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_200, __p3_200), vget_lane_u64(*(uint64x1_t *) &__reint_200, __p3_200)}; \
36890 __ret_200 = vcmlaq_rot270_f32(__s0_200, __s1_200, *(float32x4_t *) &__reint1_200); \
36786 __ret_200 = vcmlaq_rot270_f32(__s0_200, __s1_200, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_200), __p3_200), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_200), __p3_200)})); \
3689136787 __ret_200; \
3689236788})
3689336789#else
......@@ -36899,9 +36795,7 @@ uint64x2_t __reint1_200 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2
3689936795 float32x4_t __rev0_201; __rev0_201 = __builtin_shufflevector(__s0_201, __s0_201, 3, 2, 1, 0); \
3690036796 float32x4_t __rev1_201; __rev1_201 = __builtin_shufflevector(__s1_201, __s1_201, 3, 2, 1, 0); \
3690136797 float32x2_t __rev2_201; __rev2_201 = __builtin_shufflevector(__s2_201, __s2_201, 1, 0); \
36902float32x2_t __reint_201 = __rev2_201; \
36903uint64x2_t __reint1_201 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_201, __p3_201), vget_lane_u64(*(uint64x1_t *) &__reint_201, __p3_201)}; \
36904 __ret_201 = __noswap_vcmlaq_rot270_f32(__rev0_201, __rev1_201, *(float32x4_t *) &__reint1_201); \
36798 __ret_201 = __noswap_vcmlaq_rot270_f32(__rev0_201, __rev1_201, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_201), __p3_201), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_201), __p3_201)})); \
3690536799 __ret_201 = __builtin_shufflevector(__ret_201, __ret_201, 3, 2, 1, 0); \
3690636800 __ret_201; \
3690736801})
......@@ -36913,9 +36807,7 @@ uint64x2_t __reint1_201 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2
3691336807 float32x2_t __s0_202 = __p0_202; \
3691436808 float32x2_t __s1_202 = __p1_202; \
3691536809 float32x4_t __s2_202 = __p2_202; \
36916float32x4_t __reint_202 = __s2_202; \
36917uint64x1_t __reint1_202 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_202, __p3_202)}; \
36918 __ret_202 = vcmla_rot270_f32(__s0_202, __s1_202, *(float32x2_t *) &__reint1_202); \
36810 __ret_202 = vcmla_rot270_f32(__s0_202, __s1_202, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_202), __p3_202)})); \
3691936811 __ret_202; \
3692036812})
3692136813#else
......@@ -36927,9 +36819,7 @@ uint64x1_t __reint1_202 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
3692736819 float32x2_t __rev0_203; __rev0_203 = __builtin_shufflevector(__s0_203, __s0_203, 1, 0); \
3692836820 float32x2_t __rev1_203; __rev1_203 = __builtin_shufflevector(__s1_203, __s1_203, 1, 0); \
3692936821 float32x4_t __rev2_203; __rev2_203 = __builtin_shufflevector(__s2_203, __s2_203, 3, 2, 1, 0); \
36930float32x4_t __reint_203 = __rev2_203; \
36931uint64x1_t __reint1_203 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_203, __p3_203)}; \
36932 __ret_203 = __noswap_vcmla_rot270_f32(__rev0_203, __rev1_203, *(float32x2_t *) &__reint1_203); \
36822 __ret_203 = __noswap_vcmla_rot270_f32(__rev0_203, __rev1_203, __builtin_bit_cast(float32x2_t, (uint64x1_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_203), __p3_203)})); \
3693336823 __ret_203 = __builtin_shufflevector(__ret_203, __ret_203, 1, 0); \
3693436824 __ret_203; \
3693536825})
......@@ -36941,9 +36831,7 @@ uint64x1_t __reint1_203 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *)
3694136831 float32x4_t __s0_204 = __p0_204; \
3694236832 float32x4_t __s1_204 = __p1_204; \
3694336833 float32x4_t __s2_204 = __p2_204; \
36944float32x4_t __reint_204 = __s2_204; \
36945uint64x2_t __reint1_204 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_204, __p3_204), vgetq_lane_u64(*(uint64x2_t *) &__reint_204, __p3_204)}; \
36946 __ret_204 = vcmlaq_rot270_f32(__s0_204, __s1_204, *(float32x4_t *) &__reint1_204); \
36834 __ret_204 = vcmlaq_rot270_f32(__s0_204, __s1_204, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_204), __p3_204), vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_204), __p3_204)})); \
3694736835 __ret_204; \
3694836836})
3694936837#else
......@@ -36955,9 +36843,7 @@ uint64x2_t __reint1_204 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
3695536843 float32x4_t __rev0_205; __rev0_205 = __builtin_shufflevector(__s0_205, __s0_205, 3, 2, 1, 0); \
3695636844 float32x4_t __rev1_205; __rev1_205 = __builtin_shufflevector(__s1_205, __s1_205, 3, 2, 1, 0); \
3695736845 float32x4_t __rev2_205; __rev2_205 = __builtin_shufflevector(__s2_205, __s2_205, 3, 2, 1, 0); \
36958float32x4_t __reint_205 = __rev2_205; \
36959uint64x2_t __reint1_205 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_205, __p3_205), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_205, __p3_205)}; \
36960 __ret_205 = __noswap_vcmlaq_rot270_f32(__rev0_205, __rev1_205, *(float32x4_t *) &__reint1_205); \
36846 __ret_205 = __noswap_vcmlaq_rot270_f32(__rev0_205, __rev1_205, __builtin_bit_cast(float32x4_t, (uint64x2_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_205), __p3_205), __noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_205), __p3_205)})); \
3696136847 __ret_205 = __builtin_shufflevector(__ret_205, __ret_205, 3, 2, 1, 0); \
3696236848 __ret_205; \
3696336849})
......@@ -37015,9 +36901,7 @@ __ai __attribute__((target("v8.3a,neon"))) float32x2_t __noswap_vcmla_rot90_f32(
3701536901 float32x2_t __s0_206 = __p0_206; \
3701636902 float32x2_t __s1_206 = __p1_206; \
3701736903 float32x2_t __s2_206 = __p2_206; \
37018float32x2_t __reint_206 = __s2_206; \
37019uint64x1_t __reint1_206 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_206, __p3_206)}; \
37020 __ret_206 = vcmla_rot90_f32(__s0_206, __s1_206, *(float32x2_t *) &__reint1_206); \
36904 __ret_206 = vcmla_rot90_f32(__s0_206, __s1_206, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_206), __p3_206)})); \
3702136905 __ret_206; \
3702236906})
3702336907#else
......@@ -37029,9 +36913,7 @@ uint64x1_t __reint1_206 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2
3702936913 float32x2_t __rev0_207; __rev0_207 = __builtin_shufflevector(__s0_207, __s0_207, 1, 0); \
3703036914 float32x2_t __rev1_207; __rev1_207 = __builtin_shufflevector(__s1_207, __s1_207, 1, 0); \
3703136915 float32x2_t __rev2_207; __rev2_207 = __builtin_shufflevector(__s2_207, __s2_207, 1, 0); \
37032float32x2_t __reint_207 = __rev2_207; \
37033uint64x1_t __reint1_207 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_207, __p3_207)}; \
37034 __ret_207 = __noswap_vcmla_rot90_f32(__rev0_207, __rev1_207, *(float32x2_t *) &__reint1_207); \
36916 __ret_207 = __noswap_vcmla_rot90_f32(__rev0_207, __rev1_207, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_207), __p3_207)})); \
3703536917 __ret_207 = __builtin_shufflevector(__ret_207, __ret_207, 1, 0); \
3703636918 __ret_207; \
3703736919})
......@@ -37043,9 +36925,7 @@ uint64x1_t __reint1_207 = (uint64x1_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2
3704336925 float32x4_t __s0_208 = __p0_208; \
3704436926 float32x4_t __s1_208 = __p1_208; \
3704536927 float32x2_t __s2_208 = __p2_208; \
37046float32x2_t __reint_208 = __s2_208; \
37047uint64x2_t __reint1_208 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_208, __p3_208), vget_lane_u64(*(uint64x1_t *) &__reint_208, __p3_208)}; \
37048 __ret_208 = vcmlaq_rot90_f32(__s0_208, __s1_208, *(float32x4_t *) &__reint1_208); \
36928 __ret_208 = vcmlaq_rot90_f32(__s0_208, __s1_208, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_208), __p3_208), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __s2_208), __p3_208)})); \
3704936929 __ret_208; \
3705036930})
3705136931#else
......@@ -37057,9 +36937,7 @@ uint64x2_t __reint1_208 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2
3705736937 float32x4_t __rev0_209; __rev0_209 = __builtin_shufflevector(__s0_209, __s0_209, 3, 2, 1, 0); \
3705836938 float32x4_t __rev1_209; __rev1_209 = __builtin_shufflevector(__s1_209, __s1_209, 3, 2, 1, 0); \
3705936939 float32x2_t __rev2_209; __rev2_209 = __builtin_shufflevector(__s2_209, __s2_209, 1, 0); \
37060float32x2_t __reint_209 = __rev2_209; \
37061uint64x2_t __reint1_209 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_209, __p3_209), vget_lane_u64(*(uint64x1_t *) &__reint_209, __p3_209)}; \
37062 __ret_209 = __noswap_vcmlaq_rot90_f32(__rev0_209, __rev1_209, *(float32x4_t *) &__reint1_209); \
36940 __ret_209 = __noswap_vcmlaq_rot90_f32(__rev0_209, __rev1_209, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_209), __p3_209), vget_lane_u64(__builtin_bit_cast(uint64x1_t, __rev2_209), __p3_209)})); \
3706336941 __ret_209 = __builtin_shufflevector(__ret_209, __ret_209, 3, 2, 1, 0); \
3706436942 __ret_209; \
3706536943})
......@@ -37071,9 +36949,7 @@ uint64x2_t __reint1_209 = (uint64x2_t) {vget_lane_u64(*(uint64x1_t *) &__reint_2
3707136949 float32x2_t __s0_210 = __p0_210; \
3707236950 float32x2_t __s1_210 = __p1_210; \
3707336951 float32x4_t __s2_210 = __p2_210; \
37074float32x4_t __reint_210 = __s2_210; \
37075uint64x1_t __reint1_210 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_210, __p3_210)}; \
37076 __ret_210 = vcmla_rot90_f32(__s0_210, __s1_210, *(float32x2_t *) &__reint1_210); \
36952 __ret_210 = vcmla_rot90_f32(__s0_210, __s1_210, __builtin_bit_cast(float32x2_t, (uint64x1_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_210), __p3_210)})); \
3707736953 __ret_210; \
3707836954})
3707936955#else
......@@ -37085,9 +36961,7 @@ uint64x1_t __reint1_210 = (uint64x1_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
3708536961 float32x2_t __rev0_211; __rev0_211 = __builtin_shufflevector(__s0_211, __s0_211, 1, 0); \
3708636962 float32x2_t __rev1_211; __rev1_211 = __builtin_shufflevector(__s1_211, __s1_211, 1, 0); \
3708736963 float32x4_t __rev2_211; __rev2_211 = __builtin_shufflevector(__s2_211, __s2_211, 3, 2, 1, 0); \
37088float32x4_t __reint_211 = __rev2_211; \
37089uint64x1_t __reint1_211 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_211, __p3_211)}; \
37090 __ret_211 = __noswap_vcmla_rot90_f32(__rev0_211, __rev1_211, *(float32x2_t *) &__reint1_211); \
36964 __ret_211 = __noswap_vcmla_rot90_f32(__rev0_211, __rev1_211, __builtin_bit_cast(float32x2_t, (uint64x1_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_211), __p3_211)})); \
3709136965 __ret_211 = __builtin_shufflevector(__ret_211, __ret_211, 1, 0); \
3709236966 __ret_211; \
3709336967})
......@@ -37099,9 +36973,7 @@ uint64x1_t __reint1_211 = (uint64x1_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *)
3709936973 float32x4_t __s0_212 = __p0_212; \
3710036974 float32x4_t __s1_212 = __p1_212; \
3710136975 float32x4_t __s2_212 = __p2_212; \
37102float32x4_t __reint_212 = __s2_212; \
37103uint64x2_t __reint1_212 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_212, __p3_212), vgetq_lane_u64(*(uint64x2_t *) &__reint_212, __p3_212)}; \
37104 __ret_212 = vcmlaq_rot90_f32(__s0_212, __s1_212, *(float32x4_t *) &__reint1_212); \
36976 __ret_212 = vcmlaq_rot90_f32(__s0_212, __s1_212, __builtin_bit_cast(float32x4_t, (uint64x2_t) {vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_212), __p3_212), vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __s2_212), __p3_212)})); \
3710536977 __ret_212; \
3710636978})
3710736979#else
......@@ -37113,9 +36985,7 @@ uint64x2_t __reint1_212 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_
3711336985 float32x4_t __rev0_213; __rev0_213 = __builtin_shufflevector(__s0_213, __s0_213, 3, 2, 1, 0); \
3711436986 float32x4_t __rev1_213; __rev1_213 = __builtin_shufflevector(__s1_213, __s1_213, 3, 2, 1, 0); \
3711536987 float32x4_t __rev2_213; __rev2_213 = __builtin_shufflevector(__s2_213, __s2_213, 3, 2, 1, 0); \
37116float32x4_t __reint_213 = __rev2_213; \
37117uint64x2_t __reint1_213 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_213, __p3_213), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_213, __p3_213)}; \
37118 __ret_213 = __noswap_vcmlaq_rot90_f32(__rev0_213, __rev1_213, *(float32x4_t *) &__reint1_213); \
36988 __ret_213 = __noswap_vcmlaq_rot90_f32(__rev0_213, __rev1_213, __builtin_bit_cast(float32x4_t, (uint64x2_t) {__noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_213), __p3_213), __noswap_vgetq_lane_u64(__builtin_bit_cast(uint64x2_t, __rev2_213), __p3_213)})); \
3711936989 __ret_213 = __builtin_shufflevector(__ret_213, __ret_213, 3, 2, 1, 0); \
3712036990 __ret_213; \
3712136991})
......@@ -41249,33 +41119,2258 @@ __ai __attribute__((target("neon"))) float32x2_t vfms_f32(float32x2_t __p0, floa
4124941119#endif
4125041120
4125141121#endif
41252#if defined(__aarch64__) || defined(__arm64ec__)
41253__ai __attribute__((target("aes,neon"))) poly128_t vmull_p64(poly64_t __p0, poly64_t __p1) {
41254 poly128_t __ret;
41255 __ret = (poly128_t) __builtin_neon_vmull_p64(__p0, __p1);
41122#if defined(__aarch64__)
41123#ifdef __LITTLE_ENDIAN__
41124__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt1_bf16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41125 bfloat16x8_t __ret;
41126 __ret = (bfloat16x8_t) __builtin_neon_vcvt1_bf16_mf8_fpm(__p0, __p1);
41127 return __ret;
41128}
41129#else
41130__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt1_bf16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41131 bfloat16x8_t __ret;
41132 mfloat8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41133 __ret = (bfloat16x8_t) __builtin_neon_vcvt1_bf16_mf8_fpm(__rev0, __p1);
41134 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41135 return __ret;
41136}
41137#endif
41138
41139#ifdef __LITTLE_ENDIAN__
41140__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt1_f16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41141 float16x8_t __ret;
41142 __ret = (float16x8_t) __builtin_neon_vcvt1_f16_mf8_fpm(__p0, __p1);
41143 return __ret;
41144}
41145#else
41146__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt1_f16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41147 float16x8_t __ret;
41148 mfloat8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41149 __ret = (float16x8_t) __builtin_neon_vcvt1_f16_mf8_fpm(__rev0, __p1);
41150 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41151 return __ret;
41152}
41153#endif
41154
41155#ifdef __LITTLE_ENDIAN__
41156__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt1_high_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41157 bfloat16x8_t __ret;
41158 __ret = (bfloat16x8_t) __builtin_neon_vcvt1_high_bf16_mf8_fpm(__p0, __p1);
41159 return __ret;
41160}
41161#else
41162__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt1_high_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41163 bfloat16x8_t __ret;
41164 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41165 __ret = (bfloat16x8_t) __builtin_neon_vcvt1_high_bf16_mf8_fpm(__rev0, __p1);
41166 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41167 return __ret;
41168}
41169#endif
41170
41171#ifdef __LITTLE_ENDIAN__
41172__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt1_high_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41173 float16x8_t __ret;
41174 __ret = (float16x8_t) __builtin_neon_vcvt1_high_f16_mf8_fpm(__p0, __p1);
41175 return __ret;
41176}
41177#else
41178__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt1_high_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41179 float16x8_t __ret;
41180 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41181 __ret = (float16x8_t) __builtin_neon_vcvt1_high_f16_mf8_fpm(__rev0, __p1);
41182 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41183 return __ret;
41184}
41185#endif
41186
41187#ifdef __LITTLE_ENDIAN__
41188__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt1_low_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41189 bfloat16x8_t __ret;
41190 __ret = (bfloat16x8_t) __builtin_neon_vcvt1_low_bf16_mf8_fpm(__p0, __p1);
41191 return __ret;
41192}
41193#else
41194__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt1_low_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41195 bfloat16x8_t __ret;
41196 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41197 __ret = (bfloat16x8_t) __builtin_neon_vcvt1_low_bf16_mf8_fpm(__rev0, __p1);
41198 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41199 return __ret;
41200}
41201#endif
41202
41203#ifdef __LITTLE_ENDIAN__
41204__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt1_low_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41205 float16x8_t __ret;
41206 __ret = (float16x8_t) __builtin_neon_vcvt1_low_f16_mf8_fpm(__p0, __p1);
41207 return __ret;
41208}
41209#else
41210__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt1_low_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41211 float16x8_t __ret;
41212 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41213 __ret = (float16x8_t) __builtin_neon_vcvt1_low_f16_mf8_fpm(__rev0, __p1);
41214 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41215 return __ret;
41216}
41217#endif
41218
41219#ifdef __LITTLE_ENDIAN__
41220__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt2_bf16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41221 bfloat16x8_t __ret;
41222 __ret = (bfloat16x8_t) __builtin_neon_vcvt2_bf16_mf8_fpm(__p0, __p1);
41223 return __ret;
41224}
41225#else
41226__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt2_bf16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41227 bfloat16x8_t __ret;
41228 mfloat8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41229 __ret = (bfloat16x8_t) __builtin_neon_vcvt2_bf16_mf8_fpm(__rev0, __p1);
41230 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41231 return __ret;
41232}
41233#endif
41234
41235#ifdef __LITTLE_ENDIAN__
41236__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt2_f16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41237 float16x8_t __ret;
41238 __ret = (float16x8_t) __builtin_neon_vcvt2_f16_mf8_fpm(__p0, __p1);
41239 return __ret;
41240}
41241#else
41242__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt2_f16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41243 float16x8_t __ret;
41244 mfloat8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41245 __ret = (float16x8_t) __builtin_neon_vcvt2_f16_mf8_fpm(__rev0, __p1);
41246 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41247 return __ret;
41248}
41249#endif
41250
41251#ifdef __LITTLE_ENDIAN__
41252__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt2_high_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41253 bfloat16x8_t __ret;
41254 __ret = (bfloat16x8_t) __builtin_neon_vcvt2_high_bf16_mf8_fpm(__p0, __p1);
41255 return __ret;
41256}
41257#else
41258__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt2_high_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41259 bfloat16x8_t __ret;
41260 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41261 __ret = (bfloat16x8_t) __builtin_neon_vcvt2_high_bf16_mf8_fpm(__rev0, __p1);
41262 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41263 return __ret;
41264}
41265#endif
41266
41267#ifdef __LITTLE_ENDIAN__
41268__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt2_high_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41269 float16x8_t __ret;
41270 __ret = (float16x8_t) __builtin_neon_vcvt2_high_f16_mf8_fpm(__p0, __p1);
41271 return __ret;
41272}
41273#else
41274__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt2_high_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41275 float16x8_t __ret;
41276 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41277 __ret = (float16x8_t) __builtin_neon_vcvt2_high_f16_mf8_fpm(__rev0, __p1);
41278 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41279 return __ret;
41280}
41281#endif
41282
41283#ifdef __LITTLE_ENDIAN__
41284__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt2_low_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41285 bfloat16x8_t __ret;
41286 __ret = (bfloat16x8_t) __builtin_neon_vcvt2_low_bf16_mf8_fpm(__p0, __p1);
41287 return __ret;
41288}
41289#else
41290__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt2_low_bf16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41291 bfloat16x8_t __ret;
41292 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41293 __ret = (bfloat16x8_t) __builtin_neon_vcvt2_low_bf16_mf8_fpm(__rev0, __p1);
41294 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41295 return __ret;
41296}
41297#endif
41298
41299#ifdef __LITTLE_ENDIAN__
41300__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt2_low_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41301 float16x8_t __ret;
41302 __ret = (float16x8_t) __builtin_neon_vcvt2_low_f16_mf8_fpm(__p0, __p1);
41303 return __ret;
41304}
41305#else
41306__ai __attribute__((target("fp8,neon"))) float16x8_t vcvt2_low_f16_mf8_fpm(mfloat8x16_t __p0, fpm_t __p1) {
41307 float16x8_t __ret;
41308 mfloat8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41309 __ret = (float16x8_t) __builtin_neon_vcvt2_low_f16_mf8_fpm(__rev0, __p1);
41310 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41311 return __ret;
41312}
41313#endif
41314
41315#ifdef __LITTLE_ENDIAN__
41316__ai __attribute__((target("fp8,neon"))) mfloat8x16_t vcvt_high_mf8_f32_fpm(mfloat8x8_t __p0, float32x4_t __p1, float32x4_t __p2, fpm_t __p3) {
41317 mfloat8x16_t __ret;
41318 __ret = (mfloat8x16_t) __builtin_neon_vcvt_high_mf8_f32_fpm(__p0, __p1, __p2, __p3);
41319 return __ret;
41320}
41321#else
41322__ai __attribute__((target("fp8,neon"))) mfloat8x16_t vcvt_high_mf8_f32_fpm(mfloat8x8_t __p0, float32x4_t __p1, float32x4_t __p2, fpm_t __p3) {
41323 mfloat8x16_t __ret;
41324 mfloat8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41325 float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
41326 float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0);
41327 __ret = (mfloat8x16_t) __builtin_neon_vcvt_high_mf8_f32_fpm(__rev0, __rev1, __rev2, __p3);
41328 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41329 return __ret;
41330}
41331#endif
41332
41333#ifdef __LITTLE_ENDIAN__
41334__ai __attribute__((target("fp8,neon"))) mfloat8x16_t vcvtq_mf8_f16_fpm(float16x8_t __p0, float16x8_t __p1, fpm_t __p2) {
41335 mfloat8x16_t __ret;
41336 __ret = (mfloat8x16_t) __builtin_neon_vcvtq_mf8_f16_fpm((int8x16_t)__p0, (int8x16_t)__p1, __p2);
41337 return __ret;
41338}
41339#else
41340__ai __attribute__((target("fp8,neon"))) mfloat8x16_t vcvtq_mf8_f16_fpm(float16x8_t __p0, float16x8_t __p1, fpm_t __p2) {
41341 mfloat8x16_t __ret;
41342 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41343 float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
41344 __ret = (mfloat8x16_t) __builtin_neon_vcvtq_mf8_f16_fpm((int8x16_t)__rev0, (int8x16_t)__rev1, __p2);
41345 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41346 return __ret;
41347}
41348#endif
41349
41350#ifdef __LITTLE_ENDIAN__
41351__ai __attribute__((target("fp8,neon"))) mfloat8x8_t vcvt_mf8_f16_fpm(float16x4_t __p0, float16x4_t __p1, fpm_t __p2) {
41352 mfloat8x8_t __ret;
41353 __ret = (mfloat8x8_t) __builtin_neon_vcvt_mf8_f16_fpm((int8x8_t)__p0, (int8x8_t)__p1, __p2);
41354 return __ret;
41355}
41356#else
41357__ai __attribute__((target("fp8,neon"))) mfloat8x8_t vcvt_mf8_f16_fpm(float16x4_t __p0, float16x4_t __p1, fpm_t __p2) {
41358 mfloat8x8_t __ret;
41359 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41360 float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
41361 __ret = (mfloat8x8_t) __builtin_neon_vcvt_mf8_f16_fpm((int8x8_t)__rev0, (int8x8_t)__rev1, __p2);
41362 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41363 return __ret;
41364}
41365#endif
41366
41367#ifdef __LITTLE_ENDIAN__
41368__ai __attribute__((target("fp8,neon"))) mfloat8x8_t vcvt_mf8_f32_fpm(float32x4_t __p0, float32x4_t __p1, fpm_t __p2) {
41369 mfloat8x8_t __ret;
41370 __ret = (mfloat8x8_t) __builtin_neon_vcvt_mf8_f32_fpm(__p0, __p1, __p2);
41371 return __ret;
41372}
41373#else
41374__ai __attribute__((target("fp8,neon"))) mfloat8x8_t vcvt_mf8_f32_fpm(float32x4_t __p0, float32x4_t __p1, fpm_t __p2) {
41375 mfloat8x8_t __ret;
41376 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41377 float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
41378 __ret = (mfloat8x8_t) __builtin_neon_vcvt_mf8_f32_fpm(__rev0, __rev1, __p2);
41379 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41380 return __ret;
41381}
41382#endif
41383
41384#ifdef __LITTLE_ENDIAN__
41385__ai __attribute__((target("fp8,neon"))) float32x2_t vscale_f32(float32x2_t __p0, int32x2_t __p1) {
41386 float32x2_t __ret;
41387 __ret = (float32x2_t) __builtin_neon_vscale_f32((int8x8_t)__p0, (int8x8_t)__p1, 9);
41388 return __ret;
41389}
41390#else
41391__ai __attribute__((target("fp8,neon"))) float32x2_t vscale_f32(float32x2_t __p0, int32x2_t __p1) {
41392 float32x2_t __ret;
41393 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
41394 int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
41395 __ret = (float32x2_t) __builtin_neon_vscale_f32((int8x8_t)__rev0, (int8x8_t)__rev1, 9);
41396 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
41397 return __ret;
41398}
41399#endif
41400
41401#ifdef __LITTLE_ENDIAN__
41402__ai __attribute__((target("fp8,neon"))) float16x4_t vscale_f16(float16x4_t __p0, int16x4_t __p1) {
41403 float16x4_t __ret;
41404 __ret = (float16x4_t) __builtin_neon_vscale_f16((int8x8_t)__p0, (int8x8_t)__p1, 8);
41405 return __ret;
41406}
41407#else
41408__ai __attribute__((target("fp8,neon"))) float16x4_t vscale_f16(float16x4_t __p0, int16x4_t __p1) {
41409 float16x4_t __ret;
41410 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41411 int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
41412 __ret = (float16x4_t) __builtin_neon_vscale_f16((int8x8_t)__rev0, (int8x8_t)__rev1, 8);
41413 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41414 return __ret;
41415}
41416#endif
41417
41418#ifdef __LITTLE_ENDIAN__
41419__ai __attribute__((target("fp8,neon"))) float64x2_t vscaleq_f64(float64x2_t __p0, int64x2_t __p1) {
41420 float64x2_t __ret;
41421 __ret = (float64x2_t) __builtin_neon_vscaleq_f64((int8x16_t)__p0, (int8x16_t)__p1, 42);
41422 return __ret;
41423}
41424#else
41425__ai __attribute__((target("fp8,neon"))) float64x2_t vscaleq_f64(float64x2_t __p0, int64x2_t __p1) {
41426 float64x2_t __ret;
41427 float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
41428 int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
41429 __ret = (float64x2_t) __builtin_neon_vscaleq_f64((int8x16_t)__rev0, (int8x16_t)__rev1, 42);
41430 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
41431 return __ret;
41432}
41433#endif
41434
41435#ifdef __LITTLE_ENDIAN__
41436__ai __attribute__((target("fp8,neon"))) float32x4_t vscaleq_f32(float32x4_t __p0, int32x4_t __p1) {
41437 float32x4_t __ret;
41438 __ret = (float32x4_t) __builtin_neon_vscaleq_f32((int8x16_t)__p0, (int8x16_t)__p1, 41);
41439 return __ret;
41440}
41441#else
41442__ai __attribute__((target("fp8,neon"))) float32x4_t vscaleq_f32(float32x4_t __p0, int32x4_t __p1) {
41443 float32x4_t __ret;
41444 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41445 int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
41446 __ret = (float32x4_t) __builtin_neon_vscaleq_f32((int8x16_t)__rev0, (int8x16_t)__rev1, 41);
41447 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41448 return __ret;
41449}
41450#endif
41451
41452#ifdef __LITTLE_ENDIAN__
41453__ai __attribute__((target("fp8,neon"))) float16x8_t vscaleq_f16(float16x8_t __p0, int16x8_t __p1) {
41454 float16x8_t __ret;
41455 __ret = (float16x8_t) __builtin_neon_vscaleq_f16((int8x16_t)__p0, (int8x16_t)__p1, 40);
41456 return __ret;
41457}
41458#else
41459__ai __attribute__((target("fp8,neon"))) float16x8_t vscaleq_f16(float16x8_t __p0, int16x8_t __p1) {
41460 float16x8_t __ret;
41461 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41462 int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
41463 __ret = (float16x8_t) __builtin_neon_vscaleq_f16((int8x16_t)__rev0, (int8x16_t)__rev1, 40);
41464 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41465 return __ret;
41466}
41467#endif
41468
41469#ifdef __LITTLE_ENDIAN__
41470__ai __attribute__((target("fp8dot2,neon"))) float16x8_t vdotq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41471 float16x8_t __ret;
41472 __ret = (float16x8_t) __builtin_neon_vdotq_f16_mf8_fpm((int8x16_t)__p0, __p1, __p2, __p3);
41473 return __ret;
41474}
41475#else
41476__ai __attribute__((target("fp8dot2,neon"))) float16x8_t vdotq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41477 float16x8_t __ret;
41478 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41479 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41480 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41481 __ret = (float16x8_t) __builtin_neon_vdotq_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3);
41482 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41483 return __ret;
41484}
41485#endif
41486
41487#ifdef __LITTLE_ENDIAN__
41488__ai __attribute__((target("fp8dot2,neon"))) float16x4_t vdot_f16_mf8_fpm(float16x4_t __p0, mfloat8x8_t __p1, mfloat8x8_t __p2, fpm_t __p3) {
41489 float16x4_t __ret;
41490 __ret = (float16x4_t) __builtin_neon_vdot_f16_mf8_fpm((int8x8_t)__p0, __p1, __p2, __p3);
41491 return __ret;
41492}
41493#else
41494__ai __attribute__((target("fp8dot2,neon"))) float16x4_t vdot_f16_mf8_fpm(float16x4_t __p0, mfloat8x8_t __p1, mfloat8x8_t __p2, fpm_t __p3) {
41495 float16x4_t __ret;
41496 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41497 mfloat8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
41498 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
41499 __ret = (float16x4_t) __builtin_neon_vdot_f16_mf8_fpm((int8x8_t)__rev0, __rev1, __rev2, __p3);
41500 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41501 return __ret;
41502}
41503#endif
41504
41505#ifdef __LITTLE_ENDIAN__
41506#define vdotq_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41507 float16x8_t __ret; \
41508 float16x8_t __s0 = __p0; \
41509 mfloat8x16_t __s1 = __p1; \
41510 mfloat8x8_t __s2 = __p2; \
41511 fpm_t __s4 = __p4; \
41512 __ret = (float16x8_t) __builtin_neon_vdotq_lane_f16_mf8_fpm((int8x16_t)__s0, __s1, __s2, __p3, __s4); \
41513 __ret; \
41514})
41515#else
41516#define vdotq_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41517 float16x8_t __ret; \
41518 float16x8_t __s0 = __p0; \
41519 mfloat8x16_t __s1 = __p1; \
41520 mfloat8x8_t __s2 = __p2; \
41521 fpm_t __s4 = __p4; \
41522 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
41523 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41524 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41525 __ret = (float16x8_t) __builtin_neon_vdotq_lane_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3, __s4); \
41526 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
41527 __ret; \
41528})
41529#endif
41530
41531#ifdef __LITTLE_ENDIAN__
41532#define vdot_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41533 float16x4_t __ret; \
41534 float16x4_t __s0 = __p0; \
41535 mfloat8x8_t __s1 = __p1; \
41536 mfloat8x8_t __s2 = __p2; \
41537 fpm_t __s4 = __p4; \
41538 __ret = (float16x4_t) __builtin_neon_vdot_lane_f16_mf8_fpm((int8x8_t)__s0, __s1, __s2, __p3, __s4); \
41539 __ret; \
41540})
41541#else
41542#define vdot_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41543 float16x4_t __ret; \
41544 float16x4_t __s0 = __p0; \
41545 mfloat8x8_t __s1 = __p1; \
41546 mfloat8x8_t __s2 = __p2; \
41547 fpm_t __s4 = __p4; \
41548 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41549 mfloat8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
41550 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41551 __ret = (float16x4_t) __builtin_neon_vdot_lane_f16_mf8_fpm((int8x8_t)__rev0, __rev1, __rev2, __p3, __s4); \
41552 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41553 __ret; \
41554})
41555#endif
41556
41557#ifdef __LITTLE_ENDIAN__
41558#define vdotq_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41559 float16x8_t __ret; \
41560 float16x8_t __s0 = __p0; \
41561 mfloat8x16_t __s1 = __p1; \
41562 mfloat8x16_t __s2 = __p2; \
41563 fpm_t __s4 = __p4; \
41564 __ret = (float16x8_t) __builtin_neon_vdotq_laneq_f16_mf8_fpm((int8x16_t)__s0, __s1, __s2, __p3, __s4); \
41565 __ret; \
41566})
41567#else
41568#define vdotq_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41569 float16x8_t __ret; \
41570 float16x8_t __s0 = __p0; \
41571 mfloat8x16_t __s1 = __p1; \
41572 mfloat8x16_t __s2 = __p2; \
41573 fpm_t __s4 = __p4; \
41574 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
41575 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41576 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41577 __ret = (float16x8_t) __builtin_neon_vdotq_laneq_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3, __s4); \
41578 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
41579 __ret; \
41580})
41581#endif
41582
41583#ifdef __LITTLE_ENDIAN__
41584#define vdot_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41585 float16x4_t __ret; \
41586 float16x4_t __s0 = __p0; \
41587 mfloat8x8_t __s1 = __p1; \
41588 mfloat8x16_t __s2 = __p2; \
41589 fpm_t __s4 = __p4; \
41590 __ret = (float16x4_t) __builtin_neon_vdot_laneq_f16_mf8_fpm((int8x8_t)__s0, __s1, __s2, __p3, __s4); \
41591 __ret; \
41592})
41593#else
41594#define vdot_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41595 float16x4_t __ret; \
41596 float16x4_t __s0 = __p0; \
41597 mfloat8x8_t __s1 = __p1; \
41598 mfloat8x16_t __s2 = __p2; \
41599 fpm_t __s4 = __p4; \
41600 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41601 mfloat8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
41602 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41603 __ret = (float16x4_t) __builtin_neon_vdot_laneq_f16_mf8_fpm((int8x8_t)__rev0, __rev1, __rev2, __p3, __s4); \
41604 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41605 __ret; \
41606})
41607#endif
41608
41609#ifdef __LITTLE_ENDIAN__
41610__ai __attribute__((target("fp8dot4,neon"))) float32x4_t vdotq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41611 float32x4_t __ret;
41612 __ret = (float32x4_t) __builtin_neon_vdotq_f32_mf8_fpm(__p0, __p1, __p2, __p3);
41613 return __ret;
41614}
41615#else
41616__ai __attribute__((target("fp8dot4,neon"))) float32x4_t vdotq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41617 float32x4_t __ret;
41618 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41619 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41620 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41621 __ret = (float32x4_t) __builtin_neon_vdotq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3);
41622 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41623 return __ret;
41624}
41625#endif
41626
41627#ifdef __LITTLE_ENDIAN__
41628__ai __attribute__((target("fp8dot4,neon"))) float32x2_t vdot_f32_mf8_fpm(float32x2_t __p0, mfloat8x8_t __p1, mfloat8x8_t __p2, fpm_t __p3) {
41629 float32x2_t __ret;
41630 __ret = (float32x2_t) __builtin_neon_vdot_f32_mf8_fpm(__p0, __p1, __p2, __p3);
41631 return __ret;
41632}
41633#else
41634__ai __attribute__((target("fp8dot4,neon"))) float32x2_t vdot_f32_mf8_fpm(float32x2_t __p0, mfloat8x8_t __p1, mfloat8x8_t __p2, fpm_t __p3) {
41635 float32x2_t __ret;
41636 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
41637 mfloat8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
41638 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0);
41639 __ret = (float32x2_t) __builtin_neon_vdot_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3);
41640 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
41641 return __ret;
41642}
41643#endif
41644
41645#ifdef __LITTLE_ENDIAN__
41646#define vdotq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41647 float32x4_t __ret; \
41648 float32x4_t __s0 = __p0; \
41649 mfloat8x16_t __s1 = __p1; \
41650 mfloat8x8_t __s2 = __p2; \
41651 fpm_t __s4 = __p4; \
41652 __ret = (float32x4_t) __builtin_neon_vdotq_lane_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41653 __ret; \
41654})
41655#else
41656#define vdotq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41657 float32x4_t __ret; \
41658 float32x4_t __s0 = __p0; \
41659 mfloat8x16_t __s1 = __p1; \
41660 mfloat8x8_t __s2 = __p2; \
41661 fpm_t __s4 = __p4; \
41662 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41663 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41664 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41665 __ret = (float32x4_t) __builtin_neon_vdotq_lane_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41666 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41667 __ret; \
41668})
41669#endif
41670
41671#ifdef __LITTLE_ENDIAN__
41672#define vdot_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41673 float32x2_t __ret; \
41674 float32x2_t __s0 = __p0; \
41675 mfloat8x8_t __s1 = __p1; \
41676 mfloat8x8_t __s2 = __p2; \
41677 fpm_t __s4 = __p4; \
41678 __ret = (float32x2_t) __builtin_neon_vdot_lane_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41679 __ret; \
41680})
41681#else
41682#define vdot_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41683 float32x2_t __ret; \
41684 float32x2_t __s0 = __p0; \
41685 mfloat8x8_t __s1 = __p1; \
41686 mfloat8x8_t __s2 = __p2; \
41687 fpm_t __s4 = __p4; \
41688 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \
41689 mfloat8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
41690 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41691 __ret = (float32x2_t) __builtin_neon_vdot_lane_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41692 __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \
41693 __ret; \
41694})
41695#endif
41696
41697#ifdef __LITTLE_ENDIAN__
41698#define vdotq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41699 float32x4_t __ret; \
41700 float32x4_t __s0 = __p0; \
41701 mfloat8x16_t __s1 = __p1; \
41702 mfloat8x16_t __s2 = __p2; \
41703 fpm_t __s4 = __p4; \
41704 __ret = (float32x4_t) __builtin_neon_vdotq_laneq_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41705 __ret; \
41706})
41707#else
41708#define vdotq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41709 float32x4_t __ret; \
41710 float32x4_t __s0 = __p0; \
41711 mfloat8x16_t __s1 = __p1; \
41712 mfloat8x16_t __s2 = __p2; \
41713 fpm_t __s4 = __p4; \
41714 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41715 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41716 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41717 __ret = (float32x4_t) __builtin_neon_vdotq_laneq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41718 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41719 __ret; \
41720})
41721#endif
41722
41723#ifdef __LITTLE_ENDIAN__
41724#define vdot_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41725 float32x2_t __ret; \
41726 float32x2_t __s0 = __p0; \
41727 mfloat8x8_t __s1 = __p1; \
41728 mfloat8x16_t __s2 = __p2; \
41729 fpm_t __s4 = __p4; \
41730 __ret = (float32x2_t) __builtin_neon_vdot_laneq_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41731 __ret; \
41732})
41733#else
41734#define vdot_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41735 float32x2_t __ret; \
41736 float32x2_t __s0 = __p0; \
41737 mfloat8x8_t __s1 = __p1; \
41738 mfloat8x16_t __s2 = __p2; \
41739 fpm_t __s4 = __p4; \
41740 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \
41741 mfloat8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
41742 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41743 __ret = (float32x2_t) __builtin_neon_vdot_laneq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41744 __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \
41745 __ret; \
41746})
41747#endif
41748
41749#ifdef __LITTLE_ENDIAN__
41750__ai __attribute__((target("fp8fma,neon"))) float16x8_t vmlalbq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41751 float16x8_t __ret;
41752 __ret = (float16x8_t) __builtin_neon_vmlalbq_f16_mf8_fpm((int8x16_t)__p0, __p1, __p2, __p3);
41753 return __ret;
41754}
41755#else
41756__ai __attribute__((target("fp8fma,neon"))) float16x8_t vmlalbq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41757 float16x8_t __ret;
41758 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
41759 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41760 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41761 __ret = (float16x8_t) __builtin_neon_vmlalbq_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3);
41762 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
41763 return __ret;
41764}
41765#endif
41766
41767#ifdef __LITTLE_ENDIAN__
41768#define vmlalbq_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41769 float16x8_t __ret; \
41770 float16x8_t __s0 = __p0; \
41771 mfloat8x16_t __s1 = __p1; \
41772 mfloat8x8_t __s2 = __p2; \
41773 fpm_t __s4 = __p4; \
41774 __ret = (float16x8_t) __builtin_neon_vmlalbq_lane_f16_mf8_fpm((int8x16_t)__s0, __s1, __s2, __p3, __s4); \
41775 __ret; \
41776})
41777#else
41778#define vmlalbq_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41779 float16x8_t __ret; \
41780 float16x8_t __s0 = __p0; \
41781 mfloat8x16_t __s1 = __p1; \
41782 mfloat8x8_t __s2 = __p2; \
41783 fpm_t __s4 = __p4; \
41784 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
41785 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41786 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41787 __ret = (float16x8_t) __builtin_neon_vmlalbq_lane_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3, __s4); \
41788 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
41789 __ret; \
41790})
41791#endif
41792
41793#ifdef __LITTLE_ENDIAN__
41794#define vmlalbq_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41795 float16x8_t __ret; \
41796 float16x8_t __s0 = __p0; \
41797 mfloat8x16_t __s1 = __p1; \
41798 mfloat8x16_t __s2 = __p2; \
41799 fpm_t __s4 = __p4; \
41800 __ret = (float16x8_t) __builtin_neon_vmlalbq_laneq_f16_mf8_fpm((int8x16_t)__s0, __s1, __s2, __p3, __s4); \
41801 __ret; \
41802})
41803#else
41804#define vmlalbq_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41805 float16x8_t __ret; \
41806 float16x8_t __s0 = __p0; \
41807 mfloat8x16_t __s1 = __p1; \
41808 mfloat8x16_t __s2 = __p2; \
41809 fpm_t __s4 = __p4; \
41810 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
41811 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41812 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41813 __ret = (float16x8_t) __builtin_neon_vmlalbq_laneq_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3, __s4); \
41814 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
41815 __ret; \
41816})
41817#endif
41818
41819#ifdef __LITTLE_ENDIAN__
41820__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlallbbq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41821 float32x4_t __ret;
41822 __ret = (float32x4_t) __builtin_neon_vmlallbbq_f32_mf8_fpm(__p0, __p1, __p2, __p3);
41823 return __ret;
41824}
41825#else
41826__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlallbbq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41827 float32x4_t __ret;
41828 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41829 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41830 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41831 __ret = (float32x4_t) __builtin_neon_vmlallbbq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3);
41832 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41833 return __ret;
41834}
41835#endif
41836
41837#ifdef __LITTLE_ENDIAN__
41838#define vmlallbbq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41839 float32x4_t __ret; \
41840 float32x4_t __s0 = __p0; \
41841 mfloat8x16_t __s1 = __p1; \
41842 mfloat8x8_t __s2 = __p2; \
41843 fpm_t __s4 = __p4; \
41844 __ret = (float32x4_t) __builtin_neon_vmlallbbq_lane_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41845 __ret; \
41846})
41847#else
41848#define vmlallbbq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41849 float32x4_t __ret; \
41850 float32x4_t __s0 = __p0; \
41851 mfloat8x16_t __s1 = __p1; \
41852 mfloat8x8_t __s2 = __p2; \
41853 fpm_t __s4 = __p4; \
41854 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41855 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41856 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41857 __ret = (float32x4_t) __builtin_neon_vmlallbbq_lane_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41858 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41859 __ret; \
41860})
41861#endif
41862
41863#ifdef __LITTLE_ENDIAN__
41864#define vmlallbbq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41865 float32x4_t __ret; \
41866 float32x4_t __s0 = __p0; \
41867 mfloat8x16_t __s1 = __p1; \
41868 mfloat8x16_t __s2 = __p2; \
41869 fpm_t __s4 = __p4; \
41870 __ret = (float32x4_t) __builtin_neon_vmlallbbq_laneq_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41871 __ret; \
41872})
41873#else
41874#define vmlallbbq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41875 float32x4_t __ret; \
41876 float32x4_t __s0 = __p0; \
41877 mfloat8x16_t __s1 = __p1; \
41878 mfloat8x16_t __s2 = __p2; \
41879 fpm_t __s4 = __p4; \
41880 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41881 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41882 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41883 __ret = (float32x4_t) __builtin_neon_vmlallbbq_laneq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41884 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41885 __ret; \
41886})
41887#endif
41888
41889#ifdef __LITTLE_ENDIAN__
41890__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlallbtq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41891 float32x4_t __ret;
41892 __ret = (float32x4_t) __builtin_neon_vmlallbtq_f32_mf8_fpm(__p0, __p1, __p2, __p3);
41893 return __ret;
41894}
41895#else
41896__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlallbtq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41897 float32x4_t __ret;
41898 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41899 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41900 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41901 __ret = (float32x4_t) __builtin_neon_vmlallbtq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3);
41902 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41903 return __ret;
41904}
41905#endif
41906
41907#ifdef __LITTLE_ENDIAN__
41908#define vmlallbtq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41909 float32x4_t __ret; \
41910 float32x4_t __s0 = __p0; \
41911 mfloat8x16_t __s1 = __p1; \
41912 mfloat8x8_t __s2 = __p2; \
41913 fpm_t __s4 = __p4; \
41914 __ret = (float32x4_t) __builtin_neon_vmlallbtq_lane_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41915 __ret; \
41916})
41917#else
41918#define vmlallbtq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41919 float32x4_t __ret; \
41920 float32x4_t __s0 = __p0; \
41921 mfloat8x16_t __s1 = __p1; \
41922 mfloat8x8_t __s2 = __p2; \
41923 fpm_t __s4 = __p4; \
41924 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41925 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41926 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41927 __ret = (float32x4_t) __builtin_neon_vmlallbtq_lane_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41928 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41929 __ret; \
41930})
41931#endif
41932
41933#ifdef __LITTLE_ENDIAN__
41934#define vmlallbtq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41935 float32x4_t __ret; \
41936 float32x4_t __s0 = __p0; \
41937 mfloat8x16_t __s1 = __p1; \
41938 mfloat8x16_t __s2 = __p2; \
41939 fpm_t __s4 = __p4; \
41940 __ret = (float32x4_t) __builtin_neon_vmlallbtq_laneq_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41941 __ret; \
41942})
41943#else
41944#define vmlallbtq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41945 float32x4_t __ret; \
41946 float32x4_t __s0 = __p0; \
41947 mfloat8x16_t __s1 = __p1; \
41948 mfloat8x16_t __s2 = __p2; \
41949 fpm_t __s4 = __p4; \
41950 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41951 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41952 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41953 __ret = (float32x4_t) __builtin_neon_vmlallbtq_laneq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41954 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41955 __ret; \
41956})
41957#endif
41958
41959#ifdef __LITTLE_ENDIAN__
41960__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlalltbq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41961 float32x4_t __ret;
41962 __ret = (float32x4_t) __builtin_neon_vmlalltbq_f32_mf8_fpm(__p0, __p1, __p2, __p3);
41963 return __ret;
41964}
41965#else
41966__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlalltbq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
41967 float32x4_t __ret;
41968 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41969 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41970 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
41971 __ret = (float32x4_t) __builtin_neon_vmlalltbq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3);
41972 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
41973 return __ret;
41974}
41975#endif
41976
41977#ifdef __LITTLE_ENDIAN__
41978#define vmlalltbq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41979 float32x4_t __ret; \
41980 float32x4_t __s0 = __p0; \
41981 mfloat8x16_t __s1 = __p1; \
41982 mfloat8x8_t __s2 = __p2; \
41983 fpm_t __s4 = __p4; \
41984 __ret = (float32x4_t) __builtin_neon_vmlalltbq_lane_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
41985 __ret; \
41986})
41987#else
41988#define vmlalltbq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
41989 float32x4_t __ret; \
41990 float32x4_t __s0 = __p0; \
41991 mfloat8x16_t __s1 = __p1; \
41992 mfloat8x8_t __s2 = __p2; \
41993 fpm_t __s4 = __p4; \
41994 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
41995 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41996 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
41997 __ret = (float32x4_t) __builtin_neon_vmlalltbq_lane_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
41998 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
41999 __ret; \
42000})
42001#endif
42002
42003#ifdef __LITTLE_ENDIAN__
42004#define vmlalltbq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42005 float32x4_t __ret; \
42006 float32x4_t __s0 = __p0; \
42007 mfloat8x16_t __s1 = __p1; \
42008 mfloat8x16_t __s2 = __p2; \
42009 fpm_t __s4 = __p4; \
42010 __ret = (float32x4_t) __builtin_neon_vmlalltbq_laneq_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
42011 __ret; \
42012})
42013#else
42014#define vmlalltbq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42015 float32x4_t __ret; \
42016 float32x4_t __s0 = __p0; \
42017 mfloat8x16_t __s1 = __p1; \
42018 mfloat8x16_t __s2 = __p2; \
42019 fpm_t __s4 = __p4; \
42020 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42021 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42022 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42023 __ret = (float32x4_t) __builtin_neon_vmlalltbq_laneq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
42024 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
42025 __ret; \
42026})
42027#endif
42028
42029#ifdef __LITTLE_ENDIAN__
42030__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlallttq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
42031 float32x4_t __ret;
42032 __ret = (float32x4_t) __builtin_neon_vmlallttq_f32_mf8_fpm(__p0, __p1, __p2, __p3);
42033 return __ret;
42034}
42035#else
42036__ai __attribute__((target("fp8fma,neon"))) float32x4_t vmlallttq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
42037 float32x4_t __ret;
42038 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
42039 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
42040 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
42041 __ret = (float32x4_t) __builtin_neon_vmlallttq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3);
42042 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
42043 return __ret;
42044}
42045#endif
42046
42047#ifdef __LITTLE_ENDIAN__
42048#define vmlallttq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42049 float32x4_t __ret; \
42050 float32x4_t __s0 = __p0; \
42051 mfloat8x16_t __s1 = __p1; \
42052 mfloat8x8_t __s2 = __p2; \
42053 fpm_t __s4 = __p4; \
42054 __ret = (float32x4_t) __builtin_neon_vmlallttq_lane_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
42055 __ret; \
42056})
42057#else
42058#define vmlallttq_lane_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42059 float32x4_t __ret; \
42060 float32x4_t __s0 = __p0; \
42061 mfloat8x16_t __s1 = __p1; \
42062 mfloat8x8_t __s2 = __p2; \
42063 fpm_t __s4 = __p4; \
42064 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42065 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42066 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
42067 __ret = (float32x4_t) __builtin_neon_vmlallttq_lane_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
42068 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
42069 __ret; \
42070})
42071#endif
42072
42073#ifdef __LITTLE_ENDIAN__
42074#define vmlallttq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42075 float32x4_t __ret; \
42076 float32x4_t __s0 = __p0; \
42077 mfloat8x16_t __s1 = __p1; \
42078 mfloat8x16_t __s2 = __p2; \
42079 fpm_t __s4 = __p4; \
42080 __ret = (float32x4_t) __builtin_neon_vmlallttq_laneq_f32_mf8_fpm(__s0, __s1, __s2, __p3, __s4); \
42081 __ret; \
42082})
42083#else
42084#define vmlallttq_laneq_f32_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42085 float32x4_t __ret; \
42086 float32x4_t __s0 = __p0; \
42087 mfloat8x16_t __s1 = __p1; \
42088 mfloat8x16_t __s2 = __p2; \
42089 fpm_t __s4 = __p4; \
42090 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42091 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42092 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42093 __ret = (float32x4_t) __builtin_neon_vmlallttq_laneq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3, __s4); \
42094 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \
42095 __ret; \
42096})
42097#endif
42098
42099#ifdef __LITTLE_ENDIAN__
42100__ai __attribute__((target("fp8fma,neon"))) float16x8_t vmlaltq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
42101 float16x8_t __ret;
42102 __ret = (float16x8_t) __builtin_neon_vmlaltq_f16_mf8_fpm((int8x16_t)__p0, __p1, __p2, __p3);
42103 return __ret;
42104}
42105#else
42106__ai __attribute__((target("fp8fma,neon"))) float16x8_t vmlaltq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
42107 float16x8_t __ret;
42108 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
42109 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
42110 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
42111 __ret = (float16x8_t) __builtin_neon_vmlaltq_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3);
42112 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
42113 return __ret;
42114}
42115#endif
42116
42117#ifdef __LITTLE_ENDIAN__
42118#define vmlaltq_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42119 float16x8_t __ret; \
42120 float16x8_t __s0 = __p0; \
42121 mfloat8x16_t __s1 = __p1; \
42122 mfloat8x8_t __s2 = __p2; \
42123 fpm_t __s4 = __p4; \
42124 __ret = (float16x8_t) __builtin_neon_vmlaltq_lane_f16_mf8_fpm((int8x16_t)__s0, __s1, __s2, __p3, __s4); \
42125 __ret; \
42126})
42127#else
42128#define vmlaltq_lane_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42129 float16x8_t __ret; \
42130 float16x8_t __s0 = __p0; \
42131 mfloat8x16_t __s1 = __p1; \
42132 mfloat8x8_t __s2 = __p2; \
42133 fpm_t __s4 = __p4; \
42134 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42135 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42136 mfloat8x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \
42137 __ret = (float16x8_t) __builtin_neon_vmlaltq_lane_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3, __s4); \
42138 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42139 __ret; \
42140})
42141#endif
42142
42143#ifdef __LITTLE_ENDIAN__
42144#define vmlaltq_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42145 float16x8_t __ret; \
42146 float16x8_t __s0 = __p0; \
42147 mfloat8x16_t __s1 = __p1; \
42148 mfloat8x16_t __s2 = __p2; \
42149 fpm_t __s4 = __p4; \
42150 __ret = (float16x8_t) __builtin_neon_vmlaltq_laneq_f16_mf8_fpm((int8x16_t)__s0, __s1, __s2, __p3, __s4); \
42151 __ret; \
42152})
42153#else
42154#define vmlaltq_laneq_f16_mf8_fpm(__p0, __p1, __p2, __p3, __p4) __extension__ ({ \
42155 float16x8_t __ret; \
42156 float16x8_t __s0 = __p0; \
42157 mfloat8x16_t __s1 = __p1; \
42158 mfloat8x16_t __s2 = __p2; \
42159 fpm_t __s4 = __p4; \
42160 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42161 mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42162 mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42163 __ret = (float16x8_t) __builtin_neon_vmlaltq_laneq_f16_mf8_fpm((int8x16_t)__rev0, __rev1, __rev2, __p3, __s4); \
42164 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42165 __ret; \
42166})
42167#endif
42168
42169#ifdef __LITTLE_ENDIAN__
42170#define vluti2_lane_p8(__p0, __p1, __p2) __extension__ ({ \
42171 poly8x16_t __ret; \
42172 poly8x8_t __s0 = __p0; \
42173 uint8x8_t __s1 = __p1; \
42174 __ret = (poly8x16_t) __builtin_neon_vluti2_lane_p8((int8x8_t)__s0, (int8x8_t)__s1, __p2, 36); \
42175 __ret; \
42176})
42177#else
42178#define vluti2_lane_p8(__p0, __p1, __p2) __extension__ ({ \
42179 poly8x16_t __ret; \
42180 poly8x8_t __s0 = __p0; \
42181 uint8x8_t __s1 = __p1; \
42182 poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42183 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42184 __ret = (poly8x16_t) __builtin_neon_vluti2_lane_p8((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 36); \
42185 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42186 __ret; \
42187})
42188#endif
42189
42190#ifdef __LITTLE_ENDIAN__
42191#define vluti2q_lane_p8(__p0, __p1, __p2) __extension__ ({ \
42192 poly8x16_t __ret; \
42193 poly8x16_t __s0 = __p0; \
42194 uint8x8_t __s1 = __p1; \
42195 __ret = (poly8x16_t) __builtin_neon_vluti2q_lane_p8((int8x16_t)__s0, (int8x8_t)__s1, __p2, 36); \
42196 __ret; \
42197})
42198#else
42199#define vluti2q_lane_p8(__p0, __p1, __p2) __extension__ ({ \
42200 poly8x16_t __ret; \
42201 poly8x16_t __s0 = __p0; \
42202 uint8x8_t __s1 = __p1; \
42203 poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42204 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42205 __ret = (poly8x16_t) __builtin_neon_vluti2q_lane_p8((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 36); \
42206 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42207 __ret; \
42208})
42209#endif
42210
42211#ifdef __LITTLE_ENDIAN__
42212#define vluti2q_lane_u8(__p0, __p1, __p2) __extension__ ({ \
42213 uint8x16_t __ret; \
42214 uint8x16_t __s0 = __p0; \
42215 uint8x8_t __s1 = __p1; \
42216 __ret = (uint8x16_t) __builtin_neon_vluti2q_lane_u8((int8x16_t)__s0, (int8x8_t)__s1, __p2, 48); \
42217 __ret; \
42218})
42219#else
42220#define vluti2q_lane_u8(__p0, __p1, __p2) __extension__ ({ \
42221 uint8x16_t __ret; \
42222 uint8x16_t __s0 = __p0; \
42223 uint8x8_t __s1 = __p1; \
42224 uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42225 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42226 __ret = (uint8x16_t) __builtin_neon_vluti2q_lane_u8((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 48); \
42227 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42228 __ret; \
42229})
42230#endif
42231
42232#ifdef __LITTLE_ENDIAN__
42233#define vluti2q_lane_s8(__p0, __p1, __p2) __extension__ ({ \
42234 int8x16_t __ret; \
42235 int8x16_t __s0 = __p0; \
42236 uint8x8_t __s1 = __p1; \
42237 __ret = (int8x16_t) __builtin_neon_vluti2q_lane_s8((int8x16_t)__s0, (int8x8_t)__s1, __p2, 32); \
42238 __ret; \
42239})
42240#else
42241#define vluti2q_lane_s8(__p0, __p1, __p2) __extension__ ({ \
42242 int8x16_t __ret; \
42243 int8x16_t __s0 = __p0; \
42244 uint8x8_t __s1 = __p1; \
42245 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42246 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42247 __ret = (int8x16_t) __builtin_neon_vluti2q_lane_s8((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 32); \
42248 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42249 __ret; \
42250})
42251#endif
42252
42253#ifdef __LITTLE_ENDIAN__
42254#define vluti2_lane_u8(__p0, __p1, __p2) __extension__ ({ \
42255 uint8x16_t __ret; \
42256 uint8x8_t __s0 = __p0; \
42257 uint8x8_t __s1 = __p1; \
42258 __ret = (uint8x16_t) __builtin_neon_vluti2_lane_u8((int8x8_t)__s0, (int8x8_t)__s1, __p2, 48); \
42259 __ret; \
42260})
42261#else
42262#define vluti2_lane_u8(__p0, __p1, __p2) __extension__ ({ \
42263 uint8x16_t __ret; \
42264 uint8x8_t __s0 = __p0; \
42265 uint8x8_t __s1 = __p1; \
42266 uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42267 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42268 __ret = (uint8x16_t) __builtin_neon_vluti2_lane_u8((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 48); \
42269 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42270 __ret; \
42271})
42272#endif
42273
42274#ifdef __LITTLE_ENDIAN__
42275#define vluti2_lane_s8(__p0, __p1, __p2) __extension__ ({ \
42276 int8x16_t __ret; \
42277 int8x8_t __s0 = __p0; \
42278 uint8x8_t __s1 = __p1; \
42279 __ret = (int8x16_t) __builtin_neon_vluti2_lane_s8((int8x8_t)__s0, (int8x8_t)__s1, __p2, 32); \
42280 __ret; \
42281})
42282#else
42283#define vluti2_lane_s8(__p0, __p1, __p2) __extension__ ({ \
42284 int8x16_t __ret; \
42285 int8x8_t __s0 = __p0; \
42286 uint8x8_t __s1 = __p1; \
42287 int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42288 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42289 __ret = (int8x16_t) __builtin_neon_vluti2_lane_s8((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 32); \
42290 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42291 __ret; \
42292})
42293#endif
42294
42295#ifdef __LITTLE_ENDIAN__
42296#define vluti2_lane_p16(__p0, __p1, __p2) __extension__ ({ \
42297 poly16x8_t __ret; \
42298 poly16x4_t __s0 = __p0; \
42299 uint8x8_t __s1 = __p1; \
42300 __ret = (poly16x8_t) __builtin_neon_vluti2_lane_p16((int8x8_t)__s0, (int8x8_t)__s1, __p2, 37); \
42301 __ret; \
42302})
42303#else
42304#define vluti2_lane_p16(__p0, __p1, __p2) __extension__ ({ \
42305 poly16x8_t __ret; \
42306 poly16x4_t __s0 = __p0; \
42307 uint8x8_t __s1 = __p1; \
42308 poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42309 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42310 __ret = (poly16x8_t) __builtin_neon_vluti2_lane_p16((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 37); \
42311 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42312 __ret; \
42313})
42314#endif
42315
42316#ifdef __LITTLE_ENDIAN__
42317#define vluti2q_lane_p16(__p0, __p1, __p2) __extension__ ({ \
42318 poly16x8_t __ret; \
42319 poly16x8_t __s0 = __p0; \
42320 uint8x8_t __s1 = __p1; \
42321 __ret = (poly16x8_t) __builtin_neon_vluti2q_lane_p16((int8x16_t)__s0, (int8x8_t)__s1, __p2, 37); \
42322 __ret; \
42323})
42324#else
42325#define vluti2q_lane_p16(__p0, __p1, __p2) __extension__ ({ \
42326 poly16x8_t __ret; \
42327 poly16x8_t __s0 = __p0; \
42328 uint8x8_t __s1 = __p1; \
42329 poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42330 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42331 __ret = (poly16x8_t) __builtin_neon_vluti2q_lane_p16((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 37); \
42332 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42333 __ret; \
42334})
42335#endif
42336
42337#ifdef __LITTLE_ENDIAN__
42338#define vluti2q_lane_u16(__p0, __p1, __p2) __extension__ ({ \
42339 uint16x8_t __ret; \
42340 uint16x8_t __s0 = __p0; \
42341 uint8x8_t __s1 = __p1; \
42342 __ret = (uint16x8_t) __builtin_neon_vluti2q_lane_u16((int8x16_t)__s0, (int8x8_t)__s1, __p2, 49); \
42343 __ret; \
42344})
42345#else
42346#define vluti2q_lane_u16(__p0, __p1, __p2) __extension__ ({ \
42347 uint16x8_t __ret; \
42348 uint16x8_t __s0 = __p0; \
42349 uint8x8_t __s1 = __p1; \
42350 uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42351 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42352 __ret = (uint16x8_t) __builtin_neon_vluti2q_lane_u16((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 49); \
42353 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42354 __ret; \
42355})
42356#endif
42357
42358#ifdef __LITTLE_ENDIAN__
42359#define vluti2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \
42360 float16x8_t __ret; \
42361 float16x8_t __s0 = __p0; \
42362 uint8x8_t __s1 = __p1; \
42363 __ret = (float16x8_t) __builtin_neon_vluti2q_lane_f16((int8x16_t)__s0, (int8x8_t)__s1, __p2, 40); \
42364 __ret; \
42365})
42366#else
42367#define vluti2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \
42368 float16x8_t __ret; \
42369 float16x8_t __s0 = __p0; \
42370 uint8x8_t __s1 = __p1; \
42371 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42372 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42373 __ret = (float16x8_t) __builtin_neon_vluti2q_lane_f16((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 40); \
42374 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42375 __ret; \
42376})
42377#endif
42378
42379#ifdef __LITTLE_ENDIAN__
42380#define vluti2q_lane_s16(__p0, __p1, __p2) __extension__ ({ \
42381 int16x8_t __ret; \
42382 int16x8_t __s0 = __p0; \
42383 uint8x8_t __s1 = __p1; \
42384 __ret = (int16x8_t) __builtin_neon_vluti2q_lane_s16((int8x16_t)__s0, (int8x8_t)__s1, __p2, 33); \
42385 __ret; \
42386})
42387#else
42388#define vluti2q_lane_s16(__p0, __p1, __p2) __extension__ ({ \
42389 int16x8_t __ret; \
42390 int16x8_t __s0 = __p0; \
42391 uint8x8_t __s1 = __p1; \
42392 int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42393 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42394 __ret = (int16x8_t) __builtin_neon_vluti2q_lane_s16((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 33); \
42395 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42396 __ret; \
42397})
42398#endif
42399
42400#ifdef __LITTLE_ENDIAN__
42401#define vluti2_lane_u16(__p0, __p1, __p2) __extension__ ({ \
42402 uint16x8_t __ret; \
42403 uint16x4_t __s0 = __p0; \
42404 uint8x8_t __s1 = __p1; \
42405 __ret = (uint16x8_t) __builtin_neon_vluti2_lane_u16((int8x8_t)__s0, (int8x8_t)__s1, __p2, 49); \
42406 __ret; \
42407})
42408#else
42409#define vluti2_lane_u16(__p0, __p1, __p2) __extension__ ({ \
42410 uint16x8_t __ret; \
42411 uint16x4_t __s0 = __p0; \
42412 uint8x8_t __s1 = __p1; \
42413 uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42414 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42415 __ret = (uint16x8_t) __builtin_neon_vluti2_lane_u16((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 49); \
42416 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42417 __ret; \
42418})
42419#endif
42420
42421#ifdef __LITTLE_ENDIAN__
42422#define vluti2_lane_f16(__p0, __p1, __p2) __extension__ ({ \
42423 float16x8_t __ret; \
42424 float16x4_t __s0 = __p0; \
42425 uint8x8_t __s1 = __p1; \
42426 __ret = (float16x8_t) __builtin_neon_vluti2_lane_f16((int8x8_t)__s0, (int8x8_t)__s1, __p2, 40); \
42427 __ret; \
42428})
42429#else
42430#define vluti2_lane_f16(__p0, __p1, __p2) __extension__ ({ \
42431 float16x8_t __ret; \
42432 float16x4_t __s0 = __p0; \
42433 uint8x8_t __s1 = __p1; \
42434 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42435 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42436 __ret = (float16x8_t) __builtin_neon_vluti2_lane_f16((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 40); \
42437 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42438 __ret; \
42439})
42440#endif
42441
42442#ifdef __LITTLE_ENDIAN__
42443#define vluti2_lane_s16(__p0, __p1, __p2) __extension__ ({ \
42444 int16x8_t __ret; \
42445 int16x4_t __s0 = __p0; \
42446 uint8x8_t __s1 = __p1; \
42447 __ret = (int16x8_t) __builtin_neon_vluti2_lane_s16((int8x8_t)__s0, (int8x8_t)__s1, __p2, 33); \
42448 __ret; \
42449})
42450#else
42451#define vluti2_lane_s16(__p0, __p1, __p2) __extension__ ({ \
42452 int16x8_t __ret; \
42453 int16x4_t __s0 = __p0; \
42454 uint8x8_t __s1 = __p1; \
42455 int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42456 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42457 __ret = (int16x8_t) __builtin_neon_vluti2_lane_s16((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 33); \
42458 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42459 __ret; \
42460})
42461#endif
42462
42463#ifdef __LITTLE_ENDIAN__
42464#define vluti2_laneq_p8(__p0, __p1, __p2) __extension__ ({ \
42465 poly8x16_t __ret; \
42466 poly8x8_t __s0 = __p0; \
42467 uint8x16_t __s1 = __p1; \
42468 __ret = (poly8x16_t) __builtin_neon_vluti2_laneq_p8((int8x8_t)__s0, (int8x16_t)__s1, __p2, 36); \
42469 __ret; \
42470})
42471#else
42472#define vluti2_laneq_p8(__p0, __p1, __p2) __extension__ ({ \
42473 poly8x16_t __ret; \
42474 poly8x8_t __s0 = __p0; \
42475 uint8x16_t __s1 = __p1; \
42476 poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42477 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42478 __ret = (poly8x16_t) __builtin_neon_vluti2_laneq_p8((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 36); \
42479 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42480 __ret; \
42481})
42482#endif
42483
42484#ifdef __LITTLE_ENDIAN__
42485#define vluti2q_laneq_p8(__p0, __p1, __p2) __extension__ ({ \
42486 poly8x16_t __ret; \
42487 poly8x16_t __s0 = __p0; \
42488 uint8x16_t __s1 = __p1; \
42489 __ret = (poly8x16_t) __builtin_neon_vluti2q_laneq_p8((int8x16_t)__s0, (int8x16_t)__s1, __p2, 36); \
42490 __ret; \
42491})
42492#else
42493#define vluti2q_laneq_p8(__p0, __p1, __p2) __extension__ ({ \
42494 poly8x16_t __ret; \
42495 poly8x16_t __s0 = __p0; \
42496 uint8x16_t __s1 = __p1; \
42497 poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42498 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42499 __ret = (poly8x16_t) __builtin_neon_vluti2q_laneq_p8((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 36); \
42500 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42501 __ret; \
42502})
42503#endif
42504
42505#ifdef __LITTLE_ENDIAN__
42506#define vluti2q_laneq_u8(__p0, __p1, __p2) __extension__ ({ \
42507 uint8x16_t __ret; \
42508 uint8x16_t __s0 = __p0; \
42509 uint8x16_t __s1 = __p1; \
42510 __ret = (uint8x16_t) __builtin_neon_vluti2q_laneq_u8((int8x16_t)__s0, (int8x16_t)__s1, __p2, 48); \
42511 __ret; \
42512})
42513#else
42514#define vluti2q_laneq_u8(__p0, __p1, __p2) __extension__ ({ \
42515 uint8x16_t __ret; \
42516 uint8x16_t __s0 = __p0; \
42517 uint8x16_t __s1 = __p1; \
42518 uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42519 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42520 __ret = (uint8x16_t) __builtin_neon_vluti2q_laneq_u8((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 48); \
42521 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42522 __ret; \
42523})
42524#endif
42525
42526#ifdef __LITTLE_ENDIAN__
42527#define vluti2q_laneq_s8(__p0, __p1, __p2) __extension__ ({ \
42528 int8x16_t __ret; \
42529 int8x16_t __s0 = __p0; \
42530 uint8x16_t __s1 = __p1; \
42531 __ret = (int8x16_t) __builtin_neon_vluti2q_laneq_s8((int8x16_t)__s0, (int8x16_t)__s1, __p2, 32); \
42532 __ret; \
42533})
42534#else
42535#define vluti2q_laneq_s8(__p0, __p1, __p2) __extension__ ({ \
42536 int8x16_t __ret; \
42537 int8x16_t __s0 = __p0; \
42538 uint8x16_t __s1 = __p1; \
42539 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42540 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42541 __ret = (int8x16_t) __builtin_neon_vluti2q_laneq_s8((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 32); \
42542 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42543 __ret; \
42544})
42545#endif
42546
42547#ifdef __LITTLE_ENDIAN__
42548#define vluti2_laneq_u8(__p0, __p1, __p2) __extension__ ({ \
42549 uint8x16_t __ret; \
42550 uint8x8_t __s0 = __p0; \
42551 uint8x16_t __s1 = __p1; \
42552 __ret = (uint8x16_t) __builtin_neon_vluti2_laneq_u8((int8x8_t)__s0, (int8x16_t)__s1, __p2, 48); \
42553 __ret; \
42554})
42555#else
42556#define vluti2_laneq_u8(__p0, __p1, __p2) __extension__ ({ \
42557 uint8x16_t __ret; \
42558 uint8x8_t __s0 = __p0; \
42559 uint8x16_t __s1 = __p1; \
42560 uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42561 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42562 __ret = (uint8x16_t) __builtin_neon_vluti2_laneq_u8((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 48); \
42563 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42564 __ret; \
42565})
42566#endif
42567
42568#ifdef __LITTLE_ENDIAN__
42569#define vluti2_laneq_s8(__p0, __p1, __p2) __extension__ ({ \
42570 int8x16_t __ret; \
42571 int8x8_t __s0 = __p0; \
42572 uint8x16_t __s1 = __p1; \
42573 __ret = (int8x16_t) __builtin_neon_vluti2_laneq_s8((int8x8_t)__s0, (int8x16_t)__s1, __p2, 32); \
42574 __ret; \
42575})
42576#else
42577#define vluti2_laneq_s8(__p0, __p1, __p2) __extension__ ({ \
42578 int8x16_t __ret; \
42579 int8x8_t __s0 = __p0; \
42580 uint8x16_t __s1 = __p1; \
42581 int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42582 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42583 __ret = (int8x16_t) __builtin_neon_vluti2_laneq_s8((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 32); \
42584 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42585 __ret; \
42586})
42587#endif
42588
42589#ifdef __LITTLE_ENDIAN__
42590#define vluti2_laneq_p16(__p0, __p1, __p2) __extension__ ({ \
42591 poly16x8_t __ret; \
42592 poly16x4_t __s0 = __p0; \
42593 uint8x16_t __s1 = __p1; \
42594 __ret = (poly16x8_t) __builtin_neon_vluti2_laneq_p16((int8x8_t)__s0, (int8x16_t)__s1, __p2, 37); \
42595 __ret; \
42596})
42597#else
42598#define vluti2_laneq_p16(__p0, __p1, __p2) __extension__ ({ \
42599 poly16x8_t __ret; \
42600 poly16x4_t __s0 = __p0; \
42601 uint8x16_t __s1 = __p1; \
42602 poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42603 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42604 __ret = (poly16x8_t) __builtin_neon_vluti2_laneq_p16((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 37); \
42605 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42606 __ret; \
42607})
42608#endif
42609
42610#ifdef __LITTLE_ENDIAN__
42611#define vluti2q_laneq_p16(__p0, __p1, __p2) __extension__ ({ \
42612 poly16x8_t __ret; \
42613 poly16x8_t __s0 = __p0; \
42614 uint8x16_t __s1 = __p1; \
42615 __ret = (poly16x8_t) __builtin_neon_vluti2q_laneq_p16((int8x16_t)__s0, (int8x16_t)__s1, __p2, 37); \
42616 __ret; \
42617})
42618#else
42619#define vluti2q_laneq_p16(__p0, __p1, __p2) __extension__ ({ \
42620 poly16x8_t __ret; \
42621 poly16x8_t __s0 = __p0; \
42622 uint8x16_t __s1 = __p1; \
42623 poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42624 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42625 __ret = (poly16x8_t) __builtin_neon_vluti2q_laneq_p16((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 37); \
42626 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42627 __ret; \
42628})
42629#endif
42630
42631#ifdef __LITTLE_ENDIAN__
42632#define vluti2q_laneq_u16(__p0, __p1, __p2) __extension__ ({ \
42633 uint16x8_t __ret; \
42634 uint16x8_t __s0 = __p0; \
42635 uint8x16_t __s1 = __p1; \
42636 __ret = (uint16x8_t) __builtin_neon_vluti2q_laneq_u16((int8x16_t)__s0, (int8x16_t)__s1, __p2, 49); \
42637 __ret; \
42638})
42639#else
42640#define vluti2q_laneq_u16(__p0, __p1, __p2) __extension__ ({ \
42641 uint16x8_t __ret; \
42642 uint16x8_t __s0 = __p0; \
42643 uint8x16_t __s1 = __p1; \
42644 uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42645 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42646 __ret = (uint16x8_t) __builtin_neon_vluti2q_laneq_u16((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 49); \
42647 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42648 __ret; \
42649})
42650#endif
42651
42652#ifdef __LITTLE_ENDIAN__
42653#define vluti2q_laneq_f16(__p0, __p1, __p2) __extension__ ({ \
42654 float16x8_t __ret; \
42655 float16x8_t __s0 = __p0; \
42656 uint8x16_t __s1 = __p1; \
42657 __ret = (float16x8_t) __builtin_neon_vluti2q_laneq_f16((int8x16_t)__s0, (int8x16_t)__s1, __p2, 40); \
42658 __ret; \
42659})
42660#else
42661#define vluti2q_laneq_f16(__p0, __p1, __p2) __extension__ ({ \
42662 float16x8_t __ret; \
42663 float16x8_t __s0 = __p0; \
42664 uint8x16_t __s1 = __p1; \
42665 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42666 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42667 __ret = (float16x8_t) __builtin_neon_vluti2q_laneq_f16((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 40); \
42668 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42669 __ret; \
42670})
42671#endif
42672
42673#ifdef __LITTLE_ENDIAN__
42674#define vluti2q_laneq_s16(__p0, __p1, __p2) __extension__ ({ \
42675 int16x8_t __ret; \
42676 int16x8_t __s0 = __p0; \
42677 uint8x16_t __s1 = __p1; \
42678 __ret = (int16x8_t) __builtin_neon_vluti2q_laneq_s16((int8x16_t)__s0, (int8x16_t)__s1, __p2, 33); \
42679 __ret; \
42680})
42681#else
42682#define vluti2q_laneq_s16(__p0, __p1, __p2) __extension__ ({ \
42683 int16x8_t __ret; \
42684 int16x8_t __s0 = __p0; \
42685 uint8x16_t __s1 = __p1; \
42686 int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
42687 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42688 __ret = (int16x8_t) __builtin_neon_vluti2q_laneq_s16((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 33); \
42689 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42690 __ret; \
42691})
42692#endif
42693
42694#ifdef __LITTLE_ENDIAN__
42695#define vluti2_laneq_u16(__p0, __p1, __p2) __extension__ ({ \
42696 uint16x8_t __ret; \
42697 uint16x4_t __s0 = __p0; \
42698 uint8x16_t __s1 = __p1; \
42699 __ret = (uint16x8_t) __builtin_neon_vluti2_laneq_u16((int8x8_t)__s0, (int8x16_t)__s1, __p2, 49); \
42700 __ret; \
42701})
42702#else
42703#define vluti2_laneq_u16(__p0, __p1, __p2) __extension__ ({ \
42704 uint16x8_t __ret; \
42705 uint16x4_t __s0 = __p0; \
42706 uint8x16_t __s1 = __p1; \
42707 uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42708 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42709 __ret = (uint16x8_t) __builtin_neon_vluti2_laneq_u16((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 49); \
42710 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42711 __ret; \
42712})
42713#endif
42714
42715#ifdef __LITTLE_ENDIAN__
42716#define vluti2_laneq_f16(__p0, __p1, __p2) __extension__ ({ \
42717 float16x8_t __ret; \
42718 float16x4_t __s0 = __p0; \
42719 uint8x16_t __s1 = __p1; \
42720 __ret = (float16x8_t) __builtin_neon_vluti2_laneq_f16((int8x8_t)__s0, (int8x16_t)__s1, __p2, 40); \
42721 __ret; \
42722})
42723#else
42724#define vluti2_laneq_f16(__p0, __p1, __p2) __extension__ ({ \
42725 float16x8_t __ret; \
42726 float16x4_t __s0 = __p0; \
42727 uint8x16_t __s1 = __p1; \
42728 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42729 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42730 __ret = (float16x8_t) __builtin_neon_vluti2_laneq_f16((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 40); \
42731 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42732 __ret; \
42733})
42734#endif
42735
42736#ifdef __LITTLE_ENDIAN__
42737#define vluti2_laneq_s16(__p0, __p1, __p2) __extension__ ({ \
42738 int16x8_t __ret; \
42739 int16x4_t __s0 = __p0; \
42740 uint8x16_t __s1 = __p1; \
42741 __ret = (int16x8_t) __builtin_neon_vluti2_laneq_s16((int8x8_t)__s0, (int8x16_t)__s1, __p2, 33); \
42742 __ret; \
42743})
42744#else
42745#define vluti2_laneq_s16(__p0, __p1, __p2) __extension__ ({ \
42746 int16x8_t __ret; \
42747 int16x4_t __s0 = __p0; \
42748 uint8x16_t __s1 = __p1; \
42749 int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
42750 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42751 __ret = (int16x8_t) __builtin_neon_vluti2_laneq_s16((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 33); \
42752 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42753 __ret; \
42754})
42755#endif
42756
42757#ifdef __LITTLE_ENDIAN__
42758#define vluti4q_lane_p8(__p0, __p1, __p2) __extension__ ({ \
42759 poly8x16_t __ret; \
42760 poly8x16_t __s0 = __p0; \
42761 uint8x8_t __s1 = __p1; \
42762 __ret = (poly8x16_t) __builtin_neon_vluti4q_lane_p8((int8x16_t)__s0, (int8x8_t)__s1, __p2, 36); \
42763 __ret; \
42764})
42765#else
42766#define vluti4q_lane_p8(__p0, __p1, __p2) __extension__ ({ \
42767 poly8x16_t __ret; \
42768 poly8x16_t __s0 = __p0; \
42769 uint8x8_t __s1 = __p1; \
42770 poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42771 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42772 __ret = (poly8x16_t) __builtin_neon_vluti4q_lane_p8((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 36); \
42773 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42774 __ret; \
42775})
42776#endif
42777
42778#ifdef __LITTLE_ENDIAN__
42779#define vluti4q_lane_u8(__p0, __p1, __p2) __extension__ ({ \
42780 uint8x16_t __ret; \
42781 uint8x16_t __s0 = __p0; \
42782 uint8x8_t __s1 = __p1; \
42783 __ret = (uint8x16_t) __builtin_neon_vluti4q_lane_u8((int8x16_t)__s0, (int8x8_t)__s1, __p2, 48); \
42784 __ret; \
42785})
42786#else
42787#define vluti4q_lane_u8(__p0, __p1, __p2) __extension__ ({ \
42788 uint8x16_t __ret; \
42789 uint8x16_t __s0 = __p0; \
42790 uint8x8_t __s1 = __p1; \
42791 uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42792 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42793 __ret = (uint8x16_t) __builtin_neon_vluti4q_lane_u8((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 48); \
42794 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42795 __ret; \
42796})
42797#endif
42798
42799#ifdef __LITTLE_ENDIAN__
42800#define vluti4q_lane_s8(__p0, __p1, __p2) __extension__ ({ \
42801 int8x16_t __ret; \
42802 int8x16_t __s0 = __p0; \
42803 uint8x8_t __s1 = __p1; \
42804 __ret = (int8x16_t) __builtin_neon_vluti4q_lane_s8((int8x16_t)__s0, (int8x8_t)__s1, __p2, 32); \
42805 __ret; \
42806})
42807#else
42808#define vluti4q_lane_s8(__p0, __p1, __p2) __extension__ ({ \
42809 int8x16_t __ret; \
42810 int8x16_t __s0 = __p0; \
42811 uint8x8_t __s1 = __p1; \
42812 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42813 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42814 __ret = (int8x16_t) __builtin_neon_vluti4q_lane_s8((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 32); \
42815 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42816 __ret; \
42817})
42818#endif
42819
42820#ifdef __LITTLE_ENDIAN__
42821#define vluti4q_lane_p16_x2(__p0, __p1, __p2) __extension__ ({ \
42822 poly16x8_t __ret; \
42823 poly16x8x2_t __s0 = __p0; \
42824 uint8x8_t __s1 = __p1; \
42825 __ret = (poly16x8_t) __builtin_neon_vluti4q_lane_p16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x8_t)__s1, __p2, 37); \
42826 __ret; \
42827})
42828#else
42829#define vluti4q_lane_p16_x2(__p0, __p1, __p2) __extension__ ({ \
42830 poly16x8_t __ret; \
42831 poly16x8x2_t __s0 = __p0; \
42832 uint8x8_t __s1 = __p1; \
42833 poly16x8x2_t __rev0; \
42834 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
42835 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
42836 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42837 __ret = (poly16x8_t) __builtin_neon_vluti4q_lane_p16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, __p2, 37); \
42838 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42839 __ret; \
42840})
42841#endif
42842
42843#ifdef __LITTLE_ENDIAN__
42844#define vluti4q_lane_u16_x2(__p0, __p1, __p2) __extension__ ({ \
42845 uint16x8_t __ret; \
42846 uint16x8x2_t __s0 = __p0; \
42847 uint8x8_t __s1 = __p1; \
42848 __ret = (uint16x8_t) __builtin_neon_vluti4q_lane_u16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x8_t)__s1, __p2, 49); \
42849 __ret; \
42850})
42851#else
42852#define vluti4q_lane_u16_x2(__p0, __p1, __p2) __extension__ ({ \
42853 uint16x8_t __ret; \
42854 uint16x8x2_t __s0 = __p0; \
42855 uint8x8_t __s1 = __p1; \
42856 uint16x8x2_t __rev0; \
42857 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
42858 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
42859 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42860 __ret = (uint16x8_t) __builtin_neon_vluti4q_lane_u16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, __p2, 49); \
42861 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42862 __ret; \
42863})
42864#endif
42865
42866#ifdef __LITTLE_ENDIAN__
42867#define vluti4q_lane_f16_x2(__p0, __p1, __p2) __extension__ ({ \
42868 float16x8_t __ret; \
42869 float16x8x2_t __s0 = __p0; \
42870 uint8x8_t __s1 = __p1; \
42871 __ret = (float16x8_t) __builtin_neon_vluti4q_lane_f16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x8_t)__s1, __p2, 40); \
42872 __ret; \
42873})
42874#else
42875#define vluti4q_lane_f16_x2(__p0, __p1, __p2) __extension__ ({ \
42876 float16x8_t __ret; \
42877 float16x8x2_t __s0 = __p0; \
42878 uint8x8_t __s1 = __p1; \
42879 float16x8x2_t __rev0; \
42880 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
42881 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
42882 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42883 __ret = (float16x8_t) __builtin_neon_vluti4q_lane_f16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, __p2, 40); \
42884 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42885 __ret; \
42886})
42887#endif
42888
42889#ifdef __LITTLE_ENDIAN__
42890#define vluti4q_lane_s16_x2(__p0, __p1, __p2) __extension__ ({ \
42891 int16x8_t __ret; \
42892 int16x8x2_t __s0 = __p0; \
42893 uint8x8_t __s1 = __p1; \
42894 __ret = (int16x8_t) __builtin_neon_vluti4q_lane_s16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x8_t)__s1, __p2, 33); \
42895 __ret; \
42896})
42897#else
42898#define vluti4q_lane_s16_x2(__p0, __p1, __p2) __extension__ ({ \
42899 int16x8_t __ret; \
42900 int16x8x2_t __s0 = __p0; \
42901 uint8x8_t __s1 = __p1; \
42902 int16x8x2_t __rev0; \
42903 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
42904 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
42905 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
42906 __ret = (int16x8_t) __builtin_neon_vluti4q_lane_s16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, __p2, 33); \
42907 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42908 __ret; \
42909})
42910#endif
42911
42912#ifdef __LITTLE_ENDIAN__
42913#define vluti4q_laneq_p8(__p0, __p1, __p2) __extension__ ({ \
42914 poly8x16_t __ret; \
42915 poly8x16_t __s0 = __p0; \
42916 uint8x16_t __s1 = __p1; \
42917 __ret = (poly8x16_t) __builtin_neon_vluti4q_laneq_p8((int8x16_t)__s0, (int8x16_t)__s1, __p2, 36); \
42918 __ret; \
42919})
42920#else
42921#define vluti4q_laneq_p8(__p0, __p1, __p2) __extension__ ({ \
42922 poly8x16_t __ret; \
42923 poly8x16_t __s0 = __p0; \
42924 uint8x16_t __s1 = __p1; \
42925 poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42926 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42927 __ret = (poly8x16_t) __builtin_neon_vluti4q_laneq_p8((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 36); \
42928 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42929 __ret; \
42930})
42931#endif
42932
42933#ifdef __LITTLE_ENDIAN__
42934#define vluti4q_laneq_u8(__p0, __p1, __p2) __extension__ ({ \
42935 uint8x16_t __ret; \
42936 uint8x16_t __s0 = __p0; \
42937 uint8x16_t __s1 = __p1; \
42938 __ret = (uint8x16_t) __builtin_neon_vluti4q_laneq_u8((int8x16_t)__s0, (int8x16_t)__s1, __p2, 48); \
42939 __ret; \
42940})
42941#else
42942#define vluti4q_laneq_u8(__p0, __p1, __p2) __extension__ ({ \
42943 uint8x16_t __ret; \
42944 uint8x16_t __s0 = __p0; \
42945 uint8x16_t __s1 = __p1; \
42946 uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42947 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42948 __ret = (uint8x16_t) __builtin_neon_vluti4q_laneq_u8((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 48); \
42949 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42950 __ret; \
42951})
42952#endif
42953
42954#ifdef __LITTLE_ENDIAN__
42955#define vluti4q_laneq_s8(__p0, __p1, __p2) __extension__ ({ \
42956 int8x16_t __ret; \
42957 int8x16_t __s0 = __p0; \
42958 uint8x16_t __s1 = __p1; \
42959 __ret = (int8x16_t) __builtin_neon_vluti4q_laneq_s8((int8x16_t)__s0, (int8x16_t)__s1, __p2, 32); \
42960 __ret; \
42961})
42962#else
42963#define vluti4q_laneq_s8(__p0, __p1, __p2) __extension__ ({ \
42964 int8x16_t __ret; \
42965 int8x16_t __s0 = __p0; \
42966 uint8x16_t __s1 = __p1; \
42967 int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42968 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42969 __ret = (int8x16_t) __builtin_neon_vluti4q_laneq_s8((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 32); \
42970 __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42971 __ret; \
42972})
42973#endif
42974
42975#ifdef __LITTLE_ENDIAN__
42976#define vluti4q_laneq_p16_x2(__p0, __p1, __p2) __extension__ ({ \
42977 poly16x8_t __ret; \
42978 poly16x8x2_t __s0 = __p0; \
42979 uint8x16_t __s1 = __p1; \
42980 __ret = (poly16x8_t) __builtin_neon_vluti4q_laneq_p16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x16_t)__s1, __p2, 37); \
42981 __ret; \
42982})
42983#else
42984#define vluti4q_laneq_p16_x2(__p0, __p1, __p2) __extension__ ({ \
42985 poly16x8_t __ret; \
42986 poly16x8x2_t __s0 = __p0; \
42987 uint8x16_t __s1 = __p1; \
42988 poly16x8x2_t __rev0; \
42989 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
42990 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
42991 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
42992 __ret = (poly16x8_t) __builtin_neon_vluti4q_laneq_p16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, __p2, 37); \
42993 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
42994 __ret; \
42995})
42996#endif
42997
42998#ifdef __LITTLE_ENDIAN__
42999#define vluti4q_laneq_u16_x2(__p0, __p1, __p2) __extension__ ({ \
43000 uint16x8_t __ret; \
43001 uint16x8x2_t __s0 = __p0; \
43002 uint8x16_t __s1 = __p1; \
43003 __ret = (uint16x8_t) __builtin_neon_vluti4q_laneq_u16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x16_t)__s1, __p2, 49); \
43004 __ret; \
43005})
43006#else
43007#define vluti4q_laneq_u16_x2(__p0, __p1, __p2) __extension__ ({ \
43008 uint16x8_t __ret; \
43009 uint16x8x2_t __s0 = __p0; \
43010 uint8x16_t __s1 = __p1; \
43011 uint16x8x2_t __rev0; \
43012 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
43013 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
43014 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43015 __ret = (uint16x8_t) __builtin_neon_vluti4q_laneq_u16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, __p2, 49); \
43016 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43017 __ret; \
43018})
43019#endif
43020
43021#ifdef __LITTLE_ENDIAN__
43022#define vluti4q_laneq_f16_x2(__p0, __p1, __p2) __extension__ ({ \
43023 float16x8_t __ret; \
43024 float16x8x2_t __s0 = __p0; \
43025 uint8x16_t __s1 = __p1; \
43026 __ret = (float16x8_t) __builtin_neon_vluti4q_laneq_f16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x16_t)__s1, __p2, 40); \
43027 __ret; \
43028})
43029#else
43030#define vluti4q_laneq_f16_x2(__p0, __p1, __p2) __extension__ ({ \
43031 float16x8_t __ret; \
43032 float16x8x2_t __s0 = __p0; \
43033 uint8x16_t __s1 = __p1; \
43034 float16x8x2_t __rev0; \
43035 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
43036 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
43037 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43038 __ret = (float16x8_t) __builtin_neon_vluti4q_laneq_f16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, __p2, 40); \
43039 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43040 __ret; \
43041})
43042#endif
43043
43044#ifdef __LITTLE_ENDIAN__
43045#define vluti4q_laneq_s16_x2(__p0, __p1, __p2) __extension__ ({ \
43046 int16x8_t __ret; \
43047 int16x8x2_t __s0 = __p0; \
43048 uint8x16_t __s1 = __p1; \
43049 __ret = (int16x8_t) __builtin_neon_vluti4q_laneq_s16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x16_t)__s1, __p2, 33); \
43050 __ret; \
43051})
43052#else
43053#define vluti4q_laneq_s16_x2(__p0, __p1, __p2) __extension__ ({ \
43054 int16x8_t __ret; \
43055 int16x8x2_t __s0 = __p0; \
43056 uint8x16_t __s1 = __p1; \
43057 int16x8x2_t __rev0; \
43058 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
43059 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
43060 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43061 __ret = (int16x8_t) __builtin_neon_vluti4q_laneq_s16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, __p2, 33); \
43062 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43063 __ret; \
43064})
43065#endif
43066
43067#ifdef __LITTLE_ENDIAN__
43068#define vluti2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \
43069 bfloat16x8_t __ret; \
43070 bfloat16x8_t __s0 = __p0; \
43071 uint8x8_t __s1 = __p1; \
43072 __ret = (bfloat16x8_t) __builtin_neon_vluti2q_lane_bf16((int8x16_t)__s0, (int8x8_t)__s1, __p2, 43); \
43073 __ret; \
43074})
43075#else
43076#define vluti2q_lane_bf16(__p0, __p1, __p2) __extension__ ({ \
43077 bfloat16x8_t __ret; \
43078 bfloat16x8_t __s0 = __p0; \
43079 uint8x8_t __s1 = __p1; \
43080 bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
43081 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
43082 __ret = (bfloat16x8_t) __builtin_neon_vluti2q_lane_bf16((int8x16_t)__rev0, (int8x8_t)__rev1, __p2, 43); \
43083 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43084 __ret; \
43085})
43086#endif
43087
43088#ifdef __LITTLE_ENDIAN__
43089#define vluti2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \
43090 bfloat16x8_t __ret; \
43091 bfloat16x4_t __s0 = __p0; \
43092 uint8x8_t __s1 = __p1; \
43093 __ret = (bfloat16x8_t) __builtin_neon_vluti2_lane_bf16((int8x8_t)__s0, (int8x8_t)__s1, __p2, 43); \
43094 __ret; \
43095})
43096#else
43097#define vluti2_lane_bf16(__p0, __p1, __p2) __extension__ ({ \
43098 bfloat16x8_t __ret; \
43099 bfloat16x4_t __s0 = __p0; \
43100 uint8x8_t __s1 = __p1; \
43101 bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
43102 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
43103 __ret = (bfloat16x8_t) __builtin_neon_vluti2_lane_bf16((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 43); \
43104 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43105 __ret; \
43106})
43107#endif
43108
43109#ifdef __LITTLE_ENDIAN__
43110#define vluti2q_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \
43111 bfloat16x8_t __ret; \
43112 bfloat16x8_t __s0 = __p0; \
43113 uint8x16_t __s1 = __p1; \
43114 __ret = (bfloat16x8_t) __builtin_neon_vluti2q_laneq_bf16((int8x16_t)__s0, (int8x16_t)__s1, __p2, 43); \
43115 __ret; \
43116})
43117#else
43118#define vluti2q_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \
43119 bfloat16x8_t __ret; \
43120 bfloat16x8_t __s0 = __p0; \
43121 uint8x16_t __s1 = __p1; \
43122 bfloat16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \
43123 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43124 __ret = (bfloat16x8_t) __builtin_neon_vluti2q_laneq_bf16((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 43); \
43125 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43126 __ret; \
43127})
43128#endif
43129
43130#ifdef __LITTLE_ENDIAN__
43131#define vluti2_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \
43132 bfloat16x8_t __ret; \
43133 bfloat16x4_t __s0 = __p0; \
43134 uint8x16_t __s1 = __p1; \
43135 __ret = (bfloat16x8_t) __builtin_neon_vluti2_laneq_bf16((int8x8_t)__s0, (int8x16_t)__s1, __p2, 43); \
43136 __ret; \
43137})
43138#else
43139#define vluti2_laneq_bf16(__p0, __p1, __p2) __extension__ ({ \
43140 bfloat16x8_t __ret; \
43141 bfloat16x4_t __s0 = __p0; \
43142 uint8x16_t __s1 = __p1; \
43143 bfloat16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \
43144 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43145 __ret = (bfloat16x8_t) __builtin_neon_vluti2_laneq_bf16((int8x8_t)__rev0, (int8x16_t)__rev1, __p2, 43); \
43146 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43147 __ret; \
43148})
43149#endif
43150
43151#ifdef __LITTLE_ENDIAN__
43152#define vluti4q_lane_bf16_x2(__p0, __p1, __p2) __extension__ ({ \
43153 bfloat16x8_t __ret; \
43154 bfloat16x8x2_t __s0 = __p0; \
43155 uint8x8_t __s1 = __p1; \
43156 __ret = (bfloat16x8_t) __builtin_neon_vluti4q_lane_bf16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x8_t)__s1, __p2, 43); \
43157 __ret; \
43158})
43159#else
43160#define vluti4q_lane_bf16_x2(__p0, __p1, __p2) __extension__ ({ \
43161 bfloat16x8_t __ret; \
43162 bfloat16x8x2_t __s0 = __p0; \
43163 uint8x8_t __s1 = __p1; \
43164 bfloat16x8x2_t __rev0; \
43165 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
43166 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
43167 uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \
43168 __ret = (bfloat16x8_t) __builtin_neon_vluti4q_lane_bf16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, __p2, 43); \
43169 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43170 __ret; \
43171})
43172#endif
43173
43174#ifdef __LITTLE_ENDIAN__
43175#define vluti4q_laneq_bf16_x2(__p0, __p1, __p2) __extension__ ({ \
43176 bfloat16x8_t __ret; \
43177 bfloat16x8x2_t __s0 = __p0; \
43178 uint8x16_t __s1 = __p1; \
43179 __ret = (bfloat16x8_t) __builtin_neon_vluti4q_laneq_bf16_x2((int8x16_t)__s0.val[0], (int8x16_t)__s0.val[1], (int8x16_t)__s1, __p2, 43); \
43180 __ret; \
43181})
43182#else
43183#define vluti4q_laneq_bf16_x2(__p0, __p1, __p2) __extension__ ({ \
43184 bfloat16x8_t __ret; \
43185 bfloat16x8x2_t __s0 = __p0; \
43186 uint8x16_t __s1 = __p1; \
43187 bfloat16x8x2_t __rev0; \
43188 __rev0.val[0] = __builtin_shufflevector(__s0.val[0], __s0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \
43189 __rev0.val[1] = __builtin_shufflevector(__s0.val[1], __s0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \
43190 uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43191 __ret = (bfloat16x8_t) __builtin_neon_vluti4q_laneq_bf16_x2((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, __p2, 43); \
43192 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \
43193 __ret; \
43194})
43195#endif
43196
43197#ifdef __LITTLE_ENDIAN__
43198__ai __attribute__((target("neon,faminmax"))) float64x2_t vamaxq_f64(float64x2_t __p0, float64x2_t __p1) {
43199 float64x2_t __ret;
43200 __ret = (float64x2_t) __builtin_neon_vamaxq_f64((int8x16_t)__p0, (int8x16_t)__p1, 42);
4125643201 return __ret;
4125743202}
43203#else
43204__ai __attribute__((target("neon,faminmax"))) float64x2_t vamaxq_f64(float64x2_t __p0, float64x2_t __p1) {
43205 float64x2_t __ret;
43206 float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
43207 float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
43208 __ret = (float64x2_t) __builtin_neon_vamaxq_f64((int8x16_t)__rev0, (int8x16_t)__rev1, 42);
43209 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
43210 return __ret;
43211}
43212#endif
43213
4125843214#ifdef __LITTLE_ENDIAN__
41259__ai __attribute__((target("bf16,neon"))) bfloat16x8_t __a64_vcvtq_low_bf16_f32(float32x4_t __p0) {
41260 bfloat16x8_t __ret;
41261 __ret = (bfloat16x8_t) __builtin_neon___a64_vcvtq_low_bf16_f32((int8x16_t)__p0, 43);
43215__ai __attribute__((target("neon,faminmax"))) float32x4_t vamaxq_f32(float32x4_t __p0, float32x4_t __p1) {
43216 float32x4_t __ret;
43217 __ret = (float32x4_t) __builtin_neon_vamaxq_f32((int8x16_t)__p0, (int8x16_t)__p1, 41);
4126243218 return __ret;
4126343219}
4126443220#else
41265__ai __attribute__((target("bf16,neon"))) bfloat16x8_t __a64_vcvtq_low_bf16_f32(float32x4_t __p0) {
41266 bfloat16x8_t __ret;
43221__ai __attribute__((target("neon,faminmax"))) float32x4_t vamaxq_f32(float32x4_t __p0, float32x4_t __p1) {
43222 float32x4_t __ret;
4126743223 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41268 __ret = (bfloat16x8_t) __builtin_neon___a64_vcvtq_low_bf16_f32((int8x16_t)__rev0, 43);
43224 float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
43225 __ret = (float32x4_t) __builtin_neon_vamaxq_f32((int8x16_t)__rev0, (int8x16_t)__rev1, 41);
43226 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
43227 return __ret;
43228}
43229#endif
43230
43231#ifdef __LITTLE_ENDIAN__
43232__ai __attribute__((target("neon,faminmax"))) float16x8_t vamaxq_f16(float16x8_t __p0, float16x8_t __p1) {
43233 float16x8_t __ret;
43234 __ret = (float16x8_t) __builtin_neon_vamaxq_f16((int8x16_t)__p0, (int8x16_t)__p1, 40);
43235 return __ret;
43236}
43237#else
43238__ai __attribute__((target("neon,faminmax"))) float16x8_t vamaxq_f16(float16x8_t __p0, float16x8_t __p1) {
43239 float16x8_t __ret;
43240 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
43241 float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
43242 __ret = (float16x8_t) __builtin_neon_vamaxq_f16((int8x16_t)__rev0, (int8x16_t)__rev1, 40);
4126943243 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
4127043244 return __ret;
4127143245}
41272__ai __attribute__((target("bf16,neon"))) bfloat16x8_t __noswap___a64_vcvtq_low_bf16_f32(float32x4_t __p0) {
41273 bfloat16x8_t __ret;
41274 __ret = (bfloat16x8_t) __builtin_neon___a64_vcvtq_low_bf16_f32((int8x16_t)__p0, 43);
43246#endif
43247
43248#ifdef __LITTLE_ENDIAN__
43249__ai __attribute__((target("neon,faminmax"))) float32x2_t vamax_f32(float32x2_t __p0, float32x2_t __p1) {
43250 float32x2_t __ret;
43251 __ret = (float32x2_t) __builtin_neon_vamax_f32((int8x8_t)__p0, (int8x8_t)__p1, 9);
43252 return __ret;
43253}
43254#else
43255__ai __attribute__((target("neon,faminmax"))) float32x2_t vamax_f32(float32x2_t __p0, float32x2_t __p1) {
43256 float32x2_t __ret;
43257 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
43258 float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
43259 __ret = (float32x2_t) __builtin_neon_vamax_f32((int8x8_t)__rev0, (int8x8_t)__rev1, 9);
43260 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
43261 return __ret;
43262}
43263#endif
43264
43265#ifdef __LITTLE_ENDIAN__
43266__ai __attribute__((target("neon,faminmax"))) float16x4_t vamax_f16(float16x4_t __p0, float16x4_t __p1) {
43267 float16x4_t __ret;
43268 __ret = (float16x4_t) __builtin_neon_vamax_f16((int8x8_t)__p0, (int8x8_t)__p1, 8);
43269 return __ret;
43270}
43271#else
43272__ai __attribute__((target("neon,faminmax"))) float16x4_t vamax_f16(float16x4_t __p0, float16x4_t __p1) {
43273 float16x4_t __ret;
43274 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
43275 float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
43276 __ret = (float16x4_t) __builtin_neon_vamax_f16((int8x8_t)__rev0, (int8x8_t)__rev1, 8);
43277 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
43278 return __ret;
43279}
43280#endif
43281
43282#ifdef __LITTLE_ENDIAN__
43283__ai __attribute__((target("neon,faminmax"))) float64x2_t vaminq_f64(float64x2_t __p0, float64x2_t __p1) {
43284 float64x2_t __ret;
43285 __ret = (float64x2_t) __builtin_neon_vaminq_f64((int8x16_t)__p0, (int8x16_t)__p1, 42);
43286 return __ret;
43287}
43288#else
43289__ai __attribute__((target("neon,faminmax"))) float64x2_t vaminq_f64(float64x2_t __p0, float64x2_t __p1) {
43290 float64x2_t __ret;
43291 float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
43292 float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
43293 __ret = (float64x2_t) __builtin_neon_vaminq_f64((int8x16_t)__rev0, (int8x16_t)__rev1, 42);
43294 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
43295 return __ret;
43296}
43297#endif
43298
43299#ifdef __LITTLE_ENDIAN__
43300__ai __attribute__((target("neon,faminmax"))) float32x4_t vaminq_f32(float32x4_t __p0, float32x4_t __p1) {
43301 float32x4_t __ret;
43302 __ret = (float32x4_t) __builtin_neon_vaminq_f32((int8x16_t)__p0, (int8x16_t)__p1, 41);
43303 return __ret;
43304}
43305#else
43306__ai __attribute__((target("neon,faminmax"))) float32x4_t vaminq_f32(float32x4_t __p0, float32x4_t __p1) {
43307 float32x4_t __ret;
43308 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
43309 float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
43310 __ret = (float32x4_t) __builtin_neon_vaminq_f32((int8x16_t)__rev0, (int8x16_t)__rev1, 41);
43311 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
43312 return __ret;
43313}
43314#endif
43315
43316#ifdef __LITTLE_ENDIAN__
43317__ai __attribute__((target("neon,faminmax"))) float16x8_t vaminq_f16(float16x8_t __p0, float16x8_t __p1) {
43318 float16x8_t __ret;
43319 __ret = (float16x8_t) __builtin_neon_vaminq_f16((int8x16_t)__p0, (int8x16_t)__p1, 40);
43320 return __ret;
43321}
43322#else
43323__ai __attribute__((target("neon,faminmax"))) float16x8_t vaminq_f16(float16x8_t __p0, float16x8_t __p1) {
43324 float16x8_t __ret;
43325 float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0);
43326 float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0);
43327 __ret = (float16x8_t) __builtin_neon_vaminq_f16((int8x16_t)__rev0, (int8x16_t)__rev1, 40);
43328 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
43329 return __ret;
43330}
43331#endif
43332
43333#ifdef __LITTLE_ENDIAN__
43334__ai __attribute__((target("neon,faminmax"))) float32x2_t vamin_f32(float32x2_t __p0, float32x2_t __p1) {
43335 float32x2_t __ret;
43336 __ret = (float32x2_t) __builtin_neon_vamin_f32((int8x8_t)__p0, (int8x8_t)__p1, 9);
43337 return __ret;
43338}
43339#else
43340__ai __attribute__((target("neon,faminmax"))) float32x2_t vamin_f32(float32x2_t __p0, float32x2_t __p1) {
43341 float32x2_t __ret;
43342 float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0);
43343 float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0);
43344 __ret = (float32x2_t) __builtin_neon_vamin_f32((int8x8_t)__rev0, (int8x8_t)__rev1, 9);
43345 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
4127543346 return __ret;
4127643347}
4127743348#endif
4127843349
43350#ifdef __LITTLE_ENDIAN__
43351__ai __attribute__((target("neon,faminmax"))) float16x4_t vamin_f16(float16x4_t __p0, float16x4_t __p1) {
43352 float16x4_t __ret;
43353 __ret = (float16x4_t) __builtin_neon_vamin_f16((int8x8_t)__p0, (int8x8_t)__p1, 8);
43354 return __ret;
43355}
43356#else
43357__ai __attribute__((target("neon,faminmax"))) float16x4_t vamin_f16(float16x4_t __p0, float16x4_t __p1) {
43358 float16x4_t __ret;
43359 float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
43360 float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0);
43361 __ret = (float16x4_t) __builtin_neon_vamin_f16((int8x8_t)__rev0, (int8x8_t)__rev1, 8);
43362 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
43363 return __ret;
43364}
43365#endif
43366
43367#endif
43368#if defined(__aarch64__) || defined(__arm64ec__)
43369__ai __attribute__((target("aes,neon"))) poly128_t vmull_p64(poly64_t __p0, poly64_t __p1) {
43370 poly128_t __ret;
43371 __ret = (poly128_t) __builtin_neon_vmull_p64(__p0, __p1);
43372 return __ret;
43373}
4127943374#ifdef __LITTLE_ENDIAN__
4128043375#define vcopyq_lane_bf16(__p0_230, __p1_230, __p2_230, __p3_230) __extension__ ({ \
4128143376 bfloat16x8_t __ret_230; \
......@@ -41363,14 +43458,14 @@ __ai __attribute__((target("bf16,neon"))) bfloat16x8_t __noswap___a64_vcvtq_low_
4136343458#ifdef __LITTLE_ENDIAN__
4136443459__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vcvt_bf16_f32(float32x4_t __p0) {
4136543460 bfloat16x4_t __ret;
41366 __ret = vget_low_bf16(__a64_vcvtq_low_bf16_f32(__p0));
43461 __ret = (bfloat16x4_t) __builtin_neon_vcvt_bf16_f32((int8x16_t)__p0, 11);
4136743462 return __ret;
4136843463}
4136943464#else
4137043465__ai __attribute__((target("bf16,neon"))) bfloat16x4_t vcvt_bf16_f32(float32x4_t __p0) {
4137143466 bfloat16x4_t __ret;
4137243467 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41373 __ret = __noswap_vget_low_bf16(__noswap___a64_vcvtq_low_bf16_f32(__rev0));
43468 __ret = (bfloat16x4_t) __builtin_neon_vcvt_bf16_f32((int8x16_t)__rev0, 11);
4137443469 __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0);
4137543470 return __ret;
4137643471}
......@@ -41396,14 +43491,14 @@ __ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_high_bf16_f32(bfloa
4139643491#ifdef __LITTLE_ENDIAN__
4139743492__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_low_bf16_f32(float32x4_t __p0) {
4139843493 bfloat16x8_t __ret;
41399 __ret = __a64_vcvtq_low_bf16_f32(__p0);
43494 __ret = (bfloat16x8_t) __builtin_neon_vcvtq_low_bf16_f32((int8x16_t)__p0, 43);
4140043495 return __ret;
4140143496}
4140243497#else
4140343498__ai __attribute__((target("bf16,neon"))) bfloat16x8_t vcvtq_low_bf16_f32(float32x4_t __p0) {
4140443499 bfloat16x8_t __ret;
4140543500 float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0);
41406 __ret = __noswap___a64_vcvtq_low_bf16_f32(__rev0);
43501 __ret = (bfloat16x8_t) __builtin_neon_vcvtq_low_bf16_f32((int8x16_t)__rev0, 43);
4140743502 __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0);
4140843503 return __ret;
4140943504}
......@@ -41705,9 +43800,7 @@ __ai __attribute__((target("bf16,neon"))) bfloat16x4_t vreinterpret_bf16_s16(int
4170543800 uint32x4_t __s0_238 = __p0_238; \
4170643801 uint8x16_t __s1_238 = __p1_238; \
4170743802 uint8x16_t __s2_238 = __p2_238; \
41708uint8x16_t __reint_238 = __s2_238; \
41709uint32x4_t __reint1_238 = splatq_laneq_u32(*(uint32x4_t *) &__reint_238, __p3_238); \
41710 __ret_238 = vdotq_u32(__s0_238, __s1_238, *(uint8x16_t *) &__reint1_238); \
43803 __ret_238 = vdotq_u32(__s0_238, __s1_238, __builtin_bit_cast(uint8x16_t, splatq_laneq_u32(__builtin_bit_cast(uint32x4_t, __s2_238), __p3_238))); \
4171143804 __ret_238; \
4171243805})
4171343806#else
......@@ -41719,9 +43812,7 @@ uint32x4_t __reint1_238 = splatq_laneq_u32(*(uint32x4_t *) &__reint_238, __p3_23
4171943812 uint32x4_t __rev0_239; __rev0_239 = __builtin_shufflevector(__s0_239, __s0_239, 3, 2, 1, 0); \
4172043813 uint8x16_t __rev1_239; __rev1_239 = __builtin_shufflevector(__s1_239, __s1_239, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
4172143814 uint8x16_t __rev2_239; __rev2_239 = __builtin_shufflevector(__s2_239, __s2_239, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41722uint8x16_t __reint_239 = __rev2_239; \
41723uint32x4_t __reint1_239 = __noswap_splatq_laneq_u32(*(uint32x4_t *) &__reint_239, __p3_239); \
41724 __ret_239 = __noswap_vdotq_u32(__rev0_239, __rev1_239, *(uint8x16_t *) &__reint1_239); \
43815 __ret_239 = __noswap_vdotq_u32(__rev0_239, __rev1_239, __builtin_bit_cast(uint8x16_t, __noswap_splatq_laneq_u32(__builtin_bit_cast(uint32x4_t, __rev2_239), __p3_239))); \
4172543816 __ret_239 = __builtin_shufflevector(__ret_239, __ret_239, 3, 2, 1, 0); \
4172643817 __ret_239; \
4172743818})
......@@ -41733,9 +43824,7 @@ uint32x4_t __reint1_239 = __noswap_splatq_laneq_u32(*(uint32x4_t *) &__reint_239
4173343824 int32x4_t __s0_240 = __p0_240; \
4173443825 int8x16_t __s1_240 = __p1_240; \
4173543826 int8x16_t __s2_240 = __p2_240; \
41736int8x16_t __reint_240 = __s2_240; \
41737int32x4_t __reint1_240 = splatq_laneq_s32(*(int32x4_t *) &__reint_240, __p3_240); \
41738 __ret_240 = vdotq_s32(__s0_240, __s1_240, *(int8x16_t *) &__reint1_240); \
43827 __ret_240 = vdotq_s32(__s0_240, __s1_240, __builtin_bit_cast(int8x16_t, splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_240), __p3_240))); \
4173943828 __ret_240; \
4174043829})
4174143830#else
......@@ -41747,9 +43836,7 @@ int32x4_t __reint1_240 = splatq_laneq_s32(*(int32x4_t *) &__reint_240, __p3_240)
4174743836 int32x4_t __rev0_241; __rev0_241 = __builtin_shufflevector(__s0_241, __s0_241, 3, 2, 1, 0); \
4174843837 int8x16_t __rev1_241; __rev1_241 = __builtin_shufflevector(__s1_241, __s1_241, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
4174943838 int8x16_t __rev2_241; __rev2_241 = __builtin_shufflevector(__s2_241, __s2_241, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41750int8x16_t __reint_241 = __rev2_241; \
41751int32x4_t __reint1_241 = __noswap_splatq_laneq_s32(*(int32x4_t *) &__reint_241, __p3_241); \
41752 __ret_241 = __noswap_vdotq_s32(__rev0_241, __rev1_241, *(int8x16_t *) &__reint1_241); \
43839 __ret_241 = __noswap_vdotq_s32(__rev0_241, __rev1_241, __builtin_bit_cast(int8x16_t, __noswap_splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_241), __p3_241))); \
4175343840 __ret_241 = __builtin_shufflevector(__ret_241, __ret_241, 3, 2, 1, 0); \
4175443841 __ret_241; \
4175543842})
......@@ -41761,9 +43848,7 @@ int32x4_t __reint1_241 = __noswap_splatq_laneq_s32(*(int32x4_t *) &__reint_241,
4176143848 uint32x2_t __s0_242 = __p0_242; \
4176243849 uint8x8_t __s1_242 = __p1_242; \
4176343850 uint8x16_t __s2_242 = __p2_242; \
41764uint8x16_t __reint_242 = __s2_242; \
41765uint32x2_t __reint1_242 = splat_laneq_u32(*(uint32x4_t *) &__reint_242, __p3_242); \
41766 __ret_242 = vdot_u32(__s0_242, __s1_242, *(uint8x8_t *) &__reint1_242); \
43851 __ret_242 = vdot_u32(__s0_242, __s1_242, __builtin_bit_cast(uint8x8_t, splat_laneq_u32(__builtin_bit_cast(uint32x4_t, __s2_242), __p3_242))); \
4176743852 __ret_242; \
4176843853})
4176943854#else
......@@ -41775,9 +43860,7 @@ uint32x2_t __reint1_242 = splat_laneq_u32(*(uint32x4_t *) &__reint_242, __p3_242
4177543860 uint32x2_t __rev0_243; __rev0_243 = __builtin_shufflevector(__s0_243, __s0_243, 1, 0); \
4177643861 uint8x8_t __rev1_243; __rev1_243 = __builtin_shufflevector(__s1_243, __s1_243, 7, 6, 5, 4, 3, 2, 1, 0); \
4177743862 uint8x16_t __rev2_243; __rev2_243 = __builtin_shufflevector(__s2_243, __s2_243, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41778uint8x16_t __reint_243 = __rev2_243; \
41779uint32x2_t __reint1_243 = __noswap_splat_laneq_u32(*(uint32x4_t *) &__reint_243, __p3_243); \
41780 __ret_243 = __noswap_vdot_u32(__rev0_243, __rev1_243, *(uint8x8_t *) &__reint1_243); \
43863 __ret_243 = __noswap_vdot_u32(__rev0_243, __rev1_243, __builtin_bit_cast(uint8x8_t, __noswap_splat_laneq_u32(__builtin_bit_cast(uint32x4_t, __rev2_243), __p3_243))); \
4178143864 __ret_243 = __builtin_shufflevector(__ret_243, __ret_243, 1, 0); \
4178243865 __ret_243; \
4178343866})
......@@ -41789,9 +43872,7 @@ uint32x2_t __reint1_243 = __noswap_splat_laneq_u32(*(uint32x4_t *) &__reint_243,
4178943872 int32x2_t __s0_244 = __p0_244; \
4179043873 int8x8_t __s1_244 = __p1_244; \
4179143874 int8x16_t __s2_244 = __p2_244; \
41792int8x16_t __reint_244 = __s2_244; \
41793int32x2_t __reint1_244 = splat_laneq_s32(*(int32x4_t *) &__reint_244, __p3_244); \
41794 __ret_244 = vdot_s32(__s0_244, __s1_244, *(int8x8_t *) &__reint1_244); \
43875 __ret_244 = vdot_s32(__s0_244, __s1_244, __builtin_bit_cast(int8x8_t, splat_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_244), __p3_244))); \
4179543876 __ret_244; \
4179643877})
4179743878#else
......@@ -41803,9 +43884,7 @@ int32x2_t __reint1_244 = splat_laneq_s32(*(int32x4_t *) &__reint_244, __p3_244);
4180343884 int32x2_t __rev0_245; __rev0_245 = __builtin_shufflevector(__s0_245, __s0_245, 1, 0); \
4180443885 int8x8_t __rev1_245; __rev1_245 = __builtin_shufflevector(__s1_245, __s1_245, 7, 6, 5, 4, 3, 2, 1, 0); \
4180543886 int8x16_t __rev2_245; __rev2_245 = __builtin_shufflevector(__s2_245, __s2_245, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
41806int8x16_t __reint_245 = __rev2_245; \
41807int32x2_t __reint1_245 = __noswap_splat_laneq_s32(*(int32x4_t *) &__reint_245, __p3_245); \
41808 __ret_245 = __noswap_vdot_s32(__rev0_245, __rev1_245, *(int8x8_t *) &__reint1_245); \
43887 __ret_245 = __noswap_vdot_s32(__rev0_245, __rev1_245, __builtin_bit_cast(int8x8_t, __noswap_splat_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_245), __p3_245))); \
4180943888 __ret_245 = __builtin_shufflevector(__ret_245, __ret_245, 1, 0); \
4181043889 __ret_245; \
4181143890})
......@@ -43018,8 +45097,7 @@ __ai __attribute__((target("fullfp16,neon"))) float16x4_t vsqrt_f16(float16x4_t
4301845097 int32x4_t __s0_270 = __p0_270; \
4301945098 int8x16_t __s1_270 = __p1_270; \
4302045099 uint8x16_t __s2_270 = __p2_270; \
43021uint8x16_t __reint_270 = __s2_270; \
43022 __ret_270 = vusdotq_s32(__s0_270, (uint8x16_t)(splatq_laneq_s32(*(int32x4_t *) &__reint_270, __p3_270)), __s1_270); \
45100 __ret_270 = vusdotq_s32(__s0_270, (uint8x16_t)(splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_270), __p3_270)), __s1_270); \
4302345101 __ret_270; \
4302445102})
4302545103#else
......@@ -43031,8 +45109,7 @@ uint8x16_t __reint_270 = __s2_270; \
4303145109 int32x4_t __rev0_271; __rev0_271 = __builtin_shufflevector(__s0_271, __s0_271, 3, 2, 1, 0); \
4303245110 int8x16_t __rev1_271; __rev1_271 = __builtin_shufflevector(__s1_271, __s1_271, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
4303345111 uint8x16_t __rev2_271; __rev2_271 = __builtin_shufflevector(__s2_271, __s2_271, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43034uint8x16_t __reint_271 = __rev2_271; \
43035 __ret_271 = __noswap_vusdotq_s32(__rev0_271, (uint8x16_t)(__noswap_splatq_laneq_s32(*(int32x4_t *) &__reint_271, __p3_271)), __rev1_271); \
45112 __ret_271 = __noswap_vusdotq_s32(__rev0_271, (uint8x16_t)(__noswap_splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_271), __p3_271)), __rev1_271); \
4303645113 __ret_271 = __builtin_shufflevector(__ret_271, __ret_271, 3, 2, 1, 0); \
4303745114 __ret_271; \
4303845115})
......@@ -43044,8 +45121,7 @@ uint8x16_t __reint_271 = __rev2_271; \
4304445121 int32x2_t __s0_272 = __p0_272; \
4304545122 int8x8_t __s1_272 = __p1_272; \
4304645123 uint8x16_t __s2_272 = __p2_272; \
43047uint8x16_t __reint_272 = __s2_272; \
43048 __ret_272 = vusdot_s32(__s0_272, (uint8x8_t)(splat_laneq_s32(*(int32x4_t *) &__reint_272, __p3_272)), __s1_272); \
45124 __ret_272 = vusdot_s32(__s0_272, (uint8x8_t)(splat_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_272), __p3_272)), __s1_272); \
4304945125 __ret_272; \
4305045126})
4305145127#else
......@@ -43057,8 +45133,7 @@ uint8x16_t __reint_272 = __s2_272; \
4305745133 int32x2_t __rev0_273; __rev0_273 = __builtin_shufflevector(__s0_273, __s0_273, 1, 0); \
4305845134 int8x8_t __rev1_273; __rev1_273 = __builtin_shufflevector(__s1_273, __s1_273, 7, 6, 5, 4, 3, 2, 1, 0); \
4305945135 uint8x16_t __rev2_273; __rev2_273 = __builtin_shufflevector(__s2_273, __s2_273, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43060uint8x16_t __reint_273 = __rev2_273; \
43061 __ret_273 = __noswap_vusdot_s32(__rev0_273, (uint8x8_t)(__noswap_splat_laneq_s32(*(int32x4_t *) &__reint_273, __p3_273)), __rev1_273); \
45136 __ret_273 = __noswap_vusdot_s32(__rev0_273, (uint8x8_t)(__noswap_splat_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_273), __p3_273)), __rev1_273); \
4306245137 __ret_273 = __builtin_shufflevector(__ret_273, __ret_273, 1, 0); \
4306345138 __ret_273; \
4306445139})
......@@ -43070,8 +45145,7 @@ uint8x16_t __reint_273 = __rev2_273; \
4307045145 int32x4_t __s0_274 = __p0_274; \
4307145146 uint8x16_t __s1_274 = __p1_274; \
4307245147 int8x16_t __s2_274 = __p2_274; \
43073int8x16_t __reint_274 = __s2_274; \
43074 __ret_274 = vusdotq_s32(__s0_274, __s1_274, (int8x16_t)(splatq_laneq_s32(*(int32x4_t *) &__reint_274, __p3_274))); \
45148 __ret_274 = vusdotq_s32(__s0_274, __s1_274, (int8x16_t)(splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_274), __p3_274))); \
4307545149 __ret_274; \
4307645150})
4307745151#else
......@@ -43083,8 +45157,7 @@ int8x16_t __reint_274 = __s2_274; \
4308345157 int32x4_t __rev0_275; __rev0_275 = __builtin_shufflevector(__s0_275, __s0_275, 3, 2, 1, 0); \
4308445158 uint8x16_t __rev1_275; __rev1_275 = __builtin_shufflevector(__s1_275, __s1_275, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
4308545159 int8x16_t __rev2_275; __rev2_275 = __builtin_shufflevector(__s2_275, __s2_275, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43086int8x16_t __reint_275 = __rev2_275; \
43087 __ret_275 = __noswap_vusdotq_s32(__rev0_275, __rev1_275, (int8x16_t)(__noswap_splatq_laneq_s32(*(int32x4_t *) &__reint_275, __p3_275))); \
45160 __ret_275 = __noswap_vusdotq_s32(__rev0_275, __rev1_275, (int8x16_t)(__noswap_splatq_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_275), __p3_275))); \
4308845161 __ret_275 = __builtin_shufflevector(__ret_275, __ret_275, 3, 2, 1, 0); \
4308945162 __ret_275; \
4309045163})
......@@ -43096,8 +45169,7 @@ int8x16_t __reint_275 = __rev2_275; \
4309645169 int32x2_t __s0_276 = __p0_276; \
4309745170 uint8x8_t __s1_276 = __p1_276; \
4309845171 int8x16_t __s2_276 = __p2_276; \
43099int8x16_t __reint_276 = __s2_276; \
43100 __ret_276 = vusdot_s32(__s0_276, __s1_276, (int8x8_t)(splat_laneq_s32(*(int32x4_t *) &__reint_276, __p3_276))); \
45172 __ret_276 = vusdot_s32(__s0_276, __s1_276, (int8x8_t)(splat_laneq_s32(__builtin_bit_cast(int32x4_t, __s2_276), __p3_276))); \
4310145173 __ret_276; \
4310245174})
4310345175#else
......@@ -43109,8 +45181,7 @@ int8x16_t __reint_276 = __s2_276; \
4310945181 int32x2_t __rev0_277; __rev0_277 = __builtin_shufflevector(__s0_277, __s0_277, 1, 0); \
4311045182 uint8x8_t __rev1_277; __rev1_277 = __builtin_shufflevector(__s1_277, __s1_277, 7, 6, 5, 4, 3, 2, 1, 0); \
4311145183 int8x16_t __rev2_277; __rev2_277 = __builtin_shufflevector(__s2_277, __s2_277, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
43112int8x16_t __reint_277 = __rev2_277; \
43113 __ret_277 = __noswap_vusdot_s32(__rev0_277, __rev1_277, (int8x8_t)(__noswap_splat_laneq_s32(*(int32x4_t *) &__reint_277, __p3_277))); \
45184 __ret_277 = __noswap_vusdot_s32(__rev0_277, __rev1_277, (int8x8_t)(__noswap_splat_laneq_s32(__builtin_bit_cast(int32x4_t, __rev2_277), __p3_277))); \
4311445185 __ret_277 = __builtin_shufflevector(__ret_277, __ret_277, 1, 0); \
4311545186 __ret_277; \
4311645187})
......@@ -57773,6 +59844,11 @@ __ai __attribute__((target("neon"))) poly8x8_t vreinterpret_p8_s64(int64x1_t __p
5777359844 __ret = (poly8x8_t)(__p0);
5777459845 return __ret;
5777559846}
59847__ai __attribute__((target("neon"))) poly8x8_t vreinterpret_p8_mf8(mfloat8x8_t __p0) {
59848 poly8x8_t __ret;
59849 __ret = (poly8x8_t)(__p0);
59850 return __ret;
59851}
5777659852__ai __attribute__((target("neon"))) poly8x8_t vreinterpret_p8_s16(int16x4_t __p0) {
5777759853 poly8x8_t __ret;
5777859854 __ret = (poly8x8_t)(__p0);
......@@ -57838,6 +59914,11 @@ __ai __attribute__((target("neon"))) poly64x1_t vreinterpret_p64_s64(int64x1_t _
5783859914 __ret = (poly64x1_t)(__p0);
5783959915 return __ret;
5784059916}
59917__ai __attribute__((target("neon"))) poly64x1_t vreinterpret_p64_mf8(mfloat8x8_t __p0) {
59918 poly64x1_t __ret;
59919 __ret = (poly64x1_t)(__p0);
59920 return __ret;
59921}
5784159922__ai __attribute__((target("neon"))) poly64x1_t vreinterpret_p64_s16(int16x4_t __p0) {
5784259923 poly64x1_t __ret;
5784359924 __ret = (poly64x1_t)(__p0);
......@@ -57903,6 +59984,11 @@ __ai __attribute__((target("neon"))) poly16x4_t vreinterpret_p16_s64(int64x1_t _
5790359984 __ret = (poly16x4_t)(__p0);
5790459985 return __ret;
5790559986}
59987__ai __attribute__((target("neon"))) poly16x4_t vreinterpret_p16_mf8(mfloat8x8_t __p0) {
59988 poly16x4_t __ret;
59989 __ret = (poly16x4_t)(__p0);
59990 return __ret;
59991}
5790659992__ai __attribute__((target("neon"))) poly16x4_t vreinterpret_p16_s16(int16x4_t __p0) {
5790759993 poly16x4_t __ret;
5790859994 __ret = (poly16x4_t)(__p0);
......@@ -57973,6 +60059,11 @@ __ai __attribute__((target("neon"))) poly8x16_t vreinterpretq_p8_s64(int64x2_t _
5797360059 __ret = (poly8x16_t)(__p0);
5797460060 return __ret;
5797560061}
60062__ai __attribute__((target("neon"))) poly8x16_t vreinterpretq_p8_mf8(mfloat8x16_t __p0) {
60063 poly8x16_t __ret;
60064 __ret = (poly8x16_t)(__p0);
60065 return __ret;
60066}
5797660067__ai __attribute__((target("neon"))) poly8x16_t vreinterpretq_p8_s16(int16x8_t __p0) {
5797760068 poly8x16_t __ret;
5797860069 __ret = (poly8x16_t)(__p0);
......@@ -58043,6 +60134,11 @@ __ai __attribute__((target("neon"))) poly128_t vreinterpretq_p128_s64(int64x2_t
5804360134 __ret = (poly128_t)(__p0);
5804460135 return __ret;
5804560136}
60137__ai __attribute__((target("neon"))) poly128_t vreinterpretq_p128_mf8(mfloat8x16_t __p0) {
60138 poly128_t __ret;
60139 __ret = (poly128_t)(__p0);
60140 return __ret;
60141}
5804660142__ai __attribute__((target("neon"))) poly128_t vreinterpretq_p128_s16(int16x8_t __p0) {
5804760143 poly128_t __ret;
5804860144 __ret = (poly128_t)(__p0);
......@@ -58113,6 +60209,11 @@ __ai __attribute__((target("neon"))) poly64x2_t vreinterpretq_p64_s64(int64x2_t
5811360209 __ret = (poly64x2_t)(__p0);
5811460210 return __ret;
5811560211}
60212__ai __attribute__((target("neon"))) poly64x2_t vreinterpretq_p64_mf8(mfloat8x16_t __p0) {
60213 poly64x2_t __ret;
60214 __ret = (poly64x2_t)(__p0);
60215 return __ret;
60216}
5811660217__ai __attribute__((target("neon"))) poly64x2_t vreinterpretq_p64_s16(int16x8_t __p0) {
5811760218 poly64x2_t __ret;
5811860219 __ret = (poly64x2_t)(__p0);
......@@ -58183,6 +60284,11 @@ __ai __attribute__((target("neon"))) poly16x8_t vreinterpretq_p16_s64(int64x2_t
5818360284 __ret = (poly16x8_t)(__p0);
5818460285 return __ret;
5818560286}
60287__ai __attribute__((target("neon"))) poly16x8_t vreinterpretq_p16_mf8(mfloat8x16_t __p0) {
60288 poly16x8_t __ret;
60289 __ret = (poly16x8_t)(__p0);
60290 return __ret;
60291}
5818660292__ai __attribute__((target("neon"))) poly16x8_t vreinterpretq_p16_s16(int16x8_t __p0) {
5818760293 poly16x8_t __ret;
5818860294 __ret = (poly16x8_t)(__p0);
......@@ -58253,6 +60359,11 @@ __ai __attribute__((target("neon"))) uint8x16_t vreinterpretq_u8_s64(int64x2_t _
5825360359 __ret = (uint8x16_t)(__p0);
5825460360 return __ret;
5825560361}
60362__ai __attribute__((target("neon"))) uint8x16_t vreinterpretq_u8_mf8(mfloat8x16_t __p0) {
60363 uint8x16_t __ret;
60364 __ret = (uint8x16_t)(__p0);
60365 return __ret;
60366}
5825660367__ai __attribute__((target("neon"))) uint8x16_t vreinterpretq_u8_s16(int16x8_t __p0) {
5825760368 uint8x16_t __ret;
5825860369 __ret = (uint8x16_t)(__p0);
......@@ -58323,6 +60434,11 @@ __ai __attribute__((target("neon"))) uint32x4_t vreinterpretq_u32_s64(int64x2_t
5832360434 __ret = (uint32x4_t)(__p0);
5832460435 return __ret;
5832560436}
60437__ai __attribute__((target("neon"))) uint32x4_t vreinterpretq_u32_mf8(mfloat8x16_t __p0) {
60438 uint32x4_t __ret;
60439 __ret = (uint32x4_t)(__p0);
60440 return __ret;
60441}
5832660442__ai __attribute__((target("neon"))) uint32x4_t vreinterpretq_u32_s16(int16x8_t __p0) {
5832760443 uint32x4_t __ret;
5832860444 __ret = (uint32x4_t)(__p0);
......@@ -58393,6 +60509,11 @@ __ai __attribute__((target("neon"))) uint64x2_t vreinterpretq_u64_s64(int64x2_t
5839360509 __ret = (uint64x2_t)(__p0);
5839460510 return __ret;
5839560511}
60512__ai __attribute__((target("neon"))) uint64x2_t vreinterpretq_u64_mf8(mfloat8x16_t __p0) {
60513 uint64x2_t __ret;
60514 __ret = (uint64x2_t)(__p0);
60515 return __ret;
60516}
5839660517__ai __attribute__((target("neon"))) uint64x2_t vreinterpretq_u64_s16(int16x8_t __p0) {
5839760518 uint64x2_t __ret;
5839860519 __ret = (uint64x2_t)(__p0);
......@@ -58463,6 +60584,11 @@ __ai __attribute__((target("neon"))) uint16x8_t vreinterpretq_u16_s64(int64x2_t
5846360584 __ret = (uint16x8_t)(__p0);
5846460585 return __ret;
5846560586}
60587__ai __attribute__((target("neon"))) uint16x8_t vreinterpretq_u16_mf8(mfloat8x16_t __p0) {
60588 uint16x8_t __ret;
60589 __ret = (uint16x8_t)(__p0);
60590 return __ret;
60591}
5846660592__ai __attribute__((target("neon"))) uint16x8_t vreinterpretq_u16_s16(int16x8_t __p0) {
5846760593 uint16x8_t __ret;
5846860594 __ret = (uint16x8_t)(__p0);
......@@ -58533,6 +60659,11 @@ __ai __attribute__((target("neon"))) int8x16_t vreinterpretq_s8_s64(int64x2_t __
5853360659 __ret = (int8x16_t)(__p0);
5853460660 return __ret;
5853560661}
60662__ai __attribute__((target("neon"))) int8x16_t vreinterpretq_s8_mf8(mfloat8x16_t __p0) {
60663 int8x16_t __ret;
60664 __ret = (int8x16_t)(__p0);
60665 return __ret;
60666}
5853660667__ai __attribute__((target("neon"))) int8x16_t vreinterpretq_s8_s16(int16x8_t __p0) {
5853760668 int8x16_t __ret;
5853860669 __ret = (int8x16_t)(__p0);
......@@ -58603,6 +60734,11 @@ __ai __attribute__((target("neon"))) float64x2_t vreinterpretq_f64_s64(int64x2_t
5860360734 __ret = (float64x2_t)(__p0);
5860460735 return __ret;
5860560736}
60737__ai __attribute__((target("neon"))) float64x2_t vreinterpretq_f64_mf8(mfloat8x16_t __p0) {
60738 float64x2_t __ret;
60739 __ret = (float64x2_t)(__p0);
60740 return __ret;
60741}
5860660742__ai __attribute__((target("neon"))) float64x2_t vreinterpretq_f64_s16(int16x8_t __p0) {
5860760743 float64x2_t __ret;
5860860744 __ret = (float64x2_t)(__p0);
......@@ -58673,6 +60809,11 @@ __ai __attribute__((target("neon"))) float32x4_t vreinterpretq_f32_s64(int64x2_t
5867360809 __ret = (float32x4_t)(__p0);
5867460810 return __ret;
5867560811}
60812__ai __attribute__((target("neon"))) float32x4_t vreinterpretq_f32_mf8(mfloat8x16_t __p0) {
60813 float32x4_t __ret;
60814 __ret = (float32x4_t)(__p0);
60815 return __ret;
60816}
5867660817__ai __attribute__((target("neon"))) float32x4_t vreinterpretq_f32_s16(int16x8_t __p0) {
5867760818 float32x4_t __ret;
5867860819 __ret = (float32x4_t)(__p0);
......@@ -58743,6 +60884,11 @@ __ai __attribute__((target("neon"))) float16x8_t vreinterpretq_f16_s64(int64x2_t
5874360884 __ret = (float16x8_t)(__p0);
5874460885 return __ret;
5874560886}
60887__ai __attribute__((target("neon"))) float16x8_t vreinterpretq_f16_mf8(mfloat8x16_t __p0) {
60888 float16x8_t __ret;
60889 __ret = (float16x8_t)(__p0);
60890 return __ret;
60891}
5874660892__ai __attribute__((target("neon"))) float16x8_t vreinterpretq_f16_s16(int16x8_t __p0) {
5874760893 float16x8_t __ret;
5874860894 __ret = (float16x8_t)(__p0);
......@@ -58813,6 +60959,11 @@ __ai __attribute__((target("neon"))) int32x4_t vreinterpretq_s32_s64(int64x2_t _
5881360959 __ret = (int32x4_t)(__p0);
5881460960 return __ret;
5881560961}
60962__ai __attribute__((target("neon"))) int32x4_t vreinterpretq_s32_mf8(mfloat8x16_t __p0) {
60963 int32x4_t __ret;
60964 __ret = (int32x4_t)(__p0);
60965 return __ret;
60966}
5881660967__ai __attribute__((target("neon"))) int32x4_t vreinterpretq_s32_s16(int16x8_t __p0) {
5881760968 int32x4_t __ret;
5881860969 __ret = (int32x4_t)(__p0);
......@@ -58883,11 +61034,91 @@ __ai __attribute__((target("neon"))) int64x2_t vreinterpretq_s64_s32(int32x4_t _
5888361034 __ret = (int64x2_t)(__p0);
5888461035 return __ret;
5888561036}
61037__ai __attribute__((target("neon"))) int64x2_t vreinterpretq_s64_mf8(mfloat8x16_t __p0) {
61038 int64x2_t __ret;
61039 __ret = (int64x2_t)(__p0);
61040 return __ret;
61041}
5888661042__ai __attribute__((target("neon"))) int64x2_t vreinterpretq_s64_s16(int16x8_t __p0) {
5888761043 int64x2_t __ret;
5888861044 __ret = (int64x2_t)(__p0);
5888961045 return __ret;
5889061046}
61047__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_p8(poly8x16_t __p0) {
61048 mfloat8x16_t __ret;
61049 __ret = (mfloat8x16_t)(__p0);
61050 return __ret;
61051}
61052__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_p128(poly128_t __p0) {
61053 mfloat8x16_t __ret;
61054 __ret = (mfloat8x16_t)(__p0);
61055 return __ret;
61056}
61057__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_p64(poly64x2_t __p0) {
61058 mfloat8x16_t __ret;
61059 __ret = (mfloat8x16_t)(__p0);
61060 return __ret;
61061}
61062__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_p16(poly16x8_t __p0) {
61063 mfloat8x16_t __ret;
61064 __ret = (mfloat8x16_t)(__p0);
61065 return __ret;
61066}
61067__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_u8(uint8x16_t __p0) {
61068 mfloat8x16_t __ret;
61069 __ret = (mfloat8x16_t)(__p0);
61070 return __ret;
61071}
61072__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_u32(uint32x4_t __p0) {
61073 mfloat8x16_t __ret;
61074 __ret = (mfloat8x16_t)(__p0);
61075 return __ret;
61076}
61077__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_u64(uint64x2_t __p0) {
61078 mfloat8x16_t __ret;
61079 __ret = (mfloat8x16_t)(__p0);
61080 return __ret;
61081}
61082__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_u16(uint16x8_t __p0) {
61083 mfloat8x16_t __ret;
61084 __ret = (mfloat8x16_t)(__p0);
61085 return __ret;
61086}
61087__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_s8(int8x16_t __p0) {
61088 mfloat8x16_t __ret;
61089 __ret = (mfloat8x16_t)(__p0);
61090 return __ret;
61091}
61092__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_f64(float64x2_t __p0) {
61093 mfloat8x16_t __ret;
61094 __ret = (mfloat8x16_t)(__p0);
61095 return __ret;
61096}
61097__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_f32(float32x4_t __p0) {
61098 mfloat8x16_t __ret;
61099 __ret = (mfloat8x16_t)(__p0);
61100 return __ret;
61101}
61102__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_f16(float16x8_t __p0) {
61103 mfloat8x16_t __ret;
61104 __ret = (mfloat8x16_t)(__p0);
61105 return __ret;
61106}
61107__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_s32(int32x4_t __p0) {
61108 mfloat8x16_t __ret;
61109 __ret = (mfloat8x16_t)(__p0);
61110 return __ret;
61111}
61112__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_s64(int64x2_t __p0) {
61113 mfloat8x16_t __ret;
61114 __ret = (mfloat8x16_t)(__p0);
61115 return __ret;
61116}
61117__ai __attribute__((target("neon"))) mfloat8x16_t vreinterpretq_mf8_s16(int16x8_t __p0) {
61118 mfloat8x16_t __ret;
61119 __ret = (mfloat8x16_t)(__p0);
61120 return __ret;
61121}
5889161122__ai __attribute__((target("neon"))) int16x8_t vreinterpretq_s16_p8(poly8x16_t __p0) {
5889261123 int16x8_t __ret;
5889361124 __ret = (int16x8_t)(__p0);
......@@ -58958,6 +61189,11 @@ __ai __attribute__((target("neon"))) int16x8_t vreinterpretq_s16_s64(int64x2_t _
5895861189 __ret = (int16x8_t)(__p0);
5895961190 return __ret;
5896061191}
61192__ai __attribute__((target("neon"))) int16x8_t vreinterpretq_s16_mf8(mfloat8x16_t __p0) {
61193 int16x8_t __ret;
61194 __ret = (int16x8_t)(__p0);
61195 return __ret;
61196}
5896161197__ai __attribute__((target("neon"))) uint8x8_t vreinterpret_u8_p8(poly8x8_t __p0) {
5896261198 uint8x8_t __ret;
5896361199 __ret = (uint8x8_t)(__p0);
......@@ -59018,6 +61254,11 @@ __ai __attribute__((target("neon"))) uint8x8_t vreinterpret_u8_s64(int64x1_t __p
5901861254 __ret = (uint8x8_t)(__p0);
5901961255 return __ret;
5902061256}
61257__ai __attribute__((target("neon"))) uint8x8_t vreinterpret_u8_mf8(mfloat8x8_t __p0) {
61258 uint8x8_t __ret;
61259 __ret = (uint8x8_t)(__p0);
61260 return __ret;
61261}
5902161262__ai __attribute__((target("neon"))) uint8x8_t vreinterpret_u8_s16(int16x4_t __p0) {
5902261263 uint8x8_t __ret;
5902361264 __ret = (uint8x8_t)(__p0);
......@@ -59083,6 +61324,11 @@ __ai __attribute__((target("neon"))) uint32x2_t vreinterpret_u32_s64(int64x1_t _
5908361324 __ret = (uint32x2_t)(__p0);
5908461325 return __ret;
5908561326}
61327__ai __attribute__((target("neon"))) uint32x2_t vreinterpret_u32_mf8(mfloat8x8_t __p0) {
61328 uint32x2_t __ret;
61329 __ret = (uint32x2_t)(__p0);
61330 return __ret;
61331}
5908661332__ai __attribute__((target("neon"))) uint32x2_t vreinterpret_u32_s16(int16x4_t __p0) {
5908761333 uint32x2_t __ret;
5908861334 __ret = (uint32x2_t)(__p0);
......@@ -59148,6 +61394,11 @@ __ai __attribute__((target("neon"))) uint64x1_t vreinterpret_u64_s64(int64x1_t _
5914861394 __ret = (uint64x1_t)(__p0);
5914961395 return __ret;
5915061396}
61397__ai __attribute__((target("neon"))) uint64x1_t vreinterpret_u64_mf8(mfloat8x8_t __p0) {
61398 uint64x1_t __ret;
61399 __ret = (uint64x1_t)(__p0);
61400 return __ret;
61401}
5915161402__ai __attribute__((target("neon"))) uint64x1_t vreinterpret_u64_s16(int16x4_t __p0) {
5915261403 uint64x1_t __ret;
5915361404 __ret = (uint64x1_t)(__p0);
......@@ -59213,6 +61464,11 @@ __ai __attribute__((target("neon"))) uint16x4_t vreinterpret_u16_s64(int64x1_t _
5921361464 __ret = (uint16x4_t)(__p0);
5921461465 return __ret;
5921561466}
61467__ai __attribute__((target("neon"))) uint16x4_t vreinterpret_u16_mf8(mfloat8x8_t __p0) {
61468 uint16x4_t __ret;
61469 __ret = (uint16x4_t)(__p0);
61470 return __ret;
61471}
5921661472__ai __attribute__((target("neon"))) uint16x4_t vreinterpret_u16_s16(int16x4_t __p0) {
5921761473 uint16x4_t __ret;
5921861474 __ret = (uint16x4_t)(__p0);
......@@ -59278,6 +61534,11 @@ __ai __attribute__((target("neon"))) int8x8_t vreinterpret_s8_s64(int64x1_t __p0
5927861534 __ret = (int8x8_t)(__p0);
5927961535 return __ret;
5928061536}
61537__ai __attribute__((target("neon"))) int8x8_t vreinterpret_s8_mf8(mfloat8x8_t __p0) {
61538 int8x8_t __ret;
61539 __ret = (int8x8_t)(__p0);
61540 return __ret;
61541}
5928161542__ai __attribute__((target("neon"))) int8x8_t vreinterpret_s8_s16(int16x4_t __p0) {
5928261543 int8x8_t __ret;
5928361544 __ret = (int8x8_t)(__p0);
......@@ -59343,6 +61604,11 @@ __ai __attribute__((target("neon"))) float64x1_t vreinterpret_f64_s64(int64x1_t
5934361604 __ret = (float64x1_t)(__p0);
5934461605 return __ret;
5934561606}
61607__ai __attribute__((target("neon"))) float64x1_t vreinterpret_f64_mf8(mfloat8x8_t __p0) {
61608 float64x1_t __ret;
61609 __ret = (float64x1_t)(__p0);
61610 return __ret;
61611}
5934661612__ai __attribute__((target("neon"))) float64x1_t vreinterpret_f64_s16(int16x4_t __p0) {
5934761613 float64x1_t __ret;
5934861614 __ret = (float64x1_t)(__p0);
......@@ -59408,6 +61674,11 @@ __ai __attribute__((target("neon"))) float32x2_t vreinterpret_f32_s64(int64x1_t
5940861674 __ret = (float32x2_t)(__p0);
5940961675 return __ret;
5941061676}
61677__ai __attribute__((target("neon"))) float32x2_t vreinterpret_f32_mf8(mfloat8x8_t __p0) {
61678 float32x2_t __ret;
61679 __ret = (float32x2_t)(__p0);
61680 return __ret;
61681}
5941161682__ai __attribute__((target("neon"))) float32x2_t vreinterpret_f32_s16(int16x4_t __p0) {
5941261683 float32x2_t __ret;
5941361684 __ret = (float32x2_t)(__p0);
......@@ -59473,6 +61744,11 @@ __ai __attribute__((target("neon"))) float16x4_t vreinterpret_f16_s64(int64x1_t
5947361744 __ret = (float16x4_t)(__p0);
5947461745 return __ret;
5947561746}
61747__ai __attribute__((target("neon"))) float16x4_t vreinterpret_f16_mf8(mfloat8x8_t __p0) {
61748 float16x4_t __ret;
61749 __ret = (float16x4_t)(__p0);
61750 return __ret;
61751}
5947661752__ai __attribute__((target("neon"))) float16x4_t vreinterpret_f16_s16(int16x4_t __p0) {
5947761753 float16x4_t __ret;
5947861754 __ret = (float16x4_t)(__p0);
......@@ -59538,6 +61814,11 @@ __ai __attribute__((target("neon"))) int32x2_t vreinterpret_s32_s64(int64x1_t __
5953861814 __ret = (int32x2_t)(__p0);
5953961815 return __ret;
5954061816}
61817__ai __attribute__((target("neon"))) int32x2_t vreinterpret_s32_mf8(mfloat8x8_t __p0) {
61818 int32x2_t __ret;
61819 __ret = (int32x2_t)(__p0);
61820 return __ret;
61821}
5954161822__ai __attribute__((target("neon"))) int32x2_t vreinterpret_s32_s16(int16x4_t __p0) {
5954261823 int32x2_t __ret;
5954361824 __ret = (int32x2_t)(__p0);
......@@ -59603,11 +61884,86 @@ __ai __attribute__((target("neon"))) int64x1_t vreinterpret_s64_s32(int32x2_t __
5960361884 __ret = (int64x1_t)(__p0);
5960461885 return __ret;
5960561886}
61887__ai __attribute__((target("neon"))) int64x1_t vreinterpret_s64_mf8(mfloat8x8_t __p0) {
61888 int64x1_t __ret;
61889 __ret = (int64x1_t)(__p0);
61890 return __ret;
61891}
5960661892__ai __attribute__((target("neon"))) int64x1_t vreinterpret_s64_s16(int16x4_t __p0) {
5960761893 int64x1_t __ret;
5960861894 __ret = (int64x1_t)(__p0);
5960961895 return __ret;
5961061896}
61897__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_p8(poly8x8_t __p0) {
61898 mfloat8x8_t __ret;
61899 __ret = (mfloat8x8_t)(__p0);
61900 return __ret;
61901}
61902__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_p64(poly64x1_t __p0) {
61903 mfloat8x8_t __ret;
61904 __ret = (mfloat8x8_t)(__p0);
61905 return __ret;
61906}
61907__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_p16(poly16x4_t __p0) {
61908 mfloat8x8_t __ret;
61909 __ret = (mfloat8x8_t)(__p0);
61910 return __ret;
61911}
61912__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_u8(uint8x8_t __p0) {
61913 mfloat8x8_t __ret;
61914 __ret = (mfloat8x8_t)(__p0);
61915 return __ret;
61916}
61917__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_u32(uint32x2_t __p0) {
61918 mfloat8x8_t __ret;
61919 __ret = (mfloat8x8_t)(__p0);
61920 return __ret;
61921}
61922__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_u64(uint64x1_t __p0) {
61923 mfloat8x8_t __ret;
61924 __ret = (mfloat8x8_t)(__p0);
61925 return __ret;
61926}
61927__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_u16(uint16x4_t __p0) {
61928 mfloat8x8_t __ret;
61929 __ret = (mfloat8x8_t)(__p0);
61930 return __ret;
61931}
61932__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_s8(int8x8_t __p0) {
61933 mfloat8x8_t __ret;
61934 __ret = (mfloat8x8_t)(__p0);
61935 return __ret;
61936}
61937__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_f64(float64x1_t __p0) {
61938 mfloat8x8_t __ret;
61939 __ret = (mfloat8x8_t)(__p0);
61940 return __ret;
61941}
61942__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_f32(float32x2_t __p0) {
61943 mfloat8x8_t __ret;
61944 __ret = (mfloat8x8_t)(__p0);
61945 return __ret;
61946}
61947__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_f16(float16x4_t __p0) {
61948 mfloat8x8_t __ret;
61949 __ret = (mfloat8x8_t)(__p0);
61950 return __ret;
61951}
61952__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_s32(int32x2_t __p0) {
61953 mfloat8x8_t __ret;
61954 __ret = (mfloat8x8_t)(__p0);
61955 return __ret;
61956}
61957__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_s64(int64x1_t __p0) {
61958 mfloat8x8_t __ret;
61959 __ret = (mfloat8x8_t)(__p0);
61960 return __ret;
61961}
61962__ai __attribute__((target("neon"))) mfloat8x8_t vreinterpret_mf8_s16(int16x4_t __p0) {
61963 mfloat8x8_t __ret;
61964 __ret = (mfloat8x8_t)(__p0);
61965 return __ret;
61966}
5961161967__ai __attribute__((target("neon"))) int16x4_t vreinterpret_s16_p8(poly8x8_t __p0) {
5961261968 int16x4_t __ret;
5961361969 __ret = (int16x4_t)(__p0);
......@@ -59673,6 +62029,11 @@ __ai __attribute__((target("neon"))) int16x4_t vreinterpret_s16_s64(int64x1_t __
5967362029 __ret = (int16x4_t)(__p0);
5967462030 return __ret;
5967562031}
62032__ai __attribute__((target("neon"))) int16x4_t vreinterpret_s16_mf8(mfloat8x8_t __p0) {
62033 int16x4_t __ret;
62034 __ret = (int16x4_t)(__p0);
62035 return __ret;
62036}
5967662037__ai __attribute__((target("neon"))) uint64_t vrshld_u64(uint64_t __p0, int64_t __p1) {
5967762038 uint64_t __ret;
5967862039 __ret = (uint64_t) __builtin_neon_vrshld_u64(__p0, __p1);
......@@ -65616,106 +67977,6 @@ __ai __attribute__((target("v8.3a,neon"))) float64x2_t vcmlaq_f64(float64x2_t __
6561667977 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
6561767978 return __ret;
6561867979}
65619__ai __attribute__((target("v8.3a,neon"))) float64x2_t __noswap_vcmlaq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
65620 float64x2_t __ret;
65621 __ret = (float64x2_t) __builtin_neon_vcmlaq_f64((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42);
65622 return __ret;
65623}
65624#endif
65625
65626__ai __attribute__((target("v8.3a,neon"))) float64x1_t vcmla_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) {
65627 float64x1_t __ret;
65628 __ret = (float64x1_t) __builtin_neon_vcmla_f64((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10);
65629 return __ret;
65630}
65631#define vcmla_lane_f64(__p0_792, __p1_792, __p2_792, __p3_792) __extension__ ({ \
65632 float64x1_t __ret_792; \
65633 float64x1_t __s0_792 = __p0_792; \
65634 float64x1_t __s1_792 = __p1_792; \
65635 float64x1_t __s2_792 = __p2_792; \
65636float64x1_t __reint_792 = __s2_792; \
65637uint64x2_t __reint1_792 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_792, __p3_792), vgetq_lane_u64(*(uint64x2_t *) &__reint_792, __p3_792)}; \
65638 __ret_792 = vcmla_f64(__s0_792, __s1_792, *(float64x1_t *) &__reint1_792); \
65639 __ret_792; \
65640})
65641#ifdef __LITTLE_ENDIAN__
65642#define vcmlaq_lane_f64(__p0_793, __p1_793, __p2_793, __p3_793) __extension__ ({ \
65643 float64x2_t __ret_793; \
65644 float64x2_t __s0_793 = __p0_793; \
65645 float64x2_t __s1_793 = __p1_793; \
65646 float64x1_t __s2_793 = __p2_793; \
65647float64x1_t __reint_793 = __s2_793; \
65648uint64x2_t __reint1_793 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_793, __p3_793), vgetq_lane_u64(*(uint64x2_t *) &__reint_793, __p3_793)}; \
65649 __ret_793 = vcmlaq_f64(__s0_793, __s1_793, *(float64x2_t *) &__reint1_793); \
65650 __ret_793; \
65651})
65652#else
65653#define vcmlaq_lane_f64(__p0_794, __p1_794, __p2_794, __p3_794) __extension__ ({ \
65654 float64x2_t __ret_794; \
65655 float64x2_t __s0_794 = __p0_794; \
65656 float64x2_t __s1_794 = __p1_794; \
65657 float64x1_t __s2_794 = __p2_794; \
65658 float64x2_t __rev0_794; __rev0_794 = __builtin_shufflevector(__s0_794, __s0_794, 1, 0); \
65659 float64x2_t __rev1_794; __rev1_794 = __builtin_shufflevector(__s1_794, __s1_794, 1, 0); \
65660float64x1_t __reint_794 = __s2_794; \
65661uint64x2_t __reint1_794 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_794, __p3_794), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_794, __p3_794)}; \
65662 __ret_794 = __noswap_vcmlaq_f64(__rev0_794, __rev1_794, *(float64x2_t *) &__reint1_794); \
65663 __ret_794 = __builtin_shufflevector(__ret_794, __ret_794, 1, 0); \
65664 __ret_794; \
65665})
65666#endif
65667
65668#ifdef __LITTLE_ENDIAN__
65669#define vcmla_laneq_f64(__p0_795, __p1_795, __p2_795, __p3_795) __extension__ ({ \
65670 float64x1_t __ret_795; \
65671 float64x1_t __s0_795 = __p0_795; \
65672 float64x1_t __s1_795 = __p1_795; \
65673 float64x2_t __s2_795 = __p2_795; \
65674float64x2_t __reint_795 = __s2_795; \
65675uint64x2_t __reint1_795 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_795, __p3_795), vgetq_lane_u64(*(uint64x2_t *) &__reint_795, __p3_795)}; \
65676 __ret_795 = vcmla_f64(__s0_795, __s1_795, *(float64x1_t *) &__reint1_795); \
65677 __ret_795; \
65678})
65679#else
65680#define vcmla_laneq_f64(__p0_796, __p1_796, __p2_796, __p3_796) __extension__ ({ \
65681 float64x1_t __ret_796; \
65682 float64x1_t __s0_796 = __p0_796; \
65683 float64x1_t __s1_796 = __p1_796; \
65684 float64x2_t __s2_796 = __p2_796; \
65685 float64x2_t __rev2_796; __rev2_796 = __builtin_shufflevector(__s2_796, __s2_796, 1, 0); \
65686float64x2_t __reint_796 = __rev2_796; \
65687uint64x2_t __reint1_796 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_796, __p3_796), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_796, __p3_796)}; \
65688 __ret_796 = vcmla_f64(__s0_796, __s1_796, *(float64x1_t *) &__reint1_796); \
65689 __ret_796; \
65690})
65691#endif
65692
65693#ifdef __LITTLE_ENDIAN__
65694#define vcmlaq_laneq_f64(__p0_797, __p1_797, __p2_797, __p3_797) __extension__ ({ \
65695 float64x2_t __ret_797; \
65696 float64x2_t __s0_797 = __p0_797; \
65697 float64x2_t __s1_797 = __p1_797; \
65698 float64x2_t __s2_797 = __p2_797; \
65699float64x2_t __reint_797 = __s2_797; \
65700uint64x2_t __reint1_797 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_797, __p3_797), vgetq_lane_u64(*(uint64x2_t *) &__reint_797, __p3_797)}; \
65701 __ret_797 = vcmlaq_f64(__s0_797, __s1_797, *(float64x2_t *) &__reint1_797); \
65702 __ret_797; \
65703})
65704#else
65705#define vcmlaq_laneq_f64(__p0_798, __p1_798, __p2_798, __p3_798) __extension__ ({ \
65706 float64x2_t __ret_798; \
65707 float64x2_t __s0_798 = __p0_798; \
65708 float64x2_t __s1_798 = __p1_798; \
65709 float64x2_t __s2_798 = __p2_798; \
65710 float64x2_t __rev0_798; __rev0_798 = __builtin_shufflevector(__s0_798, __s0_798, 1, 0); \
65711 float64x2_t __rev1_798; __rev1_798 = __builtin_shufflevector(__s1_798, __s1_798, 1, 0); \
65712 float64x2_t __rev2_798; __rev2_798 = __builtin_shufflevector(__s2_798, __s2_798, 1, 0); \
65713float64x2_t __reint_798 = __rev2_798; \
65714uint64x2_t __reint1_798 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_798, __p3_798), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_798, __p3_798)}; \
65715 __ret_798 = __noswap_vcmlaq_f64(__rev0_798, __rev1_798, *(float64x2_t *) &__reint1_798); \
65716 __ret_798 = __builtin_shufflevector(__ret_798, __ret_798, 1, 0); \
65717 __ret_798; \
65718})
6571967980#endif
6572067981
6572167982#ifdef __LITTLE_ENDIAN__
......@@ -65734,106 +67995,6 @@ __ai __attribute__((target("v8.3a,neon"))) float64x2_t vcmlaq_rot180_f64(float64
6573467995 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
6573567996 return __ret;
6573667997}
65737__ai __attribute__((target("v8.3a,neon"))) float64x2_t __noswap_vcmlaq_rot180_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
65738 float64x2_t __ret;
65739 __ret = (float64x2_t) __builtin_neon_vcmlaq_rot180_f64((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42);
65740 return __ret;
65741}
65742#endif
65743
65744__ai __attribute__((target("v8.3a,neon"))) float64x1_t vcmla_rot180_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) {
65745 float64x1_t __ret;
65746 __ret = (float64x1_t) __builtin_neon_vcmla_rot180_f64((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10);
65747 return __ret;
65748}
65749#define vcmla_rot180_lane_f64(__p0_799, __p1_799, __p2_799, __p3_799) __extension__ ({ \
65750 float64x1_t __ret_799; \
65751 float64x1_t __s0_799 = __p0_799; \
65752 float64x1_t __s1_799 = __p1_799; \
65753 float64x1_t __s2_799 = __p2_799; \
65754float64x1_t __reint_799 = __s2_799; \
65755uint64x2_t __reint1_799 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_799, __p3_799), vgetq_lane_u64(*(uint64x2_t *) &__reint_799, __p3_799)}; \
65756 __ret_799 = vcmla_rot180_f64(__s0_799, __s1_799, *(float64x1_t *) &__reint1_799); \
65757 __ret_799; \
65758})
65759#ifdef __LITTLE_ENDIAN__
65760#define vcmlaq_rot180_lane_f64(__p0_800, __p1_800, __p2_800, __p3_800) __extension__ ({ \
65761 float64x2_t __ret_800; \
65762 float64x2_t __s0_800 = __p0_800; \
65763 float64x2_t __s1_800 = __p1_800; \
65764 float64x1_t __s2_800 = __p2_800; \
65765float64x1_t __reint_800 = __s2_800; \
65766uint64x2_t __reint1_800 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_800, __p3_800), vgetq_lane_u64(*(uint64x2_t *) &__reint_800, __p3_800)}; \
65767 __ret_800 = vcmlaq_rot180_f64(__s0_800, __s1_800, *(float64x2_t *) &__reint1_800); \
65768 __ret_800; \
65769})
65770#else
65771#define vcmlaq_rot180_lane_f64(__p0_801, __p1_801, __p2_801, __p3_801) __extension__ ({ \
65772 float64x2_t __ret_801; \
65773 float64x2_t __s0_801 = __p0_801; \
65774 float64x2_t __s1_801 = __p1_801; \
65775 float64x1_t __s2_801 = __p2_801; \
65776 float64x2_t __rev0_801; __rev0_801 = __builtin_shufflevector(__s0_801, __s0_801, 1, 0); \
65777 float64x2_t __rev1_801; __rev1_801 = __builtin_shufflevector(__s1_801, __s1_801, 1, 0); \
65778float64x1_t __reint_801 = __s2_801; \
65779uint64x2_t __reint1_801 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_801, __p3_801), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_801, __p3_801)}; \
65780 __ret_801 = __noswap_vcmlaq_rot180_f64(__rev0_801, __rev1_801, *(float64x2_t *) &__reint1_801); \
65781 __ret_801 = __builtin_shufflevector(__ret_801, __ret_801, 1, 0); \
65782 __ret_801; \
65783})
65784#endif
65785
65786#ifdef __LITTLE_ENDIAN__
65787#define vcmla_rot180_laneq_f64(__p0_802, __p1_802, __p2_802, __p3_802) __extension__ ({ \
65788 float64x1_t __ret_802; \
65789 float64x1_t __s0_802 = __p0_802; \
65790 float64x1_t __s1_802 = __p1_802; \
65791 float64x2_t __s2_802 = __p2_802; \
65792float64x2_t __reint_802 = __s2_802; \
65793uint64x2_t __reint1_802 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_802, __p3_802), vgetq_lane_u64(*(uint64x2_t *) &__reint_802, __p3_802)}; \
65794 __ret_802 = vcmla_rot180_f64(__s0_802, __s1_802, *(float64x1_t *) &__reint1_802); \
65795 __ret_802; \
65796})
65797#else
65798#define vcmla_rot180_laneq_f64(__p0_803, __p1_803, __p2_803, __p3_803) __extension__ ({ \
65799 float64x1_t __ret_803; \
65800 float64x1_t __s0_803 = __p0_803; \
65801 float64x1_t __s1_803 = __p1_803; \
65802 float64x2_t __s2_803 = __p2_803; \
65803 float64x2_t __rev2_803; __rev2_803 = __builtin_shufflevector(__s2_803, __s2_803, 1, 0); \
65804float64x2_t __reint_803 = __rev2_803; \
65805uint64x2_t __reint1_803 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_803, __p3_803), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_803, __p3_803)}; \
65806 __ret_803 = vcmla_rot180_f64(__s0_803, __s1_803, *(float64x1_t *) &__reint1_803); \
65807 __ret_803; \
65808})
65809#endif
65810
65811#ifdef __LITTLE_ENDIAN__
65812#define vcmlaq_rot180_laneq_f64(__p0_804, __p1_804, __p2_804, __p3_804) __extension__ ({ \
65813 float64x2_t __ret_804; \
65814 float64x2_t __s0_804 = __p0_804; \
65815 float64x2_t __s1_804 = __p1_804; \
65816 float64x2_t __s2_804 = __p2_804; \
65817float64x2_t __reint_804 = __s2_804; \
65818uint64x2_t __reint1_804 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_804, __p3_804), vgetq_lane_u64(*(uint64x2_t *) &__reint_804, __p3_804)}; \
65819 __ret_804 = vcmlaq_rot180_f64(__s0_804, __s1_804, *(float64x2_t *) &__reint1_804); \
65820 __ret_804; \
65821})
65822#else
65823#define vcmlaq_rot180_laneq_f64(__p0_805, __p1_805, __p2_805, __p3_805) __extension__ ({ \
65824 float64x2_t __ret_805; \
65825 float64x2_t __s0_805 = __p0_805; \
65826 float64x2_t __s1_805 = __p1_805; \
65827 float64x2_t __s2_805 = __p2_805; \
65828 float64x2_t __rev0_805; __rev0_805 = __builtin_shufflevector(__s0_805, __s0_805, 1, 0); \
65829 float64x2_t __rev1_805; __rev1_805 = __builtin_shufflevector(__s1_805, __s1_805, 1, 0); \
65830 float64x2_t __rev2_805; __rev2_805 = __builtin_shufflevector(__s2_805, __s2_805, 1, 0); \
65831float64x2_t __reint_805 = __rev2_805; \
65832uint64x2_t __reint1_805 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_805, __p3_805), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_805, __p3_805)}; \
65833 __ret_805 = __noswap_vcmlaq_rot180_f64(__rev0_805, __rev1_805, *(float64x2_t *) &__reint1_805); \
65834 __ret_805 = __builtin_shufflevector(__ret_805, __ret_805, 1, 0); \
65835 __ret_805; \
65836})
6583767998#endif
6583867999
6583968000#ifdef __LITTLE_ENDIAN__
......@@ -65852,106 +68013,6 @@ __ai __attribute__((target("v8.3a,neon"))) float64x2_t vcmlaq_rot270_f64(float64
6585268013 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
6585368014 return __ret;
6585468015}
65855__ai __attribute__((target("v8.3a,neon"))) float64x2_t __noswap_vcmlaq_rot270_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
65856 float64x2_t __ret;
65857 __ret = (float64x2_t) __builtin_neon_vcmlaq_rot270_f64((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42);
65858 return __ret;
65859}
65860#endif
65861
65862__ai __attribute__((target("v8.3a,neon"))) float64x1_t vcmla_rot270_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) {
65863 float64x1_t __ret;
65864 __ret = (float64x1_t) __builtin_neon_vcmla_rot270_f64((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10);
65865 return __ret;
65866}
65867#define vcmla_rot270_lane_f64(__p0_806, __p1_806, __p2_806, __p3_806) __extension__ ({ \
65868 float64x1_t __ret_806; \
65869 float64x1_t __s0_806 = __p0_806; \
65870 float64x1_t __s1_806 = __p1_806; \
65871 float64x1_t __s2_806 = __p2_806; \
65872float64x1_t __reint_806 = __s2_806; \
65873uint64x2_t __reint1_806 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_806, __p3_806), vgetq_lane_u64(*(uint64x2_t *) &__reint_806, __p3_806)}; \
65874 __ret_806 = vcmla_rot270_f64(__s0_806, __s1_806, *(float64x1_t *) &__reint1_806); \
65875 __ret_806; \
65876})
65877#ifdef __LITTLE_ENDIAN__
65878#define vcmlaq_rot270_lane_f64(__p0_807, __p1_807, __p2_807, __p3_807) __extension__ ({ \
65879 float64x2_t __ret_807; \
65880 float64x2_t __s0_807 = __p0_807; \
65881 float64x2_t __s1_807 = __p1_807; \
65882 float64x1_t __s2_807 = __p2_807; \
65883float64x1_t __reint_807 = __s2_807; \
65884uint64x2_t __reint1_807 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_807, __p3_807), vgetq_lane_u64(*(uint64x2_t *) &__reint_807, __p3_807)}; \
65885 __ret_807 = vcmlaq_rot270_f64(__s0_807, __s1_807, *(float64x2_t *) &__reint1_807); \
65886 __ret_807; \
65887})
65888#else
65889#define vcmlaq_rot270_lane_f64(__p0_808, __p1_808, __p2_808, __p3_808) __extension__ ({ \
65890 float64x2_t __ret_808; \
65891 float64x2_t __s0_808 = __p0_808; \
65892 float64x2_t __s1_808 = __p1_808; \
65893 float64x1_t __s2_808 = __p2_808; \
65894 float64x2_t __rev0_808; __rev0_808 = __builtin_shufflevector(__s0_808, __s0_808, 1, 0); \
65895 float64x2_t __rev1_808; __rev1_808 = __builtin_shufflevector(__s1_808, __s1_808, 1, 0); \
65896float64x1_t __reint_808 = __s2_808; \
65897uint64x2_t __reint1_808 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_808, __p3_808), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_808, __p3_808)}; \
65898 __ret_808 = __noswap_vcmlaq_rot270_f64(__rev0_808, __rev1_808, *(float64x2_t *) &__reint1_808); \
65899 __ret_808 = __builtin_shufflevector(__ret_808, __ret_808, 1, 0); \
65900 __ret_808; \
65901})
65902#endif
65903
65904#ifdef __LITTLE_ENDIAN__
65905#define vcmla_rot270_laneq_f64(__p0_809, __p1_809, __p2_809, __p3_809) __extension__ ({ \
65906 float64x1_t __ret_809; \
65907 float64x1_t __s0_809 = __p0_809; \
65908 float64x1_t __s1_809 = __p1_809; \
65909 float64x2_t __s2_809 = __p2_809; \
65910float64x2_t __reint_809 = __s2_809; \
65911uint64x2_t __reint1_809 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_809, __p3_809), vgetq_lane_u64(*(uint64x2_t *) &__reint_809, __p3_809)}; \
65912 __ret_809 = vcmla_rot270_f64(__s0_809, __s1_809, *(float64x1_t *) &__reint1_809); \
65913 __ret_809; \
65914})
65915#else
65916#define vcmla_rot270_laneq_f64(__p0_810, __p1_810, __p2_810, __p3_810) __extension__ ({ \
65917 float64x1_t __ret_810; \
65918 float64x1_t __s0_810 = __p0_810; \
65919 float64x1_t __s1_810 = __p1_810; \
65920 float64x2_t __s2_810 = __p2_810; \
65921 float64x2_t __rev2_810; __rev2_810 = __builtin_shufflevector(__s2_810, __s2_810, 1, 0); \
65922float64x2_t __reint_810 = __rev2_810; \
65923uint64x2_t __reint1_810 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_810, __p3_810), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_810, __p3_810)}; \
65924 __ret_810 = vcmla_rot270_f64(__s0_810, __s1_810, *(float64x1_t *) &__reint1_810); \
65925 __ret_810; \
65926})
65927#endif
65928
65929#ifdef __LITTLE_ENDIAN__
65930#define vcmlaq_rot270_laneq_f64(__p0_811, __p1_811, __p2_811, __p3_811) __extension__ ({ \
65931 float64x2_t __ret_811; \
65932 float64x2_t __s0_811 = __p0_811; \
65933 float64x2_t __s1_811 = __p1_811; \
65934 float64x2_t __s2_811 = __p2_811; \
65935float64x2_t __reint_811 = __s2_811; \
65936uint64x2_t __reint1_811 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_811, __p3_811), vgetq_lane_u64(*(uint64x2_t *) &__reint_811, __p3_811)}; \
65937 __ret_811 = vcmlaq_rot270_f64(__s0_811, __s1_811, *(float64x2_t *) &__reint1_811); \
65938 __ret_811; \
65939})
65940#else
65941#define vcmlaq_rot270_laneq_f64(__p0_812, __p1_812, __p2_812, __p3_812) __extension__ ({ \
65942 float64x2_t __ret_812; \
65943 float64x2_t __s0_812 = __p0_812; \
65944 float64x2_t __s1_812 = __p1_812; \
65945 float64x2_t __s2_812 = __p2_812; \
65946 float64x2_t __rev0_812; __rev0_812 = __builtin_shufflevector(__s0_812, __s0_812, 1, 0); \
65947 float64x2_t __rev1_812; __rev1_812 = __builtin_shufflevector(__s1_812, __s1_812, 1, 0); \
65948 float64x2_t __rev2_812; __rev2_812 = __builtin_shufflevector(__s2_812, __s2_812, 1, 0); \
65949float64x2_t __reint_812 = __rev2_812; \
65950uint64x2_t __reint1_812 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_812, __p3_812), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_812, __p3_812)}; \
65951 __ret_812 = __noswap_vcmlaq_rot270_f64(__rev0_812, __rev1_812, *(float64x2_t *) &__reint1_812); \
65952 __ret_812 = __builtin_shufflevector(__ret_812, __ret_812, 1, 0); \
65953 __ret_812; \
65954})
6595568016#endif
6595668017
6595768018#ifdef __LITTLE_ENDIAN__
......@@ -65970,106 +68031,6 @@ __ai __attribute__((target("v8.3a,neon"))) float64x2_t vcmlaq_rot90_f64(float64x
6597068031 __ret = __builtin_shufflevector(__ret, __ret, 1, 0);
6597168032 return __ret;
6597268033}
65973__ai __attribute__((target("v8.3a,neon"))) float64x2_t __noswap_vcmlaq_rot90_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) {
65974 float64x2_t __ret;
65975 __ret = (float64x2_t) __builtin_neon_vcmlaq_rot90_f64((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42);
65976 return __ret;
65977}
65978#endif
65979
65980__ai __attribute__((target("v8.3a,neon"))) float64x1_t vcmla_rot90_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) {
65981 float64x1_t __ret;
65982 __ret = (float64x1_t) __builtin_neon_vcmla_rot90_f64((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10);
65983 return __ret;
65984}
65985#define vcmla_rot90_lane_f64(__p0_813, __p1_813, __p2_813, __p3_813) __extension__ ({ \
65986 float64x1_t __ret_813; \
65987 float64x1_t __s0_813 = __p0_813; \
65988 float64x1_t __s1_813 = __p1_813; \
65989 float64x1_t __s2_813 = __p2_813; \
65990float64x1_t __reint_813 = __s2_813; \
65991uint64x2_t __reint1_813 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_813, __p3_813), vgetq_lane_u64(*(uint64x2_t *) &__reint_813, __p3_813)}; \
65992 __ret_813 = vcmla_rot90_f64(__s0_813, __s1_813, *(float64x1_t *) &__reint1_813); \
65993 __ret_813; \
65994})
65995#ifdef __LITTLE_ENDIAN__
65996#define vcmlaq_rot90_lane_f64(__p0_814, __p1_814, __p2_814, __p3_814) __extension__ ({ \
65997 float64x2_t __ret_814; \
65998 float64x2_t __s0_814 = __p0_814; \
65999 float64x2_t __s1_814 = __p1_814; \
66000 float64x1_t __s2_814 = __p2_814; \
66001float64x1_t __reint_814 = __s2_814; \
66002uint64x2_t __reint1_814 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_814, __p3_814), vgetq_lane_u64(*(uint64x2_t *) &__reint_814, __p3_814)}; \
66003 __ret_814 = vcmlaq_rot90_f64(__s0_814, __s1_814, *(float64x2_t *) &__reint1_814); \
66004 __ret_814; \
66005})
66006#else
66007#define vcmlaq_rot90_lane_f64(__p0_815, __p1_815, __p2_815, __p3_815) __extension__ ({ \
66008 float64x2_t __ret_815; \
66009 float64x2_t __s0_815 = __p0_815; \
66010 float64x2_t __s1_815 = __p1_815; \
66011 float64x1_t __s2_815 = __p2_815; \
66012 float64x2_t __rev0_815; __rev0_815 = __builtin_shufflevector(__s0_815, __s0_815, 1, 0); \
66013 float64x2_t __rev1_815; __rev1_815 = __builtin_shufflevector(__s1_815, __s1_815, 1, 0); \
66014float64x1_t __reint_815 = __s2_815; \
66015uint64x2_t __reint1_815 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_815, __p3_815), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_815, __p3_815)}; \
66016 __ret_815 = __noswap_vcmlaq_rot90_f64(__rev0_815, __rev1_815, *(float64x2_t *) &__reint1_815); \
66017 __ret_815 = __builtin_shufflevector(__ret_815, __ret_815, 1, 0); \
66018 __ret_815; \
66019})
66020#endif
66021
66022#ifdef __LITTLE_ENDIAN__
66023#define vcmla_rot90_laneq_f64(__p0_816, __p1_816, __p2_816, __p3_816) __extension__ ({ \
66024 float64x1_t __ret_816; \
66025 float64x1_t __s0_816 = __p0_816; \
66026 float64x1_t __s1_816 = __p1_816; \
66027 float64x2_t __s2_816 = __p2_816; \
66028float64x2_t __reint_816 = __s2_816; \
66029uint64x2_t __reint1_816 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_816, __p3_816), vgetq_lane_u64(*(uint64x2_t *) &__reint_816, __p3_816)}; \
66030 __ret_816 = vcmla_rot90_f64(__s0_816, __s1_816, *(float64x1_t *) &__reint1_816); \
66031 __ret_816; \
66032})
66033#else
66034#define vcmla_rot90_laneq_f64(__p0_817, __p1_817, __p2_817, __p3_817) __extension__ ({ \
66035 float64x1_t __ret_817; \
66036 float64x1_t __s0_817 = __p0_817; \
66037 float64x1_t __s1_817 = __p1_817; \
66038 float64x2_t __s2_817 = __p2_817; \
66039 float64x2_t __rev2_817; __rev2_817 = __builtin_shufflevector(__s2_817, __s2_817, 1, 0); \
66040float64x2_t __reint_817 = __rev2_817; \
66041uint64x2_t __reint1_817 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_817, __p3_817), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_817, __p3_817)}; \
66042 __ret_817 = vcmla_rot90_f64(__s0_817, __s1_817, *(float64x1_t *) &__reint1_817); \
66043 __ret_817; \
66044})
66045#endif
66046
66047#ifdef __LITTLE_ENDIAN__
66048#define vcmlaq_rot90_laneq_f64(__p0_818, __p1_818, __p2_818, __p3_818) __extension__ ({ \
66049 float64x2_t __ret_818; \
66050 float64x2_t __s0_818 = __p0_818; \
66051 float64x2_t __s1_818 = __p1_818; \
66052 float64x2_t __s2_818 = __p2_818; \
66053float64x2_t __reint_818 = __s2_818; \
66054uint64x2_t __reint1_818 = (uint64x2_t) {vgetq_lane_u64(*(uint64x2_t *) &__reint_818, __p3_818), vgetq_lane_u64(*(uint64x2_t *) &__reint_818, __p3_818)}; \
66055 __ret_818 = vcmlaq_rot90_f64(__s0_818, __s1_818, *(float64x2_t *) &__reint1_818); \
66056 __ret_818; \
66057})
66058#else
66059#define vcmlaq_rot90_laneq_f64(__p0_819, __p1_819, __p2_819, __p3_819) __extension__ ({ \
66060 float64x2_t __ret_819; \
66061 float64x2_t __s0_819 = __p0_819; \
66062 float64x2_t __s1_819 = __p1_819; \
66063 float64x2_t __s2_819 = __p2_819; \
66064 float64x2_t __rev0_819; __rev0_819 = __builtin_shufflevector(__s0_819, __s0_819, 1, 0); \
66065 float64x2_t __rev1_819; __rev1_819 = __builtin_shufflevector(__s1_819, __s1_819, 1, 0); \
66066 float64x2_t __rev2_819; __rev2_819 = __builtin_shufflevector(__s2_819, __s2_819, 1, 0); \
66067float64x2_t __reint_819 = __rev2_819; \
66068uint64x2_t __reint1_819 = (uint64x2_t) {__noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_819, __p3_819), __noswap_vgetq_lane_u64(*(uint64x2_t *) &__reint_819, __p3_819)}; \
66069 __ret_819 = __noswap_vcmlaq_rot90_f64(__rev0_819, __rev1_819, *(float64x2_t *) &__reint1_819); \
66070 __ret_819 = __builtin_shufflevector(__ret_819, __ret_819, 1, 0); \
66071 __ret_819; \
66072})
6607368034#endif
6607468035
6607568036#ifdef __LITTLE_ENDIAN__
......@@ -66286,237 +68247,215 @@ __ai __attribute__((target("v8.5a,neon"))) float64x1_t vrnd64z_f64(float64x1_t _
6628668247}
6628768248#endif
6628868249#ifdef __LITTLE_ENDIAN__
66289#define vbfdotq_lane_f32(__p0_820, __p1_820, __p2_820, __p3_820) __extension__ ({ \
66290 float32x4_t __ret_820; \
66291 float32x4_t __s0_820 = __p0_820; \
66292 bfloat16x8_t __s1_820 = __p1_820; \
66293 bfloat16x4_t __s2_820 = __p2_820; \
66294bfloat16x4_t __reint_820 = __s2_820; \
66295float32x4_t __reint1_820 = splatq_lane_f32(*(float32x2_t *) &__reint_820, __p3_820); \
66296 __ret_820 = vbfdotq_f32(__s0_820, __s1_820, *(bfloat16x8_t *) &__reint1_820); \
66297 __ret_820; \
68250#define vbfdotq_lane_f32(__p0_792, __p1_792, __p2_792, __p3_792) __extension__ ({ \
68251 float32x4_t __ret_792; \
68252 float32x4_t __s0_792 = __p0_792; \
68253 bfloat16x8_t __s1_792 = __p1_792; \
68254 bfloat16x4_t __s2_792 = __p2_792; \
68255 __ret_792 = vbfdotq_f32(__s0_792, __s1_792, __builtin_bit_cast(bfloat16x8_t, splatq_lane_f32(__builtin_bit_cast(float32x2_t, __s2_792), __p3_792))); \
68256 __ret_792; \
6629868257})
6629968258#else
66300#define vbfdotq_lane_f32(__p0_821, __p1_821, __p2_821, __p3_821) __extension__ ({ \
66301 float32x4_t __ret_821; \
66302 float32x4_t __s0_821 = __p0_821; \
66303 bfloat16x8_t __s1_821 = __p1_821; \
66304 bfloat16x4_t __s2_821 = __p2_821; \
66305 float32x4_t __rev0_821; __rev0_821 = __builtin_shufflevector(__s0_821, __s0_821, 3, 2, 1, 0); \
66306 bfloat16x8_t __rev1_821; __rev1_821 = __builtin_shufflevector(__s1_821, __s1_821, 7, 6, 5, 4, 3, 2, 1, 0); \
66307 bfloat16x4_t __rev2_821; __rev2_821 = __builtin_shufflevector(__s2_821, __s2_821, 3, 2, 1, 0); \
66308bfloat16x4_t __reint_821 = __rev2_821; \
66309float32x4_t __reint1_821 = __noswap_splatq_lane_f32(*(float32x2_t *) &__reint_821, __p3_821); \
66310 __ret_821 = __noswap_vbfdotq_f32(__rev0_821, __rev1_821, *(bfloat16x8_t *) &__reint1_821); \
66311 __ret_821 = __builtin_shufflevector(__ret_821, __ret_821, 3, 2, 1, 0); \
66312 __ret_821; \
68259#define vbfdotq_lane_f32(__p0_793, __p1_793, __p2_793, __p3_793) __extension__ ({ \
68260 float32x4_t __ret_793; \
68261 float32x4_t __s0_793 = __p0_793; \
68262 bfloat16x8_t __s1_793 = __p1_793; \
68263 bfloat16x4_t __s2_793 = __p2_793; \
68264 float32x4_t __rev0_793; __rev0_793 = __builtin_shufflevector(__s0_793, __s0_793, 3, 2, 1, 0); \
68265 bfloat16x8_t __rev1_793; __rev1_793 = __builtin_shufflevector(__s1_793, __s1_793, 7, 6, 5, 4, 3, 2, 1, 0); \
68266 bfloat16x4_t __rev2_793; __rev2_793 = __builtin_shufflevector(__s2_793, __s2_793, 3, 2, 1, 0); \
68267 __ret_793 = __noswap_vbfdotq_f32(__rev0_793, __rev1_793, __builtin_bit_cast(bfloat16x8_t, __noswap_splatq_lane_f32(__builtin_bit_cast(float32x2_t, __rev2_793), __p3_793))); \
68268 __ret_793 = __builtin_shufflevector(__ret_793, __ret_793, 3, 2, 1, 0); \
68269 __ret_793; \
6631368270})
6631468271#endif
6631568272
6631668273#ifdef __LITTLE_ENDIAN__
66317#define vbfdot_lane_f32(__p0_822, __p1_822, __p2_822, __p3_822) __extension__ ({ \
66318 float32x2_t __ret_822; \
66319 float32x2_t __s0_822 = __p0_822; \
66320 bfloat16x4_t __s1_822 = __p1_822; \
66321 bfloat16x4_t __s2_822 = __p2_822; \
66322bfloat16x4_t __reint_822 = __s2_822; \
66323float32x2_t __reint1_822 = splat_lane_f32(*(float32x2_t *) &__reint_822, __p3_822); \
66324 __ret_822 = vbfdot_f32(__s0_822, __s1_822, *(bfloat16x4_t *) &__reint1_822); \
66325 __ret_822; \
68274#define vbfdot_lane_f32(__p0_794, __p1_794, __p2_794, __p3_794) __extension__ ({ \
68275 float32x2_t __ret_794; \
68276 float32x2_t __s0_794 = __p0_794; \
68277 bfloat16x4_t __s1_794 = __p1_794; \
68278 bfloat16x4_t __s2_794 = __p2_794; \
68279 __ret_794 = vbfdot_f32(__s0_794, __s1_794, __builtin_bit_cast(bfloat16x4_t, splat_lane_f32(__builtin_bit_cast(float32x2_t, __s2_794), __p3_794))); \
68280 __ret_794; \
6632668281})
6632768282#else
66328#define vbfdot_lane_f32(__p0_823, __p1_823, __p2_823, __p3_823) __extension__ ({ \
66329 float32x2_t __ret_823; \
66330 float32x2_t __s0_823 = __p0_823; \
66331 bfloat16x4_t __s1_823 = __p1_823; \
66332 bfloat16x4_t __s2_823 = __p2_823; \
66333 float32x2_t __rev0_823; __rev0_823 = __builtin_shufflevector(__s0_823, __s0_823, 1, 0); \
66334 bfloat16x4_t __rev1_823; __rev1_823 = __builtin_shufflevector(__s1_823, __s1_823, 3, 2, 1, 0); \
66335 bfloat16x4_t __rev2_823; __rev2_823 = __builtin_shufflevector(__s2_823, __s2_823, 3, 2, 1, 0); \
66336bfloat16x4_t __reint_823 = __rev2_823; \
66337float32x2_t __reint1_823 = __noswap_splat_lane_f32(*(float32x2_t *) &__reint_823, __p3_823); \
66338 __ret_823 = __noswap_vbfdot_f32(__rev0_823, __rev1_823, *(bfloat16x4_t *) &__reint1_823); \
66339 __ret_823 = __builtin_shufflevector(__ret_823, __ret_823, 1, 0); \
66340 __ret_823; \
68283#define vbfdot_lane_f32(__p0_795, __p1_795, __p2_795, __p3_795) __extension__ ({ \
68284 float32x2_t __ret_795; \
68285 float32x2_t __s0_795 = __p0_795; \
68286 bfloat16x4_t __s1_795 = __p1_795; \
68287 bfloat16x4_t __s2_795 = __p2_795; \
68288 float32x2_t __rev0_795; __rev0_795 = __builtin_shufflevector(__s0_795, __s0_795, 1, 0); \
68289 bfloat16x4_t __rev1_795; __rev1_795 = __builtin_shufflevector(__s1_795, __s1_795, 3, 2, 1, 0); \
68290 bfloat16x4_t __rev2_795; __rev2_795 = __builtin_shufflevector(__s2_795, __s2_795, 3, 2, 1, 0); \
68291 __ret_795 = __noswap_vbfdot_f32(__rev0_795, __rev1_795, __builtin_bit_cast(bfloat16x4_t, __noswap_splat_lane_f32(__builtin_bit_cast(float32x2_t, __rev2_795), __p3_795))); \
68292 __ret_795 = __builtin_shufflevector(__ret_795, __ret_795, 1, 0); \
68293 __ret_795; \
6634168294})
6634268295#endif
6634368296
6634468297#ifdef __LITTLE_ENDIAN__
66345#define vbfdotq_laneq_f32(__p0_824, __p1_824, __p2_824, __p3_824) __extension__ ({ \
66346 float32x4_t __ret_824; \
66347 float32x4_t __s0_824 = __p0_824; \
66348 bfloat16x8_t __s1_824 = __p1_824; \
66349 bfloat16x8_t __s2_824 = __p2_824; \
66350bfloat16x8_t __reint_824 = __s2_824; \
66351float32x4_t __reint1_824 = splatq_laneq_f32(*(float32x4_t *) &__reint_824, __p3_824); \
66352 __ret_824 = vbfdotq_f32(__s0_824, __s1_824, *(bfloat16x8_t *) &__reint1_824); \
66353 __ret_824; \
68298#define vbfdotq_laneq_f32(__p0_796, __p1_796, __p2_796, __p3_796) __extension__ ({ \
68299 float32x4_t __ret_796; \
68300 float32x4_t __s0_796 = __p0_796; \
68301 bfloat16x8_t __s1_796 = __p1_796; \
68302 bfloat16x8_t __s2_796 = __p2_796; \
68303 __ret_796 = vbfdotq_f32(__s0_796, __s1_796, __builtin_bit_cast(bfloat16x8_t, splatq_laneq_f32(__builtin_bit_cast(float32x4_t, __s2_796), __p3_796))); \
68304 __ret_796; \
6635468305})
6635568306#else
66356#define vbfdotq_laneq_f32(__p0_825, __p1_825, __p2_825, __p3_825) __extension__ ({ \
66357 float32x4_t __ret_825; \
66358 float32x4_t __s0_825 = __p0_825; \
66359 bfloat16x8_t __s1_825 = __p1_825; \
66360 bfloat16x8_t __s2_825 = __p2_825; \
66361 float32x4_t __rev0_825; __rev0_825 = __builtin_shufflevector(__s0_825, __s0_825, 3, 2, 1, 0); \
66362 bfloat16x8_t __rev1_825; __rev1_825 = __builtin_shufflevector(__s1_825, __s1_825, 7, 6, 5, 4, 3, 2, 1, 0); \
66363 bfloat16x8_t __rev2_825; __rev2_825 = __builtin_shufflevector(__s2_825, __s2_825, 7, 6, 5, 4, 3, 2, 1, 0); \
66364bfloat16x8_t __reint_825 = __rev2_825; \
66365float32x4_t __reint1_825 = __noswap_splatq_laneq_f32(*(float32x4_t *) &__reint_825, __p3_825); \
66366 __ret_825 = __noswap_vbfdotq_f32(__rev0_825, __rev1_825, *(bfloat16x8_t *) &__reint1_825); \
66367 __ret_825 = __builtin_shufflevector(__ret_825, __ret_825, 3, 2, 1, 0); \
66368 __ret_825; \
68307#define vbfdotq_laneq_f32(__p0_797, __p1_797, __p2_797, __p3_797) __extension__ ({ \
68308 float32x4_t __ret_797; \
68309 float32x4_t __s0_797 = __p0_797; \
68310 bfloat16x8_t __s1_797 = __p1_797; \
68311 bfloat16x8_t __s2_797 = __p2_797; \
68312 float32x4_t __rev0_797; __rev0_797 = __builtin_shufflevector(__s0_797, __s0_797, 3, 2, 1, 0); \
68313 bfloat16x8_t __rev1_797; __rev1_797 = __builtin_shufflevector(__s1_797, __s1_797, 7, 6, 5, 4, 3, 2, 1, 0); \
68314 bfloat16x8_t __rev2_797; __rev2_797 = __builtin_shufflevector(__s2_797, __s2_797, 7, 6, 5, 4, 3, 2, 1, 0); \
68315 __ret_797 = __noswap_vbfdotq_f32(__rev0_797, __rev1_797, __builtin_bit_cast(bfloat16x8_t, __noswap_splatq_laneq_f32(__builtin_bit_cast(float32x4_t, __rev2_797), __p3_797))); \
68316 __ret_797 = __builtin_shufflevector(__ret_797, __ret_797, 3, 2, 1, 0); \
68317 __ret_797; \
6636968318})
6637068319#endif
6637168320
6637268321#ifdef __LITTLE_ENDIAN__
66373#define vbfdot_laneq_f32(__p0_826, __p1_826, __p2_826, __p3_826) __extension__ ({ \
66374 float32x2_t __ret_826; \
66375 float32x2_t __s0_826 = __p0_826; \
66376 bfloat16x4_t __s1_826 = __p1_826; \
66377 bfloat16x8_t __s2_826 = __p2_826; \
66378bfloat16x8_t __reint_826 = __s2_826; \
66379float32x2_t __reint1_826 = splat_laneq_f32(*(float32x4_t *) &__reint_826, __p3_826); \
66380 __ret_826 = vbfdot_f32(__s0_826, __s1_826, *(bfloat16x4_t *) &__reint1_826); \
66381 __ret_826; \
68322#define vbfdot_laneq_f32(__p0_798, __p1_798, __p2_798, __p3_798) __extension__ ({ \
68323 float32x2_t __ret_798; \
68324 float32x2_t __s0_798 = __p0_798; \
68325 bfloat16x4_t __s1_798 = __p1_798; \
68326 bfloat16x8_t __s2_798 = __p2_798; \
68327 __ret_798 = vbfdot_f32(__s0_798, __s1_798, __builtin_bit_cast(bfloat16x4_t, splat_laneq_f32(__builtin_bit_cast(float32x4_t, __s2_798), __p3_798))); \
68328 __ret_798; \
6638268329})
6638368330#else
66384#define vbfdot_laneq_f32(__p0_827, __p1_827, __p2_827, __p3_827) __extension__ ({ \
66385 float32x2_t __ret_827; \
66386 float32x2_t __s0_827 = __p0_827; \
66387 bfloat16x4_t __s1_827 = __p1_827; \
66388 bfloat16x8_t __s2_827 = __p2_827; \
66389 float32x2_t __rev0_827; __rev0_827 = __builtin_shufflevector(__s0_827, __s0_827, 1, 0); \
66390 bfloat16x4_t __rev1_827; __rev1_827 = __builtin_shufflevector(__s1_827, __s1_827, 3, 2, 1, 0); \
66391 bfloat16x8_t __rev2_827; __rev2_827 = __builtin_shufflevector(__s2_827, __s2_827, 7, 6, 5, 4, 3, 2, 1, 0); \
66392bfloat16x8_t __reint_827 = __rev2_827; \
66393float32x2_t __reint1_827 = __noswap_splat_laneq_f32(*(float32x4_t *) &__reint_827, __p3_827); \
66394 __ret_827 = __noswap_vbfdot_f32(__rev0_827, __rev1_827, *(bfloat16x4_t *) &__reint1_827); \
66395 __ret_827 = __builtin_shufflevector(__ret_827, __ret_827, 1, 0); \
66396 __ret_827; \
68331#define vbfdot_laneq_f32(__p0_799, __p1_799, __p2_799, __p3_799) __extension__ ({ \
68332 float32x2_t __ret_799; \
68333 float32x2_t __s0_799 = __p0_799; \
68334 bfloat16x4_t __s1_799 = __p1_799; \
68335 bfloat16x8_t __s2_799 = __p2_799; \
68336 float32x2_t __rev0_799; __rev0_799 = __builtin_shufflevector(__s0_799, __s0_799, 1, 0); \
68337 bfloat16x4_t __rev1_799; __rev1_799 = __builtin_shufflevector(__s1_799, __s1_799, 3, 2, 1, 0); \
68338 bfloat16x8_t __rev2_799; __rev2_799 = __builtin_shufflevector(__s2_799, __s2_799, 7, 6, 5, 4, 3, 2, 1, 0); \
68339 __ret_799 = __noswap_vbfdot_f32(__rev0_799, __rev1_799, __builtin_bit_cast(bfloat16x4_t, __noswap_splat_laneq_f32(__builtin_bit_cast(float32x4_t, __rev2_799), __p3_799))); \
68340 __ret_799 = __builtin_shufflevector(__ret_799, __ret_799, 1, 0); \
68341 __ret_799; \
6639768342})
6639868343#endif
6639968344
6640068345#ifdef __LITTLE_ENDIAN__
66401#define vbfmlalbq_lane_f32(__p0_828, __p1_828, __p2_828, __p3_828) __extension__ ({ \
66402 float32x4_t __ret_828; \
66403 float32x4_t __s0_828 = __p0_828; \
66404 bfloat16x8_t __s1_828 = __p1_828; \
66405 bfloat16x4_t __s2_828 = __p2_828; \
66406 __ret_828 = vbfmlalbq_f32(__s0_828, __s1_828, (bfloat16x8_t) {vget_lane_bf16(__s2_828, __p3_828), vget_lane_bf16(__s2_828, __p3_828), vget_lane_bf16(__s2_828, __p3_828), vget_lane_bf16(__s2_828, __p3_828), vget_lane_bf16(__s2_828, __p3_828), vget_lane_bf16(__s2_828, __p3_828), vget_lane_bf16(__s2_828, __p3_828), vget_lane_bf16(__s2_828, __p3_828)}); \
66407 __ret_828; \
68346#define vbfmlalbq_lane_f32(__p0_800, __p1_800, __p2_800, __p3_800) __extension__ ({ \
68347 float32x4_t __ret_800; \
68348 float32x4_t __s0_800 = __p0_800; \
68349 bfloat16x8_t __s1_800 = __p1_800; \
68350 bfloat16x4_t __s2_800 = __p2_800; \
68351 __ret_800 = vbfmlalbq_f32(__s0_800, __s1_800, (bfloat16x8_t) {vget_lane_bf16(__s2_800, __p3_800), vget_lane_bf16(__s2_800, __p3_800), vget_lane_bf16(__s2_800, __p3_800), vget_lane_bf16(__s2_800, __p3_800), vget_lane_bf16(__s2_800, __p3_800), vget_lane_bf16(__s2_800, __p3_800), vget_lane_bf16(__s2_800, __p3_800), vget_lane_bf16(__s2_800, __p3_800)}); \
68352 __ret_800; \
6640868353})
6640968354#else
66410#define vbfmlalbq_lane_f32(__p0_829, __p1_829, __p2_829, __p3_829) __extension__ ({ \
66411 float32x4_t __ret_829; \
66412 float32x4_t __s0_829 = __p0_829; \
66413 bfloat16x8_t __s1_829 = __p1_829; \
66414 bfloat16x4_t __s2_829 = __p2_829; \
66415 float32x4_t __rev0_829; __rev0_829 = __builtin_shufflevector(__s0_829, __s0_829, 3, 2, 1, 0); \
66416 bfloat16x8_t __rev1_829; __rev1_829 = __builtin_shufflevector(__s1_829, __s1_829, 7, 6, 5, 4, 3, 2, 1, 0); \
66417 bfloat16x4_t __rev2_829; __rev2_829 = __builtin_shufflevector(__s2_829, __s2_829, 3, 2, 1, 0); \
66418 __ret_829 = __noswap_vbfmlalbq_f32(__rev0_829, __rev1_829, (bfloat16x8_t) {__noswap_vget_lane_bf16(__rev2_829, __p3_829), __noswap_vget_lane_bf16(__rev2_829, __p3_829), __noswap_vget_lane_bf16(__rev2_829, __p3_829), __noswap_vget_lane_bf16(__rev2_829, __p3_829), __noswap_vget_lane_bf16(__rev2_829, __p3_829), __noswap_vget_lane_bf16(__rev2_829, __p3_829), __noswap_vget_lane_bf16(__rev2_829, __p3_829), __noswap_vget_lane_bf16(__rev2_829, __p3_829)}); \
66419 __ret_829 = __builtin_shufflevector(__ret_829, __ret_829, 3, 2, 1, 0); \
66420 __ret_829; \
68355#define vbfmlalbq_lane_f32(__p0_801, __p1_801, __p2_801, __p3_801) __extension__ ({ \
68356 float32x4_t __ret_801; \
68357 float32x4_t __s0_801 = __p0_801; \
68358 bfloat16x8_t __s1_801 = __p1_801; \
68359 bfloat16x4_t __s2_801 = __p2_801; \
68360 float32x4_t __rev0_801; __rev0_801 = __builtin_shufflevector(__s0_801, __s0_801, 3, 2, 1, 0); \
68361 bfloat16x8_t __rev1_801; __rev1_801 = __builtin_shufflevector(__s1_801, __s1_801, 7, 6, 5, 4, 3, 2, 1, 0); \
68362 bfloat16x4_t __rev2_801; __rev2_801 = __builtin_shufflevector(__s2_801, __s2_801, 3, 2, 1, 0); \
68363 __ret_801 = __noswap_vbfmlalbq_f32(__rev0_801, __rev1_801, (bfloat16x8_t) {__noswap_vget_lane_bf16(__rev2_801, __p3_801), __noswap_vget_lane_bf16(__rev2_801, __p3_801), __noswap_vget_lane_bf16(__rev2_801, __p3_801), __noswap_vget_lane_bf16(__rev2_801, __p3_801), __noswap_vget_lane_bf16(__rev2_801, __p3_801), __noswap_vget_lane_bf16(__rev2_801, __p3_801), __noswap_vget_lane_bf16(__rev2_801, __p3_801), __noswap_vget_lane_bf16(__rev2_801, __p3_801)}); \
68364 __ret_801 = __builtin_shufflevector(__ret_801, __ret_801, 3, 2, 1, 0); \
68365 __ret_801; \
6642168366})
6642268367#endif
6642368368
6642468369#ifdef __LITTLE_ENDIAN__
66425#define vbfmlalbq_laneq_f32(__p0_830, __p1_830, __p2_830, __p3_830) __extension__ ({ \
66426 float32x4_t __ret_830; \
66427 float32x4_t __s0_830 = __p0_830; \
66428 bfloat16x8_t __s1_830 = __p1_830; \
66429 bfloat16x8_t __s2_830 = __p2_830; \
66430 __ret_830 = vbfmlalbq_f32(__s0_830, __s1_830, (bfloat16x8_t) {vgetq_lane_bf16(__s2_830, __p3_830), vgetq_lane_bf16(__s2_830, __p3_830), vgetq_lane_bf16(__s2_830, __p3_830), vgetq_lane_bf16(__s2_830, __p3_830), vgetq_lane_bf16(__s2_830, __p3_830), vgetq_lane_bf16(__s2_830, __p3_830), vgetq_lane_bf16(__s2_830, __p3_830), vgetq_lane_bf16(__s2_830, __p3_830)}); \
66431 __ret_830; \
68370#define vbfmlalbq_laneq_f32(__p0_802, __p1_802, __p2_802, __p3_802) __extension__ ({ \
68371 float32x4_t __ret_802; \
68372 float32x4_t __s0_802 = __p0_802; \
68373 bfloat16x8_t __s1_802 = __p1_802; \
68374 bfloat16x8_t __s2_802 = __p2_802; \
68375 __ret_802 = vbfmlalbq_f32(__s0_802, __s1_802, (bfloat16x8_t) {vgetq_lane_bf16(__s2_802, __p3_802), vgetq_lane_bf16(__s2_802, __p3_802), vgetq_lane_bf16(__s2_802, __p3_802), vgetq_lane_bf16(__s2_802, __p3_802), vgetq_lane_bf16(__s2_802, __p3_802), vgetq_lane_bf16(__s2_802, __p3_802), vgetq_lane_bf16(__s2_802, __p3_802), vgetq_lane_bf16(__s2_802, __p3_802)}); \
68376 __ret_802; \
6643268377})
6643368378#else
66434#define vbfmlalbq_laneq_f32(__p0_831, __p1_831, __p2_831, __p3_831) __extension__ ({ \
66435 float32x4_t __ret_831; \
66436 float32x4_t __s0_831 = __p0_831; \
66437 bfloat16x8_t __s1_831 = __p1_831; \
66438 bfloat16x8_t __s2_831 = __p2_831; \
66439 float32x4_t __rev0_831; __rev0_831 = __builtin_shufflevector(__s0_831, __s0_831, 3, 2, 1, 0); \
66440 bfloat16x8_t __rev1_831; __rev1_831 = __builtin_shufflevector(__s1_831, __s1_831, 7, 6, 5, 4, 3, 2, 1, 0); \
66441 bfloat16x8_t __rev2_831; __rev2_831 = __builtin_shufflevector(__s2_831, __s2_831, 7, 6, 5, 4, 3, 2, 1, 0); \
66442 __ret_831 = __noswap_vbfmlalbq_f32(__rev0_831, __rev1_831, (bfloat16x8_t) {__noswap_vgetq_lane_bf16(__rev2_831, __p3_831), __noswap_vgetq_lane_bf16(__rev2_831, __p3_831), __noswap_vgetq_lane_bf16(__rev2_831, __p3_831), __noswap_vgetq_lane_bf16(__rev2_831, __p3_831), __noswap_vgetq_lane_bf16(__rev2_831, __p3_831), __noswap_vgetq_lane_bf16(__rev2_831, __p3_831), __noswap_vgetq_lane_bf16(__rev2_831, __p3_831), __noswap_vgetq_lane_bf16(__rev2_831, __p3_831)}); \
66443 __ret_831 = __builtin_shufflevector(__ret_831, __ret_831, 3, 2, 1, 0); \
66444 __ret_831; \
68379#define vbfmlalbq_laneq_f32(__p0_803, __p1_803, __p2_803, __p3_803) __extension__ ({ \
68380 float32x4_t __ret_803; \
68381 float32x4_t __s0_803 = __p0_803; \
68382 bfloat16x8_t __s1_803 = __p1_803; \
68383 bfloat16x8_t __s2_803 = __p2_803; \
68384 float32x4_t __rev0_803; __rev0_803 = __builtin_shufflevector(__s0_803, __s0_803, 3, 2, 1, 0); \
68385 bfloat16x8_t __rev1_803; __rev1_803 = __builtin_shufflevector(__s1_803, __s1_803, 7, 6, 5, 4, 3, 2, 1, 0); \
68386 bfloat16x8_t __rev2_803; __rev2_803 = __builtin_shufflevector(__s2_803, __s2_803, 7, 6, 5, 4, 3, 2, 1, 0); \
68387 __ret_803 = __noswap_vbfmlalbq_f32(__rev0_803, __rev1_803, (bfloat16x8_t) {__noswap_vgetq_lane_bf16(__rev2_803, __p3_803), __noswap_vgetq_lane_bf16(__rev2_803, __p3_803), __noswap_vgetq_lane_bf16(__rev2_803, __p3_803), __noswap_vgetq_lane_bf16(__rev2_803, __p3_803), __noswap_vgetq_lane_bf16(__rev2_803, __p3_803), __noswap_vgetq_lane_bf16(__rev2_803, __p3_803), __noswap_vgetq_lane_bf16(__rev2_803, __p3_803), __noswap_vgetq_lane_bf16(__rev2_803, __p3_803)}); \
68388 __ret_803 = __builtin_shufflevector(__ret_803, __ret_803, 3, 2, 1, 0); \
68389 __ret_803; \
6644568390})
6644668391#endif
6644768392
6644868393#ifdef __LITTLE_ENDIAN__
66449#define vbfmlaltq_lane_f32(__p0_832, __p1_832, __p2_832, __p3_832) __extension__ ({ \
66450 float32x4_t __ret_832; \
66451 float32x4_t __s0_832 = __p0_832; \
66452 bfloat16x8_t __s1_832 = __p1_832; \
66453 bfloat16x4_t __s2_832 = __p2_832; \
66454 __ret_832 = vbfmlaltq_f32(__s0_832, __s1_832, (bfloat16x8_t) {vget_lane_bf16(__s2_832, __p3_832), vget_lane_bf16(__s2_832, __p3_832), vget_lane_bf16(__s2_832, __p3_832), vget_lane_bf16(__s2_832, __p3_832), vget_lane_bf16(__s2_832, __p3_832), vget_lane_bf16(__s2_832, __p3_832), vget_lane_bf16(__s2_832, __p3_832), vget_lane_bf16(__s2_832, __p3_832)}); \
66455 __ret_832; \
68394#define vbfmlaltq_lane_f32(__p0_804, __p1_804, __p2_804, __p3_804) __extension__ ({ \
68395 float32x4_t __ret_804; \
68396 float32x4_t __s0_804 = __p0_804; \
68397 bfloat16x8_t __s1_804 = __p1_804; \
68398 bfloat16x4_t __s2_804 = __p2_804; \
68399 __ret_804 = vbfmlaltq_f32(__s0_804, __s1_804, (bfloat16x8_t) {vget_lane_bf16(__s2_804, __p3_804), vget_lane_bf16(__s2_804, __p3_804), vget_lane_bf16(__s2_804, __p3_804), vget_lane_bf16(__s2_804, __p3_804), vget_lane_bf16(__s2_804, __p3_804), vget_lane_bf16(__s2_804, __p3_804), vget_lane_bf16(__s2_804, __p3_804), vget_lane_bf16(__s2_804, __p3_804)}); \
68400 __ret_804; \
6645668401})
6645768402#else
66458#define vbfmlaltq_lane_f32(__p0_833, __p1_833, __p2_833, __p3_833) __extension__ ({ \
66459 float32x4_t __ret_833; \
66460 float32x4_t __s0_833 = __p0_833; \
66461 bfloat16x8_t __s1_833 = __p1_833; \
66462 bfloat16x4_t __s2_833 = __p2_833; \
66463 float32x4_t __rev0_833; __rev0_833 = __builtin_shufflevector(__s0_833, __s0_833, 3, 2, 1, 0); \
66464 bfloat16x8_t __rev1_833; __rev1_833 = __builtin_shufflevector(__s1_833, __s1_833, 7, 6, 5, 4, 3, 2, 1, 0); \
66465 bfloat16x4_t __rev2_833; __rev2_833 = __builtin_shufflevector(__s2_833, __s2_833, 3, 2, 1, 0); \
66466 __ret_833 = __noswap_vbfmlaltq_f32(__rev0_833, __rev1_833, (bfloat16x8_t) {__noswap_vget_lane_bf16(__rev2_833, __p3_833), __noswap_vget_lane_bf16(__rev2_833, __p3_833), __noswap_vget_lane_bf16(__rev2_833, __p3_833), __noswap_vget_lane_bf16(__rev2_833, __p3_833), __noswap_vget_lane_bf16(__rev2_833, __p3_833), __noswap_vget_lane_bf16(__rev2_833, __p3_833), __noswap_vget_lane_bf16(__rev2_833, __p3_833), __noswap_vget_lane_bf16(__rev2_833, __p3_833)}); \
66467 __ret_833 = __builtin_shufflevector(__ret_833, __ret_833, 3, 2, 1, 0); \
66468 __ret_833; \
68403#define vbfmlaltq_lane_f32(__p0_805, __p1_805, __p2_805, __p3_805) __extension__ ({ \
68404 float32x4_t __ret_805; \
68405 float32x4_t __s0_805 = __p0_805; \
68406 bfloat16x8_t __s1_805 = __p1_805; \
68407 bfloat16x4_t __s2_805 = __p2_805; \
68408 float32x4_t __rev0_805; __rev0_805 = __builtin_shufflevector(__s0_805, __s0_805, 3, 2, 1, 0); \
68409 bfloat16x8_t __rev1_805; __rev1_805 = __builtin_shufflevector(__s1_805, __s1_805, 7, 6, 5, 4, 3, 2, 1, 0); \
68410 bfloat16x4_t __rev2_805; __rev2_805 = __builtin_shufflevector(__s2_805, __s2_805, 3, 2, 1, 0); \
68411 __ret_805 = __noswap_vbfmlaltq_f32(__rev0_805, __rev1_805, (bfloat16x8_t) {__noswap_vget_lane_bf16(__rev2_805, __p3_805), __noswap_vget_lane_bf16(__rev2_805, __p3_805), __noswap_vget_lane_bf16(__rev2_805, __p3_805), __noswap_vget_lane_bf16(__rev2_805, __p3_805), __noswap_vget_lane_bf16(__rev2_805, __p3_805), __noswap_vget_lane_bf16(__rev2_805, __p3_805), __noswap_vget_lane_bf16(__rev2_805, __p3_805), __noswap_vget_lane_bf16(__rev2_805, __p3_805)}); \
68412 __ret_805 = __builtin_shufflevector(__ret_805, __ret_805, 3, 2, 1, 0); \
68413 __ret_805; \
6646968414})
6647068415#endif
6647168416
6647268417#ifdef __LITTLE_ENDIAN__
66473#define vbfmlaltq_laneq_f32(__p0_834, __p1_834, __p2_834, __p3_834) __extension__ ({ \
66474 float32x4_t __ret_834; \
66475 float32x4_t __s0_834 = __p0_834; \
66476 bfloat16x8_t __s1_834 = __p1_834; \
66477 bfloat16x8_t __s2_834 = __p2_834; \
66478 __ret_834 = vbfmlaltq_f32(__s0_834, __s1_834, (bfloat16x8_t) {vgetq_lane_bf16(__s2_834, __p3_834), vgetq_lane_bf16(__s2_834, __p3_834), vgetq_lane_bf16(__s2_834, __p3_834), vgetq_lane_bf16(__s2_834, __p3_834), vgetq_lane_bf16(__s2_834, __p3_834), vgetq_lane_bf16(__s2_834, __p3_834), vgetq_lane_bf16(__s2_834, __p3_834), vgetq_lane_bf16(__s2_834, __p3_834)}); \
66479 __ret_834; \
68418#define vbfmlaltq_laneq_f32(__p0_806, __p1_806, __p2_806, __p3_806) __extension__ ({ \
68419 float32x4_t __ret_806; \
68420 float32x4_t __s0_806 = __p0_806; \
68421 bfloat16x8_t __s1_806 = __p1_806; \
68422 bfloat16x8_t __s2_806 = __p2_806; \
68423 __ret_806 = vbfmlaltq_f32(__s0_806, __s1_806, (bfloat16x8_t) {vgetq_lane_bf16(__s2_806, __p3_806), vgetq_lane_bf16(__s2_806, __p3_806), vgetq_lane_bf16(__s2_806, __p3_806), vgetq_lane_bf16(__s2_806, __p3_806), vgetq_lane_bf16(__s2_806, __p3_806), vgetq_lane_bf16(__s2_806, __p3_806), vgetq_lane_bf16(__s2_806, __p3_806), vgetq_lane_bf16(__s2_806, __p3_806)}); \
68424 __ret_806; \
6648068425})
6648168426#else
66482#define vbfmlaltq_laneq_f32(__p0_835, __p1_835, __p2_835, __p3_835) __extension__ ({ \
66483 float32x4_t __ret_835; \
66484 float32x4_t __s0_835 = __p0_835; \
66485 bfloat16x8_t __s1_835 = __p1_835; \
66486 bfloat16x8_t __s2_835 = __p2_835; \
66487 float32x4_t __rev0_835; __rev0_835 = __builtin_shufflevector(__s0_835, __s0_835, 3, 2, 1, 0); \
66488 bfloat16x8_t __rev1_835; __rev1_835 = __builtin_shufflevector(__s1_835, __s1_835, 7, 6, 5, 4, 3, 2, 1, 0); \
66489 bfloat16x8_t __rev2_835; __rev2_835 = __builtin_shufflevector(__s2_835, __s2_835, 7, 6, 5, 4, 3, 2, 1, 0); \
66490 __ret_835 = __noswap_vbfmlaltq_f32(__rev0_835, __rev1_835, (bfloat16x8_t) {__noswap_vgetq_lane_bf16(__rev2_835, __p3_835), __noswap_vgetq_lane_bf16(__rev2_835, __p3_835), __noswap_vgetq_lane_bf16(__rev2_835, __p3_835), __noswap_vgetq_lane_bf16(__rev2_835, __p3_835), __noswap_vgetq_lane_bf16(__rev2_835, __p3_835), __noswap_vgetq_lane_bf16(__rev2_835, __p3_835), __noswap_vgetq_lane_bf16(__rev2_835, __p3_835), __noswap_vgetq_lane_bf16(__rev2_835, __p3_835)}); \
66491 __ret_835 = __builtin_shufflevector(__ret_835, __ret_835, 3, 2, 1, 0); \
66492 __ret_835; \
68427#define vbfmlaltq_laneq_f32(__p0_807, __p1_807, __p2_807, __p3_807) __extension__ ({ \
68428 float32x4_t __ret_807; \
68429 float32x4_t __s0_807 = __p0_807; \
68430 bfloat16x8_t __s1_807 = __p1_807; \
68431 bfloat16x8_t __s2_807 = __p2_807; \
68432 float32x4_t __rev0_807; __rev0_807 = __builtin_shufflevector(__s0_807, __s0_807, 3, 2, 1, 0); \
68433 bfloat16x8_t __rev1_807; __rev1_807 = __builtin_shufflevector(__s1_807, __s1_807, 7, 6, 5, 4, 3, 2, 1, 0); \
68434 bfloat16x8_t __rev2_807; __rev2_807 = __builtin_shufflevector(__s2_807, __s2_807, 7, 6, 5, 4, 3, 2, 1, 0); \
68435 __ret_807 = __noswap_vbfmlaltq_f32(__rev0_807, __rev1_807, (bfloat16x8_t) {__noswap_vgetq_lane_bf16(__rev2_807, __p3_807), __noswap_vgetq_lane_bf16(__rev2_807, __p3_807), __noswap_vgetq_lane_bf16(__rev2_807, __p3_807), __noswap_vgetq_lane_bf16(__rev2_807, __p3_807), __noswap_vgetq_lane_bf16(__rev2_807, __p3_807), __noswap_vgetq_lane_bf16(__rev2_807, __p3_807), __noswap_vgetq_lane_bf16(__rev2_807, __p3_807), __noswap_vgetq_lane_bf16(__rev2_807, __p3_807)}); \
68436 __ret_807 = __builtin_shufflevector(__ret_807, __ret_807, 3, 2, 1, 0); \
68437 __ret_807; \
6649368438})
6649468439#endif
6649568440
6649668441#ifdef __LITTLE_ENDIAN__
66497__ai __attribute__((target("bf16,neon"))) float32x4_t vcvt_f32_bf16(bfloat16x4_t __p0_836) {
66498 float32x4_t __ret_836;
66499bfloat16x4_t __reint_836 = __p0_836;
66500int32x4_t __reint1_836 = vshll_n_s16(*(int16x4_t *) &__reint_836, 16);
66501 __ret_836 = *(float32x4_t *) &__reint1_836;
66502 return __ret_836;
68442__ai __attribute__((target("bf16,neon"))) float32x4_t vcvt_f32_bf16(bfloat16x4_t __p0_808) {
68443 float32x4_t __ret_808;
68444 __ret_808 = __builtin_bit_cast(float32x4_t, vshll_n_u16(__builtin_bit_cast(uint16x4_t, __p0_808), 16));
68445 return __ret_808;
6650368446}
6650468447#else
66505__ai __attribute__((target("bf16,neon"))) float32x4_t vcvt_f32_bf16(bfloat16x4_t __p0_837) {
66506 float32x4_t __ret_837;
66507 bfloat16x4_t __rev0_837; __rev0_837 = __builtin_shufflevector(__p0_837, __p0_837, 3, 2, 1, 0);
66508bfloat16x4_t __reint_837 = __rev0_837;
66509int32x4_t __reint1_837 = __noswap_vshll_n_s16(*(int16x4_t *) &__reint_837, 16);
66510 __ret_837 = *(float32x4_t *) &__reint1_837;
66511 __ret_837 = __builtin_shufflevector(__ret_837, __ret_837, 3, 2, 1, 0);
66512 return __ret_837;
68448__ai __attribute__((target("bf16,neon"))) float32x4_t vcvt_f32_bf16(bfloat16x4_t __p0_809) {
68449 float32x4_t __ret_809;
68450 bfloat16x4_t __rev0_809; __rev0_809 = __builtin_shufflevector(__p0_809, __p0_809, 3, 2, 1, 0);
68451 __ret_809 = __builtin_bit_cast(float32x4_t, __noswap_vshll_n_u16(__builtin_bit_cast(uint16x4_t, __rev0_809), 16));
68452 __ret_809 = __builtin_shufflevector(__ret_809, __ret_809, 3, 2, 1, 0);
68453 return __ret_809;
6651368454}
66514__ai __attribute__((target("bf16,neon"))) float32x4_t __noswap_vcvt_f32_bf16(bfloat16x4_t __p0_838) {
66515 float32x4_t __ret_838;
66516bfloat16x4_t __reint_838 = __p0_838;
66517int32x4_t __reint1_838 = __noswap_vshll_n_s16(*(int16x4_t *) &__reint_838, 16);
66518 __ret_838 = *(float32x4_t *) &__reint1_838;
66519 return __ret_838;
68455__ai __attribute__((target("bf16,neon"))) float32x4_t __noswap_vcvt_f32_bf16(bfloat16x4_t __p0_810) {
68456 float32x4_t __ret_810;
68457 __ret_810 = __builtin_bit_cast(float32x4_t, __noswap_vshll_n_u16(__builtin_bit_cast(uint16x4_t, __p0_810), 16));
68458 return __ret_810;
6652068459}
6652168460#endif
6652268461
......@@ -66553,260 +68492,236 @@ __ai __attribute__((target("bf16,neon"))) float32x4_t vcvtq_low_f32_bf16(bfloat1
6655368492#endif
6655468493
6655568494#ifdef __LITTLE_ENDIAN__
66556#define vdotq_lane_u32(__p0_839, __p1_839, __p2_839, __p3_839) __extension__ ({ \
66557 uint32x4_t __ret_839; \
66558 uint32x4_t __s0_839 = __p0_839; \
66559 uint8x16_t __s1_839 = __p1_839; \
66560 uint8x8_t __s2_839 = __p2_839; \
66561uint8x8_t __reint_839 = __s2_839; \
66562uint32x4_t __reint1_839 = splatq_lane_u32(*(uint32x2_t *) &__reint_839, __p3_839); \
66563 __ret_839 = vdotq_u32(__s0_839, __s1_839, *(uint8x16_t *) &__reint1_839); \
66564 __ret_839; \
68495#define vdotq_lane_u32(__p0_811, __p1_811, __p2_811, __p3_811) __extension__ ({ \
68496 uint32x4_t __ret_811; \
68497 uint32x4_t __s0_811 = __p0_811; \
68498 uint8x16_t __s1_811 = __p1_811; \
68499 uint8x8_t __s2_811 = __p2_811; \
68500 __ret_811 = vdotq_u32(__s0_811, __s1_811, __builtin_bit_cast(uint8x16_t, splatq_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_811), __p3_811))); \
68501 __ret_811; \
6656568502})
6656668503#else
66567#define vdotq_lane_u32(__p0_840, __p1_840, __p2_840, __p3_840) __extension__ ({ \
66568 uint32x4_t __ret_840; \
66569 uint32x4_t __s0_840 = __p0_840; \
66570 uint8x16_t __s1_840 = __p1_840; \
66571 uint8x8_t __s2_840 = __p2_840; \
66572 uint32x4_t __rev0_840; __rev0_840 = __builtin_shufflevector(__s0_840, __s0_840, 3, 2, 1, 0); \
66573 uint8x16_t __rev1_840; __rev1_840 = __builtin_shufflevector(__s1_840, __s1_840, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
66574 uint8x8_t __rev2_840; __rev2_840 = __builtin_shufflevector(__s2_840, __s2_840, 7, 6, 5, 4, 3, 2, 1, 0); \
66575uint8x8_t __reint_840 = __rev2_840; \
66576uint32x4_t __reint1_840 = __noswap_splatq_lane_u32(*(uint32x2_t *) &__reint_840, __p3_840); \
66577 __ret_840 = __noswap_vdotq_u32(__rev0_840, __rev1_840, *(uint8x16_t *) &__reint1_840); \
66578 __ret_840 = __builtin_shufflevector(__ret_840, __ret_840, 3, 2, 1, 0); \
66579 __ret_840; \
68504#define vdotq_lane_u32(__p0_812, __p1_812, __p2_812, __p3_812) __extension__ ({ \
68505 uint32x4_t __ret_812; \
68506 uint32x4_t __s0_812 = __p0_812; \
68507 uint8x16_t __s1_812 = __p1_812; \
68508 uint8x8_t __s2_812 = __p2_812; \
68509 uint32x4_t __rev0_812; __rev0_812 = __builtin_shufflevector(__s0_812, __s0_812, 3, 2, 1, 0); \
68510 uint8x16_t __rev1_812; __rev1_812 = __builtin_shufflevector(__s1_812, __s1_812, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
68511 uint8x8_t __rev2_812; __rev2_812 = __builtin_shufflevector(__s2_812, __s2_812, 7, 6, 5, 4, 3, 2, 1, 0); \
68512 __ret_812 = __noswap_vdotq_u32(__rev0_812, __rev1_812, __builtin_bit_cast(uint8x16_t, __noswap_splatq_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_812), __p3_812))); \
68513 __ret_812 = __builtin_shufflevector(__ret_812, __ret_812, 3, 2, 1, 0); \
68514 __ret_812; \
6658068515})
6658168516#endif
6658268517
6658368518#ifdef __LITTLE_ENDIAN__
66584#define vdotq_lane_s32(__p0_841, __p1_841, __p2_841, __p3_841) __extension__ ({ \
66585 int32x4_t __ret_841; \
66586 int32x4_t __s0_841 = __p0_841; \
66587 int8x16_t __s1_841 = __p1_841; \
66588 int8x8_t __s2_841 = __p2_841; \
66589int8x8_t __reint_841 = __s2_841; \
66590int32x4_t __reint1_841 = splatq_lane_s32(*(int32x2_t *) &__reint_841, __p3_841); \
66591 __ret_841 = vdotq_s32(__s0_841, __s1_841, *(int8x16_t *) &__reint1_841); \
66592 __ret_841; \
68519#define vdotq_lane_s32(__p0_813, __p1_813, __p2_813, __p3_813) __extension__ ({ \
68520 int32x4_t __ret_813; \
68521 int32x4_t __s0_813 = __p0_813; \
68522 int8x16_t __s1_813 = __p1_813; \
68523 int8x8_t __s2_813 = __p2_813; \
68524 __ret_813 = vdotq_s32(__s0_813, __s1_813, __builtin_bit_cast(int8x16_t, splatq_lane_s32(__builtin_bit_cast(int32x2_t, __s2_813), __p3_813))); \
68525 __ret_813; \
6659368526})
6659468527#else
66595#define vdotq_lane_s32(__p0_842, __p1_842, __p2_842, __p3_842) __extension__ ({ \
66596 int32x4_t __ret_842; \
66597 int32x4_t __s0_842 = __p0_842; \
66598 int8x16_t __s1_842 = __p1_842; \
66599 int8x8_t __s2_842 = __p2_842; \
66600 int32x4_t __rev0_842; __rev0_842 = __builtin_shufflevector(__s0_842, __s0_842, 3, 2, 1, 0); \
66601 int8x16_t __rev1_842; __rev1_842 = __builtin_shufflevector(__s1_842, __s1_842, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
66602 int8x8_t __rev2_842; __rev2_842 = __builtin_shufflevector(__s2_842, __s2_842, 7, 6, 5, 4, 3, 2, 1, 0); \
66603int8x8_t __reint_842 = __rev2_842; \
66604int32x4_t __reint1_842 = __noswap_splatq_lane_s32(*(int32x2_t *) &__reint_842, __p3_842); \
66605 __ret_842 = __noswap_vdotq_s32(__rev0_842, __rev1_842, *(int8x16_t *) &__reint1_842); \
66606 __ret_842 = __builtin_shufflevector(__ret_842, __ret_842, 3, 2, 1, 0); \
66607 __ret_842; \
68528#define vdotq_lane_s32(__p0_814, __p1_814, __p2_814, __p3_814) __extension__ ({ \
68529 int32x4_t __ret_814; \
68530 int32x4_t __s0_814 = __p0_814; \
68531 int8x16_t __s1_814 = __p1_814; \
68532 int8x8_t __s2_814 = __p2_814; \
68533 int32x4_t __rev0_814; __rev0_814 = __builtin_shufflevector(__s0_814, __s0_814, 3, 2, 1, 0); \
68534 int8x16_t __rev1_814; __rev1_814 = __builtin_shufflevector(__s1_814, __s1_814, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
68535 int8x8_t __rev2_814; __rev2_814 = __builtin_shufflevector(__s2_814, __s2_814, 7, 6, 5, 4, 3, 2, 1, 0); \
68536 __ret_814 = __noswap_vdotq_s32(__rev0_814, __rev1_814, __builtin_bit_cast(int8x16_t, __noswap_splatq_lane_s32(__builtin_bit_cast(int32x2_t, __rev2_814), __p3_814))); \
68537 __ret_814 = __builtin_shufflevector(__ret_814, __ret_814, 3, 2, 1, 0); \
68538 __ret_814; \
6660868539})
6660968540#endif
6661068541
6661168542#ifdef __LITTLE_ENDIAN__
66612#define vdot_lane_u32(__p0_843, __p1_843, __p2_843, __p3_843) __extension__ ({ \
66613 uint32x2_t __ret_843; \
66614 uint32x2_t __s0_843 = __p0_843; \
66615 uint8x8_t __s1_843 = __p1_843; \
66616 uint8x8_t __s2_843 = __p2_843; \
66617uint8x8_t __reint_843 = __s2_843; \
66618uint32x2_t __reint1_843 = splat_lane_u32(*(uint32x2_t *) &__reint_843, __p3_843); \
66619 __ret_843 = vdot_u32(__s0_843, __s1_843, *(uint8x8_t *) &__reint1_843); \
66620 __ret_843; \
68543#define vdot_lane_u32(__p0_815, __p1_815, __p2_815, __p3_815) __extension__ ({ \
68544 uint32x2_t __ret_815; \
68545 uint32x2_t __s0_815 = __p0_815; \
68546 uint8x8_t __s1_815 = __p1_815; \
68547 uint8x8_t __s2_815 = __p2_815; \
68548 __ret_815 = vdot_u32(__s0_815, __s1_815, __builtin_bit_cast(uint8x8_t, splat_lane_u32(__builtin_bit_cast(uint32x2_t, __s2_815), __p3_815))); \
68549 __ret_815; \
6662168550})
6662268551#else
66623#define vdot_lane_u32(__p0_844, __p1_844, __p2_844, __p3_844) __extension__ ({ \
66624 uint32x2_t __ret_844; \
66625 uint32x2_t __s0_844 = __p0_844; \
66626 uint8x8_t __s1_844 = __p1_844; \
66627 uint8x8_t __s2_844 = __p2_844; \
66628 uint32x2_t __rev0_844; __rev0_844 = __builtin_shufflevector(__s0_844, __s0_844, 1, 0); \
66629 uint8x8_t __rev1_844; __rev1_844 = __builtin_shufflevector(__s1_844, __s1_844, 7, 6, 5, 4, 3, 2, 1, 0); \
66630 uint8x8_t __rev2_844; __rev2_844 = __builtin_shufflevector(__s2_844, __s2_844, 7, 6, 5, 4, 3, 2, 1, 0); \
66631uint8x8_t __reint_844 = __rev2_844; \
66632uint32x2_t __reint1_844 = __noswap_splat_lane_u32(*(uint32x2_t *) &__reint_844, __p3_844); \
66633 __ret_844 = __noswap_vdot_u32(__rev0_844, __rev1_844, *(uint8x8_t *) &__reint1_844); \
66634 __ret_844 = __builtin_shufflevector(__ret_844, __ret_844, 1, 0); \
66635 __ret_844; \
68552#define vdot_lane_u32(__p0_816, __p1_816, __p2_816, __p3_816) __extension__ ({ \
68553 uint32x2_t __ret_816; \
68554 uint32x2_t __s0_816 = __p0_816; \
68555 uint8x8_t __s1_816 = __p1_816; \
68556 uint8x8_t __s2_816 = __p2_816; \
68557 uint32x2_t __rev0_816; __rev0_816 = __builtin_shufflevector(__s0_816, __s0_816, 1, 0); \
68558 uint8x8_t __rev1_816; __rev1_816 = __builtin_shufflevector(__s1_816, __s1_816, 7, 6, 5, 4, 3, 2, 1, 0); \
68559 uint8x8_t __rev2_816; __rev2_816 = __builtin_shufflevector(__s2_816, __s2_816, 7, 6, 5, 4, 3, 2, 1, 0); \
68560 __ret_816 = __noswap_vdot_u32(__rev0_816, __rev1_816, __builtin_bit_cast(uint8x8_t, __noswap_splat_lane_u32(__builtin_bit_cast(uint32x2_t, __rev2_816), __p3_816))); \
68561 __ret_816 = __builtin_shufflevector(__ret_816, __ret_816, 1, 0); \
68562 __ret_816; \
6663668563})
6663768564#endif
6663868565
6663968566#ifdef __LITTLE_ENDIAN__
66640#define vdot_lane_s32(__p0_845, __p1_845, __p2_845, __p3_845) __extension__ ({ \
66641 int32x2_t __ret_845; \
66642 int32x2_t __s0_845 = __p0_845; \
66643 int8x8_t __s1_845 = __p1_845; \
66644 int8x8_t __s2_845 = __p2_845; \
66645int8x8_t __reint_845 = __s2_845; \
66646int32x2_t __reint1_845 = splat_lane_s32(*(int32x2_t *) &__reint_845, __p3_845); \
66647 __ret_845 = vdot_s32(__s0_845, __s1_845, *(int8x8_t *) &__reint1_845); \
66648 __ret_845; \
68567#define vdot_lane_s32(__p0_817, __p1_817, __p2_817, __p3_817) __extension__ ({ \
68568 int32x2_t __ret_817; \
68569 int32x2_t __s0_817 = __p0_817; \
68570 int8x8_t __s1_817 = __p1_817; \
68571 int8x8_t __s2_817 = __p2_817; \
68572 __ret_817 = vdot_s32(__s0_817, __s1_817, __builtin_bit_cast(int8x8_t, splat_lane_s32(__builtin_bit_cast(int32x2_t, __s2_817), __p3_817))); \
68573 __ret_817; \
6664968574})
6665068575#else
66651#define vdot_lane_s32(__p0_846, __p1_846, __p2_846, __p3_846) __extension__ ({ \
66652 int32x2_t __ret_846; \
66653 int32x2_t __s0_846 = __p0_846; \
66654 int8x8_t __s1_846 = __p1_846; \
66655 int8x8_t __s2_846 = __p2_846; \
66656 int32x2_t __rev0_846; __rev0_846 = __builtin_shufflevector(__s0_846, __s0_846, 1, 0); \
66657 int8x8_t __rev1_846; __rev1_846 = __builtin_shufflevector(__s1_846, __s1_846, 7, 6, 5, 4, 3, 2, 1, 0); \
66658 int8x8_t __rev2_846; __rev2_846 = __builtin_shufflevector(__s2_846, __s2_846, 7, 6, 5, 4, 3, 2, 1, 0); \
66659int8x8_t __reint_846 = __rev2_846; \
66660int32x2_t __reint1_846 = __noswap_splat_lane_s32(*(int32x2_t *) &__reint_846, __p3_846); \
66661 __ret_846 = __noswap_vdot_s32(__rev0_846, __rev1_846, *(int8x8_t *) &__reint1_846); \
66662 __ret_846 = __builtin_shufflevector(__ret_846, __ret_846, 1, 0); \
66663 __ret_846; \
68576#define vdot_lane_s32(__p0_818, __p1_818, __p2_818, __p3_818) __extension__ ({ \
68577 int32x2_t __ret_818; \
68578 int32x2_t __s0_818 = __p0_818; \
68579 int8x8_t __s1_818 = __p1_818; \
68580 int8x8_t __s2_818 = __p2_818; \
68581 int32x2_t __rev0_818; __rev0_818 = __builtin_shufflevector(__s0_818, __s0_818, 1, 0); \
68582 int8x8_t __rev1_818; __rev1_818 = __builtin_shufflevector(__s1_818, __s1_818, 7, 6, 5, 4, 3, 2, 1, 0); \
68583 int8x8_t __rev2_818; __rev2_818 = __builtin_shufflevector(__s2_818, __s2_818, 7, 6, 5, 4, 3, 2, 1, 0); \
68584 __ret_818 = __noswap_vdot_s32(__rev0_818, __rev1_818, __builtin_bit_cast(int8x8_t, __noswap_splat_lane_s32(__builtin_bit_cast(int32x2_t, __rev2_818), __p3_818))); \
68585 __ret_818 = __builtin_shufflevector(__ret_818, __ret_818, 1, 0); \
68586 __ret_818; \
6666468587})
6666568588#endif
6666668589
6666768590#ifdef __LITTLE_ENDIAN__
66668#define vmulq_lane_f16(__p0_847, __p1_847, __p2_847) __extension__ ({ \
66669 float16x8_t __ret_847; \
66670 float16x8_t __s0_847 = __p0_847; \
66671 float16x4_t __s1_847 = __p1_847; \
66672 __ret_847 = __s0_847 * splatq_lane_f16(__s1_847, __p2_847); \
66673 __ret_847; \
68591#define vmulq_lane_f16(__p0_819, __p1_819, __p2_819) __extension__ ({ \
68592 float16x8_t __ret_819; \
68593 float16x8_t __s0_819 = __p0_819; \
68594 float16x4_t __s1_819 = __p1_819; \
68595 __ret_819 = __s0_819 * splatq_lane_f16(__s1_819, __p2_819); \
68596 __ret_819; \
6667468597})
6667568598#else
66676#define vmulq_lane_f16(__p0_848, __p1_848, __p2_848) __extension__ ({ \
66677 float16x8_t __ret_848; \
66678 float16x8_t __s0_848 = __p0_848; \
66679 float16x4_t __s1_848 = __p1_848; \
66680 float16x8_t __rev0_848; __rev0_848 = __builtin_shufflevector(__s0_848, __s0_848, 7, 6, 5, 4, 3, 2, 1, 0); \
66681 float16x4_t __rev1_848; __rev1_848 = __builtin_shufflevector(__s1_848, __s1_848, 3, 2, 1, 0); \
66682 __ret_848 = __rev0_848 * __noswap_splatq_lane_f16(__rev1_848, __p2_848); \
66683 __ret_848 = __builtin_shufflevector(__ret_848, __ret_848, 7, 6, 5, 4, 3, 2, 1, 0); \
66684 __ret_848; \
68599#define vmulq_lane_f16(__p0_820, __p1_820, __p2_820) __extension__ ({ \
68600 float16x8_t __ret_820; \
68601 float16x8_t __s0_820 = __p0_820; \
68602 float16x4_t __s1_820 = __p1_820; \
68603 float16x8_t __rev0_820; __rev0_820 = __builtin_shufflevector(__s0_820, __s0_820, 7, 6, 5, 4, 3, 2, 1, 0); \
68604 float16x4_t __rev1_820; __rev1_820 = __builtin_shufflevector(__s1_820, __s1_820, 3, 2, 1, 0); \
68605 __ret_820 = __rev0_820 * __noswap_splatq_lane_f16(__rev1_820, __p2_820); \
68606 __ret_820 = __builtin_shufflevector(__ret_820, __ret_820, 7, 6, 5, 4, 3, 2, 1, 0); \
68607 __ret_820; \
6668568608})
6668668609#endif
6668768610
6668868611#ifdef __LITTLE_ENDIAN__
66689#define vmul_lane_f16(__p0_849, __p1_849, __p2_849) __extension__ ({ \
66690 float16x4_t __ret_849; \
66691 float16x4_t __s0_849 = __p0_849; \
66692 float16x4_t __s1_849 = __p1_849; \
66693 __ret_849 = __s0_849 * splat_lane_f16(__s1_849, __p2_849); \
66694 __ret_849; \
68612#define vmul_lane_f16(__p0_821, __p1_821, __p2_821) __extension__ ({ \
68613 float16x4_t __ret_821; \
68614 float16x4_t __s0_821 = __p0_821; \
68615 float16x4_t __s1_821 = __p1_821; \
68616 __ret_821 = __s0_821 * splat_lane_f16(__s1_821, __p2_821); \
68617 __ret_821; \
6669568618})
6669668619#else
66697#define vmul_lane_f16(__p0_850, __p1_850, __p2_850) __extension__ ({ \
66698 float16x4_t __ret_850; \
66699 float16x4_t __s0_850 = __p0_850; \
66700 float16x4_t __s1_850 = __p1_850; \
66701 float16x4_t __rev0_850; __rev0_850 = __builtin_shufflevector(__s0_850, __s0_850, 3, 2, 1, 0); \
66702 float16x4_t __rev1_850; __rev1_850 = __builtin_shufflevector(__s1_850, __s1_850, 3, 2, 1, 0); \
66703 __ret_850 = __rev0_850 * __noswap_splat_lane_f16(__rev1_850, __p2_850); \
66704 __ret_850 = __builtin_shufflevector(__ret_850, __ret_850, 3, 2, 1, 0); \
66705 __ret_850; \
68620#define vmul_lane_f16(__p0_822, __p1_822, __p2_822) __extension__ ({ \
68621 float16x4_t __ret_822; \
68622 float16x4_t __s0_822 = __p0_822; \
68623 float16x4_t __s1_822 = __p1_822; \
68624 float16x4_t __rev0_822; __rev0_822 = __builtin_shufflevector(__s0_822, __s0_822, 3, 2, 1, 0); \
68625 float16x4_t __rev1_822; __rev1_822 = __builtin_shufflevector(__s1_822, __s1_822, 3, 2, 1, 0); \
68626 __ret_822 = __rev0_822 * __noswap_splat_lane_f16(__rev1_822, __p2_822); \
68627 __ret_822 = __builtin_shufflevector(__ret_822, __ret_822, 3, 2, 1, 0); \
68628 __ret_822; \
6670668629})
6670768630#endif
6670868631
6670968632#ifdef __LITTLE_ENDIAN__
66710#define vsudotq_lane_s32(__p0_851, __p1_851, __p2_851, __p3_851) __extension__ ({ \
66711 int32x4_t __ret_851; \
66712 int32x4_t __s0_851 = __p0_851; \
66713 int8x16_t __s1_851 = __p1_851; \
66714 uint8x8_t __s2_851 = __p2_851; \
66715uint8x8_t __reint_851 = __s2_851; \
66716 __ret_851 = vusdotq_s32(__s0_851, (uint8x16_t)(splatq_lane_s32(*(int32x2_t *) &__reint_851, __p3_851)), __s1_851); \
66717 __ret_851; \
68633#define vsudotq_lane_s32(__p0_823, __p1_823, __p2_823, __p3_823) __extension__ ({ \
68634 int32x4_t __ret_823; \
68635 int32x4_t __s0_823 = __p0_823; \
68636 int8x16_t __s1_823 = __p1_823; \
68637 uint8x8_t __s2_823 = __p2_823; \
68638 __ret_823 = vusdotq_s32(__s0_823, (uint8x16_t)(splatq_lane_s32(__builtin_bit_cast(int32x2_t, __s2_823), __p3_823)), __s1_823); \
68639 __ret_823; \
6671868640})
6671968641#else
66720#define vsudotq_lane_s32(__p0_852, __p1_852, __p2_852, __p3_852) __extension__ ({ \
66721 int32x4_t __ret_852; \
66722 int32x4_t __s0_852 = __p0_852; \
66723 int8x16_t __s1_852 = __p1_852; \
66724 uint8x8_t __s2_852 = __p2_852; \
66725 int32x4_t __rev0_852; __rev0_852 = __builtin_shufflevector(__s0_852, __s0_852, 3, 2, 1, 0); \
66726 int8x16_t __rev1_852; __rev1_852 = __builtin_shufflevector(__s1_852, __s1_852, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
66727 uint8x8_t __rev2_852; __rev2_852 = __builtin_shufflevector(__s2_852, __s2_852, 7, 6, 5, 4, 3, 2, 1, 0); \
66728uint8x8_t __reint_852 = __rev2_852; \
66729 __ret_852 = __noswap_vusdotq_s32(__rev0_852, (uint8x16_t)(__noswap_splatq_lane_s32(*(int32x2_t *) &__reint_852, __p3_852)), __rev1_852); \
66730 __ret_852 = __builtin_shufflevector(__ret_852, __ret_852, 3, 2, 1, 0); \
66731 __ret_852; \
68642#define vsudotq_lane_s32(__p0_824, __p1_824, __p2_824, __p3_824) __extension__ ({ \
68643 int32x4_t __ret_824; \
68644 int32x4_t __s0_824 = __p0_824; \
68645 int8x16_t __s1_824 = __p1_824; \
68646 uint8x8_t __s2_824 = __p2_824; \
68647 int32x4_t __rev0_824; __rev0_824 = __builtin_shufflevector(__s0_824, __s0_824, 3, 2, 1, 0); \
68648 int8x16_t __rev1_824; __rev1_824 = __builtin_shufflevector(__s1_824, __s1_824, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
68649 uint8x8_t __rev2_824; __rev2_824 = __builtin_shufflevector(__s2_824, __s2_824, 7, 6, 5, 4, 3, 2, 1, 0); \
68650 __ret_824 = __noswap_vusdotq_s32(__rev0_824, (uint8x16_t)(__noswap_splatq_lane_s32(__builtin_bit_cast(int32x2_t, __rev2_824), __p3_824)), __rev1_824); \
68651 __ret_824 = __builtin_shufflevector(__ret_824, __ret_824, 3, 2, 1, 0); \
68652 __ret_824; \
6673268653})
6673368654#endif
6673468655
6673568656#ifdef __LITTLE_ENDIAN__
66736#define vsudot_lane_s32(__p0_853, __p1_853, __p2_853, __p3_853) __extension__ ({ \
66737 int32x2_t __ret_853; \
66738 int32x2_t __s0_853 = __p0_853; \
66739 int8x8_t __s1_853 = __p1_853; \
66740 uint8x8_t __s2_853 = __p2_853; \
66741uint8x8_t __reint_853 = __s2_853; \
66742 __ret_853 = vusdot_s32(__s0_853, (uint8x8_t)(splat_lane_s32(*(int32x2_t *) &__reint_853, __p3_853)), __s1_853); \
66743 __ret_853; \
68657#define vsudot_lane_s32(__p0_825, __p1_825, __p2_825, __p3_825) __extension__ ({ \
68658 int32x2_t __ret_825; \
68659 int32x2_t __s0_825 = __p0_825; \
68660 int8x8_t __s1_825 = __p1_825; \
68661 uint8x8_t __s2_825 = __p2_825; \
68662 __ret_825 = vusdot_s32(__s0_825, (uint8x8_t)(splat_lane_s32(__builtin_bit_cast(int32x2_t, __s2_825), __p3_825)), __s1_825); \
68663 __ret_825; \
6674468664})
6674568665#else
66746#define vsudot_lane_s32(__p0_854, __p1_854, __p2_854, __p3_854) __extension__ ({ \
66747 int32x2_t __ret_854; \
66748 int32x2_t __s0_854 = __p0_854; \
66749 int8x8_t __s1_854 = __p1_854; \
66750 uint8x8_t __s2_854 = __p2_854; \
66751 int32x2_t __rev0_854; __rev0_854 = __builtin_shufflevector(__s0_854, __s0_854, 1, 0); \
66752 int8x8_t __rev1_854; __rev1_854 = __builtin_shufflevector(__s1_854, __s1_854, 7, 6, 5, 4, 3, 2, 1, 0); \
66753 uint8x8_t __rev2_854; __rev2_854 = __builtin_shufflevector(__s2_854, __s2_854, 7, 6, 5, 4, 3, 2, 1, 0); \
66754uint8x8_t __reint_854 = __rev2_854; \
66755 __ret_854 = __noswap_vusdot_s32(__rev0_854, (uint8x8_t)(__noswap_splat_lane_s32(*(int32x2_t *) &__reint_854, __p3_854)), __rev1_854); \
66756 __ret_854 = __builtin_shufflevector(__ret_854, __ret_854, 1, 0); \
66757 __ret_854; \
68666#define vsudot_lane_s32(__p0_826, __p1_826, __p2_826, __p3_826) __extension__ ({ \
68667 int32x2_t __ret_826; \
68668 int32x2_t __s0_826 = __p0_826; \
68669 int8x8_t __s1_826 = __p1_826; \
68670 uint8x8_t __s2_826 = __p2_826; \
68671 int32x2_t __rev0_826; __rev0_826 = __builtin_shufflevector(__s0_826, __s0_826, 1, 0); \
68672 int8x8_t __rev1_826; __rev1_826 = __builtin_shufflevector(__s1_826, __s1_826, 7, 6, 5, 4, 3, 2, 1, 0); \
68673 uint8x8_t __rev2_826; __rev2_826 = __builtin_shufflevector(__s2_826, __s2_826, 7, 6, 5, 4, 3, 2, 1, 0); \
68674 __ret_826 = __noswap_vusdot_s32(__rev0_826, (uint8x8_t)(__noswap_splat_lane_s32(__builtin_bit_cast(int32x2_t, __rev2_826), __p3_826)), __rev1_826); \
68675 __ret_826 = __builtin_shufflevector(__ret_826, __ret_826, 1, 0); \
68676 __ret_826; \
6675868677})
6675968678#endif
6676068679
6676168680#ifdef __LITTLE_ENDIAN__
66762#define vusdotq_lane_s32(__p0_855, __p1_855, __p2_855, __p3_855) __extension__ ({ \
66763 int32x4_t __ret_855; \
66764 int32x4_t __s0_855 = __p0_855; \
66765 uint8x16_t __s1_855 = __p1_855; \
66766 int8x8_t __s2_855 = __p2_855; \
66767int8x8_t __reint_855 = __s2_855; \
66768 __ret_855 = vusdotq_s32(__s0_855, __s1_855, (int8x16_t)(splatq_lane_s32(*(int32x2_t *) &__reint_855, __p3_855))); \
66769 __ret_855; \
68681#define vusdotq_lane_s32(__p0_827, __p1_827, __p2_827, __p3_827) __extension__ ({ \
68682 int32x4_t __ret_827; \
68683 int32x4_t __s0_827 = __p0_827; \
68684 uint8x16_t __s1_827 = __p1_827; \
68685 int8x8_t __s2_827 = __p2_827; \
68686 __ret_827 = vusdotq_s32(__s0_827, __s1_827, (int8x16_t)(splatq_lane_s32(__builtin_bit_cast(int32x2_t, __s2_827), __p3_827))); \
68687 __ret_827; \
6677068688})
6677168689#else
66772#define vusdotq_lane_s32(__p0_856, __p1_856, __p2_856, __p3_856) __extension__ ({ \
66773 int32x4_t __ret_856; \
66774 int32x4_t __s0_856 = __p0_856; \
66775 uint8x16_t __s1_856 = __p1_856; \
66776 int8x8_t __s2_856 = __p2_856; \
66777 int32x4_t __rev0_856; __rev0_856 = __builtin_shufflevector(__s0_856, __s0_856, 3, 2, 1, 0); \
66778 uint8x16_t __rev1_856; __rev1_856 = __builtin_shufflevector(__s1_856, __s1_856, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
66779 int8x8_t __rev2_856; __rev2_856 = __builtin_shufflevector(__s2_856, __s2_856, 7, 6, 5, 4, 3, 2, 1, 0); \
66780int8x8_t __reint_856 = __rev2_856; \
66781 __ret_856 = __noswap_vusdotq_s32(__rev0_856, __rev1_856, (int8x16_t)(__noswap_splatq_lane_s32(*(int32x2_t *) &__reint_856, __p3_856))); \
66782 __ret_856 = __builtin_shufflevector(__ret_856, __ret_856, 3, 2, 1, 0); \
66783 __ret_856; \
68690#define vusdotq_lane_s32(__p0_828, __p1_828, __p2_828, __p3_828) __extension__ ({ \
68691 int32x4_t __ret_828; \
68692 int32x4_t __s0_828 = __p0_828; \
68693 uint8x16_t __s1_828 = __p1_828; \
68694 int8x8_t __s2_828 = __p2_828; \
68695 int32x4_t __rev0_828; __rev0_828 = __builtin_shufflevector(__s0_828, __s0_828, 3, 2, 1, 0); \
68696 uint8x16_t __rev1_828; __rev1_828 = __builtin_shufflevector(__s1_828, __s1_828, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \
68697 int8x8_t __rev2_828; __rev2_828 = __builtin_shufflevector(__s2_828, __s2_828, 7, 6, 5, 4, 3, 2, 1, 0); \
68698 __ret_828 = __noswap_vusdotq_s32(__rev0_828, __rev1_828, (int8x16_t)(__noswap_splatq_lane_s32(__builtin_bit_cast(int32x2_t, __rev2_828), __p3_828))); \
68699 __ret_828 = __builtin_shufflevector(__ret_828, __ret_828, 3, 2, 1, 0); \
68700 __ret_828; \
6678468701})
6678568702#endif
6678668703
6678768704#ifdef __LITTLE_ENDIAN__
66788#define vusdot_lane_s32(__p0_857, __p1_857, __p2_857, __p3_857) __extension__ ({ \
66789 int32x2_t __ret_857; \
66790 int32x2_t __s0_857 = __p0_857; \
66791 uint8x8_t __s1_857 = __p1_857; \
66792 int8x8_t __s2_857 = __p2_857; \
66793int8x8_t __reint_857 = __s2_857; \
66794 __ret_857 = vusdot_s32(__s0_857, __s1_857, (int8x8_t)(splat_lane_s32(*(int32x2_t *) &__reint_857, __p3_857))); \
66795 __ret_857; \
68705#define vusdot_lane_s32(__p0_829, __p1_829, __p2_829, __p3_829) __extension__ ({ \
68706 int32x2_t __ret_829; \
68707 int32x2_t __s0_829 = __p0_829; \
68708 uint8x8_t __s1_829 = __p1_829; \
68709 int8x8_t __s2_829 = __p2_829; \
68710 __ret_829 = vusdot_s32(__s0_829, __s1_829, (int8x8_t)(splat_lane_s32(__builtin_bit_cast(int32x2_t, __s2_829), __p3_829))); \
68711 __ret_829; \
6679668712})
6679768713#else
66798#define vusdot_lane_s32(__p0_858, __p1_858, __p2_858, __p3_858) __extension__ ({ \
66799 int32x2_t __ret_858; \
66800 int32x2_t __s0_858 = __p0_858; \
66801 uint8x8_t __s1_858 = __p1_858; \
66802 int8x8_t __s2_858 = __p2_858; \
66803 int32x2_t __rev0_858; __rev0_858 = __builtin_shufflevector(__s0_858, __s0_858, 1, 0); \
66804 uint8x8_t __rev1_858; __rev1_858 = __builtin_shufflevector(__s1_858, __s1_858, 7, 6, 5, 4, 3, 2, 1, 0); \
66805 int8x8_t __rev2_858; __rev2_858 = __builtin_shufflevector(__s2_858, __s2_858, 7, 6, 5, 4, 3, 2, 1, 0); \
66806int8x8_t __reint_858 = __rev2_858; \
66807 __ret_858 = __noswap_vusdot_s32(__rev0_858, __rev1_858, (int8x8_t)(__noswap_splat_lane_s32(*(int32x2_t *) &__reint_858, __p3_858))); \
66808 __ret_858 = __builtin_shufflevector(__ret_858, __ret_858, 1, 0); \
66809 __ret_858; \
68714#define vusdot_lane_s32(__p0_830, __p1_830, __p2_830, __p3_830) __extension__ ({ \
68715 int32x2_t __ret_830; \
68716 int32x2_t __s0_830 = __p0_830; \
68717 uint8x8_t __s1_830 = __p1_830; \
68718 int8x8_t __s2_830 = __p2_830; \
68719 int32x2_t __rev0_830; __rev0_830 = __builtin_shufflevector(__s0_830, __s0_830, 1, 0); \
68720 uint8x8_t __rev1_830; __rev1_830 = __builtin_shufflevector(__s1_830, __s1_830, 7, 6, 5, 4, 3, 2, 1, 0); \
68721 int8x8_t __rev2_830; __rev2_830 = __builtin_shufflevector(__s2_830, __s2_830, 7, 6, 5, 4, 3, 2, 1, 0); \
68722 __ret_830 = __noswap_vusdot_s32(__rev0_830, __rev1_830, (int8x8_t)(__noswap_splat_lane_s32(__builtin_bit_cast(int32x2_t, __rev2_830), __p3_830))); \
68723 __ret_830 = __builtin_shufflevector(__ret_830, __ret_830, 1, 0); \
68724 __ret_830; \
6681068725})
6681168726#endif
6681268727
......@@ -67363,60 +69278,48 @@ __ai __attribute__((target("neon"))) int32x4_t vaddw_s16(int32x4_t __p0, int16x4
6736369278#endif
6736469279
6736569280#ifdef __LITTLE_ENDIAN__
67366#define vget_lane_f16(__p0_859, __p1_859) __extension__ ({ \
67367 float16_t __ret_859; \
67368 float16x4_t __s0_859 = __p0_859; \
67369float16x4_t __reint_859 = __s0_859; \
67370int16_t __reint1_859 = vget_lane_s16(*(int16x4_t *) &__reint_859, __p1_859); \
67371 __ret_859 = *(float16_t *) &__reint1_859; \
67372 __ret_859; \
69281#define vget_lane_f16(__p0_831, __p1_831) __extension__ ({ \
69282 float16_t __ret_831; \
69283 float16x4_t __s0_831 = __p0_831; \
69284 __ret_831 = __builtin_bit_cast(float16_t, vget_lane_s16(__builtin_bit_cast(int16x4_t, __s0_831), __p1_831)); \
69285 __ret_831; \
6737369286})
6737469287#else
67375#define vget_lane_f16(__p0_860, __p1_860) __extension__ ({ \
67376 float16_t __ret_860; \
67377 float16x4_t __s0_860 = __p0_860; \
67378 float16x4_t __rev0_860; __rev0_860 = __builtin_shufflevector(__s0_860, __s0_860, 3, 2, 1, 0); \
67379float16x4_t __reint_860 = __rev0_860; \
67380int16_t __reint1_860 = __noswap_vget_lane_s16(*(int16x4_t *) &__reint_860, __p1_860); \
67381 __ret_860 = *(float16_t *) &__reint1_860; \
67382 __ret_860; \
69288#define vget_lane_f16(__p0_832, __p1_832) __extension__ ({ \
69289 float16_t __ret_832; \
69290 float16x4_t __s0_832 = __p0_832; \
69291 float16x4_t __rev0_832; __rev0_832 = __builtin_shufflevector(__s0_832, __s0_832, 3, 2, 1, 0); \
69292 __ret_832 = __builtin_bit_cast(float16_t, __noswap_vget_lane_s16(__builtin_bit_cast(int16x4_t, __rev0_832), __p1_832)); \
69293 __ret_832; \
6738369294})
67384#define __noswap_vget_lane_f16(__p0_861, __p1_861) __extension__ ({ \
67385 float16_t __ret_861; \
67386 float16x4_t __s0_861 = __p0_861; \
67387float16x4_t __reint_861 = __s0_861; \
67388int16_t __reint1_861 = __noswap_vget_lane_s16(*(int16x4_t *) &__reint_861, __p1_861); \
67389 __ret_861 = *(float16_t *) &__reint1_861; \
67390 __ret_861; \
69295#define __noswap_vget_lane_f16(__p0_833, __p1_833) __extension__ ({ \
69296 float16_t __ret_833; \
69297 float16x4_t __s0_833 = __p0_833; \
69298 __ret_833 = __builtin_bit_cast(float16_t, __noswap_vget_lane_s16(__builtin_bit_cast(int16x4_t, __s0_833), __p1_833)); \
69299 __ret_833; \
6739169300})
6739269301#endif
6739369302
6739469303#ifdef __LITTLE_ENDIAN__
67395#define vgetq_lane_f16(__p0_862, __p1_862) __extension__ ({ \
67396 float16_t __ret_862; \
67397 float16x8_t __s0_862 = __p0_862; \
67398float16x8_t __reint_862 = __s0_862; \
67399int16_t __reint1_862 = vgetq_lane_s16(*(int16x8_t *) &__reint_862, __p1_862); \
67400 __ret_862 = *(float16_t *) &__reint1_862; \
67401 __ret_862; \
69304#define vgetq_lane_f16(__p0_834, __p1_834) __extension__ ({ \
69305 float16_t __ret_834; \
69306 float16x8_t __s0_834 = __p0_834; \
69307 __ret_834 = __builtin_bit_cast(float16_t, vgetq_lane_s16(__builtin_bit_cast(int16x8_t, __s0_834), __p1_834)); \
69308 __ret_834; \
6740269309})
6740369310#else
67404#define vgetq_lane_f16(__p0_863, __p1_863) __extension__ ({ \
67405 float16_t __ret_863; \
67406 float16x8_t __s0_863 = __p0_863; \
67407 float16x8_t __rev0_863; __rev0_863 = __builtin_shufflevector(__s0_863, __s0_863, 7, 6, 5, 4, 3, 2, 1, 0); \
67408float16x8_t __reint_863 = __rev0_863; \
67409int16_t __reint1_863 = __noswap_vgetq_lane_s16(*(int16x8_t *) &__reint_863, __p1_863); \
67410 __ret_863 = *(float16_t *) &__reint1_863; \
67411 __ret_863; \
69311#define vgetq_lane_f16(__p0_835, __p1_835) __extension__ ({ \
69312 float16_t __ret_835; \
69313 float16x8_t __s0_835 = __p0_835; \
69314 float16x8_t __rev0_835; __rev0_835 = __builtin_shufflevector(__s0_835, __s0_835, 7, 6, 5, 4, 3, 2, 1, 0); \
69315 __ret_835 = __builtin_bit_cast(float16_t, __noswap_vgetq_lane_s16(__builtin_bit_cast(int16x8_t, __rev0_835), __p1_835)); \
69316 __ret_835; \
6741269317})
67413#define __noswap_vgetq_lane_f16(__p0_864, __p1_864) __extension__ ({ \
67414 float16_t __ret_864; \
67415 float16x8_t __s0_864 = __p0_864; \
67416float16x8_t __reint_864 = __s0_864; \
67417int16_t __reint1_864 = __noswap_vgetq_lane_s16(*(int16x8_t *) &__reint_864, __p1_864); \
67418 __ret_864 = *(float16_t *) &__reint1_864; \
67419 __ret_864; \
69318#define __noswap_vgetq_lane_f16(__p0_836, __p1_836) __extension__ ({ \
69319 float16_t __ret_836; \
69320 float16x8_t __s0_836 = __p0_836; \
69321 __ret_836 = __builtin_bit_cast(float16_t, __noswap_vgetq_lane_s16(__builtin_bit_cast(int16x8_t, __s0_836), __p1_836)); \
69322 __ret_836; \
6742069323})
6742169324#endif
6742269325
......@@ -67559,98 +69462,98 @@ __ai __attribute__((target("neon"))) int32x4_t __noswap_vmlal_s16(int32x4_t __p0
6755969462#endif
6756069463
6756169464#ifdef __LITTLE_ENDIAN__
67562#define vmlal_lane_u32(__p0_865, __p1_865, __p2_865, __p3_865) __extension__ ({ \
67563 uint64x2_t __ret_865; \
67564 uint64x2_t __s0_865 = __p0_865; \
67565 uint32x2_t __s1_865 = __p1_865; \
67566 uint32x2_t __s2_865 = __p2_865; \
67567 __ret_865 = __s0_865 + vmull_u32(__s1_865, splat_lane_u32(__s2_865, __p3_865)); \
67568 __ret_865; \
69465#define vmlal_lane_u32(__p0_837, __p1_837, __p2_837, __p3_837) __extension__ ({ \
69466 uint64x2_t __ret_837; \
69467 uint64x2_t __s0_837 = __p0_837; \
69468 uint32x2_t __s1_837 = __p1_837; \
69469 uint32x2_t __s2_837 = __p2_837; \
69470 __ret_837 = __s0_837 + vmull_u32(__s1_837, splat_lane_u32(__s2_837, __p3_837)); \
69471 __ret_837; \
6756969472})
6757069473#else
67571#define vmlal_lane_u32(__p0_866, __p1_866, __p2_866, __p3_866) __extension__ ({ \
67572 uint64x2_t __ret_866; \
67573 uint64x2_t __s0_866 = __p0_866; \
67574 uint32x2_t __s1_866 = __p1_866; \
67575 uint32x2_t __s2_866 = __p2_866; \
67576 uint64x2_t __rev0_866; __rev0_866 = __builtin_shufflevector(__s0_866, __s0_866, 1, 0); \
67577 uint32x2_t __rev1_866; __rev1_866 = __builtin_shufflevector(__s1_866, __s1_866, 1, 0); \
67578 uint32x2_t __rev2_866; __rev2_866 = __builtin_shufflevector(__s2_866, __s2_866, 1, 0); \
67579 __ret_866 = __rev0_866 + __noswap_vmull_u32(__rev1_866, __noswap_splat_lane_u32(__rev2_866, __p3_866)); \
67580 __ret_866 = __builtin_shufflevector(__ret_866, __ret_866, 1, 0); \
67581 __ret_866; \
69474#define vmlal_lane_u32(__p0_838, __p1_838, __p2_838, __p3_838) __extension__ ({ \
69475 uint64x2_t __ret_838; \
69476 uint64x2_t __s0_838 = __p0_838; \
69477 uint32x2_t __s1_838 = __p1_838; \
69478 uint32x2_t __s2_838 = __p2_838; \
69479 uint64x2_t __rev0_838; __rev0_838 = __builtin_shufflevector(__s0_838, __s0_838, 1, 0); \
69480 uint32x2_t __rev1_838; __rev1_838 = __builtin_shufflevector(__s1_838, __s1_838, 1, 0); \
69481 uint32x2_t __rev2_838; __rev2_838 = __builtin_shufflevector(__s2_838, __s2_838, 1, 0); \
69482 __ret_838 = __rev0_838 + __noswap_vmull_u32(__rev1_838, __noswap_splat_lane_u32(__rev2_838, __p3_838)); \
69483 __ret_838 = __builtin_shufflevector(__ret_838, __ret_838, 1, 0); \
69484 __ret_838; \
6758269485})
6758369486#endif
6758469487
6758569488#ifdef __LITTLE_ENDIAN__
67586#define vmlal_lane_u16(__p0_867, __p1_867, __p2_867, __p3_867) __extension__ ({ \
67587 uint32x4_t __ret_867; \
67588 uint32x4_t __s0_867 = __p0_867; \
67589 uint16x4_t __s1_867 = __p1_867; \
67590 uint16x4_t __s2_867 = __p2_867; \
67591 __ret_867 = __s0_867 + vmull_u16(__s1_867, splat_lane_u16(__s2_867, __p3_867)); \
67592 __ret_867; \
69489#define vmlal_lane_u16(__p0_839, __p1_839, __p2_839, __p3_839) __extension__ ({ \
69490 uint32x4_t __ret_839; \
69491 uint32x4_t __s0_839 = __p0_839; \
69492 uint16x4_t __s1_839 = __p1_839; \
69493 uint16x4_t __s2_839 = __p2_839; \
69494 __ret_839 = __s0_839 + vmull_u16(__s1_839, splat_lane_u16(__s2_839, __p3_839)); \
69495 __ret_839; \
6759369496})
6759469497#else
67595#define vmlal_lane_u16(__p0_868, __p1_868, __p2_868, __p3_868) __extension__ ({ \
67596 uint32x4_t __ret_868; \
67597 uint32x4_t __s0_868 = __p0_868; \
67598 uint16x4_t __s1_868 = __p1_868; \
67599 uint16x4_t __s2_868 = __p2_868; \
67600 uint32x4_t __rev0_868; __rev0_868 = __builtin_shufflevector(__s0_868, __s0_868, 3, 2, 1, 0); \
67601 uint16x4_t __rev1_868; __rev1_868 = __builtin_shufflevector(__s1_868, __s1_868, 3, 2, 1, 0); \
67602 uint16x4_t __rev2_868; __rev2_868 = __builtin_shufflevector(__s2_868, __s2_868, 3, 2, 1, 0); \
67603 __ret_868 = __rev0_868 + __noswap_vmull_u16(__rev1_868, __noswap_splat_lane_u16(__rev2_868, __p3_868)); \
67604 __ret_868 = __builtin_shufflevector(__ret_868, __ret_868, 3, 2, 1, 0); \
67605 __ret_868; \
69498#define vmlal_lane_u16(__p0_840, __p1_840, __p2_840, __p3_840) __extension__ ({ \
69499 uint32x4_t __ret_840; \
69500 uint32x4_t __s0_840 = __p0_840; \
69501 uint16x4_t __s1_840 = __p1_840; \
69502 uint16x4_t __s2_840 = __p2_840; \
69503 uint32x4_t __rev0_840; __rev0_840 = __builtin_shufflevector(__s0_840, __s0_840, 3, 2, 1, 0); \
69504 uint16x4_t __rev1_840; __rev1_840 = __builtin_shufflevector(__s1_840, __s1_840, 3, 2, 1, 0); \
69505 uint16x4_t __rev2_840; __rev2_840 = __builtin_shufflevector(__s2_840, __s2_840, 3, 2, 1, 0); \
69506 __ret_840 = __rev0_840 + __noswap_vmull_u16(__rev1_840, __noswap_splat_lane_u16(__rev2_840, __p3_840)); \
69507 __ret_840 = __builtin_shufflevector(__ret_840, __ret_840, 3, 2, 1, 0); \
69508 __ret_840; \
6760669509})
6760769510#endif
6760869511
6760969512#ifdef __LITTLE_ENDIAN__
67610#define vmlal_lane_s32(__p0_869, __p1_869, __p2_869, __p3_869) __extension__ ({ \
67611 int64x2_t __ret_869; \
67612 int64x2_t __s0_869 = __p0_869; \
67613 int32x2_t __s1_869 = __p1_869; \
67614 int32x2_t __s2_869 = __p2_869; \
67615 __ret_869 = __s0_869 + vmull_s32(__s1_869, splat_lane_s32(__s2_869, __p3_869)); \
67616 __ret_869; \
69513#define vmlal_lane_s32(__p0_841, __p1_841, __p2_841, __p3_841) __extension__ ({ \
69514 int64x2_t __ret_841; \
69515 int64x2_t __s0_841 = __p0_841; \
69516 int32x2_t __s1_841 = __p1_841; \
69517 int32x2_t __s2_841 = __p2_841; \
69518 __ret_841 = __s0_841 + vmull_s32(__s1_841, splat_lane_s32(__s2_841, __p3_841)); \
69519 __ret_841; \
6761769520})
6761869521#else
67619#define vmlal_lane_s32(__p0_870, __p1_870, __p2_870, __p3_870) __extension__ ({ \
67620 int64x2_t __ret_870; \
67621 int64x2_t __s0_870 = __p0_870; \
67622 int32x2_t __s1_870 = __p1_870; \
67623 int32x2_t __s2_870 = __p2_870; \
67624 int64x2_t __rev0_870; __rev0_870 = __builtin_shufflevector(__s0_870, __s0_870, 1, 0); \
67625 int32x2_t __rev1_870; __rev1_870 = __builtin_shufflevector(__s1_870, __s1_870, 1, 0); \
67626 int32x2_t __rev2_870; __rev2_870 = __builtin_shufflevector(__s2_870, __s2_870, 1, 0); \
67627 __ret_870 = __rev0_870 + __noswap_vmull_s32(__rev1_870, __noswap_splat_lane_s32(__rev2_870, __p3_870)); \
67628 __ret_870 = __builtin_shufflevector(__ret_870, __ret_870, 1, 0); \
67629 __ret_870; \
69522#define vmlal_lane_s32(__p0_842, __p1_842, __p2_842, __p3_842) __extension__ ({ \
69523 int64x2_t __ret_842; \
69524 int64x2_t __s0_842 = __p0_842; \
69525 int32x2_t __s1_842 = __p1_842; \
69526 int32x2_t __s2_842 = __p2_842; \
69527 int64x2_t __rev0_842; __rev0_842 = __builtin_shufflevector(__s0_842, __s0_842, 1, 0); \
69528 int32x2_t __rev1_842; __rev1_842 = __builtin_shufflevector(__s1_842, __s1_842, 1, 0); \
69529 int32x2_t __rev2_842; __rev2_842 = __builtin_shufflevector(__s2_842, __s2_842, 1, 0); \
69530 __ret_842 = __rev0_842 + __noswap_vmull_s32(__rev1_842, __noswap_splat_lane_s32(__rev2_842, __p3_842)); \
69531 __ret_842 = __builtin_shufflevector(__ret_842, __ret_842, 1, 0); \
69532 __ret_842; \
6763069533})
6763169534#endif
6763269535
6763369536#ifdef __LITTLE_ENDIAN__
67634#define vmlal_lane_s16(__p0_871, __p1_871, __p2_871, __p3_871) __extension__ ({ \
67635 int32x4_t __ret_871; \
67636 int32x4_t __s0_871 = __p0_871; \
67637 int16x4_t __s1_871 = __p1_871; \
67638 int16x4_t __s2_871 = __p2_871; \
67639 __ret_871 = __s0_871 + vmull_s16(__s1_871, splat_lane_s16(__s2_871, __p3_871)); \
67640 __ret_871; \
69537#define vmlal_lane_s16(__p0_843, __p1_843, __p2_843, __p3_843) __extension__ ({ \
69538 int32x4_t __ret_843; \
69539 int32x4_t __s0_843 = __p0_843; \
69540 int16x4_t __s1_843 = __p1_843; \
69541 int16x4_t __s2_843 = __p2_843; \
69542 __ret_843 = __s0_843 + vmull_s16(__s1_843, splat_lane_s16(__s2_843, __p3_843)); \
69543 __ret_843; \
6764169544})
6764269545#else
67643#define vmlal_lane_s16(__p0_872, __p1_872, __p2_872, __p3_872) __extension__ ({ \
67644 int32x4_t __ret_872; \
67645 int32x4_t __s0_872 = __p0_872; \
67646 int16x4_t __s1_872 = __p1_872; \
67647 int16x4_t __s2_872 = __p2_872; \
67648 int32x4_t __rev0_872; __rev0_872 = __builtin_shufflevector(__s0_872, __s0_872, 3, 2, 1, 0); \
67649 int16x4_t __rev1_872; __rev1_872 = __builtin_shufflevector(__s1_872, __s1_872, 3, 2, 1, 0); \
67650 int16x4_t __rev2_872; __rev2_872 = __builtin_shufflevector(__s2_872, __s2_872, 3, 2, 1, 0); \
67651 __ret_872 = __rev0_872 + __noswap_vmull_s16(__rev1_872, __noswap_splat_lane_s16(__rev2_872, __p3_872)); \
67652 __ret_872 = __builtin_shufflevector(__ret_872, __ret_872, 3, 2, 1, 0); \
67653 __ret_872; \
69546#define vmlal_lane_s16(__p0_844, __p1_844, __p2_844, __p3_844) __extension__ ({ \
69547 int32x4_t __ret_844; \
69548 int32x4_t __s0_844 = __p0_844; \
69549 int16x4_t __s1_844 = __p1_844; \
69550 int16x4_t __s2_844 = __p2_844; \
69551 int32x4_t __rev0_844; __rev0_844 = __builtin_shufflevector(__s0_844, __s0_844, 3, 2, 1, 0); \
69552 int16x4_t __rev1_844; __rev1_844 = __builtin_shufflevector(__s1_844, __s1_844, 3, 2, 1, 0); \
69553 int16x4_t __rev2_844; __rev2_844 = __builtin_shufflevector(__s2_844, __s2_844, 3, 2, 1, 0); \
69554 __ret_844 = __rev0_844 + __noswap_vmull_s16(__rev1_844, __noswap_splat_lane_s16(__rev2_844, __p3_844)); \
69555 __ret_844 = __builtin_shufflevector(__ret_844, __ret_844, 3, 2, 1, 0); \
69556 __ret_844; \
6765469557})
6765569558#endif
6765669559
......@@ -67881,98 +69784,98 @@ __ai __attribute__((target("neon"))) int32x4_t __noswap_vmlsl_s16(int32x4_t __p0
6788169784#endif
6788269785
6788369786#ifdef __LITTLE_ENDIAN__
67884#define vmlsl_lane_u32(__p0_873, __p1_873, __p2_873, __p3_873) __extension__ ({ \
67885 uint64x2_t __ret_873; \
67886 uint64x2_t __s0_873 = __p0_873; \
67887 uint32x2_t __s1_873 = __p1_873; \
67888 uint32x2_t __s2_873 = __p2_873; \
67889 __ret_873 = __s0_873 - vmull_u32(__s1_873, splat_lane_u32(__s2_873, __p3_873)); \
67890 __ret_873; \
69787#define vmlsl_lane_u32(__p0_845, __p1_845, __p2_845, __p3_845) __extension__ ({ \
69788 uint64x2_t __ret_845; \
69789 uint64x2_t __s0_845 = __p0_845; \
69790 uint32x2_t __s1_845 = __p1_845; \
69791 uint32x2_t __s2_845 = __p2_845; \
69792 __ret_845 = __s0_845 - vmull_u32(__s1_845, splat_lane_u32(__s2_845, __p3_845)); \
69793 __ret_845; \
6789169794})
6789269795#else
67893#define vmlsl_lane_u32(__p0_874, __p1_874, __p2_874, __p3_874) __extension__ ({ \
67894 uint64x2_t __ret_874; \
67895 uint64x2_t __s0_874 = __p0_874; \
67896 uint32x2_t __s1_874 = __p1_874; \
67897 uint32x2_t __s2_874 = __p2_874; \
67898 uint64x2_t __rev0_874; __rev0_874 = __builtin_shufflevector(__s0_874, __s0_874, 1, 0); \
67899 uint32x2_t __rev1_874; __rev1_874 = __builtin_shufflevector(__s1_874, __s1_874, 1, 0); \
67900 uint32x2_t __rev2_874; __rev2_874 = __builtin_shufflevector(__s2_874, __s2_874, 1, 0); \
67901 __ret_874 = __rev0_874 - __noswap_vmull_u32(__rev1_874, __noswap_splat_lane_u32(__rev2_874, __p3_874)); \
67902 __ret_874 = __builtin_shufflevector(__ret_874, __ret_874, 1, 0); \
67903 __ret_874; \
69796#define vmlsl_lane_u32(__p0_846, __p1_846, __p2_846, __p3_846) __extension__ ({ \
69797 uint64x2_t __ret_846; \
69798 uint64x2_t __s0_846 = __p0_846; \
69799 uint32x2_t __s1_846 = __p1_846; \
69800 uint32x2_t __s2_846 = __p2_846; \
69801 uint64x2_t __rev0_846; __rev0_846 = __builtin_shufflevector(__s0_846, __s0_846, 1, 0); \
69802 uint32x2_t __rev1_846; __rev1_846 = __builtin_shufflevector(__s1_846, __s1_846, 1, 0); \
69803 uint32x2_t __rev2_846; __rev2_846 = __builtin_shufflevector(__s2_846, __s2_846, 1, 0); \
69804 __ret_846 = __rev0_846 - __noswap_vmull_u32(__rev1_846, __noswap_splat_lane_u32(__rev2_846, __p3_846)); \
69805 __ret_846 = __builtin_shufflevector(__ret_846, __ret_846, 1, 0); \
69806 __ret_846; \
6790469807})
6790569808#endif
6790669809
6790769810#ifdef __LITTLE_ENDIAN__
67908#define vmlsl_lane_u16(__p0_875, __p1_875, __p2_875, __p3_875) __extension__ ({ \
67909 uint32x4_t __ret_875; \
67910 uint32x4_t __s0_875 = __p0_875; \
67911 uint16x4_t __s1_875 = __p1_875; \
67912 uint16x4_t __s2_875 = __p2_875; \
67913 __ret_875 = __s0_875 - vmull_u16(__s1_875, splat_lane_u16(__s2_875, __p3_875)); \
67914 __ret_875; \
69811#define vmlsl_lane_u16(__p0_847, __p1_847, __p2_847, __p3_847) __extension__ ({ \
69812 uint32x4_t __ret_847; \
69813 uint32x4_t __s0_847 = __p0_847; \
69814 uint16x4_t __s1_847 = __p1_847; \
69815 uint16x4_t __s2_847 = __p2_847; \
69816 __ret_847 = __s0_847 - vmull_u16(__s1_847, splat_lane_u16(__s2_847, __p3_847)); \
69817 __ret_847; \
6791569818})
6791669819#else
67917#define vmlsl_lane_u16(__p0_876, __p1_876, __p2_876, __p3_876) __extension__ ({ \
67918 uint32x4_t __ret_876; \
67919 uint32x4_t __s0_876 = __p0_876; \
67920 uint16x4_t __s1_876 = __p1_876; \
67921 uint16x4_t __s2_876 = __p2_876; \
67922 uint32x4_t __rev0_876; __rev0_876 = __builtin_shufflevector(__s0_876, __s0_876, 3, 2, 1, 0); \
67923 uint16x4_t __rev1_876; __rev1_876 = __builtin_shufflevector(__s1_876, __s1_876, 3, 2, 1, 0); \
67924 uint16x4_t __rev2_876; __rev2_876 = __builtin_shufflevector(__s2_876, __s2_876, 3, 2, 1, 0); \
67925 __ret_876 = __rev0_876 - __noswap_vmull_u16(__rev1_876, __noswap_splat_lane_u16(__rev2_876, __p3_876)); \
67926 __ret_876 = __builtin_shufflevector(__ret_876, __ret_876, 3, 2, 1, 0); \
67927 __ret_876; \
69820#define vmlsl_lane_u16(__p0_848, __p1_848, __p2_848, __p3_848) __extension__ ({ \
69821 uint32x4_t __ret_848; \
69822 uint32x4_t __s0_848 = __p0_848; \
69823 uint16x4_t __s1_848 = __p1_848; \
69824 uint16x4_t __s2_848 = __p2_848; \
69825 uint32x4_t __rev0_848; __rev0_848 = __builtin_shufflevector(__s0_848, __s0_848, 3, 2, 1, 0); \
69826 uint16x4_t __rev1_848; __rev1_848 = __builtin_shufflevector(__s1_848, __s1_848, 3, 2, 1, 0); \
69827 uint16x4_t __rev2_848; __rev2_848 = __builtin_shufflevector(__s2_848, __s2_848, 3, 2, 1, 0); \
69828 __ret_848 = __rev0_848 - __noswap_vmull_u16(__rev1_848, __noswap_splat_lane_u16(__rev2_848, __p3_848)); \
69829 __ret_848 = __builtin_shufflevector(__ret_848, __ret_848, 3, 2, 1, 0); \
69830 __ret_848; \
6792869831})
6792969832#endif
6793069833
6793169834#ifdef __LITTLE_ENDIAN__
67932#define vmlsl_lane_s32(__p0_877, __p1_877, __p2_877, __p3_877) __extension__ ({ \
67933 int64x2_t __ret_877; \
67934 int64x2_t __s0_877 = __p0_877; \
67935 int32x2_t __s1_877 = __p1_877; \
67936 int32x2_t __s2_877 = __p2_877; \
67937 __ret_877 = __s0_877 - vmull_s32(__s1_877, splat_lane_s32(__s2_877, __p3_877)); \
67938 __ret_877; \
69835#define vmlsl_lane_s32(__p0_849, __p1_849, __p2_849, __p3_849) __extension__ ({ \
69836 int64x2_t __ret_849; \
69837 int64x2_t __s0_849 = __p0_849; \
69838 int32x2_t __s1_849 = __p1_849; \
69839 int32x2_t __s2_849 = __p2_849; \
69840 __ret_849 = __s0_849 - vmull_s32(__s1_849, splat_lane_s32(__s2_849, __p3_849)); \
69841 __ret_849; \
6793969842})
6794069843#else
67941#define vmlsl_lane_s32(__p0_878, __p1_878, __p2_878, __p3_878) __extension__ ({ \
67942 int64x2_t __ret_878; \
67943 int64x2_t __s0_878 = __p0_878; \
67944 int32x2_t __s1_878 = __p1_878; \
67945 int32x2_t __s2_878 = __p2_878; \
67946 int64x2_t __rev0_878; __rev0_878 = __builtin_shufflevector(__s0_878, __s0_878, 1, 0); \
67947 int32x2_t __rev1_878; __rev1_878 = __builtin_shufflevector(__s1_878, __s1_878, 1, 0); \
67948 int32x2_t __rev2_878; __rev2_878 = __builtin_shufflevector(__s2_878, __s2_878, 1, 0); \
67949 __ret_878 = __rev0_878 - __noswap_vmull_s32(__rev1_878, __noswap_splat_lane_s32(__rev2_878, __p3_878)); \
67950 __ret_878 = __builtin_shufflevector(__ret_878, __ret_878, 1, 0); \
67951 __ret_878; \
69844#define vmlsl_lane_s32(__p0_850, __p1_850, __p2_850, __p3_850) __extension__ ({ \
69845 int64x2_t __ret_850; \
69846 int64x2_t __s0_850 = __p0_850; \
69847 int32x2_t __s1_850 = __p1_850; \
69848 int32x2_t __s2_850 = __p2_850; \
69849 int64x2_t __rev0_850; __rev0_850 = __builtin_shufflevector(__s0_850, __s0_850, 1, 0); \
69850 int32x2_t __rev1_850; __rev1_850 = __builtin_shufflevector(__s1_850, __s1_850, 1, 0); \
69851 int32x2_t __rev2_850; __rev2_850 = __builtin_shufflevector(__s2_850, __s2_850, 1, 0); \
69852 __ret_850 = __rev0_850 - __noswap_vmull_s32(__rev1_850, __noswap_splat_lane_s32(__rev2_850, __p3_850)); \
69853 __ret_850 = __builtin_shufflevector(__ret_850, __ret_850, 1, 0); \
69854 __ret_850; \
6795269855})
6795369856#endif
6795469857
6795569858#ifdef __LITTLE_ENDIAN__
67956#define vmlsl_lane_s16(__p0_879, __p1_879, __p2_879, __p3_879) __extension__ ({ \
67957 int32x4_t __ret_879; \
67958 int32x4_t __s0_879 = __p0_879; \
67959 int16x4_t __s1_879 = __p1_879; \
67960 int16x4_t __s2_879 = __p2_879; \
67961 __ret_879 = __s0_879 - vmull_s16(__s1_879, splat_lane_s16(__s2_879, __p3_879)); \
67962 __ret_879; \
69859#define vmlsl_lane_s16(__p0_851, __p1_851, __p2_851, __p3_851) __extension__ ({ \
69860 int32x4_t __ret_851; \
69861 int32x4_t __s0_851 = __p0_851; \
69862 int16x4_t __s1_851 = __p1_851; \
69863 int16x4_t __s2_851 = __p2_851; \
69864 __ret_851 = __s0_851 - vmull_s16(__s1_851, splat_lane_s16(__s2_851, __p3_851)); \
69865 __ret_851; \
6796369866})
6796469867#else
67965#define vmlsl_lane_s16(__p0_880, __p1_880, __p2_880, __p3_880) __extension__ ({ \
67966 int32x4_t __ret_880; \
67967 int32x4_t __s0_880 = __p0_880; \
67968 int16x4_t __s1_880 = __p1_880; \
67969 int16x4_t __s2_880 = __p2_880; \
67970 int32x4_t __rev0_880; __rev0_880 = __builtin_shufflevector(__s0_880, __s0_880, 3, 2, 1, 0); \
67971 int16x4_t __rev1_880; __rev1_880 = __builtin_shufflevector(__s1_880, __s1_880, 3, 2, 1, 0); \
67972 int16x4_t __rev2_880; __rev2_880 = __builtin_shufflevector(__s2_880, __s2_880, 3, 2, 1, 0); \
67973 __ret_880 = __rev0_880 - __noswap_vmull_s16(__rev1_880, __noswap_splat_lane_s16(__rev2_880, __p3_880)); \
67974 __ret_880 = __builtin_shufflevector(__ret_880, __ret_880, 3, 2, 1, 0); \
67975 __ret_880; \
69868#define vmlsl_lane_s16(__p0_852, __p1_852, __p2_852, __p3_852) __extension__ ({ \
69869 int32x4_t __ret_852; \
69870 int32x4_t __s0_852 = __p0_852; \
69871 int16x4_t __s1_852 = __p1_852; \
69872 int16x4_t __s2_852 = __p2_852; \
69873 int32x4_t __rev0_852; __rev0_852 = __builtin_shufflevector(__s0_852, __s0_852, 3, 2, 1, 0); \
69874 int16x4_t __rev1_852; __rev1_852 = __builtin_shufflevector(__s1_852, __s1_852, 3, 2, 1, 0); \
69875 int16x4_t __rev2_852; __rev2_852 = __builtin_shufflevector(__s2_852, __s2_852, 3, 2, 1, 0); \
69876 __ret_852 = __rev0_852 - __noswap_vmull_s16(__rev1_852, __noswap_splat_lane_s16(__rev2_852, __p3_852)); \
69877 __ret_852 = __builtin_shufflevector(__ret_852, __ret_852, 3, 2, 1, 0); \
69878 __ret_852; \
6797669879})
6797769880#endif
6797869881
......@@ -68065,54 +69968,42 @@ __ai __attribute__((target("neon"))) int32x4_t __noswap_vmlsl_n_s16(int32x4_t __
6806569968#endif
6806669969
6806769970#ifdef __LITTLE_ENDIAN__
68068#define vset_lane_f16(__p0_881, __p1_881, __p2_881) __extension__ ({ \
68069 float16x4_t __ret_881; \
68070 float16_t __s0_881 = __p0_881; \
68071 float16x4_t __s1_881 = __p1_881; \
68072float16_t __reint_881 = __s0_881; \
68073float16x4_t __reint1_881 = __s1_881; \
68074int16x4_t __reint2_881 = vset_lane_s16(*(int16_t *) &__reint_881, *(int16x4_t *) &__reint1_881, __p2_881); \
68075 __ret_881 = *(float16x4_t *) &__reint2_881; \
68076 __ret_881; \
69971#define vset_lane_f16(__p0_853, __p1_853, __p2_853) __extension__ ({ \
69972 float16x4_t __ret_853; \
69973 float16_t __s0_853 = __p0_853; \
69974 float16x4_t __s1_853 = __p1_853; \
69975 __ret_853 = __builtin_bit_cast(float16x4_t, vset_lane_s16(__builtin_bit_cast(int16_t, __s0_853), __builtin_bit_cast(int16x4_t, __s1_853), __p2_853)); \
69976 __ret_853; \
6807769977})
6807869978#else
68079#define vset_lane_f16(__p0_882, __p1_882, __p2_882) __extension__ ({ \
68080 float16x4_t __ret_882; \
68081 float16_t __s0_882 = __p0_882; \
68082 float16x4_t __s1_882 = __p1_882; \
68083 float16x4_t __rev1_882; __rev1_882 = __builtin_shufflevector(__s1_882, __s1_882, 3, 2, 1, 0); \
68084float16_t __reint_882 = __s0_882; \
68085float16x4_t __reint1_882 = __rev1_882; \
68086int16x4_t __reint2_882 = __noswap_vset_lane_s16(*(int16_t *) &__reint_882, *(int16x4_t *) &__reint1_882, __p2_882); \
68087 __ret_882 = *(float16x4_t *) &__reint2_882; \
68088 __ret_882 = __builtin_shufflevector(__ret_882, __ret_882, 3, 2, 1, 0); \
68089 __ret_882; \
69979#define vset_lane_f16(__p0_854, __p1_854, __p2_854) __extension__ ({ \
69980 float16x4_t __ret_854; \
69981 float16_t __s0_854 = __p0_854; \
69982 float16x4_t __s1_854 = __p1_854; \
69983 float16x4_t __rev1_854; __rev1_854 = __builtin_shufflevector(__s1_854, __s1_854, 3, 2, 1, 0); \
69984 __ret_854 = __builtin_bit_cast(float16x4_t, __noswap_vset_lane_s16(__builtin_bit_cast(int16_t, __s0_854), __builtin_bit_cast(int16x4_t, __rev1_854), __p2_854)); \
69985 __ret_854 = __builtin_shufflevector(__ret_854, __ret_854, 3, 2, 1, 0); \
69986 __ret_854; \
6809069987})
6809169988#endif
6809269989
6809369990#ifdef __LITTLE_ENDIAN__
68094#define vsetq_lane_f16(__p0_883, __p1_883, __p2_883) __extension__ ({ \
68095 float16x8_t __ret_883; \
68096 float16_t __s0_883 = __p0_883; \
68097 float16x8_t __s1_883 = __p1_883; \
68098float16_t __reint_883 = __s0_883; \
68099float16x8_t __reint1_883 = __s1_883; \
68100int16x8_t __reint2_883 = vsetq_lane_s16(*(int16_t *) &__reint_883, *(int16x8_t *) &__reint1_883, __p2_883); \
68101 __ret_883 = *(float16x8_t *) &__reint2_883; \
68102 __ret_883; \
69991#define vsetq_lane_f16(__p0_855, __p1_855, __p2_855) __extension__ ({ \
69992 float16x8_t __ret_855; \
69993 float16_t __s0_855 = __p0_855; \
69994 float16x8_t __s1_855 = __p1_855; \
69995 __ret_855 = __builtin_bit_cast(float16x8_t, vsetq_lane_s16(__builtin_bit_cast(int16_t, __s0_855), __builtin_bit_cast(int16x8_t, __s1_855), __p2_855)); \
69996 __ret_855; \
6810369997})
6810469998#else
68105#define vsetq_lane_f16(__p0_884, __p1_884, __p2_884) __extension__ ({ \
68106 float16x8_t __ret_884; \
68107 float16_t __s0_884 = __p0_884; \
68108 float16x8_t __s1_884 = __p1_884; \
68109 float16x8_t __rev1_884; __rev1_884 = __builtin_shufflevector(__s1_884, __s1_884, 7, 6, 5, 4, 3, 2, 1, 0); \
68110float16_t __reint_884 = __s0_884; \
68111float16x8_t __reint1_884 = __rev1_884; \
68112int16x8_t __reint2_884 = __noswap_vsetq_lane_s16(*(int16_t *) &__reint_884, *(int16x8_t *) &__reint1_884, __p2_884); \
68113 __ret_884 = *(float16x8_t *) &__reint2_884; \
68114 __ret_884 = __builtin_shufflevector(__ret_884, __ret_884, 7, 6, 5, 4, 3, 2, 1, 0); \
68115 __ret_884; \
69999#define vsetq_lane_f16(__p0_856, __p1_856, __p2_856) __extension__ ({ \
70000 float16x8_t __ret_856; \
70001 float16_t __s0_856 = __p0_856; \
70002 float16x8_t __s1_856 = __p1_856; \
70003 float16x8_t __rev1_856; __rev1_856 = __builtin_shufflevector(__s1_856, __s1_856, 7, 6, 5, 4, 3, 2, 1, 0); \
70004 __ret_856 = __builtin_bit_cast(float16x8_t, __noswap_vsetq_lane_s16(__builtin_bit_cast(int16_t, __s0_856), __builtin_bit_cast(int16x8_t, __rev1_856), __p2_856)); \
70005 __ret_856 = __builtin_shufflevector(__ret_856, __ret_856, 7, 6, 5, 4, 3, 2, 1, 0); \
70006 __ret_856; \
6811670007})
6811770008#endif
6811870009
......@@ -68134,424 +70025,424 @@ __ai __attribute__((target("aes,neon"))) poly128_t vmull_high_p64(poly64x2_t __p
6813470025#endif
6813570026
6813670027#ifdef __LITTLE_ENDIAN__
68137#define vfmlalq_lane_high_f16(__p0_885, __p1_885, __p2_885, __p3_885) __extension__ ({ \
68138 float32x4_t __ret_885; \
68139 float32x4_t __s0_885 = __p0_885; \
68140 float16x8_t __s1_885 = __p1_885; \
68141 float16x4_t __s2_885 = __p2_885; \
68142 __ret_885 = vfmlalq_high_f16(__s0_885, __s1_885, (float16x8_t) {vget_lane_f16(__s2_885, __p3_885), vget_lane_f16(__s2_885, __p3_885), vget_lane_f16(__s2_885, __p3_885), vget_lane_f16(__s2_885, __p3_885), vget_lane_f16(__s2_885, __p3_885), vget_lane_f16(__s2_885, __p3_885), vget_lane_f16(__s2_885, __p3_885), vget_lane_f16(__s2_885, __p3_885)}); \
68143 __ret_885; \
70028#define vfmlalq_lane_high_f16(__p0_857, __p1_857, __p2_857, __p3_857) __extension__ ({ \
70029 float32x4_t __ret_857; \
70030 float32x4_t __s0_857 = __p0_857; \
70031 float16x8_t __s1_857 = __p1_857; \
70032 float16x4_t __s2_857 = __p2_857; \
70033 __ret_857 = vfmlalq_high_f16(__s0_857, __s1_857, (float16x8_t) {vget_lane_f16(__s2_857, __p3_857), vget_lane_f16(__s2_857, __p3_857), vget_lane_f16(__s2_857, __p3_857), vget_lane_f16(__s2_857, __p3_857), vget_lane_f16(__s2_857, __p3_857), vget_lane_f16(__s2_857, __p3_857), vget_lane_f16(__s2_857, __p3_857), vget_lane_f16(__s2_857, __p3_857)}); \
70034 __ret_857; \
6814470035})
6814570036#else
68146#define vfmlalq_lane_high_f16(__p0_886, __p1_886, __p2_886, __p3_886) __extension__ ({ \
68147 float32x4_t __ret_886; \
68148 float32x4_t __s0_886 = __p0_886; \
68149 float16x8_t __s1_886 = __p1_886; \
68150 float16x4_t __s2_886 = __p2_886; \
68151 float32x4_t __rev0_886; __rev0_886 = __builtin_shufflevector(__s0_886, __s0_886, 3, 2, 1, 0); \
68152 float16x8_t __rev1_886; __rev1_886 = __builtin_shufflevector(__s1_886, __s1_886, 7, 6, 5, 4, 3, 2, 1, 0); \
68153 float16x4_t __rev2_886; __rev2_886 = __builtin_shufflevector(__s2_886, __s2_886, 3, 2, 1, 0); \
68154 __ret_886 = __noswap_vfmlalq_high_f16(__rev0_886, __rev1_886, (float16x8_t) {__noswap_vget_lane_f16(__rev2_886, __p3_886), __noswap_vget_lane_f16(__rev2_886, __p3_886), __noswap_vget_lane_f16(__rev2_886, __p3_886), __noswap_vget_lane_f16(__rev2_886, __p3_886), __noswap_vget_lane_f16(__rev2_886, __p3_886), __noswap_vget_lane_f16(__rev2_886, __p3_886), __noswap_vget_lane_f16(__rev2_886, __p3_886), __noswap_vget_lane_f16(__rev2_886, __p3_886)}); \
68155 __ret_886 = __builtin_shufflevector(__ret_886, __ret_886, 3, 2, 1, 0); \
68156 __ret_886; \
70037#define vfmlalq_lane_high_f16(__p0_858, __p1_858, __p2_858, __p3_858) __extension__ ({ \
70038 float32x4_t __ret_858; \
70039 float32x4_t __s0_858 = __p0_858; \
70040 float16x8_t __s1_858 = __p1_858; \
70041 float16x4_t __s2_858 = __p2_858; \
70042 float32x4_t __rev0_858; __rev0_858 = __builtin_shufflevector(__s0_858, __s0_858, 3, 2, 1, 0); \
70043 float16x8_t __rev1_858; __rev1_858 = __builtin_shufflevector(__s1_858, __s1_858, 7, 6, 5, 4, 3, 2, 1, 0); \
70044 float16x4_t __rev2_858; __rev2_858 = __builtin_shufflevector(__s2_858, __s2_858, 3, 2, 1, 0); \
70045 __ret_858 = __noswap_vfmlalq_high_f16(__rev0_858, __rev1_858, (float16x8_t) {__noswap_vget_lane_f16(__rev2_858, __p3_858), __noswap_vget_lane_f16(__rev2_858, __p3_858), __noswap_vget_lane_f16(__rev2_858, __p3_858), __noswap_vget_lane_f16(__rev2_858, __p3_858), __noswap_vget_lane_f16(__rev2_858, __p3_858), __noswap_vget_lane_f16(__rev2_858, __p3_858), __noswap_vget_lane_f16(__rev2_858, __p3_858), __noswap_vget_lane_f16(__rev2_858, __p3_858)}); \
70046 __ret_858 = __builtin_shufflevector(__ret_858, __ret_858, 3, 2, 1, 0); \
70047 __ret_858; \
6815770048})
6815870049#endif
6815970050
6816070051#ifdef __LITTLE_ENDIAN__
68161#define vfmlal_lane_high_f16(__p0_887, __p1_887, __p2_887, __p3_887) __extension__ ({ \
68162 float32x2_t __ret_887; \
68163 float32x2_t __s0_887 = __p0_887; \
68164 float16x4_t __s1_887 = __p1_887; \
68165 float16x4_t __s2_887 = __p2_887; \
68166 __ret_887 = vfmlal_high_f16(__s0_887, __s1_887, (float16x4_t) {vget_lane_f16(__s2_887, __p3_887), vget_lane_f16(__s2_887, __p3_887), vget_lane_f16(__s2_887, __p3_887), vget_lane_f16(__s2_887, __p3_887)}); \
68167 __ret_887; \
70052#define vfmlal_lane_high_f16(__p0_859, __p1_859, __p2_859, __p3_859) __extension__ ({ \
70053 float32x2_t __ret_859; \
70054 float32x2_t __s0_859 = __p0_859; \
70055 float16x4_t __s1_859 = __p1_859; \
70056 float16x4_t __s2_859 = __p2_859; \
70057 __ret_859 = vfmlal_high_f16(__s0_859, __s1_859, (float16x4_t) {vget_lane_f16(__s2_859, __p3_859), vget_lane_f16(__s2_859, __p3_859), vget_lane_f16(__s2_859, __p3_859), vget_lane_f16(__s2_859, __p3_859)}); \
70058 __ret_859; \
6816870059})
6816970060#else
68170#define vfmlal_lane_high_f16(__p0_888, __p1_888, __p2_888, __p3_888) __extension__ ({ \
68171 float32x2_t __ret_888; \
68172 float32x2_t __s0_888 = __p0_888; \
68173 float16x4_t __s1_888 = __p1_888; \
68174 float16x4_t __s2_888 = __p2_888; \
68175 float32x2_t __rev0_888; __rev0_888 = __builtin_shufflevector(__s0_888, __s0_888, 1, 0); \
68176 float16x4_t __rev1_888; __rev1_888 = __builtin_shufflevector(__s1_888, __s1_888, 3, 2, 1, 0); \
68177 float16x4_t __rev2_888; __rev2_888 = __builtin_shufflevector(__s2_888, __s2_888, 3, 2, 1, 0); \
68178 __ret_888 = __noswap_vfmlal_high_f16(__rev0_888, __rev1_888, (float16x4_t) {__noswap_vget_lane_f16(__rev2_888, __p3_888), __noswap_vget_lane_f16(__rev2_888, __p3_888), __noswap_vget_lane_f16(__rev2_888, __p3_888), __noswap_vget_lane_f16(__rev2_888, __p3_888)}); \
68179 __ret_888 = __builtin_shufflevector(__ret_888, __ret_888, 1, 0); \
68180 __ret_888; \
70061#define vfmlal_lane_high_f16(__p0_860, __p1_860, __p2_860, __p3_860) __extension__ ({ \
70062 float32x2_t __ret_860; \
70063 float32x2_t __s0_860 = __p0_860; \
70064 float16x4_t __s1_860 = __p1_860; \
70065 float16x4_t __s2_860 = __p2_860; \
70066 float32x2_t __rev0_860; __rev0_860 = __builtin_shufflevector(__s0_860, __s0_860, 1, 0); \
70067 float16x4_t __rev1_860; __rev1_860 = __builtin_shufflevector(__s1_860, __s1_860, 3, 2, 1, 0); \
70068 float16x4_t __rev2_860; __rev2_860 = __builtin_shufflevector(__s2_860, __s2_860, 3, 2, 1, 0); \
70069 __ret_860 = __noswap_vfmlal_high_f16(__rev0_860, __rev1_860, (float16x4_t) {__noswap_vget_lane_f16(__rev2_860, __p3_860), __noswap_vget_lane_f16(__rev2_860, __p3_860), __noswap_vget_lane_f16(__rev2_860, __p3_860), __noswap_vget_lane_f16(__rev2_860, __p3_860)}); \
70070 __ret_860 = __builtin_shufflevector(__ret_860, __ret_860, 1, 0); \
70071 __ret_860; \
6818170072})
6818270073#endif
6818370074
6818470075#ifdef __LITTLE_ENDIAN__
68185#define vfmlalq_lane_low_f16(__p0_889, __p1_889, __p2_889, __p3_889) __extension__ ({ \
68186 float32x4_t __ret_889; \
68187 float32x4_t __s0_889 = __p0_889; \
68188 float16x8_t __s1_889 = __p1_889; \
68189 float16x4_t __s2_889 = __p2_889; \
68190 __ret_889 = vfmlalq_low_f16(__s0_889, __s1_889, (float16x8_t) {vget_lane_f16(__s2_889, __p3_889), vget_lane_f16(__s2_889, __p3_889), vget_lane_f16(__s2_889, __p3_889), vget_lane_f16(__s2_889, __p3_889), vget_lane_f16(__s2_889, __p3_889), vget_lane_f16(__s2_889, __p3_889), vget_lane_f16(__s2_889, __p3_889), vget_lane_f16(__s2_889, __p3_889)}); \
68191 __ret_889; \
70076#define vfmlalq_lane_low_f16(__p0_861, __p1_861, __p2_861, __p3_861) __extension__ ({ \
70077 float32x4_t __ret_861; \
70078 float32x4_t __s0_861 = __p0_861; \
70079 float16x8_t __s1_861 = __p1_861; \
70080 float16x4_t __s2_861 = __p2_861; \
70081 __ret_861 = vfmlalq_low_f16(__s0_861, __s1_861, (float16x8_t) {vget_lane_f16(__s2_861, __p3_861), vget_lane_f16(__s2_861, __p3_861), vget_lane_f16(__s2_861, __p3_861), vget_lane_f16(__s2_861, __p3_861), vget_lane_f16(__s2_861, __p3_861), vget_lane_f16(__s2_861, __p3_861), vget_lane_f16(__s2_861, __p3_861), vget_lane_f16(__s2_861, __p3_861)}); \
70082 __ret_861; \
6819270083})
6819370084#else
68194#define vfmlalq_lane_low_f16(__p0_890, __p1_890, __p2_890, __p3_890) __extension__ ({ \
68195 float32x4_t __ret_890; \
68196 float32x4_t __s0_890 = __p0_890; \
68197 float16x8_t __s1_890 = __p1_890; \
68198 float16x4_t __s2_890 = __p2_890; \
68199 float32x4_t __rev0_890; __rev0_890 = __builtin_shufflevector(__s0_890, __s0_890, 3, 2, 1, 0); \
68200 float16x8_t __rev1_890; __rev1_890 = __builtin_shufflevector(__s1_890, __s1_890, 7, 6, 5, 4, 3, 2, 1, 0); \
68201 float16x4_t __rev2_890; __rev2_890 = __builtin_shufflevector(__s2_890, __s2_890, 3, 2, 1, 0); \
68202 __ret_890 = __noswap_vfmlalq_low_f16(__rev0_890, __rev1_890, (float16x8_t) {__noswap_vget_lane_f16(__rev2_890, __p3_890), __noswap_vget_lane_f16(__rev2_890, __p3_890), __noswap_vget_lane_f16(__rev2_890, __p3_890), __noswap_vget_lane_f16(__rev2_890, __p3_890), __noswap_vget_lane_f16(__rev2_890, __p3_890), __noswap_vget_lane_f16(__rev2_890, __p3_890), __noswap_vget_lane_f16(__rev2_890, __p3_890), __noswap_vget_lane_f16(__rev2_890, __p3_890)}); \
68203 __ret_890 = __builtin_shufflevector(__ret_890, __ret_890, 3, 2, 1, 0); \
68204 __ret_890; \
70085#define vfmlalq_lane_low_f16(__p0_862, __p1_862, __p2_862, __p3_862) __extension__ ({ \
70086 float32x4_t __ret_862; \
70087 float32x4_t __s0_862 = __p0_862; \
70088 float16x8_t __s1_862 = __p1_862; \
70089 float16x4_t __s2_862 = __p2_862; \
70090 float32x4_t __rev0_862; __rev0_862 = __builtin_shufflevector(__s0_862, __s0_862, 3, 2, 1, 0); \
70091 float16x8_t __rev1_862; __rev1_862 = __builtin_shufflevector(__s1_862, __s1_862, 7, 6, 5, 4, 3, 2, 1, 0); \
70092 float16x4_t __rev2_862; __rev2_862 = __builtin_shufflevector(__s2_862, __s2_862, 3, 2, 1, 0); \
70093 __ret_862 = __noswap_vfmlalq_low_f16(__rev0_862, __rev1_862, (float16x8_t) {__noswap_vget_lane_f16(__rev2_862, __p3_862), __noswap_vget_lane_f16(__rev2_862, __p3_862), __noswap_vget_lane_f16(__rev2_862, __p3_862), __noswap_vget_lane_f16(__rev2_862, __p3_862), __noswap_vget_lane_f16(__rev2_862, __p3_862), __noswap_vget_lane_f16(__rev2_862, __p3_862), __noswap_vget_lane_f16(__rev2_862, __p3_862), __noswap_vget_lane_f16(__rev2_862, __p3_862)}); \
70094 __ret_862 = __builtin_shufflevector(__ret_862, __ret_862, 3, 2, 1, 0); \
70095 __ret_862; \
6820570096})
6820670097#endif
6820770098
6820870099#ifdef __LITTLE_ENDIAN__
68209#define vfmlal_lane_low_f16(__p0_891, __p1_891, __p2_891, __p3_891) __extension__ ({ \
68210 float32x2_t __ret_891; \
68211 float32x2_t __s0_891 = __p0_891; \
68212 float16x4_t __s1_891 = __p1_891; \
68213 float16x4_t __s2_891 = __p2_891; \
68214 __ret_891 = vfmlal_low_f16(__s0_891, __s1_891, (float16x4_t) {vget_lane_f16(__s2_891, __p3_891), vget_lane_f16(__s2_891, __p3_891), vget_lane_f16(__s2_891, __p3_891), vget_lane_f16(__s2_891, __p3_891)}); \
68215 __ret_891; \
70100#define vfmlal_lane_low_f16(__p0_863, __p1_863, __p2_863, __p3_863) __extension__ ({ \
70101 float32x2_t __ret_863; \
70102 float32x2_t __s0_863 = __p0_863; \
70103 float16x4_t __s1_863 = __p1_863; \
70104 float16x4_t __s2_863 = __p2_863; \
70105 __ret_863 = vfmlal_low_f16(__s0_863, __s1_863, (float16x4_t) {vget_lane_f16(__s2_863, __p3_863), vget_lane_f16(__s2_863, __p3_863), vget_lane_f16(__s2_863, __p3_863), vget_lane_f16(__s2_863, __p3_863)}); \
70106 __ret_863; \
6821670107})
6821770108#else
68218#define vfmlal_lane_low_f16(__p0_892, __p1_892, __p2_892, __p3_892) __extension__ ({ \
68219 float32x2_t __ret_892; \
68220 float32x2_t __s0_892 = __p0_892; \
68221 float16x4_t __s1_892 = __p1_892; \
68222 float16x4_t __s2_892 = __p2_892; \
68223 float32x2_t __rev0_892; __rev0_892 = __builtin_shufflevector(__s0_892, __s0_892, 1, 0); \
68224 float16x4_t __rev1_892; __rev1_892 = __builtin_shufflevector(__s1_892, __s1_892, 3, 2, 1, 0); \
68225 float16x4_t __rev2_892; __rev2_892 = __builtin_shufflevector(__s2_892, __s2_892, 3, 2, 1, 0); \
68226 __ret_892 = __noswap_vfmlal_low_f16(__rev0_892, __rev1_892, (float16x4_t) {__noswap_vget_lane_f16(__rev2_892, __p3_892), __noswap_vget_lane_f16(__rev2_892, __p3_892), __noswap_vget_lane_f16(__rev2_892, __p3_892), __noswap_vget_lane_f16(__rev2_892, __p3_892)}); \
68227 __ret_892 = __builtin_shufflevector(__ret_892, __ret_892, 1, 0); \
68228 __ret_892; \
70109#define vfmlal_lane_low_f16(__p0_864, __p1_864, __p2_864, __p3_864) __extension__ ({ \
70110 float32x2_t __ret_864; \
70111 float32x2_t __s0_864 = __p0_864; \
70112 float16x4_t __s1_864 = __p1_864; \
70113 float16x4_t __s2_864 = __p2_864; \
70114 float32x2_t __rev0_864; __rev0_864 = __builtin_shufflevector(__s0_864, __s0_864, 1, 0); \
70115 float16x4_t __rev1_864; __rev1_864 = __builtin_shufflevector(__s1_864, __s1_864, 3, 2, 1, 0); \
70116 float16x4_t __rev2_864; __rev2_864 = __builtin_shufflevector(__s2_864, __s2_864, 3, 2, 1, 0); \
70117 __ret_864 = __noswap_vfmlal_low_f16(__rev0_864, __rev1_864, (float16x4_t) {__noswap_vget_lane_f16(__rev2_864, __p3_864), __noswap_vget_lane_f16(__rev2_864, __p3_864), __noswap_vget_lane_f16(__rev2_864, __p3_864), __noswap_vget_lane_f16(__rev2_864, __p3_864)}); \
70118 __ret_864 = __builtin_shufflevector(__ret_864, __ret_864, 1, 0); \
70119 __ret_864; \
6822970120})
6823070121#endif
6823170122
6823270123#ifdef __LITTLE_ENDIAN__
68233#define vfmlalq_laneq_high_f16(__p0_893, __p1_893, __p2_893, __p3_893) __extension__ ({ \
68234 float32x4_t __ret_893; \
68235 float32x4_t __s0_893 = __p0_893; \
68236 float16x8_t __s1_893 = __p1_893; \
68237 float16x8_t __s2_893 = __p2_893; \
68238 __ret_893 = vfmlalq_high_f16(__s0_893, __s1_893, (float16x8_t) {vgetq_lane_f16(__s2_893, __p3_893), vgetq_lane_f16(__s2_893, __p3_893), vgetq_lane_f16(__s2_893, __p3_893), vgetq_lane_f16(__s2_893, __p3_893), vgetq_lane_f16(__s2_893, __p3_893), vgetq_lane_f16(__s2_893, __p3_893), vgetq_lane_f16(__s2_893, __p3_893), vgetq_lane_f16(__s2_893, __p3_893)}); \
68239 __ret_893; \
70124#define vfmlalq_laneq_high_f16(__p0_865, __p1_865, __p2_865, __p3_865) __extension__ ({ \
70125 float32x4_t __ret_865; \
70126 float32x4_t __s0_865 = __p0_865; \
70127 float16x8_t __s1_865 = __p1_865; \
70128 float16x8_t __s2_865 = __p2_865; \
70129 __ret_865 = vfmlalq_high_f16(__s0_865, __s1_865, (float16x8_t) {vgetq_lane_f16(__s2_865, __p3_865), vgetq_lane_f16(__s2_865, __p3_865), vgetq_lane_f16(__s2_865, __p3_865), vgetq_lane_f16(__s2_865, __p3_865), vgetq_lane_f16(__s2_865, __p3_865), vgetq_lane_f16(__s2_865, __p3_865), vgetq_lane_f16(__s2_865, __p3_865), vgetq_lane_f16(__s2_865, __p3_865)}); \
70130 __ret_865; \
6824070131})
6824170132#else
68242#define vfmlalq_laneq_high_f16(__p0_894, __p1_894, __p2_894, __p3_894) __extension__ ({ \
68243 float32x4_t __ret_894; \
68244 float32x4_t __s0_894 = __p0_894; \
68245 float16x8_t __s1_894 = __p1_894; \
68246 float16x8_t __s2_894 = __p2_894; \
68247 float32x4_t __rev0_894; __rev0_894 = __builtin_shufflevector(__s0_894, __s0_894, 3, 2, 1, 0); \
68248 float16x8_t __rev1_894; __rev1_894 = __builtin_shufflevector(__s1_894, __s1_894, 7, 6, 5, 4, 3, 2, 1, 0); \
68249 float16x8_t __rev2_894; __rev2_894 = __builtin_shufflevector(__s2_894, __s2_894, 7, 6, 5, 4, 3, 2, 1, 0); \
68250 __ret_894 = __noswap_vfmlalq_high_f16(__rev0_894, __rev1_894, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_894, __p3_894), __noswap_vgetq_lane_f16(__rev2_894, __p3_894), __noswap_vgetq_lane_f16(__rev2_894, __p3_894), __noswap_vgetq_lane_f16(__rev2_894, __p3_894), __noswap_vgetq_lane_f16(__rev2_894, __p3_894), __noswap_vgetq_lane_f16(__rev2_894, __p3_894), __noswap_vgetq_lane_f16(__rev2_894, __p3_894), __noswap_vgetq_lane_f16(__rev2_894, __p3_894)}); \
68251 __ret_894 = __builtin_shufflevector(__ret_894, __ret_894, 3, 2, 1, 0); \
68252 __ret_894; \
70133#define vfmlalq_laneq_high_f16(__p0_866, __p1_866, __p2_866, __p3_866) __extension__ ({ \
70134 float32x4_t __ret_866; \
70135 float32x4_t __s0_866 = __p0_866; \
70136 float16x8_t __s1_866 = __p1_866; \
70137 float16x8_t __s2_866 = __p2_866; \
70138 float32x4_t __rev0_866; __rev0_866 = __builtin_shufflevector(__s0_866, __s0_866, 3, 2, 1, 0); \
70139 float16x8_t __rev1_866; __rev1_866 = __builtin_shufflevector(__s1_866, __s1_866, 7, 6, 5, 4, 3, 2, 1, 0); \
70140 float16x8_t __rev2_866; __rev2_866 = __builtin_shufflevector(__s2_866, __s2_866, 7, 6, 5, 4, 3, 2, 1, 0); \
70141 __ret_866 = __noswap_vfmlalq_high_f16(__rev0_866, __rev1_866, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_866, __p3_866), __noswap_vgetq_lane_f16(__rev2_866, __p3_866), __noswap_vgetq_lane_f16(__rev2_866, __p3_866), __noswap_vgetq_lane_f16(__rev2_866, __p3_866), __noswap_vgetq_lane_f16(__rev2_866, __p3_866), __noswap_vgetq_lane_f16(__rev2_866, __p3_866), __noswap_vgetq_lane_f16(__rev2_866, __p3_866), __noswap_vgetq_lane_f16(__rev2_866, __p3_866)}); \
70142 __ret_866 = __builtin_shufflevector(__ret_866, __ret_866, 3, 2, 1, 0); \
70143 __ret_866; \
6825370144})
6825470145#endif
6825570146
6825670147#ifdef __LITTLE_ENDIAN__
68257#define vfmlal_laneq_high_f16(__p0_895, __p1_895, __p2_895, __p3_895) __extension__ ({ \
68258 float32x2_t __ret_895; \
68259 float32x2_t __s0_895 = __p0_895; \
68260 float16x4_t __s1_895 = __p1_895; \
68261 float16x8_t __s2_895 = __p2_895; \
68262 __ret_895 = vfmlal_high_f16(__s0_895, __s1_895, (float16x4_t) {vgetq_lane_f16(__s2_895, __p3_895), vgetq_lane_f16(__s2_895, __p3_895), vgetq_lane_f16(__s2_895, __p3_895), vgetq_lane_f16(__s2_895, __p3_895)}); \
68263 __ret_895; \
70148#define vfmlal_laneq_high_f16(__p0_867, __p1_867, __p2_867, __p3_867) __extension__ ({ \
70149 float32x2_t __ret_867; \
70150 float32x2_t __s0_867 = __p0_867; \
70151 float16x4_t __s1_867 = __p1_867; \
70152 float16x8_t __s2_867 = __p2_867; \
70153 __ret_867 = vfmlal_high_f16(__s0_867, __s1_867, (float16x4_t) {vgetq_lane_f16(__s2_867, __p3_867), vgetq_lane_f16(__s2_867, __p3_867), vgetq_lane_f16(__s2_867, __p3_867), vgetq_lane_f16(__s2_867, __p3_867)}); \
70154 __ret_867; \
6826470155})
6826570156#else
68266#define vfmlal_laneq_high_f16(__p0_896, __p1_896, __p2_896, __p3_896) __extension__ ({ \
68267 float32x2_t __ret_896; \
68268 float32x2_t __s0_896 = __p0_896; \
68269 float16x4_t __s1_896 = __p1_896; \
68270 float16x8_t __s2_896 = __p2_896; \
68271 float32x2_t __rev0_896; __rev0_896 = __builtin_shufflevector(__s0_896, __s0_896, 1, 0); \
68272 float16x4_t __rev1_896; __rev1_896 = __builtin_shufflevector(__s1_896, __s1_896, 3, 2, 1, 0); \
68273 float16x8_t __rev2_896; __rev2_896 = __builtin_shufflevector(__s2_896, __s2_896, 7, 6, 5, 4, 3, 2, 1, 0); \
68274 __ret_896 = __noswap_vfmlal_high_f16(__rev0_896, __rev1_896, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_896, __p3_896), __noswap_vgetq_lane_f16(__rev2_896, __p3_896), __noswap_vgetq_lane_f16(__rev2_896, __p3_896), __noswap_vgetq_lane_f16(__rev2_896, __p3_896)}); \
68275 __ret_896 = __builtin_shufflevector(__ret_896, __ret_896, 1, 0); \
68276 __ret_896; \
70157#define vfmlal_laneq_high_f16(__p0_868, __p1_868, __p2_868, __p3_868) __extension__ ({ \
70158 float32x2_t __ret_868; \
70159 float32x2_t __s0_868 = __p0_868; \
70160 float16x4_t __s1_868 = __p1_868; \
70161 float16x8_t __s2_868 = __p2_868; \
70162 float32x2_t __rev0_868; __rev0_868 = __builtin_shufflevector(__s0_868, __s0_868, 1, 0); \
70163 float16x4_t __rev1_868; __rev1_868 = __builtin_shufflevector(__s1_868, __s1_868, 3, 2, 1, 0); \
70164 float16x8_t __rev2_868; __rev2_868 = __builtin_shufflevector(__s2_868, __s2_868, 7, 6, 5, 4, 3, 2, 1, 0); \
70165 __ret_868 = __noswap_vfmlal_high_f16(__rev0_868, __rev1_868, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_868, __p3_868), __noswap_vgetq_lane_f16(__rev2_868, __p3_868), __noswap_vgetq_lane_f16(__rev2_868, __p3_868), __noswap_vgetq_lane_f16(__rev2_868, __p3_868)}); \
70166 __ret_868 = __builtin_shufflevector(__ret_868, __ret_868, 1, 0); \
70167 __ret_868; \
6827770168})
6827870169#endif
6827970170
6828070171#ifdef __LITTLE_ENDIAN__
68281#define vfmlalq_laneq_low_f16(__p0_897, __p1_897, __p2_897, __p3_897) __extension__ ({ \
68282 float32x4_t __ret_897; \
68283 float32x4_t __s0_897 = __p0_897; \
68284 float16x8_t __s1_897 = __p1_897; \
68285 float16x8_t __s2_897 = __p2_897; \
68286 __ret_897 = vfmlalq_low_f16(__s0_897, __s1_897, (float16x8_t) {vgetq_lane_f16(__s2_897, __p3_897), vgetq_lane_f16(__s2_897, __p3_897), vgetq_lane_f16(__s2_897, __p3_897), vgetq_lane_f16(__s2_897, __p3_897), vgetq_lane_f16(__s2_897, __p3_897), vgetq_lane_f16(__s2_897, __p3_897), vgetq_lane_f16(__s2_897, __p3_897), vgetq_lane_f16(__s2_897, __p3_897)}); \
68287 __ret_897; \
70172#define vfmlalq_laneq_low_f16(__p0_869, __p1_869, __p2_869, __p3_869) __extension__ ({ \
70173 float32x4_t __ret_869; \
70174 float32x4_t __s0_869 = __p0_869; \
70175 float16x8_t __s1_869 = __p1_869; \
70176 float16x8_t __s2_869 = __p2_869; \
70177 __ret_869 = vfmlalq_low_f16(__s0_869, __s1_869, (float16x8_t) {vgetq_lane_f16(__s2_869, __p3_869), vgetq_lane_f16(__s2_869, __p3_869), vgetq_lane_f16(__s2_869, __p3_869), vgetq_lane_f16(__s2_869, __p3_869), vgetq_lane_f16(__s2_869, __p3_869), vgetq_lane_f16(__s2_869, __p3_869), vgetq_lane_f16(__s2_869, __p3_869), vgetq_lane_f16(__s2_869, __p3_869)}); \
70178 __ret_869; \
6828870179})
6828970180#else
68290#define vfmlalq_laneq_low_f16(__p0_898, __p1_898, __p2_898, __p3_898) __extension__ ({ \
68291 float32x4_t __ret_898; \
68292 float32x4_t __s0_898 = __p0_898; \
68293 float16x8_t __s1_898 = __p1_898; \
68294 float16x8_t __s2_898 = __p2_898; \
68295 float32x4_t __rev0_898; __rev0_898 = __builtin_shufflevector(__s0_898, __s0_898, 3, 2, 1, 0); \
68296 float16x8_t __rev1_898; __rev1_898 = __builtin_shufflevector(__s1_898, __s1_898, 7, 6, 5, 4, 3, 2, 1, 0); \
68297 float16x8_t __rev2_898; __rev2_898 = __builtin_shufflevector(__s2_898, __s2_898, 7, 6, 5, 4, 3, 2, 1, 0); \
68298 __ret_898 = __noswap_vfmlalq_low_f16(__rev0_898, __rev1_898, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_898, __p3_898), __noswap_vgetq_lane_f16(__rev2_898, __p3_898), __noswap_vgetq_lane_f16(__rev2_898, __p3_898), __noswap_vgetq_lane_f16(__rev2_898, __p3_898), __noswap_vgetq_lane_f16(__rev2_898, __p3_898), __noswap_vgetq_lane_f16(__rev2_898, __p3_898), __noswap_vgetq_lane_f16(__rev2_898, __p3_898), __noswap_vgetq_lane_f16(__rev2_898, __p3_898)}); \
68299 __ret_898 = __builtin_shufflevector(__ret_898, __ret_898, 3, 2, 1, 0); \
68300 __ret_898; \
70181#define vfmlalq_laneq_low_f16(__p0_870, __p1_870, __p2_870, __p3_870) __extension__ ({ \
70182 float32x4_t __ret_870; \
70183 float32x4_t __s0_870 = __p0_870; \
70184 float16x8_t __s1_870 = __p1_870; \
70185 float16x8_t __s2_870 = __p2_870; \
70186 float32x4_t __rev0_870; __rev0_870 = __builtin_shufflevector(__s0_870, __s0_870, 3, 2, 1, 0); \
70187 float16x8_t __rev1_870; __rev1_870 = __builtin_shufflevector(__s1_870, __s1_870, 7, 6, 5, 4, 3, 2, 1, 0); \
70188 float16x8_t __rev2_870; __rev2_870 = __builtin_shufflevector(__s2_870, __s2_870, 7, 6, 5, 4, 3, 2, 1, 0); \
70189 __ret_870 = __noswap_vfmlalq_low_f16(__rev0_870, __rev1_870, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_870, __p3_870), __noswap_vgetq_lane_f16(__rev2_870, __p3_870), __noswap_vgetq_lane_f16(__rev2_870, __p3_870), __noswap_vgetq_lane_f16(__rev2_870, __p3_870), __noswap_vgetq_lane_f16(__rev2_870, __p3_870), __noswap_vgetq_lane_f16(__rev2_870, __p3_870), __noswap_vgetq_lane_f16(__rev2_870, __p3_870), __noswap_vgetq_lane_f16(__rev2_870, __p3_870)}); \
70190 __ret_870 = __builtin_shufflevector(__ret_870, __ret_870, 3, 2, 1, 0); \
70191 __ret_870; \
6830170192})
6830270193#endif
6830370194
6830470195#ifdef __LITTLE_ENDIAN__
68305#define vfmlal_laneq_low_f16(__p0_899, __p1_899, __p2_899, __p3_899) __extension__ ({ \
68306 float32x2_t __ret_899; \
68307 float32x2_t __s0_899 = __p0_899; \
68308 float16x4_t __s1_899 = __p1_899; \
68309 float16x8_t __s2_899 = __p2_899; \
68310 __ret_899 = vfmlal_low_f16(__s0_899, __s1_899, (float16x4_t) {vgetq_lane_f16(__s2_899, __p3_899), vgetq_lane_f16(__s2_899, __p3_899), vgetq_lane_f16(__s2_899, __p3_899), vgetq_lane_f16(__s2_899, __p3_899)}); \
68311 __ret_899; \
70196#define vfmlal_laneq_low_f16(__p0_871, __p1_871, __p2_871, __p3_871) __extension__ ({ \
70197 float32x2_t __ret_871; \
70198 float32x2_t __s0_871 = __p0_871; \
70199 float16x4_t __s1_871 = __p1_871; \
70200 float16x8_t __s2_871 = __p2_871; \
70201 __ret_871 = vfmlal_low_f16(__s0_871, __s1_871, (float16x4_t) {vgetq_lane_f16(__s2_871, __p3_871), vgetq_lane_f16(__s2_871, __p3_871), vgetq_lane_f16(__s2_871, __p3_871), vgetq_lane_f16(__s2_871, __p3_871)}); \
70202 __ret_871; \
6831270203})
6831370204#else
68314#define vfmlal_laneq_low_f16(__p0_900, __p1_900, __p2_900, __p3_900) __extension__ ({ \
68315 float32x2_t __ret_900; \
68316 float32x2_t __s0_900 = __p0_900; \
68317 float16x4_t __s1_900 = __p1_900; \
68318 float16x8_t __s2_900 = __p2_900; \
68319 float32x2_t __rev0_900; __rev0_900 = __builtin_shufflevector(__s0_900, __s0_900, 1, 0); \
68320 float16x4_t __rev1_900; __rev1_900 = __builtin_shufflevector(__s1_900, __s1_900, 3, 2, 1, 0); \
68321 float16x8_t __rev2_900; __rev2_900 = __builtin_shufflevector(__s2_900, __s2_900, 7, 6, 5, 4, 3, 2, 1, 0); \
68322 __ret_900 = __noswap_vfmlal_low_f16(__rev0_900, __rev1_900, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_900, __p3_900), __noswap_vgetq_lane_f16(__rev2_900, __p3_900), __noswap_vgetq_lane_f16(__rev2_900, __p3_900), __noswap_vgetq_lane_f16(__rev2_900, __p3_900)}); \
68323 __ret_900 = __builtin_shufflevector(__ret_900, __ret_900, 1, 0); \
68324 __ret_900; \
70205#define vfmlal_laneq_low_f16(__p0_872, __p1_872, __p2_872, __p3_872) __extension__ ({ \
70206 float32x2_t __ret_872; \
70207 float32x2_t __s0_872 = __p0_872; \
70208 float16x4_t __s1_872 = __p1_872; \
70209 float16x8_t __s2_872 = __p2_872; \
70210 float32x2_t __rev0_872; __rev0_872 = __builtin_shufflevector(__s0_872, __s0_872, 1, 0); \
70211 float16x4_t __rev1_872; __rev1_872 = __builtin_shufflevector(__s1_872, __s1_872, 3, 2, 1, 0); \
70212 float16x8_t __rev2_872; __rev2_872 = __builtin_shufflevector(__s2_872, __s2_872, 7, 6, 5, 4, 3, 2, 1, 0); \
70213 __ret_872 = __noswap_vfmlal_low_f16(__rev0_872, __rev1_872, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_872, __p3_872), __noswap_vgetq_lane_f16(__rev2_872, __p3_872), __noswap_vgetq_lane_f16(__rev2_872, __p3_872), __noswap_vgetq_lane_f16(__rev2_872, __p3_872)}); \
70214 __ret_872 = __builtin_shufflevector(__ret_872, __ret_872, 1, 0); \
70215 __ret_872; \
6832570216})
6832670217#endif
6832770218
6832870219#ifdef __LITTLE_ENDIAN__
68329#define vfmlslq_lane_high_f16(__p0_901, __p1_901, __p2_901, __p3_901) __extension__ ({ \
68330 float32x4_t __ret_901; \
68331 float32x4_t __s0_901 = __p0_901; \
68332 float16x8_t __s1_901 = __p1_901; \
68333 float16x4_t __s2_901 = __p2_901; \
68334 __ret_901 = vfmlslq_high_f16(__s0_901, __s1_901, (float16x8_t) {vget_lane_f16(__s2_901, __p3_901), vget_lane_f16(__s2_901, __p3_901), vget_lane_f16(__s2_901, __p3_901), vget_lane_f16(__s2_901, __p3_901), vget_lane_f16(__s2_901, __p3_901), vget_lane_f16(__s2_901, __p3_901), vget_lane_f16(__s2_901, __p3_901), vget_lane_f16(__s2_901, __p3_901)}); \
68335 __ret_901; \
70220#define vfmlslq_lane_high_f16(__p0_873, __p1_873, __p2_873, __p3_873) __extension__ ({ \
70221 float32x4_t __ret_873; \
70222 float32x4_t __s0_873 = __p0_873; \
70223 float16x8_t __s1_873 = __p1_873; \
70224 float16x4_t __s2_873 = __p2_873; \
70225 __ret_873 = vfmlslq_high_f16(__s0_873, __s1_873, (float16x8_t) {vget_lane_f16(__s2_873, __p3_873), vget_lane_f16(__s2_873, __p3_873), vget_lane_f16(__s2_873, __p3_873), vget_lane_f16(__s2_873, __p3_873), vget_lane_f16(__s2_873, __p3_873), vget_lane_f16(__s2_873, __p3_873), vget_lane_f16(__s2_873, __p3_873), vget_lane_f16(__s2_873, __p3_873)}); \
70226 __ret_873; \
6833670227})
6833770228#else
68338#define vfmlslq_lane_high_f16(__p0_902, __p1_902, __p2_902, __p3_902) __extension__ ({ \
68339 float32x4_t __ret_902; \
68340 float32x4_t __s0_902 = __p0_902; \
68341 float16x8_t __s1_902 = __p1_902; \
68342 float16x4_t __s2_902 = __p2_902; \
68343 float32x4_t __rev0_902; __rev0_902 = __builtin_shufflevector(__s0_902, __s0_902, 3, 2, 1, 0); \
68344 float16x8_t __rev1_902; __rev1_902 = __builtin_shufflevector(__s1_902, __s1_902, 7, 6, 5, 4, 3, 2, 1, 0); \
68345 float16x4_t __rev2_902; __rev2_902 = __builtin_shufflevector(__s2_902, __s2_902, 3, 2, 1, 0); \
68346 __ret_902 = __noswap_vfmlslq_high_f16(__rev0_902, __rev1_902, (float16x8_t) {__noswap_vget_lane_f16(__rev2_902, __p3_902), __noswap_vget_lane_f16(__rev2_902, __p3_902), __noswap_vget_lane_f16(__rev2_902, __p3_902), __noswap_vget_lane_f16(__rev2_902, __p3_902), __noswap_vget_lane_f16(__rev2_902, __p3_902), __noswap_vget_lane_f16(__rev2_902, __p3_902), __noswap_vget_lane_f16(__rev2_902, __p3_902), __noswap_vget_lane_f16(__rev2_902, __p3_902)}); \
68347 __ret_902 = __builtin_shufflevector(__ret_902, __ret_902, 3, 2, 1, 0); \
68348 __ret_902; \
70229#define vfmlslq_lane_high_f16(__p0_874, __p1_874, __p2_874, __p3_874) __extension__ ({ \
70230 float32x4_t __ret_874; \
70231 float32x4_t __s0_874 = __p0_874; \
70232 float16x8_t __s1_874 = __p1_874; \
70233 float16x4_t __s2_874 = __p2_874; \
70234 float32x4_t __rev0_874; __rev0_874 = __builtin_shufflevector(__s0_874, __s0_874, 3, 2, 1, 0); \
70235 float16x8_t __rev1_874; __rev1_874 = __builtin_shufflevector(__s1_874, __s1_874, 7, 6, 5, 4, 3, 2, 1, 0); \
70236 float16x4_t __rev2_874; __rev2_874 = __builtin_shufflevector(__s2_874, __s2_874, 3, 2, 1, 0); \
70237 __ret_874 = __noswap_vfmlslq_high_f16(__rev0_874, __rev1_874, (float16x8_t) {__noswap_vget_lane_f16(__rev2_874, __p3_874), __noswap_vget_lane_f16(__rev2_874, __p3_874), __noswap_vget_lane_f16(__rev2_874, __p3_874), __noswap_vget_lane_f16(__rev2_874, __p3_874), __noswap_vget_lane_f16(__rev2_874, __p3_874), __noswap_vget_lane_f16(__rev2_874, __p3_874), __noswap_vget_lane_f16(__rev2_874, __p3_874), __noswap_vget_lane_f16(__rev2_874, __p3_874)}); \
70238 __ret_874 = __builtin_shufflevector(__ret_874, __ret_874, 3, 2, 1, 0); \
70239 __ret_874; \
6834970240})
6835070241#endif
6835170242
6835270243#ifdef __LITTLE_ENDIAN__
68353#define vfmlsl_lane_high_f16(__p0_903, __p1_903, __p2_903, __p3_903) __extension__ ({ \
68354 float32x2_t __ret_903; \
68355 float32x2_t __s0_903 = __p0_903; \
68356 float16x4_t __s1_903 = __p1_903; \
68357 float16x4_t __s2_903 = __p2_903; \
68358 __ret_903 = vfmlsl_high_f16(__s0_903, __s1_903, (float16x4_t) {vget_lane_f16(__s2_903, __p3_903), vget_lane_f16(__s2_903, __p3_903), vget_lane_f16(__s2_903, __p3_903), vget_lane_f16(__s2_903, __p3_903)}); \
68359 __ret_903; \
70244#define vfmlsl_lane_high_f16(__p0_875, __p1_875, __p2_875, __p3_875) __extension__ ({ \
70245 float32x2_t __ret_875; \
70246 float32x2_t __s0_875 = __p0_875; \
70247 float16x4_t __s1_875 = __p1_875; \
70248 float16x4_t __s2_875 = __p2_875; \
70249 __ret_875 = vfmlsl_high_f16(__s0_875, __s1_875, (float16x4_t) {vget_lane_f16(__s2_875, __p3_875), vget_lane_f16(__s2_875, __p3_875), vget_lane_f16(__s2_875, __p3_875), vget_lane_f16(__s2_875, __p3_875)}); \
70250 __ret_875; \
6836070251})
6836170252#else
68362#define vfmlsl_lane_high_f16(__p0_904, __p1_904, __p2_904, __p3_904) __extension__ ({ \
68363 float32x2_t __ret_904; \
68364 float32x2_t __s0_904 = __p0_904; \
68365 float16x4_t __s1_904 = __p1_904; \
68366 float16x4_t __s2_904 = __p2_904; \
68367 float32x2_t __rev0_904; __rev0_904 = __builtin_shufflevector(__s0_904, __s0_904, 1, 0); \
68368 float16x4_t __rev1_904; __rev1_904 = __builtin_shufflevector(__s1_904, __s1_904, 3, 2, 1, 0); \
68369 float16x4_t __rev2_904; __rev2_904 = __builtin_shufflevector(__s2_904, __s2_904, 3, 2, 1, 0); \
68370 __ret_904 = __noswap_vfmlsl_high_f16(__rev0_904, __rev1_904, (float16x4_t) {__noswap_vget_lane_f16(__rev2_904, __p3_904), __noswap_vget_lane_f16(__rev2_904, __p3_904), __noswap_vget_lane_f16(__rev2_904, __p3_904), __noswap_vget_lane_f16(__rev2_904, __p3_904)}); \
68371 __ret_904 = __builtin_shufflevector(__ret_904, __ret_904, 1, 0); \
68372 __ret_904; \
70253#define vfmlsl_lane_high_f16(__p0_876, __p1_876, __p2_876, __p3_876) __extension__ ({ \
70254 float32x2_t __ret_876; \
70255 float32x2_t __s0_876 = __p0_876; \
70256 float16x4_t __s1_876 = __p1_876; \
70257 float16x4_t __s2_876 = __p2_876; \
70258 float32x2_t __rev0_876; __rev0_876 = __builtin_shufflevector(__s0_876, __s0_876, 1, 0); \
70259 float16x4_t __rev1_876; __rev1_876 = __builtin_shufflevector(__s1_876, __s1_876, 3, 2, 1, 0); \
70260 float16x4_t __rev2_876; __rev2_876 = __builtin_shufflevector(__s2_876, __s2_876, 3, 2, 1, 0); \
70261 __ret_876 = __noswap_vfmlsl_high_f16(__rev0_876, __rev1_876, (float16x4_t) {__noswap_vget_lane_f16(__rev2_876, __p3_876), __noswap_vget_lane_f16(__rev2_876, __p3_876), __noswap_vget_lane_f16(__rev2_876, __p3_876), __noswap_vget_lane_f16(__rev2_876, __p3_876)}); \
70262 __ret_876 = __builtin_shufflevector(__ret_876, __ret_876, 1, 0); \
70263 __ret_876; \
6837370264})
6837470265#endif
6837570266
6837670267#ifdef __LITTLE_ENDIAN__
68377#define vfmlslq_lane_low_f16(__p0_905, __p1_905, __p2_905, __p3_905) __extension__ ({ \
68378 float32x4_t __ret_905; \
68379 float32x4_t __s0_905 = __p0_905; \
68380 float16x8_t __s1_905 = __p1_905; \
68381 float16x4_t __s2_905 = __p2_905; \
68382 __ret_905 = vfmlslq_low_f16(__s0_905, __s1_905, (float16x8_t) {vget_lane_f16(__s2_905, __p3_905), vget_lane_f16(__s2_905, __p3_905), vget_lane_f16(__s2_905, __p3_905), vget_lane_f16(__s2_905, __p3_905), vget_lane_f16(__s2_905, __p3_905), vget_lane_f16(__s2_905, __p3_905), vget_lane_f16(__s2_905, __p3_905), vget_lane_f16(__s2_905, __p3_905)}); \
68383 __ret_905; \
70268#define vfmlslq_lane_low_f16(__p0_877, __p1_877, __p2_877, __p3_877) __extension__ ({ \
70269 float32x4_t __ret_877; \
70270 float32x4_t __s0_877 = __p0_877; \
70271 float16x8_t __s1_877 = __p1_877; \
70272 float16x4_t __s2_877 = __p2_877; \
70273 __ret_877 = vfmlslq_low_f16(__s0_877, __s1_877, (float16x8_t) {vget_lane_f16(__s2_877, __p3_877), vget_lane_f16(__s2_877, __p3_877), vget_lane_f16(__s2_877, __p3_877), vget_lane_f16(__s2_877, __p3_877), vget_lane_f16(__s2_877, __p3_877), vget_lane_f16(__s2_877, __p3_877), vget_lane_f16(__s2_877, __p3_877), vget_lane_f16(__s2_877, __p3_877)}); \
70274 __ret_877; \
6838470275})
6838570276#else
68386#define vfmlslq_lane_low_f16(__p0_906, __p1_906, __p2_906, __p3_906) __extension__ ({ \
68387 float32x4_t __ret_906; \
68388 float32x4_t __s0_906 = __p0_906; \
68389 float16x8_t __s1_906 = __p1_906; \
68390 float16x4_t __s2_906 = __p2_906; \
68391 float32x4_t __rev0_906; __rev0_906 = __builtin_shufflevector(__s0_906, __s0_906, 3, 2, 1, 0); \
68392 float16x8_t __rev1_906; __rev1_906 = __builtin_shufflevector(__s1_906, __s1_906, 7, 6, 5, 4, 3, 2, 1, 0); \
68393 float16x4_t __rev2_906; __rev2_906 = __builtin_shufflevector(__s2_906, __s2_906, 3, 2, 1, 0); \
68394 __ret_906 = __noswap_vfmlslq_low_f16(__rev0_906, __rev1_906, (float16x8_t) {__noswap_vget_lane_f16(__rev2_906, __p3_906), __noswap_vget_lane_f16(__rev2_906, __p3_906), __noswap_vget_lane_f16(__rev2_906, __p3_906), __noswap_vget_lane_f16(__rev2_906, __p3_906), __noswap_vget_lane_f16(__rev2_906, __p3_906), __noswap_vget_lane_f16(__rev2_906, __p3_906), __noswap_vget_lane_f16(__rev2_906, __p3_906), __noswap_vget_lane_f16(__rev2_906, __p3_906)}); \
68395 __ret_906 = __builtin_shufflevector(__ret_906, __ret_906, 3, 2, 1, 0); \
68396 __ret_906; \
70277#define vfmlslq_lane_low_f16(__p0_878, __p1_878, __p2_878, __p3_878) __extension__ ({ \
70278 float32x4_t __ret_878; \
70279 float32x4_t __s0_878 = __p0_878; \
70280 float16x8_t __s1_878 = __p1_878; \
70281 float16x4_t __s2_878 = __p2_878; \
70282 float32x4_t __rev0_878; __rev0_878 = __builtin_shufflevector(__s0_878, __s0_878, 3, 2, 1, 0); \
70283 float16x8_t __rev1_878; __rev1_878 = __builtin_shufflevector(__s1_878, __s1_878, 7, 6, 5, 4, 3, 2, 1, 0); \
70284 float16x4_t __rev2_878; __rev2_878 = __builtin_shufflevector(__s2_878, __s2_878, 3, 2, 1, 0); \
70285 __ret_878 = __noswap_vfmlslq_low_f16(__rev0_878, __rev1_878, (float16x8_t) {__noswap_vget_lane_f16(__rev2_878, __p3_878), __noswap_vget_lane_f16(__rev2_878, __p3_878), __noswap_vget_lane_f16(__rev2_878, __p3_878), __noswap_vget_lane_f16(__rev2_878, __p3_878), __noswap_vget_lane_f16(__rev2_878, __p3_878), __noswap_vget_lane_f16(__rev2_878, __p3_878), __noswap_vget_lane_f16(__rev2_878, __p3_878), __noswap_vget_lane_f16(__rev2_878, __p3_878)}); \
70286 __ret_878 = __builtin_shufflevector(__ret_878, __ret_878, 3, 2, 1, 0); \
70287 __ret_878; \
6839770288})
6839870289#endif
6839970290
6840070291#ifdef __LITTLE_ENDIAN__
68401#define vfmlsl_lane_low_f16(__p0_907, __p1_907, __p2_907, __p3_907) __extension__ ({ \
68402 float32x2_t __ret_907; \
68403 float32x2_t __s0_907 = __p0_907; \
68404 float16x4_t __s1_907 = __p1_907; \
68405 float16x4_t __s2_907 = __p2_907; \
68406 __ret_907 = vfmlsl_low_f16(__s0_907, __s1_907, (float16x4_t) {vget_lane_f16(__s2_907, __p3_907), vget_lane_f16(__s2_907, __p3_907), vget_lane_f16(__s2_907, __p3_907), vget_lane_f16(__s2_907, __p3_907)}); \
68407 __ret_907; \
70292#define vfmlsl_lane_low_f16(__p0_879, __p1_879, __p2_879, __p3_879) __extension__ ({ \
70293 float32x2_t __ret_879; \
70294 float32x2_t __s0_879 = __p0_879; \
70295 float16x4_t __s1_879 = __p1_879; \
70296 float16x4_t __s2_879 = __p2_879; \
70297 __ret_879 = vfmlsl_low_f16(__s0_879, __s1_879, (float16x4_t) {vget_lane_f16(__s2_879, __p3_879), vget_lane_f16(__s2_879, __p3_879), vget_lane_f16(__s2_879, __p3_879), vget_lane_f16(__s2_879, __p3_879)}); \
70298 __ret_879; \
6840870299})
6840970300#else
68410#define vfmlsl_lane_low_f16(__p0_908, __p1_908, __p2_908, __p3_908) __extension__ ({ \
68411 float32x2_t __ret_908; \
68412 float32x2_t __s0_908 = __p0_908; \
68413 float16x4_t __s1_908 = __p1_908; \
68414 float16x4_t __s2_908 = __p2_908; \
68415 float32x2_t __rev0_908; __rev0_908 = __builtin_shufflevector(__s0_908, __s0_908, 1, 0); \
68416 float16x4_t __rev1_908; __rev1_908 = __builtin_shufflevector(__s1_908, __s1_908, 3, 2, 1, 0); \
68417 float16x4_t __rev2_908; __rev2_908 = __builtin_shufflevector(__s2_908, __s2_908, 3, 2, 1, 0); \
68418 __ret_908 = __noswap_vfmlsl_low_f16(__rev0_908, __rev1_908, (float16x4_t) {__noswap_vget_lane_f16(__rev2_908, __p3_908), __noswap_vget_lane_f16(__rev2_908, __p3_908), __noswap_vget_lane_f16(__rev2_908, __p3_908), __noswap_vget_lane_f16(__rev2_908, __p3_908)}); \
68419 __ret_908 = __builtin_shufflevector(__ret_908, __ret_908, 1, 0); \
68420 __ret_908; \
70301#define vfmlsl_lane_low_f16(__p0_880, __p1_880, __p2_880, __p3_880) __extension__ ({ \
70302 float32x2_t __ret_880; \
70303 float32x2_t __s0_880 = __p0_880; \
70304 float16x4_t __s1_880 = __p1_880; \
70305 float16x4_t __s2_880 = __p2_880; \
70306 float32x2_t __rev0_880; __rev0_880 = __builtin_shufflevector(__s0_880, __s0_880, 1, 0); \
70307 float16x4_t __rev1_880; __rev1_880 = __builtin_shufflevector(__s1_880, __s1_880, 3, 2, 1, 0); \
70308 float16x4_t __rev2_880; __rev2_880 = __builtin_shufflevector(__s2_880, __s2_880, 3, 2, 1, 0); \
70309 __ret_880 = __noswap_vfmlsl_low_f16(__rev0_880, __rev1_880, (float16x4_t) {__noswap_vget_lane_f16(__rev2_880, __p3_880), __noswap_vget_lane_f16(__rev2_880, __p3_880), __noswap_vget_lane_f16(__rev2_880, __p3_880), __noswap_vget_lane_f16(__rev2_880, __p3_880)}); \
70310 __ret_880 = __builtin_shufflevector(__ret_880, __ret_880, 1, 0); \
70311 __ret_880; \
6842170312})
6842270313#endif
6842370314
6842470315#ifdef __LITTLE_ENDIAN__
68425#define vfmlslq_laneq_high_f16(__p0_909, __p1_909, __p2_909, __p3_909) __extension__ ({ \
68426 float32x4_t __ret_909; \
68427 float32x4_t __s0_909 = __p0_909; \
68428 float16x8_t __s1_909 = __p1_909; \
68429 float16x8_t __s2_909 = __p2_909; \
68430 __ret_909 = vfmlslq_high_f16(__s0_909, __s1_909, (float16x8_t) {vgetq_lane_f16(__s2_909, __p3_909), vgetq_lane_f16(__s2_909, __p3_909), vgetq_lane_f16(__s2_909, __p3_909), vgetq_lane_f16(__s2_909, __p3_909), vgetq_lane_f16(__s2_909, __p3_909), vgetq_lane_f16(__s2_909, __p3_909), vgetq_lane_f16(__s2_909, __p3_909), vgetq_lane_f16(__s2_909, __p3_909)}); \
68431 __ret_909; \
70316#define vfmlslq_laneq_high_f16(__p0_881, __p1_881, __p2_881, __p3_881) __extension__ ({ \
70317 float32x4_t __ret_881; \
70318 float32x4_t __s0_881 = __p0_881; \
70319 float16x8_t __s1_881 = __p1_881; \
70320 float16x8_t __s2_881 = __p2_881; \
70321 __ret_881 = vfmlslq_high_f16(__s0_881, __s1_881, (float16x8_t) {vgetq_lane_f16(__s2_881, __p3_881), vgetq_lane_f16(__s2_881, __p3_881), vgetq_lane_f16(__s2_881, __p3_881), vgetq_lane_f16(__s2_881, __p3_881), vgetq_lane_f16(__s2_881, __p3_881), vgetq_lane_f16(__s2_881, __p3_881), vgetq_lane_f16(__s2_881, __p3_881), vgetq_lane_f16(__s2_881, __p3_881)}); \
70322 __ret_881; \
6843270323})
6843370324#else
68434#define vfmlslq_laneq_high_f16(__p0_910, __p1_910, __p2_910, __p3_910) __extension__ ({ \
68435 float32x4_t __ret_910; \
68436 float32x4_t __s0_910 = __p0_910; \
68437 float16x8_t __s1_910 = __p1_910; \
68438 float16x8_t __s2_910 = __p2_910; \
68439 float32x4_t __rev0_910; __rev0_910 = __builtin_shufflevector(__s0_910, __s0_910, 3, 2, 1, 0); \
68440 float16x8_t __rev1_910; __rev1_910 = __builtin_shufflevector(__s1_910, __s1_910, 7, 6, 5, 4, 3, 2, 1, 0); \
68441 float16x8_t __rev2_910; __rev2_910 = __builtin_shufflevector(__s2_910, __s2_910, 7, 6, 5, 4, 3, 2, 1, 0); \
68442 __ret_910 = __noswap_vfmlslq_high_f16(__rev0_910, __rev1_910, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_910, __p3_910), __noswap_vgetq_lane_f16(__rev2_910, __p3_910), __noswap_vgetq_lane_f16(__rev2_910, __p3_910), __noswap_vgetq_lane_f16(__rev2_910, __p3_910), __noswap_vgetq_lane_f16(__rev2_910, __p3_910), __noswap_vgetq_lane_f16(__rev2_910, __p3_910), __noswap_vgetq_lane_f16(__rev2_910, __p3_910), __noswap_vgetq_lane_f16(__rev2_910, __p3_910)}); \
68443 __ret_910 = __builtin_shufflevector(__ret_910, __ret_910, 3, 2, 1, 0); \
68444 __ret_910; \
70325#define vfmlslq_laneq_high_f16(__p0_882, __p1_882, __p2_882, __p3_882) __extension__ ({ \
70326 float32x4_t __ret_882; \
70327 float32x4_t __s0_882 = __p0_882; \
70328 float16x8_t __s1_882 = __p1_882; \
70329 float16x8_t __s2_882 = __p2_882; \
70330 float32x4_t __rev0_882; __rev0_882 = __builtin_shufflevector(__s0_882, __s0_882, 3, 2, 1, 0); \
70331 float16x8_t __rev1_882; __rev1_882 = __builtin_shufflevector(__s1_882, __s1_882, 7, 6, 5, 4, 3, 2, 1, 0); \
70332 float16x8_t __rev2_882; __rev2_882 = __builtin_shufflevector(__s2_882, __s2_882, 7, 6, 5, 4, 3, 2, 1, 0); \
70333 __ret_882 = __noswap_vfmlslq_high_f16(__rev0_882, __rev1_882, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_882, __p3_882), __noswap_vgetq_lane_f16(__rev2_882, __p3_882), __noswap_vgetq_lane_f16(__rev2_882, __p3_882), __noswap_vgetq_lane_f16(__rev2_882, __p3_882), __noswap_vgetq_lane_f16(__rev2_882, __p3_882), __noswap_vgetq_lane_f16(__rev2_882, __p3_882), __noswap_vgetq_lane_f16(__rev2_882, __p3_882), __noswap_vgetq_lane_f16(__rev2_882, __p3_882)}); \
70334 __ret_882 = __builtin_shufflevector(__ret_882, __ret_882, 3, 2, 1, 0); \
70335 __ret_882; \
6844570336})
6844670337#endif
6844770338
6844870339#ifdef __LITTLE_ENDIAN__
68449#define vfmlsl_laneq_high_f16(__p0_911, __p1_911, __p2_911, __p3_911) __extension__ ({ \
68450 float32x2_t __ret_911; \
68451 float32x2_t __s0_911 = __p0_911; \
68452 float16x4_t __s1_911 = __p1_911; \
68453 float16x8_t __s2_911 = __p2_911; \
68454 __ret_911 = vfmlsl_high_f16(__s0_911, __s1_911, (float16x4_t) {vgetq_lane_f16(__s2_911, __p3_911), vgetq_lane_f16(__s2_911, __p3_911), vgetq_lane_f16(__s2_911, __p3_911), vgetq_lane_f16(__s2_911, __p3_911)}); \
68455 __ret_911; \
70340#define vfmlsl_laneq_high_f16(__p0_883, __p1_883, __p2_883, __p3_883) __extension__ ({ \
70341 float32x2_t __ret_883; \
70342 float32x2_t __s0_883 = __p0_883; \
70343 float16x4_t __s1_883 = __p1_883; \
70344 float16x8_t __s2_883 = __p2_883; \
70345 __ret_883 = vfmlsl_high_f16(__s0_883, __s1_883, (float16x4_t) {vgetq_lane_f16(__s2_883, __p3_883), vgetq_lane_f16(__s2_883, __p3_883), vgetq_lane_f16(__s2_883, __p3_883), vgetq_lane_f16(__s2_883, __p3_883)}); \
70346 __ret_883; \
6845670347})
6845770348#else
68458#define vfmlsl_laneq_high_f16(__p0_912, __p1_912, __p2_912, __p3_912) __extension__ ({ \
68459 float32x2_t __ret_912; \
68460 float32x2_t __s0_912 = __p0_912; \
68461 float16x4_t __s1_912 = __p1_912; \
68462 float16x8_t __s2_912 = __p2_912; \
68463 float32x2_t __rev0_912; __rev0_912 = __builtin_shufflevector(__s0_912, __s0_912, 1, 0); \
68464 float16x4_t __rev1_912; __rev1_912 = __builtin_shufflevector(__s1_912, __s1_912, 3, 2, 1, 0); \
68465 float16x8_t __rev2_912; __rev2_912 = __builtin_shufflevector(__s2_912, __s2_912, 7, 6, 5, 4, 3, 2, 1, 0); \
68466 __ret_912 = __noswap_vfmlsl_high_f16(__rev0_912, __rev1_912, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_912, __p3_912), __noswap_vgetq_lane_f16(__rev2_912, __p3_912), __noswap_vgetq_lane_f16(__rev2_912, __p3_912), __noswap_vgetq_lane_f16(__rev2_912, __p3_912)}); \
68467 __ret_912 = __builtin_shufflevector(__ret_912, __ret_912, 1, 0); \
68468 __ret_912; \
70349#define vfmlsl_laneq_high_f16(__p0_884, __p1_884, __p2_884, __p3_884) __extension__ ({ \
70350 float32x2_t __ret_884; \
70351 float32x2_t __s0_884 = __p0_884; \
70352 float16x4_t __s1_884 = __p1_884; \
70353 float16x8_t __s2_884 = __p2_884; \
70354 float32x2_t __rev0_884; __rev0_884 = __builtin_shufflevector(__s0_884, __s0_884, 1, 0); \
70355 float16x4_t __rev1_884; __rev1_884 = __builtin_shufflevector(__s1_884, __s1_884, 3, 2, 1, 0); \
70356 float16x8_t __rev2_884; __rev2_884 = __builtin_shufflevector(__s2_884, __s2_884, 7, 6, 5, 4, 3, 2, 1, 0); \
70357 __ret_884 = __noswap_vfmlsl_high_f16(__rev0_884, __rev1_884, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_884, __p3_884), __noswap_vgetq_lane_f16(__rev2_884, __p3_884), __noswap_vgetq_lane_f16(__rev2_884, __p3_884), __noswap_vgetq_lane_f16(__rev2_884, __p3_884)}); \
70358 __ret_884 = __builtin_shufflevector(__ret_884, __ret_884, 1, 0); \
70359 __ret_884; \
6846970360})
6847070361#endif
6847170362
6847270363#ifdef __LITTLE_ENDIAN__
68473#define vfmlslq_laneq_low_f16(__p0_913, __p1_913, __p2_913, __p3_913) __extension__ ({ \
68474 float32x4_t __ret_913; \
68475 float32x4_t __s0_913 = __p0_913; \
68476 float16x8_t __s1_913 = __p1_913; \
68477 float16x8_t __s2_913 = __p2_913; \
68478 __ret_913 = vfmlslq_low_f16(__s0_913, __s1_913, (float16x8_t) {vgetq_lane_f16(__s2_913, __p3_913), vgetq_lane_f16(__s2_913, __p3_913), vgetq_lane_f16(__s2_913, __p3_913), vgetq_lane_f16(__s2_913, __p3_913), vgetq_lane_f16(__s2_913, __p3_913), vgetq_lane_f16(__s2_913, __p3_913), vgetq_lane_f16(__s2_913, __p3_913), vgetq_lane_f16(__s2_913, __p3_913)}); \
68479 __ret_913; \
70364#define vfmlslq_laneq_low_f16(__p0_885, __p1_885, __p2_885, __p3_885) __extension__ ({ \
70365 float32x4_t __ret_885; \
70366 float32x4_t __s0_885 = __p0_885; \
70367 float16x8_t __s1_885 = __p1_885; \
70368 float16x8_t __s2_885 = __p2_885; \
70369 __ret_885 = vfmlslq_low_f16(__s0_885, __s1_885, (float16x8_t) {vgetq_lane_f16(__s2_885, __p3_885), vgetq_lane_f16(__s2_885, __p3_885), vgetq_lane_f16(__s2_885, __p3_885), vgetq_lane_f16(__s2_885, __p3_885), vgetq_lane_f16(__s2_885, __p3_885), vgetq_lane_f16(__s2_885, __p3_885), vgetq_lane_f16(__s2_885, __p3_885), vgetq_lane_f16(__s2_885, __p3_885)}); \
70370 __ret_885; \
6848070371})
6848170372#else
68482#define vfmlslq_laneq_low_f16(__p0_914, __p1_914, __p2_914, __p3_914) __extension__ ({ \
68483 float32x4_t __ret_914; \
68484 float32x4_t __s0_914 = __p0_914; \
68485 float16x8_t __s1_914 = __p1_914; \
68486 float16x8_t __s2_914 = __p2_914; \
68487 float32x4_t __rev0_914; __rev0_914 = __builtin_shufflevector(__s0_914, __s0_914, 3, 2, 1, 0); \
68488 float16x8_t __rev1_914; __rev1_914 = __builtin_shufflevector(__s1_914, __s1_914, 7, 6, 5, 4, 3, 2, 1, 0); \
68489 float16x8_t __rev2_914; __rev2_914 = __builtin_shufflevector(__s2_914, __s2_914, 7, 6, 5, 4, 3, 2, 1, 0); \
68490 __ret_914 = __noswap_vfmlslq_low_f16(__rev0_914, __rev1_914, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_914, __p3_914), __noswap_vgetq_lane_f16(__rev2_914, __p3_914), __noswap_vgetq_lane_f16(__rev2_914, __p3_914), __noswap_vgetq_lane_f16(__rev2_914, __p3_914), __noswap_vgetq_lane_f16(__rev2_914, __p3_914), __noswap_vgetq_lane_f16(__rev2_914, __p3_914), __noswap_vgetq_lane_f16(__rev2_914, __p3_914), __noswap_vgetq_lane_f16(__rev2_914, __p3_914)}); \
68491 __ret_914 = __builtin_shufflevector(__ret_914, __ret_914, 3, 2, 1, 0); \
68492 __ret_914; \
70373#define vfmlslq_laneq_low_f16(__p0_886, __p1_886, __p2_886, __p3_886) __extension__ ({ \
70374 float32x4_t __ret_886; \
70375 float32x4_t __s0_886 = __p0_886; \
70376 float16x8_t __s1_886 = __p1_886; \
70377 float16x8_t __s2_886 = __p2_886; \
70378 float32x4_t __rev0_886; __rev0_886 = __builtin_shufflevector(__s0_886, __s0_886, 3, 2, 1, 0); \
70379 float16x8_t __rev1_886; __rev1_886 = __builtin_shufflevector(__s1_886, __s1_886, 7, 6, 5, 4, 3, 2, 1, 0); \
70380 float16x8_t __rev2_886; __rev2_886 = __builtin_shufflevector(__s2_886, __s2_886, 7, 6, 5, 4, 3, 2, 1, 0); \
70381 __ret_886 = __noswap_vfmlslq_low_f16(__rev0_886, __rev1_886, (float16x8_t) {__noswap_vgetq_lane_f16(__rev2_886, __p3_886), __noswap_vgetq_lane_f16(__rev2_886, __p3_886), __noswap_vgetq_lane_f16(__rev2_886, __p3_886), __noswap_vgetq_lane_f16(__rev2_886, __p3_886), __noswap_vgetq_lane_f16(__rev2_886, __p3_886), __noswap_vgetq_lane_f16(__rev2_886, __p3_886), __noswap_vgetq_lane_f16(__rev2_886, __p3_886), __noswap_vgetq_lane_f16(__rev2_886, __p3_886)}); \
70382 __ret_886 = __builtin_shufflevector(__ret_886, __ret_886, 3, 2, 1, 0); \
70383 __ret_886; \
6849370384})
6849470385#endif
6849570386
6849670387#ifdef __LITTLE_ENDIAN__
68497#define vfmlsl_laneq_low_f16(__p0_915, __p1_915, __p2_915, __p3_915) __extension__ ({ \
68498 float32x2_t __ret_915; \
68499 float32x2_t __s0_915 = __p0_915; \
68500 float16x4_t __s1_915 = __p1_915; \
68501 float16x8_t __s2_915 = __p2_915; \
68502 __ret_915 = vfmlsl_low_f16(__s0_915, __s1_915, (float16x4_t) {vgetq_lane_f16(__s2_915, __p3_915), vgetq_lane_f16(__s2_915, __p3_915), vgetq_lane_f16(__s2_915, __p3_915), vgetq_lane_f16(__s2_915, __p3_915)}); \
68503 __ret_915; \
70388#define vfmlsl_laneq_low_f16(__p0_887, __p1_887, __p2_887, __p3_887) __extension__ ({ \
70389 float32x2_t __ret_887; \
70390 float32x2_t __s0_887 = __p0_887; \
70391 float16x4_t __s1_887 = __p1_887; \
70392 float16x8_t __s2_887 = __p2_887; \
70393 __ret_887 = vfmlsl_low_f16(__s0_887, __s1_887, (float16x4_t) {vgetq_lane_f16(__s2_887, __p3_887), vgetq_lane_f16(__s2_887, __p3_887), vgetq_lane_f16(__s2_887, __p3_887), vgetq_lane_f16(__s2_887, __p3_887)}); \
70394 __ret_887; \
6850470395})
6850570396#else
68506#define vfmlsl_laneq_low_f16(__p0_916, __p1_916, __p2_916, __p3_916) __extension__ ({ \
68507 float32x2_t __ret_916; \
68508 float32x2_t __s0_916 = __p0_916; \
68509 float16x4_t __s1_916 = __p1_916; \
68510 float16x8_t __s2_916 = __p2_916; \
68511 float32x2_t __rev0_916; __rev0_916 = __builtin_shufflevector(__s0_916, __s0_916, 1, 0); \
68512 float16x4_t __rev1_916; __rev1_916 = __builtin_shufflevector(__s1_916, __s1_916, 3, 2, 1, 0); \
68513 float16x8_t __rev2_916; __rev2_916 = __builtin_shufflevector(__s2_916, __s2_916, 7, 6, 5, 4, 3, 2, 1, 0); \
68514 __ret_916 = __noswap_vfmlsl_low_f16(__rev0_916, __rev1_916, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_916, __p3_916), __noswap_vgetq_lane_f16(__rev2_916, __p3_916), __noswap_vgetq_lane_f16(__rev2_916, __p3_916), __noswap_vgetq_lane_f16(__rev2_916, __p3_916)}); \
68515 __ret_916 = __builtin_shufflevector(__ret_916, __ret_916, 1, 0); \
68516 __ret_916; \
70397#define vfmlsl_laneq_low_f16(__p0_888, __p1_888, __p2_888, __p3_888) __extension__ ({ \
70398 float32x2_t __ret_888; \
70399 float32x2_t __s0_888 = __p0_888; \
70400 float16x4_t __s1_888 = __p1_888; \
70401 float16x8_t __s2_888 = __p2_888; \
70402 float32x2_t __rev0_888; __rev0_888 = __builtin_shufflevector(__s0_888, __s0_888, 1, 0); \
70403 float16x4_t __rev1_888; __rev1_888 = __builtin_shufflevector(__s1_888, __s1_888, 3, 2, 1, 0); \
70404 float16x8_t __rev2_888; __rev2_888 = __builtin_shufflevector(__s2_888, __s2_888, 7, 6, 5, 4, 3, 2, 1, 0); \
70405 __ret_888 = __noswap_vfmlsl_low_f16(__rev0_888, __rev1_888, (float16x4_t) {__noswap_vgetq_lane_f16(__rev2_888, __p3_888), __noswap_vgetq_lane_f16(__rev2_888, __p3_888), __noswap_vgetq_lane_f16(__rev2_888, __p3_888), __noswap_vgetq_lane_f16(__rev2_888, __p3_888)}); \
70406 __ret_888 = __builtin_shufflevector(__ret_888, __ret_888, 1, 0); \
70407 __ret_888; \
6851770408})
6851870409#endif
6851970410
6852070411#ifdef __LITTLE_ENDIAN__
68521#define vmulh_lane_f16(__p0_917, __p1_917, __p2_917) __extension__ ({ \
68522 float16_t __ret_917; \
68523 float16_t __s0_917 = __p0_917; \
68524 float16x4_t __s1_917 = __p1_917; \
68525 __ret_917 = __s0_917 * vget_lane_f16(__s1_917, __p2_917); \
68526 __ret_917; \
70412#define vmulh_lane_f16(__p0_889, __p1_889, __p2_889) __extension__ ({ \
70413 float16_t __ret_889; \
70414 float16_t __s0_889 = __p0_889; \
70415 float16x4_t __s1_889 = __p1_889; \
70416 __ret_889 = __s0_889 * vget_lane_f16(__s1_889, __p2_889); \
70417 __ret_889; \
6852770418})
6852870419#else
68529#define vmulh_lane_f16(__p0_918, __p1_918, __p2_918) __extension__ ({ \
68530 float16_t __ret_918; \
68531 float16_t __s0_918 = __p0_918; \
68532 float16x4_t __s1_918 = __p1_918; \
68533 float16x4_t __rev1_918; __rev1_918 = __builtin_shufflevector(__s1_918, __s1_918, 3, 2, 1, 0); \
68534 __ret_918 = __s0_918 * __noswap_vget_lane_f16(__rev1_918, __p2_918); \
68535 __ret_918; \
70420#define vmulh_lane_f16(__p0_890, __p1_890, __p2_890) __extension__ ({ \
70421 float16_t __ret_890; \
70422 float16_t __s0_890 = __p0_890; \
70423 float16x4_t __s1_890 = __p1_890; \
70424 float16x4_t __rev1_890; __rev1_890 = __builtin_shufflevector(__s1_890, __s1_890, 3, 2, 1, 0); \
70425 __ret_890 = __s0_890 * __noswap_vget_lane_f16(__rev1_890, __p2_890); \
70426 __ret_890; \
6853670427})
6853770428#endif
6853870429
6853970430#ifdef __LITTLE_ENDIAN__
68540#define vmulh_laneq_f16(__p0_919, __p1_919, __p2_919) __extension__ ({ \
68541 float16_t __ret_919; \
68542 float16_t __s0_919 = __p0_919; \
68543 float16x8_t __s1_919 = __p1_919; \
68544 __ret_919 = __s0_919 * vgetq_lane_f16(__s1_919, __p2_919); \
68545 __ret_919; \
70431#define vmulh_laneq_f16(__p0_891, __p1_891, __p2_891) __extension__ ({ \
70432 float16_t __ret_891; \
70433 float16_t __s0_891 = __p0_891; \
70434 float16x8_t __s1_891 = __p1_891; \
70435 __ret_891 = __s0_891 * vgetq_lane_f16(__s1_891, __p2_891); \
70436 __ret_891; \
6854670437})
6854770438#else
68548#define vmulh_laneq_f16(__p0_920, __p1_920, __p2_920) __extension__ ({ \
68549 float16_t __ret_920; \
68550 float16_t __s0_920 = __p0_920; \
68551 float16x8_t __s1_920 = __p1_920; \
68552 float16x8_t __rev1_920; __rev1_920 = __builtin_shufflevector(__s1_920, __s1_920, 7, 6, 5, 4, 3, 2, 1, 0); \
68553 __ret_920 = __s0_920 * __noswap_vgetq_lane_f16(__rev1_920, __p2_920); \
68554 __ret_920; \
70439#define vmulh_laneq_f16(__p0_892, __p1_892, __p2_892) __extension__ ({ \
70440 float16_t __ret_892; \
70441 float16_t __s0_892 = __p0_892; \
70442 float16x8_t __s1_892 = __p1_892; \
70443 float16x8_t __rev1_892; __rev1_892 = __builtin_shufflevector(__s1_892, __s1_892, 7, 6, 5, 4, 3, 2, 1, 0); \
70444 __ret_892 = __s0_892 * __noswap_vgetq_lane_f16(__rev1_892, __p2_892); \
70445 __ret_892; \
6855570446})
6855670447#endif
6855770448
......@@ -68862,136 +70753,136 @@ __ai __attribute__((target("neon"))) int32x4_t vaddw_high_s16(int32x4_t __p0, in
6886270753#endif
6886370754
6886470755#ifdef __LITTLE_ENDIAN__
68865#define vcopyq_lane_p64(__p0_921, __p1_921, __p2_921, __p3_921) __extension__ ({ \
68866 poly64x2_t __ret_921; \
68867 poly64x2_t __s0_921 = __p0_921; \
68868 poly64x1_t __s2_921 = __p2_921; \
68869 __ret_921 = vsetq_lane_p64(vget_lane_p64(__s2_921, __p3_921), __s0_921, __p1_921); \
68870 __ret_921; \
70756#define vcopyq_lane_p64(__p0_893, __p1_893, __p2_893, __p3_893) __extension__ ({ \
70757 poly64x2_t __ret_893; \
70758 poly64x2_t __s0_893 = __p0_893; \
70759 poly64x1_t __s2_893 = __p2_893; \
70760 __ret_893 = vsetq_lane_p64(vget_lane_p64(__s2_893, __p3_893), __s0_893, __p1_893); \
70761 __ret_893; \
6887170762})
6887270763#else
68873#define vcopyq_lane_p64(__p0_922, __p1_922, __p2_922, __p3_922) __extension__ ({ \
68874 poly64x2_t __ret_922; \
68875 poly64x2_t __s0_922 = __p0_922; \
68876 poly64x1_t __s2_922 = __p2_922; \
68877 poly64x2_t __rev0_922; __rev0_922 = __builtin_shufflevector(__s0_922, __s0_922, 1, 0); \
68878 __ret_922 = __noswap_vsetq_lane_p64(vget_lane_p64(__s2_922, __p3_922), __rev0_922, __p1_922); \
68879 __ret_922 = __builtin_shufflevector(__ret_922, __ret_922, 1, 0); \
68880 __ret_922; \
70764#define vcopyq_lane_p64(__p0_894, __p1_894, __p2_894, __p3_894) __extension__ ({ \
70765 poly64x2_t __ret_894; \
70766 poly64x2_t __s0_894 = __p0_894; \
70767 poly64x1_t __s2_894 = __p2_894; \
70768 poly64x2_t __rev0_894; __rev0_894 = __builtin_shufflevector(__s0_894, __s0_894, 1, 0); \
70769 __ret_894 = __noswap_vsetq_lane_p64(vget_lane_p64(__s2_894, __p3_894), __rev0_894, __p1_894); \
70770 __ret_894 = __builtin_shufflevector(__ret_894, __ret_894, 1, 0); \
70771 __ret_894; \
6888170772})
6888270773#endif
6888370774
6888470775#ifdef __LITTLE_ENDIAN__
68885#define vcopyq_lane_f64(__p0_923, __p1_923, __p2_923, __p3_923) __extension__ ({ \
68886 float64x2_t __ret_923; \
68887 float64x2_t __s0_923 = __p0_923; \
68888 float64x1_t __s2_923 = __p2_923; \
68889 __ret_923 = vsetq_lane_f64(vget_lane_f64(__s2_923, __p3_923), __s0_923, __p1_923); \
68890 __ret_923; \
70776#define vcopyq_lane_f64(__p0_895, __p1_895, __p2_895, __p3_895) __extension__ ({ \
70777 float64x2_t __ret_895; \
70778 float64x2_t __s0_895 = __p0_895; \
70779 float64x1_t __s2_895 = __p2_895; \
70780 __ret_895 = vsetq_lane_f64(vget_lane_f64(__s2_895, __p3_895), __s0_895, __p1_895); \
70781 __ret_895; \
6889170782})
6889270783#else
68893#define vcopyq_lane_f64(__p0_924, __p1_924, __p2_924, __p3_924) __extension__ ({ \
68894 float64x2_t __ret_924; \
68895 float64x2_t __s0_924 = __p0_924; \
68896 float64x1_t __s2_924 = __p2_924; \
68897 float64x2_t __rev0_924; __rev0_924 = __builtin_shufflevector(__s0_924, __s0_924, 1, 0); \
68898 __ret_924 = __noswap_vsetq_lane_f64(vget_lane_f64(__s2_924, __p3_924), __rev0_924, __p1_924); \
68899 __ret_924 = __builtin_shufflevector(__ret_924, __ret_924, 1, 0); \
68900 __ret_924; \
70784#define vcopyq_lane_f64(__p0_896, __p1_896, __p2_896, __p3_896) __extension__ ({ \
70785 float64x2_t __ret_896; \
70786 float64x2_t __s0_896 = __p0_896; \
70787 float64x1_t __s2_896 = __p2_896; \
70788 float64x2_t __rev0_896; __rev0_896 = __builtin_shufflevector(__s0_896, __s0_896, 1, 0); \
70789 __ret_896 = __noswap_vsetq_lane_f64(vget_lane_f64(__s2_896, __p3_896), __rev0_896, __p1_896); \
70790 __ret_896 = __builtin_shufflevector(__ret_896, __ret_896, 1, 0); \
70791 __ret_896; \
6890170792})
6890270793#endif
6890370794
68904#define vcopy_lane_p64(__p0_925, __p1_925, __p2_925, __p3_925) __extension__ ({ \
68905 poly64x1_t __ret_925; \
68906 poly64x1_t __s0_925 = __p0_925; \
68907 poly64x1_t __s2_925 = __p2_925; \
68908 __ret_925 = vset_lane_p64(vget_lane_p64(__s2_925, __p3_925), __s0_925, __p1_925); \
68909 __ret_925; \
70795#define vcopy_lane_p64(__p0_897, __p1_897, __p2_897, __p3_897) __extension__ ({ \
70796 poly64x1_t __ret_897; \
70797 poly64x1_t __s0_897 = __p0_897; \
70798 poly64x1_t __s2_897 = __p2_897; \
70799 __ret_897 = vset_lane_p64(vget_lane_p64(__s2_897, __p3_897), __s0_897, __p1_897); \
70800 __ret_897; \
6891070801})
68911#define vcopy_lane_f64(__p0_926, __p1_926, __p2_926, __p3_926) __extension__ ({ \
68912 float64x1_t __ret_926; \
68913 float64x1_t __s0_926 = __p0_926; \
68914 float64x1_t __s2_926 = __p2_926; \
68915 __ret_926 = vset_lane_f64(vget_lane_f64(__s2_926, __p3_926), __s0_926, __p1_926); \
68916 __ret_926; \
70802#define vcopy_lane_f64(__p0_898, __p1_898, __p2_898, __p3_898) __extension__ ({ \
70803 float64x1_t __ret_898; \
70804 float64x1_t __s0_898 = __p0_898; \
70805 float64x1_t __s2_898 = __p2_898; \
70806 __ret_898 = vset_lane_f64(vget_lane_f64(__s2_898, __p3_898), __s0_898, __p1_898); \
70807 __ret_898; \
6891770808})
6891870809#ifdef __LITTLE_ENDIAN__
68919#define vcopyq_laneq_p64(__p0_927, __p1_927, __p2_927, __p3_927) __extension__ ({ \
68920 poly64x2_t __ret_927; \
68921 poly64x2_t __s0_927 = __p0_927; \
68922 poly64x2_t __s2_927 = __p2_927; \
68923 __ret_927 = vsetq_lane_p64(vgetq_lane_p64(__s2_927, __p3_927), __s0_927, __p1_927); \
68924 __ret_927; \
70810#define vcopyq_laneq_p64(__p0_899, __p1_899, __p2_899, __p3_899) __extension__ ({ \
70811 poly64x2_t __ret_899; \
70812 poly64x2_t __s0_899 = __p0_899; \
70813 poly64x2_t __s2_899 = __p2_899; \
70814 __ret_899 = vsetq_lane_p64(vgetq_lane_p64(__s2_899, __p3_899), __s0_899, __p1_899); \
70815 __ret_899; \
6892570816})
6892670817#else
68927#define vcopyq_laneq_p64(__p0_928, __p1_928, __p2_928, __p3_928) __extension__ ({ \
68928 poly64x2_t __ret_928; \
68929 poly64x2_t __s0_928 = __p0_928; \
68930 poly64x2_t __s2_928 = __p2_928; \
68931 poly64x2_t __rev0_928; __rev0_928 = __builtin_shufflevector(__s0_928, __s0_928, 1, 0); \
68932 poly64x2_t __rev2_928; __rev2_928 = __builtin_shufflevector(__s2_928, __s2_928, 1, 0); \
68933 __ret_928 = __noswap_vsetq_lane_p64(__noswap_vgetq_lane_p64(__rev2_928, __p3_928), __rev0_928, __p1_928); \
68934 __ret_928 = __builtin_shufflevector(__ret_928, __ret_928, 1, 0); \
68935 __ret_928; \
70818#define vcopyq_laneq_p64(__p0_900, __p1_900, __p2_900, __p3_900) __extension__ ({ \
70819 poly64x2_t __ret_900; \
70820 poly64x2_t __s0_900 = __p0_900; \
70821 poly64x2_t __s2_900 = __p2_900; \
70822 poly64x2_t __rev0_900; __rev0_900 = __builtin_shufflevector(__s0_900, __s0_900, 1, 0); \
70823 poly64x2_t __rev2_900; __rev2_900 = __builtin_shufflevector(__s2_900, __s2_900, 1, 0); \
70824 __ret_900 = __noswap_vsetq_lane_p64(__noswap_vgetq_lane_p64(__rev2_900, __p3_900), __rev0_900, __p1_900); \
70825 __ret_900 = __builtin_shufflevector(__ret_900, __ret_900, 1, 0); \
70826 __ret_900; \
6893670827})
6893770828#endif
6893870829
6893970830#ifdef __LITTLE_ENDIAN__
68940#define vcopyq_laneq_f64(__p0_929, __p1_929, __p2_929, __p3_929) __extension__ ({ \
68941 float64x2_t __ret_929; \
68942 float64x2_t __s0_929 = __p0_929; \
68943 float64x2_t __s2_929 = __p2_929; \
68944 __ret_929 = vsetq_lane_f64(vgetq_lane_f64(__s2_929, __p3_929), __s0_929, __p1_929); \
68945 __ret_929; \
70831#define vcopyq_laneq_f64(__p0_901, __p1_901, __p2_901, __p3_901) __extension__ ({ \
70832 float64x2_t __ret_901; \
70833 float64x2_t __s0_901 = __p0_901; \
70834 float64x2_t __s2_901 = __p2_901; \
70835 __ret_901 = vsetq_lane_f64(vgetq_lane_f64(__s2_901, __p3_901), __s0_901, __p1_901); \
70836 __ret_901; \
6894670837})
6894770838#else
68948#define vcopyq_laneq_f64(__p0_930, __p1_930, __p2_930, __p3_930) __extension__ ({ \
68949 float64x2_t __ret_930; \
68950 float64x2_t __s0_930 = __p0_930; \
68951 float64x2_t __s2_930 = __p2_930; \
68952 float64x2_t __rev0_930; __rev0_930 = __builtin_shufflevector(__s0_930, __s0_930, 1, 0); \
68953 float64x2_t __rev2_930; __rev2_930 = __builtin_shufflevector(__s2_930, __s2_930, 1, 0); \
68954 __ret_930 = __noswap_vsetq_lane_f64(__noswap_vgetq_lane_f64(__rev2_930, __p3_930), __rev0_930, __p1_930); \
68955 __ret_930 = __builtin_shufflevector(__ret_930, __ret_930, 1, 0); \
68956 __ret_930; \
70839#define vcopyq_laneq_f64(__p0_902, __p1_902, __p2_902, __p3_902) __extension__ ({ \
70840 float64x2_t __ret_902; \
70841 float64x2_t __s0_902 = __p0_902; \
70842 float64x2_t __s2_902 = __p2_902; \
70843 float64x2_t __rev0_902; __rev0_902 = __builtin_shufflevector(__s0_902, __s0_902, 1, 0); \
70844 float64x2_t __rev2_902; __rev2_902 = __builtin_shufflevector(__s2_902, __s2_902, 1, 0); \
70845 __ret_902 = __noswap_vsetq_lane_f64(__noswap_vgetq_lane_f64(__rev2_902, __p3_902), __rev0_902, __p1_902); \
70846 __ret_902 = __builtin_shufflevector(__ret_902, __ret_902, 1, 0); \
70847 __ret_902; \
6895770848})
6895870849#endif
6895970850
6896070851#ifdef __LITTLE_ENDIAN__
68961#define vcopy_laneq_p64(__p0_931, __p1_931, __p2_931, __p3_931) __extension__ ({ \
68962 poly64x1_t __ret_931; \
68963 poly64x1_t __s0_931 = __p0_931; \
68964 poly64x2_t __s2_931 = __p2_931; \
68965 __ret_931 = vset_lane_p64(vgetq_lane_p64(__s2_931, __p3_931), __s0_931, __p1_931); \
68966 __ret_931; \
70852#define vcopy_laneq_p64(__p0_903, __p1_903, __p2_903, __p3_903) __extension__ ({ \
70853 poly64x1_t __ret_903; \
70854 poly64x1_t __s0_903 = __p0_903; \
70855 poly64x2_t __s2_903 = __p2_903; \
70856 __ret_903 = vset_lane_p64(vgetq_lane_p64(__s2_903, __p3_903), __s0_903, __p1_903); \
70857 __ret_903; \
6896770858})
6896870859#else
68969#define vcopy_laneq_p64(__p0_932, __p1_932, __p2_932, __p3_932) __extension__ ({ \
68970 poly64x1_t __ret_932; \
68971 poly64x1_t __s0_932 = __p0_932; \
68972 poly64x2_t __s2_932 = __p2_932; \
68973 poly64x2_t __rev2_932; __rev2_932 = __builtin_shufflevector(__s2_932, __s2_932, 1, 0); \
68974 __ret_932 = vset_lane_p64(__noswap_vgetq_lane_p64(__rev2_932, __p3_932), __s0_932, __p1_932); \
68975 __ret_932; \
70860#define vcopy_laneq_p64(__p0_904, __p1_904, __p2_904, __p3_904) __extension__ ({ \
70861 poly64x1_t __ret_904; \
70862 poly64x1_t __s0_904 = __p0_904; \
70863 poly64x2_t __s2_904 = __p2_904; \
70864 poly64x2_t __rev2_904; __rev2_904 = __builtin_shufflevector(__s2_904, __s2_904, 1, 0); \
70865 __ret_904 = vset_lane_p64(__noswap_vgetq_lane_p64(__rev2_904, __p3_904), __s0_904, __p1_904); \
70866 __ret_904; \
6897670867})
6897770868#endif
6897870869
6897970870#ifdef __LITTLE_ENDIAN__
68980#define vcopy_laneq_f64(__p0_933, __p1_933, __p2_933, __p3_933) __extension__ ({ \
68981 float64x1_t __ret_933; \
68982 float64x1_t __s0_933 = __p0_933; \
68983 float64x2_t __s2_933 = __p2_933; \
68984 __ret_933 = vset_lane_f64(vgetq_lane_f64(__s2_933, __p3_933), __s0_933, __p1_933); \
68985 __ret_933; \
70871#define vcopy_laneq_f64(__p0_905, __p1_905, __p2_905, __p3_905) __extension__ ({ \
70872 float64x1_t __ret_905; \
70873 float64x1_t __s0_905 = __p0_905; \
70874 float64x2_t __s2_905 = __p2_905; \
70875 __ret_905 = vset_lane_f64(vgetq_lane_f64(__s2_905, __p3_905), __s0_905, __p1_905); \
70876 __ret_905; \
6898670877})
6898770878#else
68988#define vcopy_laneq_f64(__p0_934, __p1_934, __p2_934, __p3_934) __extension__ ({ \
68989 float64x1_t __ret_934; \
68990 float64x1_t __s0_934 = __p0_934; \
68991 float64x2_t __s2_934 = __p2_934; \
68992 float64x2_t __rev2_934; __rev2_934 = __builtin_shufflevector(__s2_934, __s2_934, 1, 0); \
68993 __ret_934 = vset_lane_f64(__noswap_vgetq_lane_f64(__rev2_934, __p3_934), __s0_934, __p1_934); \
68994 __ret_934; \
70879#define vcopy_laneq_f64(__p0_906, __p1_906, __p2_906, __p3_906) __extension__ ({ \
70880 float64x1_t __ret_906; \
70881 float64x1_t __s0_906 = __p0_906; \
70882 float64x2_t __s2_906 = __p2_906; \
70883 float64x2_t __rev2_906; __rev2_906 = __builtin_shufflevector(__s2_906, __s2_906, 1, 0); \
70884 __ret_906 = vset_lane_f64(__noswap_vgetq_lane_f64(__rev2_906, __p3_906), __s0_906, __p1_906); \
70885 __ret_906; \
6899570886})
6899670887#endif
6899770888
......@@ -69347,38 +71238,38 @@ __ai __attribute__((target("neon"))) int32x4_t vmlsl_high_n_s16(int32x4_t __p0,
6934771238}
6934871239#endif
6934971240
69350#define vmulx_lane_f64(__p0_935, __p1_935, __p2_935) __extension__ ({ \
69351 float64x1_t __ret_935; \
69352 float64x1_t __s0_935 = __p0_935; \
69353 float64x1_t __s1_935 = __p1_935; \
69354 float64_t __x_935 = vget_lane_f64(__s0_935, 0); \
69355 float64_t __y_935 = vget_lane_f64(__s1_935, __p2_935); \
69356 float64_t __z_935 = vmulxd_f64(__x_935, __y_935); \
69357 __ret_935 = vset_lane_f64(__z_935, __s0_935, __p2_935); \
69358 __ret_935; \
71241#define vmulx_lane_f64(__p0_907, __p1_907, __p2_907) __extension__ ({ \
71242 float64x1_t __ret_907; \
71243 float64x1_t __s0_907 = __p0_907; \
71244 float64x1_t __s1_907 = __p1_907; \
71245 float64_t __x_907 = vget_lane_f64(__s0_907, 0); \
71246 float64_t __y_907 = vget_lane_f64(__s1_907, __p2_907); \
71247 float64_t __z_907 = vmulxd_f64(__x_907, __y_907); \
71248 __ret_907 = vset_lane_f64(__z_907, __s0_907, __p2_907); \
71249 __ret_907; \
6935971250})
6936071251#ifdef __LITTLE_ENDIAN__
69361#define vmulx_laneq_f64(__p0_936, __p1_936, __p2_936) __extension__ ({ \
69362 float64x1_t __ret_936; \
69363 float64x1_t __s0_936 = __p0_936; \
69364 float64x2_t __s1_936 = __p1_936; \
69365 float64_t __x_936 = vget_lane_f64(__s0_936, 0); \
69366 float64_t __y_936 = vgetq_lane_f64(__s1_936, __p2_936); \
69367 float64_t __z_936 = vmulxd_f64(__x_936, __y_936); \
69368 __ret_936 = vset_lane_f64(__z_936, __s0_936, 0); \
69369 __ret_936; \
71252#define vmulx_laneq_f64(__p0_908, __p1_908, __p2_908) __extension__ ({ \
71253 float64x1_t __ret_908; \
71254 float64x1_t __s0_908 = __p0_908; \
71255 float64x2_t __s1_908 = __p1_908; \
71256 float64_t __x_908 = vget_lane_f64(__s0_908, 0); \
71257 float64_t __y_908 = vgetq_lane_f64(__s1_908, __p2_908); \
71258 float64_t __z_908 = vmulxd_f64(__x_908, __y_908); \
71259 __ret_908 = vset_lane_f64(__z_908, __s0_908, 0); \
71260 __ret_908; \
6937071261})
6937171262#else
69372#define vmulx_laneq_f64(__p0_937, __p1_937, __p2_937) __extension__ ({ \
69373 float64x1_t __ret_937; \
69374 float64x1_t __s0_937 = __p0_937; \
69375 float64x2_t __s1_937 = __p1_937; \
69376 float64x2_t __rev1_937; __rev1_937 = __builtin_shufflevector(__s1_937, __s1_937, 1, 0); \
69377 float64_t __x_937 = vget_lane_f64(__s0_937, 0); \
69378 float64_t __y_937 = __noswap_vgetq_lane_f64(__rev1_937, __p2_937); \
69379 float64_t __z_937 = vmulxd_f64(__x_937, __y_937); \
69380 __ret_937 = vset_lane_f64(__z_937, __s0_937, 0); \
69381 __ret_937; \
71263#define vmulx_laneq_f64(__p0_909, __p1_909, __p2_909) __extension__ ({ \
71264 float64x1_t __ret_909; \
71265 float64x1_t __s0_909 = __p0_909; \
71266 float64x2_t __s1_909 = __p1_909; \
71267 float64x2_t __rev1_909; __rev1_909 = __builtin_shufflevector(__s1_909, __s1_909, 1, 0); \
71268 float64_t __x_909 = vget_lane_f64(__s0_909, 0); \
71269 float64_t __y_909 = __noswap_vgetq_lane_f64(__rev1_909, __p2_909); \
71270 float64_t __z_909 = vmulxd_f64(__x_909, __y_909); \
71271 __ret_909 = vset_lane_f64(__z_909, __s0_909, 0); \
71272 __ret_909; \
6938271273})
6938371274#endif
6938471275
lib/include/arm_sme.h+322-94
......@@ -35,12 +35,6 @@ __ai bool __arm_has_sme(void) __arm_streaming_compatible {
3535 return x0 & (1ULL << 63);
3636}
3737
38__ai bool __arm_in_streaming_mode(void) __arm_streaming_compatible {
39 uint64_t x0, x1;
40 __builtin_arm_get_sme_state(&x0, &x1);
41 return x0 & 1;
42}
43
4438void *__arm_sc_memcpy(void *dest, const void *src, size_t n) __arm_streaming_compatible;
4539void *__arm_sc_memmove(void *dest, const void *src, size_t n) __arm_streaming_compatible;
4640void *__arm_sc_memset(void *s, int c, size_t n) __arm_streaming_compatible;
......@@ -48,6 +42,8 @@ void *__arm_sc_memchr(void *s, int c, size_t n) __arm_streaming_compatible;
4842
4943__ai __attribute__((target("sme"))) void svundef_za(void) __arm_streaming_compatible __arm_out("za") { }
5044
45__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme___arm_in_streaming_mode)))
46bool __arm_in_streaming_mode(void);
5147__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_u32_m)))
5248void svaddha_za32_u32_m(uint64_t, svbool_t, svbool_t, svuint32_t);
5349__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_s32_m)))
......@@ -604,6 +600,94 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_u8_
604600void svwrite_ver_za8_m(uint64_t, uint32_t, svbool_t, svuint8_t);
605601__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_s8_m)))
606602void svwrite_ver_za8_m(uint64_t, uint32_t, svbool_t, svint8_t);
603__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x2)))
604void svadd_za16_f16_vg1x2(uint32_t, svfloat16x2_t);
605__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x4)))
606void svadd_za16_f16_vg1x4(uint32_t, svfloat16x4_t);
607__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x2)))
608void svsub_za16_f16_vg1x2(uint32_t, svfloat16x2_t);
609__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x4)))
610void svsub_za16_f16_vg1x4(uint32_t, svfloat16x4_t);
611__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x2)))
612void svadd_za16_vg1x2(uint32_t, svfloat16x2_t);
613__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x4)))
614void svadd_za16_vg1x4(uint32_t, svfloat16x4_t);
615__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x2)))
616void svsub_za16_vg1x2(uint32_t, svfloat16x2_t);
617__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x4)))
618void svsub_za16_vg1x4(uint32_t, svfloat16x4_t);
619__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x2)))
620void svadd_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t);
621__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x4)))
622void svadd_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t);
623__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x2)))
624void svmla_single_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
625__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x4)))
626void svmla_single_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
627__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x2)))
628void svmla_lane_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
629__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x4)))
630void svmla_lane_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
631__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x2)))
632void svmla_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
633__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x4)))
634void svmla_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
635__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x2)))
636void svmls_single_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
637__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x4)))
638void svmls_single_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
639__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x2)))
640void svmls_lane_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
641__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x4)))
642void svmls_lane_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
643__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x2)))
644void svmls_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
645__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x4)))
646void svmls_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
647__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_bf16_m)))
648void svmopa_za16_bf16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
649__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za16_bf16_m)))
650void svmops_za16_bf16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
651__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x2)))
652void svsub_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t);
653__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x4)))
654void svsub_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t);
655__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x2)))
656void svadd_za16_vg1x2(uint32_t, svbfloat16x2_t);
657__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x4)))
658void svadd_za16_vg1x4(uint32_t, svbfloat16x4_t);
659__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x2)))
660void svmla_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
661__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x4)))
662void svmla_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
663__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x2)))
664void svmla_lane_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
665__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x4)))
666void svmla_lane_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
667__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x2)))
668void svmla_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
669__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x4)))
670void svmla_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
671__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x2)))
672void svmls_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
673__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x4)))
674void svmls_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
675__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x2)))
676void svmls_lane_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
677__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x4)))
678void svmls_lane_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
679__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x2)))
680void svmls_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
681__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x4)))
682void svmls_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
683__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_bf16_m)))
684void svmopa_za16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
685__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za16_bf16_m)))
686void svmops_za16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
687__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x2)))
688void svsub_za16_vg1x2(uint32_t, svbfloat16x2_t);
689__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x4)))
690void svsub_za16_vg1x4(uint32_t, svbfloat16x4_t);
607691__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_f16_vg1x2)))
608692void svmla_single_za16_f16_vg1x2(uint32_t, svfloat16x2_t, svfloat16_t);
609693__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_f16_vg1x4)))
......@@ -660,22 +744,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_f16_m))
660744void svmopa_za16_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t);
661745__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za16_f16_m)))
662746void svmops_za16_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t);
663__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x2)))
664void svadd_za16_f16_vg1x2(uint32_t, svfloat16x2_t);
665__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x4)))
666void svadd_za16_f16_vg1x4(uint32_t, svfloat16x4_t);
667__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x2)))
668void svsub_za16_f16_vg1x2(uint32_t, svfloat16x2_t);
669__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x4)))
670void svsub_za16_f16_vg1x4(uint32_t, svfloat16x4_t);
671__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x2)))
672void svadd_za16_vg1x2(uint32_t, svfloat16x2_t);
673__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_f16_vg1x4)))
674void svadd_za16_vg1x4(uint32_t, svfloat16x4_t);
675__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x2)))
676void svsub_za16_vg1x2(uint32_t, svfloat16x2_t);
677__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_f16_vg1x4)))
678void svsub_za16_vg1x4(uint32_t, svfloat16x4_t);
679747__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_f64_m)))
680748void svmopa_za64_f64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);
681749__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_f64_m)))
......@@ -684,6 +752,138 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_f64_m))
684752void svmopa_za64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);
685753__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_f64_m)))
686754void svmops_za64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);
755__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za16_mf8_vg1x2_fpm)))
756void svdot_single_za16_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
757__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za16_mf8_vg1x4_fpm)))
758void svdot_single_za16_mf8_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
759__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za16_mf8_vg1x2_fpm)))
760void svdot_lane_za16_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
761__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za16_mf8_vg1x4_fpm)))
762void svdot_lane_za16_mf8_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
763__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za16_mf8_vg1x2_fpm)))
764void svdot_za16_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
765__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za16_mf8_vg1x4_fpm)))
766void svdot_za16_mf8_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
767__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x1_fpm)))
768void svmla_single_za16_mf8_vg2x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
769__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x2_fpm)))
770void svmla_single_za16_mf8_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
771__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x4_fpm)))
772void svmla_single_za16_mf8_vg2x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
773__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_vg2x1_fpm)))
774void svmla_lane_za16_mf8_vg2x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
775__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_vg2x2_fpm)))
776void svmla_lane_za16_mf8_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
777__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_vg2x4_fpm)))
778void svmla_lane_za16_mf8_vg2x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
779__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_mf8_vg2x2_fpm)))
780void svmla_za16_mf8_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
781__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_mf8_vg2x4_fpm)))
782void svmla_za16_mf8_vg2x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
783__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_mf8_m_fpm)))
784void svmopa_za16_mf8_m_fpm(uint64_t, svbool_t, svbool_t, svmfloat8_t, svmfloat8_t, fpm_t);
785__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svvdot_lane_za16_mf8_vg1x2_fpm)))
786void svvdot_lane_za16_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
787__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za16_mf8_vg1x2_fpm)))
788void svdot_za16_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
789__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za16_mf8_vg1x4_fpm)))
790void svdot_za16_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
791__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za16_mf8_vg1x2_fpm)))
792void svdot_lane_za16_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
793__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za16_mf8_vg1x4_fpm)))
794void svdot_lane_za16_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
795__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za16_mf8_vg1x2_fpm)))
796void svdot_za16_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
797__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za16_mf8_vg1x4_fpm)))
798void svdot_za16_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
799__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x1_fpm)))
800void svmla_za16_vg2x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
801__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x2_fpm)))
802void svmla_za16_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
803__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x4_fpm)))
804void svmla_za16_vg2x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
805__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_vg2x1_fpm)))
806void svmla_lane_za16_vg2x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
807__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_vg2x2_fpm)))
808void svmla_lane_za16_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
809__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_vg2x4_fpm)))
810void svmla_lane_za16_vg2x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
811__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_mf8_vg2x2_fpm)))
812void svmla_za16_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
813__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_mf8_vg2x4_fpm)))
814void svmla_za16_vg2x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
815__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_mf8_m_fpm)))
816void svmopa_za16_m_fpm(uint64_t, svbool_t, svbool_t, svmfloat8_t, svmfloat8_t, fpm_t);
817__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svvdot_lane_za16_mf8_vg1x2_fpm)))
818void svvdot_lane_za16_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
819__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za32_mf8_vg1x2_fpm)))
820void svdot_single_za32_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
821__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za32_mf8_vg1x4_fpm)))
822void svdot_single_za32_mf8_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
823__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za32_mf8_vg1x2_fpm)))
824void svdot_lane_za32_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
825__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za32_mf8_vg1x4_fpm)))
826void svdot_lane_za32_mf8_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
827__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za32_mf8_vg1x2_fpm)))
828void svdot_za32_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
829__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za32_mf8_vg1x4_fpm)))
830void svdot_za32_mf8_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
831__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x1_fpm)))
832void svmla_single_za32_mf8_vg4x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
833__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x2_fpm)))
834void svmla_single_za32_mf8_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
835__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x4_fpm)))
836void svmla_single_za32_mf8_vg4x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
837__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_vg4x1_fpm)))
838void svmla_lane_za32_mf8_vg4x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
839__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_vg4x2_fpm)))
840void svmla_lane_za32_mf8_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
841__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_vg4x4_fpm)))
842void svmla_lane_za32_mf8_vg4x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
843__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za32_mf8_vg4x2_fpm)))
844void svmla_za32_mf8_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
845__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za32_mf8_vg4x4_fpm)))
846void svmla_za32_mf8_vg4x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
847__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_mf8_m_fpm)))
848void svmopa_za32_mf8_m_fpm(uint64_t, svbool_t, svbool_t, svmfloat8_t, svmfloat8_t, fpm_t);
849__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svvdotb_lane_za32_mf8_vg1x4_fpm)))
850void svvdotb_lane_za32_mf8_vg1x4_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
851__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svvdott_lane_za32_mf8_vg1x4_fpm)))
852void svvdott_lane_za32_mf8_vg1x4_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
853__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za32_mf8_vg1x2_fpm)))
854void svdot_za32_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
855__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_single_za32_mf8_vg1x4_fpm)))
856void svdot_za32_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
857__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za32_mf8_vg1x2_fpm)))
858void svdot_lane_za32_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
859__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_lane_za32_mf8_vg1x4_fpm)))
860void svdot_lane_za32_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
861__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za32_mf8_vg1x2_fpm)))
862void svdot_za32_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
863__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za32_mf8_vg1x4_fpm)))
864void svdot_za32_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
865__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x1_fpm)))
866void svmla_za32_vg4x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
867__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x2_fpm)))
868void svmla_za32_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
869__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x4_fpm)))
870void svmla_za32_vg4x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, fpm_t);
871__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_vg4x1_fpm)))
872void svmla_lane_za32_vg4x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
873__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_vg4x2_fpm)))
874void svmla_lane_za32_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
875__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_vg4x4_fpm)))
876void svmla_lane_za32_vg4x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
877__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za32_mf8_vg4x2_fpm)))
878void svmla_za32_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
879__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za32_mf8_vg4x4_fpm)))
880void svmla_za32_vg4x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
881__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_mf8_m_fpm)))
882void svmopa_za32_m_fpm(uint64_t, svbool_t, svbool_t, svmfloat8_t, svmfloat8_t, fpm_t);
883__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svvdotb_lane_za32_mf8_vg1x4_fpm)))
884void svvdotb_lane_za32_vg1x4_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
885__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svvdott_lane_za32_mf8_vg1x4_fpm)))
886void svvdott_lane_za32_vg1x4_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
687887__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_u64_m)))
688888void svaddha_za64_u64_m(uint64_t, svbool_t, svbool_t, svuint64_t);
689889__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_s64_m)))
......@@ -732,6 +932,106 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za64_u16_m
732932void svusmopa_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t);
733933__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za64_u16_m)))
734934void svusmops_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t);
935__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti4_zt_u8_x4)))
936svuint8x4_t svluti4_zt_u8_x4(uint64_t, svuint8x2_t);
937__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svluti4_zt_s8_x4)))
938svint8x4_t svluti4_zt_s8_x4(uint64_t, svuint8x2_t);
939__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u8)))
940void svwrite_lane_zt_u8(uint64_t, svuint8_t, uint64_t);
941__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u32)))
942void svwrite_lane_zt_u32(uint64_t, svuint32_t, uint64_t);
943__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u64)))
944void svwrite_lane_zt_u64(uint64_t, svuint64_t, uint64_t);
945__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u16)))
946void svwrite_lane_zt_u16(uint64_t, svuint16_t, uint64_t);
947__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_bf16)))
948void svwrite_lane_zt_bf16(uint64_t, svbfloat16_t, uint64_t);
949__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s8)))
950void svwrite_lane_zt_s8(uint64_t, svint8_t, uint64_t);
951__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_f64)))
952void svwrite_lane_zt_f64(uint64_t, svfloat64_t, uint64_t);
953__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_f32)))
954void svwrite_lane_zt_f32(uint64_t, svfloat32_t, uint64_t);
955__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_f16)))
956void svwrite_lane_zt_f16(uint64_t, svfloat16_t, uint64_t);
957__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s32)))
958void svwrite_lane_zt_s32(uint64_t, svint32_t, uint64_t);
959__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s64)))
960void svwrite_lane_zt_s64(uint64_t, svint64_t, uint64_t);
961__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s16)))
962void svwrite_lane_zt_s16(uint64_t, svint16_t, uint64_t);
963__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u8)))
964void svwrite_zt_u8(uint64_t, svuint8_t);
965__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u32)))
966void svwrite_zt_u32(uint64_t, svuint32_t);
967__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u64)))
968void svwrite_zt_u64(uint64_t, svuint64_t);
969__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u16)))
970void svwrite_zt_u16(uint64_t, svuint16_t);
971__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_bf16)))
972void svwrite_zt_bf16(uint64_t, svbfloat16_t);
973__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s8)))
974void svwrite_zt_s8(uint64_t, svint8_t);
975__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_f64)))
976void svwrite_zt_f64(uint64_t, svfloat64_t);
977__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_f32)))
978void svwrite_zt_f32(uint64_t, svfloat32_t);
979__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_f16)))
980void svwrite_zt_f16(uint64_t, svfloat16_t);
981__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s32)))
982void svwrite_zt_s32(uint64_t, svint32_t);
983__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s64)))
984void svwrite_zt_s64(uint64_t, svint64_t);
985__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s16)))
986void svwrite_zt_s16(uint64_t, svint16_t);
987__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u8)))
988void svwrite_lane_zt(uint64_t, svuint8_t, uint64_t);
989__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u32)))
990void svwrite_lane_zt(uint64_t, svuint32_t, uint64_t);
991__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u64)))
992void svwrite_lane_zt(uint64_t, svuint64_t, uint64_t);
993__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_u16)))
994void svwrite_lane_zt(uint64_t, svuint16_t, uint64_t);
995__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_bf16)))
996void svwrite_lane_zt(uint64_t, svbfloat16_t, uint64_t);
997__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s8)))
998void svwrite_lane_zt(uint64_t, svint8_t, uint64_t);
999__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_f64)))
1000void svwrite_lane_zt(uint64_t, svfloat64_t, uint64_t);
1001__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_f32)))
1002void svwrite_lane_zt(uint64_t, svfloat32_t, uint64_t);
1003__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_f16)))
1004void svwrite_lane_zt(uint64_t, svfloat16_t, uint64_t);
1005__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s32)))
1006void svwrite_lane_zt(uint64_t, svint32_t, uint64_t);
1007__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s64)))
1008void svwrite_lane_zt(uint64_t, svint64_t, uint64_t);
1009__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_lane_zt_s16)))
1010void svwrite_lane_zt(uint64_t, svint16_t, uint64_t);
1011__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u8)))
1012void svwrite_zt(uint64_t, svuint8_t);
1013__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u32)))
1014void svwrite_zt(uint64_t, svuint32_t);
1015__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u64)))
1016void svwrite_zt(uint64_t, svuint64_t);
1017__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_u16)))
1018void svwrite_zt(uint64_t, svuint16_t);
1019__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_bf16)))
1020void svwrite_zt(uint64_t, svbfloat16_t);
1021__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s8)))
1022void svwrite_zt(uint64_t, svint8_t);
1023__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_f64)))
1024void svwrite_zt(uint64_t, svfloat64_t);
1025__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_f32)))
1026void svwrite_zt(uint64_t, svfloat32_t);
1027__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_f16)))
1028void svwrite_zt(uint64_t, svfloat16_t);
1029__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s32)))
1030void svwrite_zt(uint64_t, svint32_t);
1031__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s64)))
1032void svwrite_zt(uint64_t, svint64_t);
1033__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_zt_s16)))
1034void svwrite_zt(uint64_t, svint16_t);
7351035__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_write_single_za32_u32_vg1x2)))
7361036void svadd_write_single_za32_u32_vg1x2(uint32_t, svuint32x2_t, svuint32_t);
7371037__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_write_single_za32_s32_vg1x2)))
......@@ -2138,78 +2438,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_za8_u8_vg1x
21382438void svwrite_za8_vg1x4(uint32_t, svuint8x4_t);
21392439__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_za8_s8_vg1x4)))
21402440void svwrite_za8_vg1x4(uint32_t, svint8x4_t);
2141__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x2)))
2142void svadd_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t);
2143__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x4)))
2144void svadd_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t);
2145__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x2)))
2146void svmla_single_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
2147__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x4)))
2148void svmla_single_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
2149__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x2)))
2150void svmla_lane_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
2151__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x4)))
2152void svmla_lane_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
2153__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x2)))
2154void svmla_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
2155__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x4)))
2156void svmla_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
2157__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x2)))
2158void svmls_single_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
2159__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x4)))
2160void svmls_single_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
2161__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x2)))
2162void svmls_lane_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
2163__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x4)))
2164void svmls_lane_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
2165__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x2)))
2166void svmls_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
2167__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x4)))
2168void svmls_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
2169__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_bf16_m)))
2170void svmopa_za16_bf16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
2171__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za16_bf16_m)))
2172void svmops_za16_bf16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
2173__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x2)))
2174void svsub_za16_bf16_vg1x2(uint32_t, svbfloat16x2_t);
2175__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x4)))
2176void svsub_za16_bf16_vg1x4(uint32_t, svbfloat16x4_t);
2177__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x2)))
2178void svadd_za16_vg1x2(uint32_t, svbfloat16x2_t);
2179__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za16_bf16_vg1x4)))
2180void svadd_za16_vg1x4(uint32_t, svbfloat16x4_t);
2181__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x2)))
2182void svmla_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
2183__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_bf16_vg1x4)))
2184void svmla_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
2185__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x2)))
2186void svmla_lane_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
2187__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_bf16_vg1x4)))
2188void svmla_lane_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
2189__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x2)))
2190void svmla_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
2191__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_bf16_vg1x4)))
2192void svmla_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
2193__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x2)))
2194void svmls_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t);
2195__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_single_za16_bf16_vg1x4)))
2196void svmls_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t);
2197__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x2)))
2198void svmls_lane_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16_t, uint64_t);
2199__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_lane_za16_bf16_vg1x4)))
2200void svmls_lane_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16_t, uint64_t);
2201__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x2)))
2202void svmls_za16_vg1x2(uint32_t, svbfloat16x2_t, svbfloat16x2_t);
2203__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmls_za16_bf16_vg1x4)))
2204void svmls_za16_vg1x4(uint32_t, svbfloat16x4_t, svbfloat16x4_t);
2205__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za16_bf16_m)))
2206void svmopa_za16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
2207__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za16_bf16_m)))
2208void svmops_za16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
2209__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x2)))
2210void svsub_za16_vg1x2(uint32_t, svbfloat16x2_t);
2211__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsub_za16_bf16_vg1x4)))
2212void svsub_za16_vg1x4(uint32_t, svbfloat16x4_t);
22132441__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za64_f64_vg1x2)))
22142442void svadd_za64_f64_vg1x2(uint32_t, svfloat64x2_t);
22152443__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svadd_za64_f64_vg1x4)))
lib/include/arm_sve.h+1105-84
......@@ -38,6 +38,8 @@ typedef __SVFloat16_t svfloat16_t;
3838typedef __SVBfloat16_t svbfloat16_t;
3939#include <arm_bf16.h>
4040#include <arm_vector_types.h>
41typedef __SVMfloat8_t svmfloat8_t;
42
4143typedef __SVFloat32_t svfloat32_t;
4244typedef __SVFloat64_t svfloat64_t;
4345typedef __clang_svint8x2_t svint8x2_t;
......@@ -80,6 +82,9 @@ typedef __clang_svboolx4_t svboolx4_t;
8082typedef __clang_svbfloat16x2_t svbfloat16x2_t;
8183typedef __clang_svbfloat16x3_t svbfloat16x3_t;
8284typedef __clang_svbfloat16x4_t svbfloat16x4_t;
85typedef __clang_svmfloat8x2_t svmfloat8x2_t;
86typedef __clang_svmfloat8x3_t svmfloat8x3_t;
87typedef __clang_svmfloat8x4_t svmfloat8x4_t;
8388typedef __SVCount_t svcount_t;
8489
8590enum svpattern
......@@ -128,6 +133,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8))
128133svint8_t svreinterpret_s8_s8(svint8_t op);
129134__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8)))
130135svint8_t svreinterpret_s8_u8(svuint8_t op);
136__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8)))
137svint8_t svreinterpret_s8_mf8(svmfloat8_t op);
131138__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16)))
132139svint8_t svreinterpret_s8_s16(svint16_t op);
133140__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16)))
......@@ -152,6 +159,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8))
152159svuint8_t svreinterpret_u8_s8(svint8_t op);
153160__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8)))
154161svuint8_t svreinterpret_u8_u8(svuint8_t op);
162__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8)))
163svuint8_t svreinterpret_u8_mf8(svmfloat8_t op);
155164__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16)))
156165svuint8_t svreinterpret_u8_s16(svint16_t op);
157166__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16)))
......@@ -172,10 +181,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32)
172181svuint8_t svreinterpret_u8_f32(svfloat32_t op);
173182__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64)))
174183svuint8_t svreinterpret_u8_f64(svfloat64_t op);
184__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8)))
185svmfloat8_t svreinterpret_mf8_s8(svint8_t op);
186__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8)))
187svmfloat8_t svreinterpret_mf8_u8(svuint8_t op);
188__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8)))
189svmfloat8_t svreinterpret_mf8_mf8(svmfloat8_t op);
190__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16)))
191svmfloat8_t svreinterpret_mf8_s16(svint16_t op);
192__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16)))
193svmfloat8_t svreinterpret_mf8_u16(svuint16_t op);
194__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32)))
195svmfloat8_t svreinterpret_mf8_s32(svint32_t op);
196__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32)))
197svmfloat8_t svreinterpret_mf8_u32(svuint32_t op);
198__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64)))
199svmfloat8_t svreinterpret_mf8_s64(svint64_t op);
200__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64)))
201svmfloat8_t svreinterpret_mf8_u64(svuint64_t op);
202__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16)))
203svmfloat8_t svreinterpret_mf8_f16(svfloat16_t op);
204__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16)))
205svmfloat8_t svreinterpret_mf8_bf16(svbfloat16_t op);
206__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32)))
207svmfloat8_t svreinterpret_mf8_f32(svfloat32_t op);
208__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64)))
209svmfloat8_t svreinterpret_mf8_f64(svfloat64_t op);
175210__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8)))
176211svint16_t svreinterpret_s16_s8(svint8_t op);
177212__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8)))
178213svint16_t svreinterpret_s16_u8(svuint8_t op);
214__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8)))
215svint16_t svreinterpret_s16_mf8(svmfloat8_t op);
179216__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16)))
180217svint16_t svreinterpret_s16_s16(svint16_t op);
181218__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16)))
......@@ -200,6 +237,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8)
200237svuint16_t svreinterpret_u16_s8(svint8_t op);
201238__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8)))
202239svuint16_t svreinterpret_u16_u8(svuint8_t op);
240__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8)))
241svuint16_t svreinterpret_u16_mf8(svmfloat8_t op);
203242__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16)))
204243svuint16_t svreinterpret_u16_s16(svint16_t op);
205244__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16)))
......@@ -224,6 +263,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8)
224263svint32_t svreinterpret_s32_s8(svint8_t op);
225264__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8)))
226265svint32_t svreinterpret_s32_u8(svuint8_t op);
266__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8)))
267svint32_t svreinterpret_s32_mf8(svmfloat8_t op);
227268__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16)))
228269svint32_t svreinterpret_s32_s16(svint16_t op);
229270__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16)))
......@@ -248,6 +289,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8)
248289svuint32_t svreinterpret_u32_s8(svint8_t op);
249290__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8)))
250291svuint32_t svreinterpret_u32_u8(svuint8_t op);
292__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8)))
293svuint32_t svreinterpret_u32_mf8(svmfloat8_t op);
251294__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16)))
252295svuint32_t svreinterpret_u32_s16(svint16_t op);
253296__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16)))
......@@ -272,6 +315,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8)
272315svint64_t svreinterpret_s64_s8(svint8_t op);
273316__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8)))
274317svint64_t svreinterpret_s64_u8(svuint8_t op);
318__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8)))
319svint64_t svreinterpret_s64_mf8(svmfloat8_t op);
275320__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16)))
276321svint64_t svreinterpret_s64_s16(svint16_t op);
277322__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16)))
......@@ -296,6 +341,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8)
296341svuint64_t svreinterpret_u64_s8(svint8_t op);
297342__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8)))
298343svuint64_t svreinterpret_u64_u8(svuint8_t op);
344__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8)))
345svuint64_t svreinterpret_u64_mf8(svmfloat8_t op);
299346__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16)))
300347svuint64_t svreinterpret_u64_s16(svint16_t op);
301348__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16)))
......@@ -320,6 +367,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8)
320367svfloat16_t svreinterpret_f16_s8(svint8_t op);
321368__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8)))
322369svfloat16_t svreinterpret_f16_u8(svuint8_t op);
370__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8)))
371svfloat16_t svreinterpret_f16_mf8(svmfloat8_t op);
323372__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16)))
324373svfloat16_t svreinterpret_f16_s16(svint16_t op);
325374__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16)))
......@@ -344,6 +393,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
344393svbfloat16_t svreinterpret_bf16_s8(svint8_t op);
345394__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8)))
346395svbfloat16_t svreinterpret_bf16_u8(svuint8_t op);
396__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8)))
397svbfloat16_t svreinterpret_bf16_mf8(svmfloat8_t op);
347398__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16)))
348399svbfloat16_t svreinterpret_bf16_s16(svint16_t op);
349400__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16)))
......@@ -368,6 +419,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8)
368419svfloat32_t svreinterpret_f32_s8(svint8_t op);
369420__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8)))
370421svfloat32_t svreinterpret_f32_u8(svuint8_t op);
422__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8)))
423svfloat32_t svreinterpret_f32_mf8(svmfloat8_t op);
371424__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16)))
372425svfloat32_t svreinterpret_f32_s16(svint16_t op);
373426__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16)))
......@@ -392,6 +445,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8)
392445svfloat64_t svreinterpret_f64_s8(svint8_t op);
393446__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8)))
394447svfloat64_t svreinterpret_f64_u8(svuint8_t op);
448__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8)))
449svfloat64_t svreinterpret_f64_mf8(svmfloat8_t op);
395450__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16)))
396451svfloat64_t svreinterpret_f64_s16(svint16_t op);
397452__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16)))
......@@ -416,6 +471,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8))
416471svint8_t svreinterpret_s8(svint8_t op);
417472__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8)))
418473svint8_t svreinterpret_s8(svuint8_t op);
474__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8)))
475svint8_t svreinterpret_s8(svmfloat8_t op);
419476__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16)))
420477svint8_t svreinterpret_s8(svint16_t op);
421478__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16)))
......@@ -440,6 +497,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8))
440497svuint8_t svreinterpret_u8(svint8_t op);
441498__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8)))
442499svuint8_t svreinterpret_u8(svuint8_t op);
500__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8)))
501svuint8_t svreinterpret_u8(svmfloat8_t op);
443502__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16)))
444503svuint8_t svreinterpret_u8(svint16_t op);
445504__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16)))
......@@ -460,10 +519,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32)
460519svuint8_t svreinterpret_u8(svfloat32_t op);
461520__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64)))
462521svuint8_t svreinterpret_u8(svfloat64_t op);
522__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8)))
523svmfloat8_t svreinterpret_mf8(svint8_t op);
524__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8)))
525svmfloat8_t svreinterpret_mf8(svuint8_t op);
526__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8)))
527svmfloat8_t svreinterpret_mf8(svmfloat8_t op);
528__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16)))
529svmfloat8_t svreinterpret_mf8(svint16_t op);
530__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16)))
531svmfloat8_t svreinterpret_mf8(svuint16_t op);
532__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32)))
533svmfloat8_t svreinterpret_mf8(svint32_t op);
534__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32)))
535svmfloat8_t svreinterpret_mf8(svuint32_t op);
536__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64)))
537svmfloat8_t svreinterpret_mf8(svint64_t op);
538__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64)))
539svmfloat8_t svreinterpret_mf8(svuint64_t op);
540__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16)))
541svmfloat8_t svreinterpret_mf8(svfloat16_t op);
542__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16)))
543svmfloat8_t svreinterpret_mf8(svbfloat16_t op);
544__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32)))
545svmfloat8_t svreinterpret_mf8(svfloat32_t op);
546__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64)))
547svmfloat8_t svreinterpret_mf8(svfloat64_t op);
463548__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8)))
464549svint16_t svreinterpret_s16(svint8_t op);
465550__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8)))
466551svint16_t svreinterpret_s16(svuint8_t op);
552__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8)))
553svint16_t svreinterpret_s16(svmfloat8_t op);
467554__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16)))
468555svint16_t svreinterpret_s16(svint16_t op);
469556__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16)))
......@@ -488,6 +575,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8)
488575svuint16_t svreinterpret_u16(svint8_t op);
489576__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8)))
490577svuint16_t svreinterpret_u16(svuint8_t op);
578__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8)))
579svuint16_t svreinterpret_u16(svmfloat8_t op);
491580__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16)))
492581svuint16_t svreinterpret_u16(svint16_t op);
493582__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16)))
......@@ -512,6 +601,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8)
512601svint32_t svreinterpret_s32(svint8_t op);
513602__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8)))
514603svint32_t svreinterpret_s32(svuint8_t op);
604__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8)))
605svint32_t svreinterpret_s32(svmfloat8_t op);
515606__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16)))
516607svint32_t svreinterpret_s32(svint16_t op);
517608__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16)))
......@@ -536,6 +627,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8)
536627svuint32_t svreinterpret_u32(svint8_t op);
537628__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8)))
538629svuint32_t svreinterpret_u32(svuint8_t op);
630__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8)))
631svuint32_t svreinterpret_u32(svmfloat8_t op);
539632__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16)))
540633svuint32_t svreinterpret_u32(svint16_t op);
541634__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16)))
......@@ -560,6 +653,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8)
560653svint64_t svreinterpret_s64(svint8_t op);
561654__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8)))
562655svint64_t svreinterpret_s64(svuint8_t op);
656__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8)))
657svint64_t svreinterpret_s64(svmfloat8_t op);
563658__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16)))
564659svint64_t svreinterpret_s64(svint16_t op);
565660__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16)))
......@@ -584,6 +679,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8)
584679svuint64_t svreinterpret_u64(svint8_t op);
585680__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8)))
586681svuint64_t svreinterpret_u64(svuint8_t op);
682__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8)))
683svuint64_t svreinterpret_u64(svmfloat8_t op);
587684__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16)))
588685svuint64_t svreinterpret_u64(svint16_t op);
589686__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16)))
......@@ -608,6 +705,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8)
608705svfloat16_t svreinterpret_f16(svint8_t op);
609706__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8)))
610707svfloat16_t svreinterpret_f16(svuint8_t op);
708__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8)))
709svfloat16_t svreinterpret_f16(svmfloat8_t op);
611710__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16)))
612711svfloat16_t svreinterpret_f16(svint16_t op);
613712__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16)))
......@@ -632,6 +731,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
632731svbfloat16_t svreinterpret_bf16(svint8_t op);
633732__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8)))
634733svbfloat16_t svreinterpret_bf16(svuint8_t op);
734__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8)))
735svbfloat16_t svreinterpret_bf16(svmfloat8_t op);
635736__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16)))
636737svbfloat16_t svreinterpret_bf16(svint16_t op);
637738__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16)))
......@@ -656,6 +757,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8)
656757svfloat32_t svreinterpret_f32(svint8_t op);
657758__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8)))
658759svfloat32_t svreinterpret_f32(svuint8_t op);
760__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8)))
761svfloat32_t svreinterpret_f32(svmfloat8_t op);
659762__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16)))
660763svfloat32_t svreinterpret_f32(svint16_t op);
661764__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16)))
......@@ -680,6 +783,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8)
680783svfloat64_t svreinterpret_f64(svint8_t op);
681784__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8)))
682785svfloat64_t svreinterpret_f64(svuint8_t op);
786__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8)))
787svfloat64_t svreinterpret_f64(svmfloat8_t op);
683788__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16)))
684789svfloat64_t svreinterpret_f64(svint16_t op);
685790__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16)))
......@@ -704,6 +809,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x
704809svint8x2_t svreinterpret_s8_s8_x2(svint8x2_t op);
705810__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x2)))
706811svint8x2_t svreinterpret_s8_u8_x2(svuint8x2_t op);
812__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8_x2)))
813svint8x2_t svreinterpret_s8_mf8_x2(svmfloat8x2_t op);
707814__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x2)))
708815svint8x2_t svreinterpret_s8_s16_x2(svint16x2_t op);
709816__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x2)))
......@@ -728,6 +835,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x
728835svuint8x2_t svreinterpret_u8_s8_x2(svint8x2_t op);
729836__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x2)))
730837svuint8x2_t svreinterpret_u8_u8_x2(svuint8x2_t op);
838__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8_x2)))
839svuint8x2_t svreinterpret_u8_mf8_x2(svmfloat8x2_t op);
731840__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x2)))
732841svuint8x2_t svreinterpret_u8_s16_x2(svint16x2_t op);
733842__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x2)))
......@@ -748,10 +857,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_
748857svuint8x2_t svreinterpret_u8_f32_x2(svfloat32x2_t op);
749858__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x2)))
750859svuint8x2_t svreinterpret_u8_f64_x2(svfloat64x2_t op);
860__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8_x2)))
861svmfloat8x2_t svreinterpret_mf8_s8_x2(svint8x2_t op);
862__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8_x2)))
863svmfloat8x2_t svreinterpret_mf8_u8_x2(svuint8x2_t op);
864__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8_x2)))
865svmfloat8x2_t svreinterpret_mf8_mf8_x2(svmfloat8x2_t op);
866__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16_x2)))
867svmfloat8x2_t svreinterpret_mf8_s16_x2(svint16x2_t op);
868__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16_x2)))
869svmfloat8x2_t svreinterpret_mf8_u16_x2(svuint16x2_t op);
870__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32_x2)))
871svmfloat8x2_t svreinterpret_mf8_s32_x2(svint32x2_t op);
872__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32_x2)))
873svmfloat8x2_t svreinterpret_mf8_u32_x2(svuint32x2_t op);
874__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64_x2)))
875svmfloat8x2_t svreinterpret_mf8_s64_x2(svint64x2_t op);
876__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64_x2)))
877svmfloat8x2_t svreinterpret_mf8_u64_x2(svuint64x2_t op);
878__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16_x2)))
879svmfloat8x2_t svreinterpret_mf8_f16_x2(svfloat16x2_t op);
880__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16_x2)))
881svmfloat8x2_t svreinterpret_mf8_bf16_x2(svbfloat16x2_t op);
882__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32_x2)))
883svmfloat8x2_t svreinterpret_mf8_f32_x2(svfloat32x2_t op);
884__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64_x2)))
885svmfloat8x2_t svreinterpret_mf8_f64_x2(svfloat64x2_t op);
751886__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x2)))
752887svint16x2_t svreinterpret_s16_s8_x2(svint8x2_t op);
753888__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x2)))
754889svint16x2_t svreinterpret_s16_u8_x2(svuint8x2_t op);
890__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8_x2)))
891svint16x2_t svreinterpret_s16_mf8_x2(svmfloat8x2_t op);
755892__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x2)))
756893svint16x2_t svreinterpret_s16_s16_x2(svint16x2_t op);
757894__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x2)))
......@@ -776,6 +913,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_
776913svuint16x2_t svreinterpret_u16_s8_x2(svint8x2_t op);
777914__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x2)))
778915svuint16x2_t svreinterpret_u16_u8_x2(svuint8x2_t op);
916__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8_x2)))
917svuint16x2_t svreinterpret_u16_mf8_x2(svmfloat8x2_t op);
779918__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x2)))
780919svuint16x2_t svreinterpret_u16_s16_x2(svint16x2_t op);
781920__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x2)))
......@@ -800,6 +939,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_
800939svint32x2_t svreinterpret_s32_s8_x2(svint8x2_t op);
801940__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x2)))
802941svint32x2_t svreinterpret_s32_u8_x2(svuint8x2_t op);
942__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8_x2)))
943svint32x2_t svreinterpret_s32_mf8_x2(svmfloat8x2_t op);
803944__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x2)))
804945svint32x2_t svreinterpret_s32_s16_x2(svint16x2_t op);
805946__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x2)))
......@@ -824,6 +965,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_
824965svuint32x2_t svreinterpret_u32_s8_x2(svint8x2_t op);
825966__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x2)))
826967svuint32x2_t svreinterpret_u32_u8_x2(svuint8x2_t op);
968__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8_x2)))
969svuint32x2_t svreinterpret_u32_mf8_x2(svmfloat8x2_t op);
827970__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x2)))
828971svuint32x2_t svreinterpret_u32_s16_x2(svint16x2_t op);
829972__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x2)))
......@@ -848,6 +991,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_
848991svint64x2_t svreinterpret_s64_s8_x2(svint8x2_t op);
849992__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x2)))
850993svint64x2_t svreinterpret_s64_u8_x2(svuint8x2_t op);
994__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8_x2)))
995svint64x2_t svreinterpret_s64_mf8_x2(svmfloat8x2_t op);
851996__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x2)))
852997svint64x2_t svreinterpret_s64_s16_x2(svint16x2_t op);
853998__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x2)))
......@@ -872,6 +1017,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_
8721017svuint64x2_t svreinterpret_u64_s8_x2(svint8x2_t op);
8731018__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x2)))
8741019svuint64x2_t svreinterpret_u64_u8_x2(svuint8x2_t op);
1020__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8_x2)))
1021svuint64x2_t svreinterpret_u64_mf8_x2(svmfloat8x2_t op);
8751022__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x2)))
8761023svuint64x2_t svreinterpret_u64_s16_x2(svint16x2_t op);
8771024__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x2)))
......@@ -896,6 +1043,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_
8961043svfloat16x2_t svreinterpret_f16_s8_x2(svint8x2_t op);
8971044__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x2)))
8981045svfloat16x2_t svreinterpret_f16_u8_x2(svuint8x2_t op);
1046__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8_x2)))
1047svfloat16x2_t svreinterpret_f16_mf8_x2(svmfloat8x2_t op);
8991048__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x2)))
9001049svfloat16x2_t svreinterpret_f16_s16_x2(svint16x2_t op);
9011050__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x2)))
......@@ -920,6 +1069,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
9201069svbfloat16x2_t svreinterpret_bf16_s8_x2(svint8x2_t op);
9211070__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x2)))
9221071svbfloat16x2_t svreinterpret_bf16_u8_x2(svuint8x2_t op);
1072__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8_x2)))
1073svbfloat16x2_t svreinterpret_bf16_mf8_x2(svmfloat8x2_t op);
9231074__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x2)))
9241075svbfloat16x2_t svreinterpret_bf16_s16_x2(svint16x2_t op);
9251076__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x2)))
......@@ -944,6 +1095,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_
9441095svfloat32x2_t svreinterpret_f32_s8_x2(svint8x2_t op);
9451096__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x2)))
9461097svfloat32x2_t svreinterpret_f32_u8_x2(svuint8x2_t op);
1098__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8_x2)))
1099svfloat32x2_t svreinterpret_f32_mf8_x2(svmfloat8x2_t op);
9471100__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x2)))
9481101svfloat32x2_t svreinterpret_f32_s16_x2(svint16x2_t op);
9491102__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x2)))
......@@ -968,6 +1121,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_
9681121svfloat64x2_t svreinterpret_f64_s8_x2(svint8x2_t op);
9691122__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x2)))
9701123svfloat64x2_t svreinterpret_f64_u8_x2(svuint8x2_t op);
1124__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8_x2)))
1125svfloat64x2_t svreinterpret_f64_mf8_x2(svmfloat8x2_t op);
9711126__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x2)))
9721127svfloat64x2_t svreinterpret_f64_s16_x2(svint16x2_t op);
9731128__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x2)))
......@@ -992,6 +1147,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x
9921147svint8x2_t svreinterpret_s8(svint8x2_t op);
9931148__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x2)))
9941149svint8x2_t svreinterpret_s8(svuint8x2_t op);
1150__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8_x2)))
1151svint8x2_t svreinterpret_s8(svmfloat8x2_t op);
9951152__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x2)))
9961153svint8x2_t svreinterpret_s8(svint16x2_t op);
9971154__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x2)))
......@@ -1016,6 +1173,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x
10161173svuint8x2_t svreinterpret_u8(svint8x2_t op);
10171174__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x2)))
10181175svuint8x2_t svreinterpret_u8(svuint8x2_t op);
1176__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8_x2)))
1177svuint8x2_t svreinterpret_u8(svmfloat8x2_t op);
10191178__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x2)))
10201179svuint8x2_t svreinterpret_u8(svint16x2_t op);
10211180__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x2)))
......@@ -1036,10 +1195,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_
10361195svuint8x2_t svreinterpret_u8(svfloat32x2_t op);
10371196__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x2)))
10381197svuint8x2_t svreinterpret_u8(svfloat64x2_t op);
1198__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8_x2)))
1199svmfloat8x2_t svreinterpret_mf8(svint8x2_t op);
1200__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8_x2)))
1201svmfloat8x2_t svreinterpret_mf8(svuint8x2_t op);
1202__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8_x2)))
1203svmfloat8x2_t svreinterpret_mf8(svmfloat8x2_t op);
1204__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16_x2)))
1205svmfloat8x2_t svreinterpret_mf8(svint16x2_t op);
1206__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16_x2)))
1207svmfloat8x2_t svreinterpret_mf8(svuint16x2_t op);
1208__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32_x2)))
1209svmfloat8x2_t svreinterpret_mf8(svint32x2_t op);
1210__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32_x2)))
1211svmfloat8x2_t svreinterpret_mf8(svuint32x2_t op);
1212__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64_x2)))
1213svmfloat8x2_t svreinterpret_mf8(svint64x2_t op);
1214__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64_x2)))
1215svmfloat8x2_t svreinterpret_mf8(svuint64x2_t op);
1216__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16_x2)))
1217svmfloat8x2_t svreinterpret_mf8(svfloat16x2_t op);
1218__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16_x2)))
1219svmfloat8x2_t svreinterpret_mf8(svbfloat16x2_t op);
1220__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32_x2)))
1221svmfloat8x2_t svreinterpret_mf8(svfloat32x2_t op);
1222__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64_x2)))
1223svmfloat8x2_t svreinterpret_mf8(svfloat64x2_t op);
10391224__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x2)))
10401225svint16x2_t svreinterpret_s16(svint8x2_t op);
10411226__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x2)))
10421227svint16x2_t svreinterpret_s16(svuint8x2_t op);
1228__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8_x2)))
1229svint16x2_t svreinterpret_s16(svmfloat8x2_t op);
10431230__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x2)))
10441231svint16x2_t svreinterpret_s16(svint16x2_t op);
10451232__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x2)))
......@@ -1064,6 +1251,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_
10641251svuint16x2_t svreinterpret_u16(svint8x2_t op);
10651252__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x2)))
10661253svuint16x2_t svreinterpret_u16(svuint8x2_t op);
1254__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8_x2)))
1255svuint16x2_t svreinterpret_u16(svmfloat8x2_t op);
10671256__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x2)))
10681257svuint16x2_t svreinterpret_u16(svint16x2_t op);
10691258__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x2)))
......@@ -1088,6 +1277,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_
10881277svint32x2_t svreinterpret_s32(svint8x2_t op);
10891278__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x2)))
10901279svint32x2_t svreinterpret_s32(svuint8x2_t op);
1280__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8_x2)))
1281svint32x2_t svreinterpret_s32(svmfloat8x2_t op);
10911282__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x2)))
10921283svint32x2_t svreinterpret_s32(svint16x2_t op);
10931284__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x2)))
......@@ -1112,6 +1303,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_
11121303svuint32x2_t svreinterpret_u32(svint8x2_t op);
11131304__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x2)))
11141305svuint32x2_t svreinterpret_u32(svuint8x2_t op);
1306__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8_x2)))
1307svuint32x2_t svreinterpret_u32(svmfloat8x2_t op);
11151308__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x2)))
11161309svuint32x2_t svreinterpret_u32(svint16x2_t op);
11171310__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x2)))
......@@ -1136,6 +1329,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_
11361329svint64x2_t svreinterpret_s64(svint8x2_t op);
11371330__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x2)))
11381331svint64x2_t svreinterpret_s64(svuint8x2_t op);
1332__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8_x2)))
1333svint64x2_t svreinterpret_s64(svmfloat8x2_t op);
11391334__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x2)))
11401335svint64x2_t svreinterpret_s64(svint16x2_t op);
11411336__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x2)))
......@@ -1160,6 +1355,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_
11601355svuint64x2_t svreinterpret_u64(svint8x2_t op);
11611356__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x2)))
11621357svuint64x2_t svreinterpret_u64(svuint8x2_t op);
1358__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8_x2)))
1359svuint64x2_t svreinterpret_u64(svmfloat8x2_t op);
11631360__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x2)))
11641361svuint64x2_t svreinterpret_u64(svint16x2_t op);
11651362__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x2)))
......@@ -1184,6 +1381,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_
11841381svfloat16x2_t svreinterpret_f16(svint8x2_t op);
11851382__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x2)))
11861383svfloat16x2_t svreinterpret_f16(svuint8x2_t op);
1384__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8_x2)))
1385svfloat16x2_t svreinterpret_f16(svmfloat8x2_t op);
11871386__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x2)))
11881387svfloat16x2_t svreinterpret_f16(svint16x2_t op);
11891388__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x2)))
......@@ -1208,6 +1407,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
12081407svbfloat16x2_t svreinterpret_bf16(svint8x2_t op);
12091408__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x2)))
12101409svbfloat16x2_t svreinterpret_bf16(svuint8x2_t op);
1410__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8_x2)))
1411svbfloat16x2_t svreinterpret_bf16(svmfloat8x2_t op);
12111412__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x2)))
12121413svbfloat16x2_t svreinterpret_bf16(svint16x2_t op);
12131414__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x2)))
......@@ -1232,6 +1433,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_
12321433svfloat32x2_t svreinterpret_f32(svint8x2_t op);
12331434__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x2)))
12341435svfloat32x2_t svreinterpret_f32(svuint8x2_t op);
1436__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8_x2)))
1437svfloat32x2_t svreinterpret_f32(svmfloat8x2_t op);
12351438__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x2)))
12361439svfloat32x2_t svreinterpret_f32(svint16x2_t op);
12371440__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x2)))
......@@ -1256,6 +1459,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_
12561459svfloat64x2_t svreinterpret_f64(svint8x2_t op);
12571460__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x2)))
12581461svfloat64x2_t svreinterpret_f64(svuint8x2_t op);
1462__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8_x2)))
1463svfloat64x2_t svreinterpret_f64(svmfloat8x2_t op);
12591464__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x2)))
12601465svfloat64x2_t svreinterpret_f64(svint16x2_t op);
12611466__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x2)))
......@@ -1280,6 +1485,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x
12801485svint8x3_t svreinterpret_s8_s8_x3(svint8x3_t op);
12811486__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x3)))
12821487svint8x3_t svreinterpret_s8_u8_x3(svuint8x3_t op);
1488__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8_x3)))
1489svint8x3_t svreinterpret_s8_mf8_x3(svmfloat8x3_t op);
12831490__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x3)))
12841491svint8x3_t svreinterpret_s8_s16_x3(svint16x3_t op);
12851492__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x3)))
......@@ -1304,6 +1511,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x
13041511svuint8x3_t svreinterpret_u8_s8_x3(svint8x3_t op);
13051512__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x3)))
13061513svuint8x3_t svreinterpret_u8_u8_x3(svuint8x3_t op);
1514__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8_x3)))
1515svuint8x3_t svreinterpret_u8_mf8_x3(svmfloat8x3_t op);
13071516__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x3)))
13081517svuint8x3_t svreinterpret_u8_s16_x3(svint16x3_t op);
13091518__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x3)))
......@@ -1324,10 +1533,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_
13241533svuint8x3_t svreinterpret_u8_f32_x3(svfloat32x3_t op);
13251534__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x3)))
13261535svuint8x3_t svreinterpret_u8_f64_x3(svfloat64x3_t op);
1536__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8_x3)))
1537svmfloat8x3_t svreinterpret_mf8_s8_x3(svint8x3_t op);
1538__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8_x3)))
1539svmfloat8x3_t svreinterpret_mf8_u8_x3(svuint8x3_t op);
1540__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8_x3)))
1541svmfloat8x3_t svreinterpret_mf8_mf8_x3(svmfloat8x3_t op);
1542__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16_x3)))
1543svmfloat8x3_t svreinterpret_mf8_s16_x3(svint16x3_t op);
1544__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16_x3)))
1545svmfloat8x3_t svreinterpret_mf8_u16_x3(svuint16x3_t op);
1546__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32_x3)))
1547svmfloat8x3_t svreinterpret_mf8_s32_x3(svint32x3_t op);
1548__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32_x3)))
1549svmfloat8x3_t svreinterpret_mf8_u32_x3(svuint32x3_t op);
1550__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64_x3)))
1551svmfloat8x3_t svreinterpret_mf8_s64_x3(svint64x3_t op);
1552__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64_x3)))
1553svmfloat8x3_t svreinterpret_mf8_u64_x3(svuint64x3_t op);
1554__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16_x3)))
1555svmfloat8x3_t svreinterpret_mf8_f16_x3(svfloat16x3_t op);
1556__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16_x3)))
1557svmfloat8x3_t svreinterpret_mf8_bf16_x3(svbfloat16x3_t op);
1558__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32_x3)))
1559svmfloat8x3_t svreinterpret_mf8_f32_x3(svfloat32x3_t op);
1560__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64_x3)))
1561svmfloat8x3_t svreinterpret_mf8_f64_x3(svfloat64x3_t op);
13271562__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x3)))
13281563svint16x3_t svreinterpret_s16_s8_x3(svint8x3_t op);
13291564__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x3)))
13301565svint16x3_t svreinterpret_s16_u8_x3(svuint8x3_t op);
1566__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8_x3)))
1567svint16x3_t svreinterpret_s16_mf8_x3(svmfloat8x3_t op);
13311568__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x3)))
13321569svint16x3_t svreinterpret_s16_s16_x3(svint16x3_t op);
13331570__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x3)))
......@@ -1352,6 +1589,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_
13521589svuint16x3_t svreinterpret_u16_s8_x3(svint8x3_t op);
13531590__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x3)))
13541591svuint16x3_t svreinterpret_u16_u8_x3(svuint8x3_t op);
1592__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8_x3)))
1593svuint16x3_t svreinterpret_u16_mf8_x3(svmfloat8x3_t op);
13551594__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x3)))
13561595svuint16x3_t svreinterpret_u16_s16_x3(svint16x3_t op);
13571596__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x3)))
......@@ -1376,6 +1615,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_
13761615svint32x3_t svreinterpret_s32_s8_x3(svint8x3_t op);
13771616__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x3)))
13781617svint32x3_t svreinterpret_s32_u8_x3(svuint8x3_t op);
1618__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8_x3)))
1619svint32x3_t svreinterpret_s32_mf8_x3(svmfloat8x3_t op);
13791620__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x3)))
13801621svint32x3_t svreinterpret_s32_s16_x3(svint16x3_t op);
13811622__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x3)))
......@@ -1400,6 +1641,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_
14001641svuint32x3_t svreinterpret_u32_s8_x3(svint8x3_t op);
14011642__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x3)))
14021643svuint32x3_t svreinterpret_u32_u8_x3(svuint8x3_t op);
1644__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8_x3)))
1645svuint32x3_t svreinterpret_u32_mf8_x3(svmfloat8x3_t op);
14031646__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x3)))
14041647svuint32x3_t svreinterpret_u32_s16_x3(svint16x3_t op);
14051648__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x3)))
......@@ -1424,6 +1667,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_
14241667svint64x3_t svreinterpret_s64_s8_x3(svint8x3_t op);
14251668__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x3)))
14261669svint64x3_t svreinterpret_s64_u8_x3(svuint8x3_t op);
1670__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8_x3)))
1671svint64x3_t svreinterpret_s64_mf8_x3(svmfloat8x3_t op);
14271672__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x3)))
14281673svint64x3_t svreinterpret_s64_s16_x3(svint16x3_t op);
14291674__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x3)))
......@@ -1448,6 +1693,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_
14481693svuint64x3_t svreinterpret_u64_s8_x3(svint8x3_t op);
14491694__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x3)))
14501695svuint64x3_t svreinterpret_u64_u8_x3(svuint8x3_t op);
1696__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8_x3)))
1697svuint64x3_t svreinterpret_u64_mf8_x3(svmfloat8x3_t op);
14511698__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x3)))
14521699svuint64x3_t svreinterpret_u64_s16_x3(svint16x3_t op);
14531700__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x3)))
......@@ -1472,6 +1719,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_
14721719svfloat16x3_t svreinterpret_f16_s8_x3(svint8x3_t op);
14731720__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x3)))
14741721svfloat16x3_t svreinterpret_f16_u8_x3(svuint8x3_t op);
1722__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8_x3)))
1723svfloat16x3_t svreinterpret_f16_mf8_x3(svmfloat8x3_t op);
14751724__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x3)))
14761725svfloat16x3_t svreinterpret_f16_s16_x3(svint16x3_t op);
14771726__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x3)))
......@@ -1496,6 +1745,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
14961745svbfloat16x3_t svreinterpret_bf16_s8_x3(svint8x3_t op);
14971746__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x3)))
14981747svbfloat16x3_t svreinterpret_bf16_u8_x3(svuint8x3_t op);
1748__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8_x3)))
1749svbfloat16x3_t svreinterpret_bf16_mf8_x3(svmfloat8x3_t op);
14991750__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x3)))
15001751svbfloat16x3_t svreinterpret_bf16_s16_x3(svint16x3_t op);
15011752__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x3)))
......@@ -1520,6 +1771,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_
15201771svfloat32x3_t svreinterpret_f32_s8_x3(svint8x3_t op);
15211772__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x3)))
15221773svfloat32x3_t svreinterpret_f32_u8_x3(svuint8x3_t op);
1774__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8_x3)))
1775svfloat32x3_t svreinterpret_f32_mf8_x3(svmfloat8x3_t op);
15231776__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x3)))
15241777svfloat32x3_t svreinterpret_f32_s16_x3(svint16x3_t op);
15251778__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x3)))
......@@ -1544,6 +1797,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_
15441797svfloat64x3_t svreinterpret_f64_s8_x3(svint8x3_t op);
15451798__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x3)))
15461799svfloat64x3_t svreinterpret_f64_u8_x3(svuint8x3_t op);
1800__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8_x3)))
1801svfloat64x3_t svreinterpret_f64_mf8_x3(svmfloat8x3_t op);
15471802__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x3)))
15481803svfloat64x3_t svreinterpret_f64_s16_x3(svint16x3_t op);
15491804__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x3)))
......@@ -1568,6 +1823,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x
15681823svint8x3_t svreinterpret_s8(svint8x3_t op);
15691824__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x3)))
15701825svint8x3_t svreinterpret_s8(svuint8x3_t op);
1826__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8_x3)))
1827svint8x3_t svreinterpret_s8(svmfloat8x3_t op);
15711828__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x3)))
15721829svint8x3_t svreinterpret_s8(svint16x3_t op);
15731830__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x3)))
......@@ -1592,6 +1849,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x
15921849svuint8x3_t svreinterpret_u8(svint8x3_t op);
15931850__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x3)))
15941851svuint8x3_t svreinterpret_u8(svuint8x3_t op);
1852__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8_x3)))
1853svuint8x3_t svreinterpret_u8(svmfloat8x3_t op);
15951854__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x3)))
15961855svuint8x3_t svreinterpret_u8(svint16x3_t op);
15971856__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x3)))
......@@ -1612,10 +1871,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_
16121871svuint8x3_t svreinterpret_u8(svfloat32x3_t op);
16131872__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x3)))
16141873svuint8x3_t svreinterpret_u8(svfloat64x3_t op);
1874__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8_x3)))
1875svmfloat8x3_t svreinterpret_mf8(svint8x3_t op);
1876__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8_x3)))
1877svmfloat8x3_t svreinterpret_mf8(svuint8x3_t op);
1878__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8_x3)))
1879svmfloat8x3_t svreinterpret_mf8(svmfloat8x3_t op);
1880__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16_x3)))
1881svmfloat8x3_t svreinterpret_mf8(svint16x3_t op);
1882__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16_x3)))
1883svmfloat8x3_t svreinterpret_mf8(svuint16x3_t op);
1884__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32_x3)))
1885svmfloat8x3_t svreinterpret_mf8(svint32x3_t op);
1886__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32_x3)))
1887svmfloat8x3_t svreinterpret_mf8(svuint32x3_t op);
1888__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64_x3)))
1889svmfloat8x3_t svreinterpret_mf8(svint64x3_t op);
1890__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64_x3)))
1891svmfloat8x3_t svreinterpret_mf8(svuint64x3_t op);
1892__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16_x3)))
1893svmfloat8x3_t svreinterpret_mf8(svfloat16x3_t op);
1894__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16_x3)))
1895svmfloat8x3_t svreinterpret_mf8(svbfloat16x3_t op);
1896__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32_x3)))
1897svmfloat8x3_t svreinterpret_mf8(svfloat32x3_t op);
1898__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64_x3)))
1899svmfloat8x3_t svreinterpret_mf8(svfloat64x3_t op);
16151900__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x3)))
16161901svint16x3_t svreinterpret_s16(svint8x3_t op);
16171902__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x3)))
16181903svint16x3_t svreinterpret_s16(svuint8x3_t op);
1904__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8_x3)))
1905svint16x3_t svreinterpret_s16(svmfloat8x3_t op);
16191906__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x3)))
16201907svint16x3_t svreinterpret_s16(svint16x3_t op);
16211908__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x3)))
......@@ -1640,6 +1927,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_
16401927svuint16x3_t svreinterpret_u16(svint8x3_t op);
16411928__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x3)))
16421929svuint16x3_t svreinterpret_u16(svuint8x3_t op);
1930__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8_x3)))
1931svuint16x3_t svreinterpret_u16(svmfloat8x3_t op);
16431932__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x3)))
16441933svuint16x3_t svreinterpret_u16(svint16x3_t op);
16451934__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x3)))
......@@ -1664,6 +1953,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_
16641953svint32x3_t svreinterpret_s32(svint8x3_t op);
16651954__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x3)))
16661955svint32x3_t svreinterpret_s32(svuint8x3_t op);
1956__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8_x3)))
1957svint32x3_t svreinterpret_s32(svmfloat8x3_t op);
16671958__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x3)))
16681959svint32x3_t svreinterpret_s32(svint16x3_t op);
16691960__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x3)))
......@@ -1688,6 +1979,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_
16881979svuint32x3_t svreinterpret_u32(svint8x3_t op);
16891980__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x3)))
16901981svuint32x3_t svreinterpret_u32(svuint8x3_t op);
1982__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8_x3)))
1983svuint32x3_t svreinterpret_u32(svmfloat8x3_t op);
16911984__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x3)))
16921985svuint32x3_t svreinterpret_u32(svint16x3_t op);
16931986__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x3)))
......@@ -1712,6 +2005,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_
17122005svint64x3_t svreinterpret_s64(svint8x3_t op);
17132006__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x3)))
17142007svint64x3_t svreinterpret_s64(svuint8x3_t op);
2008__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8_x3)))
2009svint64x3_t svreinterpret_s64(svmfloat8x3_t op);
17152010__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x3)))
17162011svint64x3_t svreinterpret_s64(svint16x3_t op);
17172012__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x3)))
......@@ -1736,6 +2031,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_
17362031svuint64x3_t svreinterpret_u64(svint8x3_t op);
17372032__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x3)))
17382033svuint64x3_t svreinterpret_u64(svuint8x3_t op);
2034__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8_x3)))
2035svuint64x3_t svreinterpret_u64(svmfloat8x3_t op);
17392036__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x3)))
17402037svuint64x3_t svreinterpret_u64(svint16x3_t op);
17412038__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x3)))
......@@ -1760,6 +2057,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_
17602057svfloat16x3_t svreinterpret_f16(svint8x3_t op);
17612058__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x3)))
17622059svfloat16x3_t svreinterpret_f16(svuint8x3_t op);
2060__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8_x3)))
2061svfloat16x3_t svreinterpret_f16(svmfloat8x3_t op);
17632062__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x3)))
17642063svfloat16x3_t svreinterpret_f16(svint16x3_t op);
17652064__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x3)))
......@@ -1784,6 +2083,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
17842083svbfloat16x3_t svreinterpret_bf16(svint8x3_t op);
17852084__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x3)))
17862085svbfloat16x3_t svreinterpret_bf16(svuint8x3_t op);
2086__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8_x3)))
2087svbfloat16x3_t svreinterpret_bf16(svmfloat8x3_t op);
17872088__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x3)))
17882089svbfloat16x3_t svreinterpret_bf16(svint16x3_t op);
17892090__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x3)))
......@@ -1808,6 +2109,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_
18082109svfloat32x3_t svreinterpret_f32(svint8x3_t op);
18092110__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x3)))
18102111svfloat32x3_t svreinterpret_f32(svuint8x3_t op);
2112__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8_x3)))
2113svfloat32x3_t svreinterpret_f32(svmfloat8x3_t op);
18112114__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x3)))
18122115svfloat32x3_t svreinterpret_f32(svint16x3_t op);
18132116__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x3)))
......@@ -1832,6 +2135,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_
18322135svfloat64x3_t svreinterpret_f64(svint8x3_t op);
18332136__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x3)))
18342137svfloat64x3_t svreinterpret_f64(svuint8x3_t op);
2138__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8_x3)))
2139svfloat64x3_t svreinterpret_f64(svmfloat8x3_t op);
18352140__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x3)))
18362141svfloat64x3_t svreinterpret_f64(svint16x3_t op);
18372142__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x3)))
......@@ -1856,6 +2161,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x
18562161svint8x4_t svreinterpret_s8_s8_x4(svint8x4_t op);
18572162__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x4)))
18582163svint8x4_t svreinterpret_s8_u8_x4(svuint8x4_t op);
2164__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8_x4)))
2165svint8x4_t svreinterpret_s8_mf8_x4(svmfloat8x4_t op);
18592166__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x4)))
18602167svint8x4_t svreinterpret_s8_s16_x4(svint16x4_t op);
18612168__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x4)))
......@@ -1880,6 +2187,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x
18802187svuint8x4_t svreinterpret_u8_s8_x4(svint8x4_t op);
18812188__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x4)))
18822189svuint8x4_t svreinterpret_u8_u8_x4(svuint8x4_t op);
2190__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8_x4)))
2191svuint8x4_t svreinterpret_u8_mf8_x4(svmfloat8x4_t op);
18832192__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x4)))
18842193svuint8x4_t svreinterpret_u8_s16_x4(svint16x4_t op);
18852194__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x4)))
......@@ -1900,10 +2209,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_
19002209svuint8x4_t svreinterpret_u8_f32_x4(svfloat32x4_t op);
19012210__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x4)))
19022211svuint8x4_t svreinterpret_u8_f64_x4(svfloat64x4_t op);
2212__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8_x4)))
2213svmfloat8x4_t svreinterpret_mf8_s8_x4(svint8x4_t op);
2214__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8_x4)))
2215svmfloat8x4_t svreinterpret_mf8_u8_x4(svuint8x4_t op);
2216__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8_x4)))
2217svmfloat8x4_t svreinterpret_mf8_mf8_x4(svmfloat8x4_t op);
2218__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16_x4)))
2219svmfloat8x4_t svreinterpret_mf8_s16_x4(svint16x4_t op);
2220__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16_x4)))
2221svmfloat8x4_t svreinterpret_mf8_u16_x4(svuint16x4_t op);
2222__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32_x4)))
2223svmfloat8x4_t svreinterpret_mf8_s32_x4(svint32x4_t op);
2224__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32_x4)))
2225svmfloat8x4_t svreinterpret_mf8_u32_x4(svuint32x4_t op);
2226__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64_x4)))
2227svmfloat8x4_t svreinterpret_mf8_s64_x4(svint64x4_t op);
2228__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64_x4)))
2229svmfloat8x4_t svreinterpret_mf8_u64_x4(svuint64x4_t op);
2230__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16_x4)))
2231svmfloat8x4_t svreinterpret_mf8_f16_x4(svfloat16x4_t op);
2232__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16_x4)))
2233svmfloat8x4_t svreinterpret_mf8_bf16_x4(svbfloat16x4_t op);
2234__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32_x4)))
2235svmfloat8x4_t svreinterpret_mf8_f32_x4(svfloat32x4_t op);
2236__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64_x4)))
2237svmfloat8x4_t svreinterpret_mf8_f64_x4(svfloat64x4_t op);
19032238__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x4)))
19042239svint16x4_t svreinterpret_s16_s8_x4(svint8x4_t op);
19052240__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x4)))
19062241svint16x4_t svreinterpret_s16_u8_x4(svuint8x4_t op);
2242__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8_x4)))
2243svint16x4_t svreinterpret_s16_mf8_x4(svmfloat8x4_t op);
19072244__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x4)))
19082245svint16x4_t svreinterpret_s16_s16_x4(svint16x4_t op);
19092246__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x4)))
......@@ -1928,6 +2265,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_
19282265svuint16x4_t svreinterpret_u16_s8_x4(svint8x4_t op);
19292266__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x4)))
19302267svuint16x4_t svreinterpret_u16_u8_x4(svuint8x4_t op);
2268__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8_x4)))
2269svuint16x4_t svreinterpret_u16_mf8_x4(svmfloat8x4_t op);
19312270__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x4)))
19322271svuint16x4_t svreinterpret_u16_s16_x4(svint16x4_t op);
19332272__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x4)))
......@@ -1952,6 +2291,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_
19522291svint32x4_t svreinterpret_s32_s8_x4(svint8x4_t op);
19532292__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x4)))
19542293svint32x4_t svreinterpret_s32_u8_x4(svuint8x4_t op);
2294__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8_x4)))
2295svint32x4_t svreinterpret_s32_mf8_x4(svmfloat8x4_t op);
19552296__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x4)))
19562297svint32x4_t svreinterpret_s32_s16_x4(svint16x4_t op);
19572298__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x4)))
......@@ -1976,6 +2317,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_
19762317svuint32x4_t svreinterpret_u32_s8_x4(svint8x4_t op);
19772318__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x4)))
19782319svuint32x4_t svreinterpret_u32_u8_x4(svuint8x4_t op);
2320__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8_x4)))
2321svuint32x4_t svreinterpret_u32_mf8_x4(svmfloat8x4_t op);
19792322__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x4)))
19802323svuint32x4_t svreinterpret_u32_s16_x4(svint16x4_t op);
19812324__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x4)))
......@@ -2000,6 +2343,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_
20002343svint64x4_t svreinterpret_s64_s8_x4(svint8x4_t op);
20012344__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x4)))
20022345svint64x4_t svreinterpret_s64_u8_x4(svuint8x4_t op);
2346__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8_x4)))
2347svint64x4_t svreinterpret_s64_mf8_x4(svmfloat8x4_t op);
20032348__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x4)))
20042349svint64x4_t svreinterpret_s64_s16_x4(svint16x4_t op);
20052350__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x4)))
......@@ -2024,6 +2369,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_
20242369svuint64x4_t svreinterpret_u64_s8_x4(svint8x4_t op);
20252370__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x4)))
20262371svuint64x4_t svreinterpret_u64_u8_x4(svuint8x4_t op);
2372__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8_x4)))
2373svuint64x4_t svreinterpret_u64_mf8_x4(svmfloat8x4_t op);
20272374__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x4)))
20282375svuint64x4_t svreinterpret_u64_s16_x4(svint16x4_t op);
20292376__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x4)))
......@@ -2048,6 +2395,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_
20482395svfloat16x4_t svreinterpret_f16_s8_x4(svint8x4_t op);
20492396__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x4)))
20502397svfloat16x4_t svreinterpret_f16_u8_x4(svuint8x4_t op);
2398__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8_x4)))
2399svfloat16x4_t svreinterpret_f16_mf8_x4(svmfloat8x4_t op);
20512400__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x4)))
20522401svfloat16x4_t svreinterpret_f16_s16_x4(svint16x4_t op);
20532402__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x4)))
......@@ -2072,6 +2421,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
20722421svbfloat16x4_t svreinterpret_bf16_s8_x4(svint8x4_t op);
20732422__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x4)))
20742423svbfloat16x4_t svreinterpret_bf16_u8_x4(svuint8x4_t op);
2424__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8_x4)))
2425svbfloat16x4_t svreinterpret_bf16_mf8_x4(svmfloat8x4_t op);
20752426__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x4)))
20762427svbfloat16x4_t svreinterpret_bf16_s16_x4(svint16x4_t op);
20772428__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x4)))
......@@ -2096,6 +2447,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_
20962447svfloat32x4_t svreinterpret_f32_s8_x4(svint8x4_t op);
20972448__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x4)))
20982449svfloat32x4_t svreinterpret_f32_u8_x4(svuint8x4_t op);
2450__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8_x4)))
2451svfloat32x4_t svreinterpret_f32_mf8_x4(svmfloat8x4_t op);
20992452__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x4)))
21002453svfloat32x4_t svreinterpret_f32_s16_x4(svint16x4_t op);
21012454__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x4)))
......@@ -2120,6 +2473,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_
21202473svfloat64x4_t svreinterpret_f64_s8_x4(svint8x4_t op);
21212474__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x4)))
21222475svfloat64x4_t svreinterpret_f64_u8_x4(svuint8x4_t op);
2476__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8_x4)))
2477svfloat64x4_t svreinterpret_f64_mf8_x4(svmfloat8x4_t op);
21232478__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x4)))
21242479svfloat64x4_t svreinterpret_f64_s16_x4(svint16x4_t op);
21252480__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x4)))
......@@ -2144,6 +2499,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s8_x
21442499svint8x4_t svreinterpret_s8(svint8x4_t op);
21452500__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u8_x4)))
21462501svint8x4_t svreinterpret_s8(svuint8x4_t op);
2502__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_mf8_x4)))
2503svint8x4_t svreinterpret_s8(svmfloat8x4_t op);
21472504__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_s16_x4)))
21482505svint8x4_t svreinterpret_s8(svint16x4_t op);
21492506__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s8_u16_x4)))
......@@ -2168,6 +2525,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s8_x
21682525svuint8x4_t svreinterpret_u8(svint8x4_t op);
21692526__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u8_x4)))
21702527svuint8x4_t svreinterpret_u8(svuint8x4_t op);
2528__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_mf8_x4)))
2529svuint8x4_t svreinterpret_u8(svmfloat8x4_t op);
21712530__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_s16_x4)))
21722531svuint8x4_t svreinterpret_u8(svint16x4_t op);
21732532__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_u16_x4)))
......@@ -2188,10 +2547,38 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f32_
21882547svuint8x4_t svreinterpret_u8(svfloat32x4_t op);
21892548__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u8_f64_x4)))
21902549svuint8x4_t svreinterpret_u8(svfloat64x4_t op);
2550__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s8_x4)))
2551svmfloat8x4_t svreinterpret_mf8(svint8x4_t op);
2552__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u8_x4)))
2553svmfloat8x4_t svreinterpret_mf8(svuint8x4_t op);
2554__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_mf8_x4)))
2555svmfloat8x4_t svreinterpret_mf8(svmfloat8x4_t op);
2556__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s16_x4)))
2557svmfloat8x4_t svreinterpret_mf8(svint16x4_t op);
2558__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u16_x4)))
2559svmfloat8x4_t svreinterpret_mf8(svuint16x4_t op);
2560__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s32_x4)))
2561svmfloat8x4_t svreinterpret_mf8(svint32x4_t op);
2562__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u32_x4)))
2563svmfloat8x4_t svreinterpret_mf8(svuint32x4_t op);
2564__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_s64_x4)))
2565svmfloat8x4_t svreinterpret_mf8(svint64x4_t op);
2566__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_u64_x4)))
2567svmfloat8x4_t svreinterpret_mf8(svuint64x4_t op);
2568__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f16_x4)))
2569svmfloat8x4_t svreinterpret_mf8(svfloat16x4_t op);
2570__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_bf16_x4)))
2571svmfloat8x4_t svreinterpret_mf8(svbfloat16x4_t op);
2572__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f32_x4)))
2573svmfloat8x4_t svreinterpret_mf8(svfloat32x4_t op);
2574__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_mf8_f64_x4)))
2575svmfloat8x4_t svreinterpret_mf8(svfloat64x4_t op);
21912576__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s8_x4)))
21922577svint16x4_t svreinterpret_s16(svint8x4_t op);
21932578__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u8_x4)))
21942579svint16x4_t svreinterpret_s16(svuint8x4_t op);
2580__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_mf8_x4)))
2581svint16x4_t svreinterpret_s16(svmfloat8x4_t op);
21952582__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_s16_x4)))
21962583svint16x4_t svreinterpret_s16(svint16x4_t op);
21972584__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s16_u16_x4)))
......@@ -2216,6 +2603,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s8_
22162603svuint16x4_t svreinterpret_u16(svint8x4_t op);
22172604__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u8_x4)))
22182605svuint16x4_t svreinterpret_u16(svuint8x4_t op);
2606__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_mf8_x4)))
2607svuint16x4_t svreinterpret_u16(svmfloat8x4_t op);
22192608__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_s16_x4)))
22202609svuint16x4_t svreinterpret_u16(svint16x4_t op);
22212610__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u16_u16_x4)))
......@@ -2240,6 +2629,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s8_
22402629svint32x4_t svreinterpret_s32(svint8x4_t op);
22412630__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u8_x4)))
22422631svint32x4_t svreinterpret_s32(svuint8x4_t op);
2632__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_mf8_x4)))
2633svint32x4_t svreinterpret_s32(svmfloat8x4_t op);
22432634__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_s16_x4)))
22442635svint32x4_t svreinterpret_s32(svint16x4_t op);
22452636__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s32_u16_x4)))
......@@ -2264,6 +2655,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s8_
22642655svuint32x4_t svreinterpret_u32(svint8x4_t op);
22652656__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u8_x4)))
22662657svuint32x4_t svreinterpret_u32(svuint8x4_t op);
2658__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_mf8_x4)))
2659svuint32x4_t svreinterpret_u32(svmfloat8x4_t op);
22672660__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_s16_x4)))
22682661svuint32x4_t svreinterpret_u32(svint16x4_t op);
22692662__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u32_u16_x4)))
......@@ -2288,6 +2681,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s8_
22882681svint64x4_t svreinterpret_s64(svint8x4_t op);
22892682__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u8_x4)))
22902683svint64x4_t svreinterpret_s64(svuint8x4_t op);
2684__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_mf8_x4)))
2685svint64x4_t svreinterpret_s64(svmfloat8x4_t op);
22912686__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_s16_x4)))
22922687svint64x4_t svreinterpret_s64(svint16x4_t op);
22932688__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_s64_u16_x4)))
......@@ -2312,6 +2707,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s8_
23122707svuint64x4_t svreinterpret_u64(svint8x4_t op);
23132708__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u8_x4)))
23142709svuint64x4_t svreinterpret_u64(svuint8x4_t op);
2710__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_mf8_x4)))
2711svuint64x4_t svreinterpret_u64(svmfloat8x4_t op);
23152712__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_s16_x4)))
23162713svuint64x4_t svreinterpret_u64(svint16x4_t op);
23172714__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_u64_u16_x4)))
......@@ -2336,6 +2733,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s8_
23362733svfloat16x4_t svreinterpret_f16(svint8x4_t op);
23372734__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u8_x4)))
23382735svfloat16x4_t svreinterpret_f16(svuint8x4_t op);
2736__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_mf8_x4)))
2737svfloat16x4_t svreinterpret_f16(svmfloat8x4_t op);
23392738__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_s16_x4)))
23402739svfloat16x4_t svreinterpret_f16(svint16x4_t op);
23412740__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f16_u16_x4)))
......@@ -2360,6 +2759,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s8
23602759svbfloat16x4_t svreinterpret_bf16(svint8x4_t op);
23612760__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u8_x4)))
23622761svbfloat16x4_t svreinterpret_bf16(svuint8x4_t op);
2762__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_mf8_x4)))
2763svbfloat16x4_t svreinterpret_bf16(svmfloat8x4_t op);
23632764__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_s16_x4)))
23642765svbfloat16x4_t svreinterpret_bf16(svint16x4_t op);
23652766__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_bf16_u16_x4)))
......@@ -2384,6 +2785,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s8_
23842785svfloat32x4_t svreinterpret_f32(svint8x4_t op);
23852786__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u8_x4)))
23862787svfloat32x4_t svreinterpret_f32(svuint8x4_t op);
2788__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_mf8_x4)))
2789svfloat32x4_t svreinterpret_f32(svmfloat8x4_t op);
23872790__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_s16_x4)))
23882791svfloat32x4_t svreinterpret_f32(svint16x4_t op);
23892792__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f32_u16_x4)))
......@@ -2408,6 +2811,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s8_
24082811svfloat64x4_t svreinterpret_f64(svint8x4_t op);
24092812__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u8_x4)))
24102813svfloat64x4_t svreinterpret_f64(svuint8x4_t op);
2814__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_mf8_x4)))
2815svfloat64x4_t svreinterpret_f64(svmfloat8x4_t op);
24112816__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_s16_x4)))
24122817svfloat64x4_t svreinterpret_f64(svint16x4_t op);
24132818__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_reinterpret_f64_u16_x4)))
......@@ -3956,6 +4361,150 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq_s64_x4)))
39564361svint64x4_t svzipq(svint64x4_t);
39574362__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq_s16_x4)))
39584363svint16x4_t svzipq(svint16x4_t);
4364__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x2)))
4365svfloat64x2_t svamax_f64_x2(svfloat64x2_t, svfloat64x2_t);
4366__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x2)))
4367svfloat32x2_t svamax_f32_x2(svfloat32x2_t, svfloat32x2_t);
4368__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x2)))
4369svfloat16x2_t svamax_f16_x2(svfloat16x2_t, svfloat16x2_t);
4370__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x4)))
4371svfloat64x4_t svamax_f64_x4(svfloat64x4_t, svfloat64x4_t);
4372__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x4)))
4373svfloat32x4_t svamax_f32_x4(svfloat32x4_t, svfloat32x4_t);
4374__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x4)))
4375svfloat16x4_t svamax_f16_x4(svfloat16x4_t, svfloat16x4_t);
4376__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x2)))
4377svfloat64x2_t svamin_f64_x2(svfloat64x2_t, svfloat64x2_t);
4378__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x2)))
4379svfloat32x2_t svamin_f32_x2(svfloat32x2_t, svfloat32x2_t);
4380__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x2)))
4381svfloat16x2_t svamin_f16_x2(svfloat16x2_t, svfloat16x2_t);
4382__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x4)))
4383svfloat64x4_t svamin_f64_x4(svfloat64x4_t, svfloat64x4_t);
4384__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x4)))
4385svfloat32x4_t svamin_f32_x4(svfloat32x4_t, svfloat32x4_t);
4386__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x4)))
4387svfloat16x4_t svamin_f16_x4(svfloat16x4_t, svfloat16x4_t);
4388__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x2)))
4389svfloat64x2_t svamax(svfloat64x2_t, svfloat64x2_t);
4390__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x2)))
4391svfloat32x2_t svamax(svfloat32x2_t, svfloat32x2_t);
4392__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x2)))
4393svfloat16x2_t svamax(svfloat16x2_t, svfloat16x2_t);
4394__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x4)))
4395svfloat64x4_t svamax(svfloat64x4_t, svfloat64x4_t);
4396__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x4)))
4397svfloat32x4_t svamax(svfloat32x4_t, svfloat32x4_t);
4398__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x4)))
4399svfloat16x4_t svamax(svfloat16x4_t, svfloat16x4_t);
4400__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x2)))
4401svfloat64x2_t svamin(svfloat64x2_t, svfloat64x2_t);
4402__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x2)))
4403svfloat32x2_t svamin(svfloat32x2_t, svfloat32x2_t);
4404__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x2)))
4405svfloat16x2_t svamin(svfloat16x2_t, svfloat16x2_t);
4406__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x4)))
4407svfloat64x4_t svamin(svfloat64x4_t, svfloat64x4_t);
4408__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x4)))
4409svfloat32x4_t svamin(svfloat32x4_t, svfloat32x4_t);
4410__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x4)))
4411svfloat16x4_t svamin(svfloat16x4_t, svfloat16x4_t);
4412__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_bf16_mf8_x2_fpm)))
4413svbfloat16x2_t svcvt1_bf16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4414__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_f16_mf8_x2_fpm)))
4415svfloat16x2_t svcvt1_f16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4416__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_bf16_mf8_x2_fpm)))
4417svbfloat16x2_t svcvt2_bf16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4418__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_f16_mf8_x2_fpm)))
4419svfloat16x2_t svcvt2_f16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4420__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_mf8_bf16_x2_fpm)))
4421svmfloat8_t svcvt_mf8_bf16_x2_fpm(svbfloat16x2_t, fpm_t);
4422__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_mf8_f16_x2_fpm)))
4423svmfloat8_t svcvt_mf8_f16_x2_fpm(svfloat16x2_t, fpm_t);
4424__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_mf8_f32_x4_fpm)))
4425svmfloat8_t svcvt_mf8_f32_x4_fpm(svfloat32x4_t, fpm_t);
4426__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl1_bf16_mf8_x2_fpm)))
4427svbfloat16x2_t svcvtl1_bf16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4428__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl1_f16_mf8_x2_fpm)))
4429svfloat16x2_t svcvtl1_f16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4430__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl2_bf16_mf8_x2_fpm)))
4431svbfloat16x2_t svcvtl2_bf16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4432__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl2_f16_mf8_x2_fpm)))
4433svfloat16x2_t svcvtl2_f16_mf8_x2_fpm(svmfloat8_t, fpm_t);
4434__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_f32_x4_fpm)))
4435svmfloat8_t svcvtn_mf8_f32_x4_fpm(svfloat32x4_t, fpm_t);
4436__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f64_x2)))
4437svfloat64x2_t svscale_single_f64_x2(svfloat64x2_t, svint64_t);
4438__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f32_x2)))
4439svfloat32x2_t svscale_single_f32_x2(svfloat32x2_t, svint32_t);
4440__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f16_x2)))
4441svfloat16x2_t svscale_single_f16_x2(svfloat16x2_t, svint16_t);
4442__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f64_x4)))
4443svfloat64x4_t svscale_single_f64_x4(svfloat64x4_t, svint64_t);
4444__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f32_x4)))
4445svfloat32x4_t svscale_single_f32_x4(svfloat32x4_t, svint32_t);
4446__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f16_x4)))
4447svfloat16x4_t svscale_single_f16_x4(svfloat16x4_t, svint16_t);
4448__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f64_x2)))
4449svfloat64x2_t svscale_f64_x2(svfloat64x2_t, svint64x2_t);
4450__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f32_x2)))
4451svfloat32x2_t svscale_f32_x2(svfloat32x2_t, svint32x2_t);
4452__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f16_x2)))
4453svfloat16x2_t svscale_f16_x2(svfloat16x2_t, svint16x2_t);
4454__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f64_x4)))
4455svfloat64x4_t svscale_f64_x4(svfloat64x4_t, svint64x4_t);
4456__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f32_x4)))
4457svfloat32x4_t svscale_f32_x4(svfloat32x4_t, svint32x4_t);
4458__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f16_x4)))
4459svfloat16x4_t svscale_f16_x4(svfloat16x4_t, svint16x4_t);
4460__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_bf16_mf8_x2_fpm)))
4461svbfloat16x2_t svcvt1_bf16_x2_fpm(svmfloat8_t, fpm_t);
4462__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_f16_mf8_x2_fpm)))
4463svfloat16x2_t svcvt1_f16_x2_fpm(svmfloat8_t, fpm_t);
4464__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_bf16_mf8_x2_fpm)))
4465svbfloat16x2_t svcvt2_bf16_x2_fpm(svmfloat8_t, fpm_t);
4466__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_f16_mf8_x2_fpm)))
4467svfloat16x2_t svcvt2_f16_x2_fpm(svmfloat8_t, fpm_t);
4468__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_mf8_bf16_x2_fpm)))
4469svmfloat8_t svcvt_mf8_fpm(svbfloat16x2_t, fpm_t);
4470__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_mf8_f16_x2_fpm)))
4471svmfloat8_t svcvt_mf8_fpm(svfloat16x2_t, fpm_t);
4472__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_mf8_f32_x4_fpm)))
4473svmfloat8_t svcvt_mf8_fpm(svfloat32x4_t, fpm_t);
4474__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl1_bf16_mf8_x2_fpm)))
4475svbfloat16x2_t svcvtl1_bf16_x2_fpm(svmfloat8_t, fpm_t);
4476__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl1_f16_mf8_x2_fpm)))
4477svfloat16x2_t svcvtl1_f16_x2_fpm(svmfloat8_t, fpm_t);
4478__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl2_bf16_mf8_x2_fpm)))
4479svbfloat16x2_t svcvtl2_bf16_x2_fpm(svmfloat8_t, fpm_t);
4480__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtl2_f16_mf8_x2_fpm)))
4481svfloat16x2_t svcvtl2_f16_x2_fpm(svmfloat8_t, fpm_t);
4482__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_f32_x4_fpm)))
4483svmfloat8_t svcvtn_mf8_fpm(svfloat32x4_t, fpm_t);
4484__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f64_x2)))
4485svfloat64x2_t svscale(svfloat64x2_t, svint64_t);
4486__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f32_x2)))
4487svfloat32x2_t svscale(svfloat32x2_t, svint32_t);
4488__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f16_x2)))
4489svfloat16x2_t svscale(svfloat16x2_t, svint16_t);
4490__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f64_x4)))
4491svfloat64x4_t svscale(svfloat64x4_t, svint64_t);
4492__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f32_x4)))
4493svfloat32x4_t svscale(svfloat32x4_t, svint32_t);
4494__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_f16_x4)))
4495svfloat16x4_t svscale(svfloat16x4_t, svint16_t);
4496__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f64_x2)))
4497svfloat64x2_t svscale(svfloat64x2_t, svint64x2_t);
4498__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f32_x2)))
4499svfloat32x2_t svscale(svfloat32x2_t, svint32x2_t);
4500__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f16_x2)))
4501svfloat16x2_t svscale(svfloat16x2_t, svint16x2_t);
4502__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f64_x4)))
4503svfloat64x4_t svscale(svfloat64x4_t, svint64x4_t);
4504__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f32_x4)))
4505svfloat32x4_t svscale(svfloat32x4_t, svint32x4_t);
4506__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_f16_x4)))
4507svfloat16x4_t svscale(svfloat16x4_t, svint16x4_t);
39594508__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_single_bf16_x2)))
39604509svbfloat16x2_t svclamp_single_bf16_x2(svbfloat16x2_t, svbfloat16_t, svbfloat16_t);
39614510__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_single_bf16_x4)))
......@@ -4028,6 +4577,12 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_x2)))
40284577svbfloat16x2_t svminnm(svbfloat16x2_t, svbfloat16x2_t);
40294578__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_x4)))
40304579svbfloat16x4_t svminnm(svbfloat16x4_t, svbfloat16x4_t);
4580__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f64)))
4581float64_t svadda_f64(svbool_t, float64_t, svfloat64_t);
4582__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f32)))
4583float32_t svadda_f32(svbool_t, float32_t, svfloat32_t);
4584__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f16)))
4585float16_t svadda_f16(svbool_t, float16_t, svfloat16_t);
40314586__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrb_u32base_u32offset)))
40324587svuint32_t svadrb_u32base_u32offset(svuint32_t, svuint32_t);
40334588__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrb_u64base_u64offset)))
......@@ -5280,6 +5835,12 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtssel_f16)))
52805835svfloat16_t svtssel_f16(svfloat16_t, svuint16_t);
52815836__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwrffr)))
52825837void svwrffr(svbool_t);
5838__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f64)))
5839float64_t svadda(svbool_t, float64_t, svfloat64_t);
5840__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f32)))
5841float32_t svadda(svbool_t, float32_t, svfloat32_t);
5842__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f16)))
5843float16_t svadda(svbool_t, float16_t, svfloat16_t);
52835844__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrb_u32base_u32offset)))
52845845svuint32_t svadrb_offset(svuint32_t, svuint32_t);
52855846__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrb_u64base_u64offset)))
......@@ -7948,6 +8509,406 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u6
79488509void svstnt1w_scatter_offset(svbool_t, int32_t *, svuint64_t, svint64_t);
79498510__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64offset_u64)))
79508511void svstnt1w_scatter_offset(svbool_t, uint32_t *, svuint64_t, svuint64_t);
8512__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbl2_bf16)))
8513svbfloat16_t svtbl2_bf16(svbfloat16x2_t, svuint16_t);
8514__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbx_bf16)))
8515svbfloat16_t svtbx_bf16(svbfloat16_t, svbfloat16_t, svuint16_t);
8516__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilerw_bf16)))
8517svbool_t svwhilerw_bf16(bfloat16_t const *, bfloat16_t const *);
8518__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilewr_bf16)))
8519svbool_t svwhilewr_bf16(bfloat16_t const *, bfloat16_t const *);
8520__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbl2_bf16)))
8521svbfloat16_t svtbl2(svbfloat16x2_t, svuint16_t);
8522__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbx_bf16)))
8523svbfloat16_t svtbx(svbfloat16_t, svbfloat16_t, svuint16_t);
8524__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilerw_bf16)))
8525svbool_t svwhilerw(bfloat16_t const *, bfloat16_t const *);
8526__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilewr_bf16)))
8527svbool_t svwhilewr(bfloat16_t const *, bfloat16_t const *);
8528__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_m)))
8529svfloat64_t svamax_n_f64_m(svbool_t, svfloat64_t, float64_t);
8530__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_m)))
8531svfloat32_t svamax_n_f32_m(svbool_t, svfloat32_t, float32_t);
8532__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_m)))
8533svfloat16_t svamax_n_f16_m(svbool_t, svfloat16_t, float16_t);
8534__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_x)))
8535svfloat64_t svamax_n_f64_x(svbool_t, svfloat64_t, float64_t);
8536__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_x)))
8537svfloat32_t svamax_n_f32_x(svbool_t, svfloat32_t, float32_t);
8538__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_x)))
8539svfloat16_t svamax_n_f16_x(svbool_t, svfloat16_t, float16_t);
8540__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_z)))
8541svfloat64_t svamax_n_f64_z(svbool_t, svfloat64_t, float64_t);
8542__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_z)))
8543svfloat32_t svamax_n_f32_z(svbool_t, svfloat32_t, float32_t);
8544__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_z)))
8545svfloat16_t svamax_n_f16_z(svbool_t, svfloat16_t, float16_t);
8546__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_m)))
8547svfloat64_t svamax_f64_m(svbool_t, svfloat64_t, svfloat64_t);
8548__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_m)))
8549svfloat32_t svamax_f32_m(svbool_t, svfloat32_t, svfloat32_t);
8550__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_m)))
8551svfloat16_t svamax_f16_m(svbool_t, svfloat16_t, svfloat16_t);
8552__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x)))
8553svfloat64_t svamax_f64_x(svbool_t, svfloat64_t, svfloat64_t);
8554__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x)))
8555svfloat32_t svamax_f32_x(svbool_t, svfloat32_t, svfloat32_t);
8556__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x)))
8557svfloat16_t svamax_f16_x(svbool_t, svfloat16_t, svfloat16_t);
8558__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_z)))
8559svfloat64_t svamax_f64_z(svbool_t, svfloat64_t, svfloat64_t);
8560__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_z)))
8561svfloat32_t svamax_f32_z(svbool_t, svfloat32_t, svfloat32_t);
8562__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_z)))
8563svfloat16_t svamax_f16_z(svbool_t, svfloat16_t, svfloat16_t);
8564__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_m)))
8565svfloat64_t svamin_n_f64_m(svbool_t, svfloat64_t, float64_t);
8566__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_m)))
8567svfloat32_t svamin_n_f32_m(svbool_t, svfloat32_t, float32_t);
8568__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_m)))
8569svfloat16_t svamin_n_f16_m(svbool_t, svfloat16_t, float16_t);
8570__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_x)))
8571svfloat64_t svamin_n_f64_x(svbool_t, svfloat64_t, float64_t);
8572__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_x)))
8573svfloat32_t svamin_n_f32_x(svbool_t, svfloat32_t, float32_t);
8574__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_x)))
8575svfloat16_t svamin_n_f16_x(svbool_t, svfloat16_t, float16_t);
8576__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_z)))
8577svfloat64_t svamin_n_f64_z(svbool_t, svfloat64_t, float64_t);
8578__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_z)))
8579svfloat32_t svamin_n_f32_z(svbool_t, svfloat32_t, float32_t);
8580__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_z)))
8581svfloat16_t svamin_n_f16_z(svbool_t, svfloat16_t, float16_t);
8582__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_m)))
8583svfloat64_t svamin_f64_m(svbool_t, svfloat64_t, svfloat64_t);
8584__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_m)))
8585svfloat32_t svamin_f32_m(svbool_t, svfloat32_t, svfloat32_t);
8586__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_m)))
8587svfloat16_t svamin_f16_m(svbool_t, svfloat16_t, svfloat16_t);
8588__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x)))
8589svfloat64_t svamin_f64_x(svbool_t, svfloat64_t, svfloat64_t);
8590__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x)))
8591svfloat32_t svamin_f32_x(svbool_t, svfloat32_t, svfloat32_t);
8592__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x)))
8593svfloat16_t svamin_f16_x(svbool_t, svfloat16_t, svfloat16_t);
8594__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_z)))
8595svfloat64_t svamin_f64_z(svbool_t, svfloat64_t, svfloat64_t);
8596__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_z)))
8597svfloat32_t svamin_f32_z(svbool_t, svfloat32_t, svfloat32_t);
8598__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_z)))
8599svfloat16_t svamin_f16_z(svbool_t, svfloat16_t, svfloat16_t);
8600__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_m)))
8601svfloat64_t svamax_m(svbool_t, svfloat64_t, float64_t);
8602__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_m)))
8603svfloat32_t svamax_m(svbool_t, svfloat32_t, float32_t);
8604__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_m)))
8605svfloat16_t svamax_m(svbool_t, svfloat16_t, float16_t);
8606__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_x)))
8607svfloat64_t svamax_x(svbool_t, svfloat64_t, float64_t);
8608__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_x)))
8609svfloat32_t svamax_x(svbool_t, svfloat32_t, float32_t);
8610__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_x)))
8611svfloat16_t svamax_x(svbool_t, svfloat16_t, float16_t);
8612__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_z)))
8613svfloat64_t svamax_z(svbool_t, svfloat64_t, float64_t);
8614__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_z)))
8615svfloat32_t svamax_z(svbool_t, svfloat32_t, float32_t);
8616__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_z)))
8617svfloat16_t svamax_z(svbool_t, svfloat16_t, float16_t);
8618__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_m)))
8619svfloat64_t svamax_m(svbool_t, svfloat64_t, svfloat64_t);
8620__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_m)))
8621svfloat32_t svamax_m(svbool_t, svfloat32_t, svfloat32_t);
8622__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_m)))
8623svfloat16_t svamax_m(svbool_t, svfloat16_t, svfloat16_t);
8624__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x)))
8625svfloat64_t svamax_x(svbool_t, svfloat64_t, svfloat64_t);
8626__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x)))
8627svfloat32_t svamax_x(svbool_t, svfloat32_t, svfloat32_t);
8628__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x)))
8629svfloat16_t svamax_x(svbool_t, svfloat16_t, svfloat16_t);
8630__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_z)))
8631svfloat64_t svamax_z(svbool_t, svfloat64_t, svfloat64_t);
8632__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_z)))
8633svfloat32_t svamax_z(svbool_t, svfloat32_t, svfloat32_t);
8634__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_z)))
8635svfloat16_t svamax_z(svbool_t, svfloat16_t, svfloat16_t);
8636__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_m)))
8637svfloat64_t svamin_m(svbool_t, svfloat64_t, float64_t);
8638__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_m)))
8639svfloat32_t svamin_m(svbool_t, svfloat32_t, float32_t);
8640__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_m)))
8641svfloat16_t svamin_m(svbool_t, svfloat16_t, float16_t);
8642__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_x)))
8643svfloat64_t svamin_x(svbool_t, svfloat64_t, float64_t);
8644__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_x)))
8645svfloat32_t svamin_x(svbool_t, svfloat32_t, float32_t);
8646__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_x)))
8647svfloat16_t svamin_x(svbool_t, svfloat16_t, float16_t);
8648__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_z)))
8649svfloat64_t svamin_z(svbool_t, svfloat64_t, float64_t);
8650__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_z)))
8651svfloat32_t svamin_z(svbool_t, svfloat32_t, float32_t);
8652__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_z)))
8653svfloat16_t svamin_z(svbool_t, svfloat16_t, float16_t);
8654__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_m)))
8655svfloat64_t svamin_m(svbool_t, svfloat64_t, svfloat64_t);
8656__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_m)))
8657svfloat32_t svamin_m(svbool_t, svfloat32_t, svfloat32_t);
8658__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_m)))
8659svfloat16_t svamin_m(svbool_t, svfloat16_t, svfloat16_t);
8660__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x)))
8661svfloat64_t svamin_x(svbool_t, svfloat64_t, svfloat64_t);
8662__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x)))
8663svfloat32_t svamin_x(svbool_t, svfloat32_t, svfloat32_t);
8664__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x)))
8665svfloat16_t svamin_x(svbool_t, svfloat16_t, svfloat16_t);
8666__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_z)))
8667svfloat64_t svamin_z(svbool_t, svfloat64_t, svfloat64_t);
8668__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_z)))
8669svfloat32_t svamin_z(svbool_t, svfloat32_t, svfloat32_t);
8670__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_z)))
8671svfloat16_t svamin_z(svbool_t, svfloat16_t, svfloat16_t);
8672__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f16_mf8_fpm)))
8673svfloat16_t svdot_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
8674__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f16_mf8_fpm)))
8675svfloat16_t svdot_n_f16_mf8_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
8676__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f16_mf8_fpm)))
8677svfloat16_t svdot_lane_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8678__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f16_mf8_fpm)))
8679svfloat16_t svdot_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
8680__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f16_mf8_fpm)))
8681svfloat16_t svdot_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
8682__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f16_mf8_fpm)))
8683svfloat16_t svdot_lane_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8684__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f32_mf8_fpm)))
8685svfloat32_t svdot_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8686__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f32_mf8_fpm)))
8687svfloat32_t svdot_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8688__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f32_mf8_fpm)))
8689svfloat32_t svdot_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8690__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f32_mf8_fpm)))
8691svfloat32_t svdot_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8692__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f32_mf8_fpm)))
8693svfloat32_t svdot_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8694__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f32_mf8_fpm)))
8695svfloat32_t svdot_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8696__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_f16_mf8_fpm)))
8697svfloat16_t svmlalb_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
8698__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_n_f16_mf8_fpm)))
8699svfloat16_t svmlalb_n_f16_mf8_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
8700__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_lane_f16_mf8_fpm)))
8701svfloat16_t svmlalb_lane_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8702__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_f32_mf8_fpm)))
8703svfloat32_t svmlallbb_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8704__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_n_f32_mf8_fpm)))
8705svfloat32_t svmlallbb_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8706__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_lane_f32_mf8_fpm)))
8707svfloat32_t svmlallbb_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8708__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_f32_mf8_fpm)))
8709svfloat32_t svmlallbt_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8710__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_n_f32_mf8_fpm)))
8711svfloat32_t svmlallbt_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8712__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_lane_f32_mf8_fpm)))
8713svfloat32_t svmlallbt_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8714__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_f32_mf8_fpm)))
8715svfloat32_t svmlalltb_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8716__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_n_f32_mf8_fpm)))
8717svfloat32_t svmlalltb_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8718__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_lane_f32_mf8_fpm)))
8719svfloat32_t svmlalltb_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8720__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_f32_mf8_fpm)))
8721svfloat32_t svmlalltt_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8722__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_n_f32_mf8_fpm)))
8723svfloat32_t svmlalltt_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8724__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_lane_f32_mf8_fpm)))
8725svfloat32_t svmlalltt_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8726__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_f16_mf8_fpm)))
8727svfloat16_t svmlalt_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
8728__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_n_f16_mf8_fpm)))
8729svfloat16_t svmlalt_n_f16_mf8_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
8730__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_lane_f16_mf8_fpm)))
8731svfloat16_t svmlalt_lane_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8732__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_f16_mf8_fpm)))
8733svfloat16_t svmlalb_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
8734__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_n_f16_mf8_fpm)))
8735svfloat16_t svmlalb_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
8736__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_lane_f16_mf8_fpm)))
8737svfloat16_t svmlalb_lane_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8738__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_f32_mf8_fpm)))
8739svfloat32_t svmlallbb_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8740__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_n_f32_mf8_fpm)))
8741svfloat32_t svmlallbb_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8742__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_lane_f32_mf8_fpm)))
8743svfloat32_t svmlallbb_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8744__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_f32_mf8_fpm)))
8745svfloat32_t svmlallbt_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8746__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_n_f32_mf8_fpm)))
8747svfloat32_t svmlallbt_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8748__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_lane_f32_mf8_fpm)))
8749svfloat32_t svmlallbt_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8750__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_f32_mf8_fpm)))
8751svfloat32_t svmlalltb_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8752__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_n_f32_mf8_fpm)))
8753svfloat32_t svmlalltb_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8754__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_lane_f32_mf8_fpm)))
8755svfloat32_t svmlalltb_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8756__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_f32_mf8_fpm)))
8757svfloat32_t svmlalltt_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
8758__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_n_f32_mf8_fpm)))
8759svfloat32_t svmlalltt_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
8760__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_lane_f32_mf8_fpm)))
8761svfloat32_t svmlalltt_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8762__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_f16_mf8_fpm)))
8763svfloat16_t svmlalt_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
8764__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_n_f16_mf8_fpm)))
8765svfloat16_t svmlalt_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
8766__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_lane_f16_mf8_fpm)))
8767svfloat16_t svmlalt_lane_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
8768__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_bf16_mf8_fpm)))
8769svbfloat16_t svcvt1_bf16_mf8_fpm(svmfloat8_t, fpm_t);
8770__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_f16_mf8_fpm)))
8771svfloat16_t svcvt1_f16_mf8_fpm(svmfloat8_t, fpm_t);
8772__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_bf16_mf8_fpm)))
8773svbfloat16_t svcvt2_bf16_mf8_fpm(svmfloat8_t, fpm_t);
8774__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_f16_mf8_fpm)))
8775svfloat16_t svcvt2_f16_mf8_fpm(svmfloat8_t, fpm_t);
8776__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_bf16_mf8_fpm)))
8777svbfloat16_t svcvtlt1_bf16_mf8_fpm(svmfloat8_t, fpm_t);
8778__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_f16_mf8_fpm)))
8779svfloat16_t svcvtlt1_f16_mf8_fpm(svmfloat8_t, fpm_t);
8780__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_bf16_mf8_fpm)))
8781svbfloat16_t svcvtlt2_bf16_mf8_fpm(svmfloat8_t, fpm_t);
8782__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_f16_mf8_fpm)))
8783svfloat16_t svcvtlt2_f16_mf8_fpm(svmfloat8_t, fpm_t);
8784__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_bf16_x2_fpm)))
8785svmfloat8_t svcvtn_mf8_bf16_x2_fpm(svbfloat16x2_t, fpm_t);
8786__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_f16_x2_fpm)))
8787svmfloat8_t svcvtn_mf8_f16_x2_fpm(svfloat16x2_t, fpm_t);
8788__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnb_mf8_f32_x2_fpm)))
8789svmfloat8_t svcvtnb_mf8_f32_x2_fpm(svfloat32x2_t, fpm_t);
8790__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_mf8_f32_x2_fpm)))
8791svmfloat8_t svcvtnt_mf8_f32_x2_fpm(svmfloat8_t, svfloat32x2_t, fpm_t);
8792__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_bf16_mf8_fpm)))
8793svbfloat16_t svcvt1_bf16_fpm(svmfloat8_t, fpm_t);
8794__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_f16_mf8_fpm)))
8795svfloat16_t svcvt1_f16_fpm(svmfloat8_t, fpm_t);
8796__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_bf16_mf8_fpm)))
8797svbfloat16_t svcvt2_bf16_fpm(svmfloat8_t, fpm_t);
8798__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_f16_mf8_fpm)))
8799svfloat16_t svcvt2_f16_fpm(svmfloat8_t, fpm_t);
8800__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_bf16_mf8_fpm)))
8801svbfloat16_t svcvtlt1_bf16_fpm(svmfloat8_t, fpm_t);
8802__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_f16_mf8_fpm)))
8803svfloat16_t svcvtlt1_f16_fpm(svmfloat8_t, fpm_t);
8804__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_bf16_mf8_fpm)))
8805svbfloat16_t svcvtlt2_bf16_fpm(svmfloat8_t, fpm_t);
8806__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_f16_mf8_fpm)))
8807svfloat16_t svcvtlt2_f16_fpm(svmfloat8_t, fpm_t);
8808__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_bf16_x2_fpm)))
8809svmfloat8_t svcvtn_mf8_fpm(svbfloat16x2_t, fpm_t);
8810__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_f16_x2_fpm)))
8811svmfloat8_t svcvtn_mf8_fpm(svfloat16x2_t, fpm_t);
8812__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnb_mf8_f32_x2_fpm)))
8813svmfloat8_t svcvtnb_mf8_fpm(svfloat32x2_t, fpm_t);
8814__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_mf8_f32_x2_fpm)))
8815svmfloat8_t svcvtnt_mf8_fpm(svmfloat8_t, svfloat32x2_t, fpm_t);
8816__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_bf16)))
8817svbfloat16_t svluti2_lane_bf16(svbfloat16_t, svuint8_t, uint64_t);
8818__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16)))
8819svbfloat16_t svluti4_lane_bf16(svbfloat16_t, svuint8_t, uint64_t);
8820__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16_x2)))
8821svbfloat16_t svluti4_lane_bf16_x2(svbfloat16x2_t, svuint8_t, uint64_t);
8822__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_bf16)))
8823svbfloat16_t svluti2_lane(svbfloat16_t, svuint8_t, uint64_t);
8824__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16)))
8825svbfloat16_t svluti4_lane(svbfloat16_t, svuint8_t, uint64_t);
8826__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16_x2)))
8827svbfloat16_t svluti4_lane(svbfloat16x2_t, svuint8_t, uint64_t);
8828__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u8)))
8829svuint8_t svluti2_lane_u8(svuint8_t, svuint8_t, uint64_t);
8830__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s8)))
8831svint8_t svluti2_lane_s8(svint8_t, svuint8_t, uint64_t);
8832__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u16)))
8833svuint16_t svluti2_lane_u16(svuint16_t, svuint8_t, uint64_t);
8834__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_f16)))
8835svfloat16_t svluti2_lane_f16(svfloat16_t, svuint8_t, uint64_t);
8836__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s16)))
8837svint16_t svluti2_lane_s16(svint16_t, svuint8_t, uint64_t);
8838__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u8)))
8839svuint8_t svluti4_lane_u8(svuint8_t, svuint8_t, uint64_t);
8840__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s8)))
8841svint8_t svluti4_lane_s8(svint8_t, svuint8_t, uint64_t);
8842__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16)))
8843svuint16_t svluti4_lane_u16(svuint16_t, svuint8_t, uint64_t);
8844__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16)))
8845svfloat16_t svluti4_lane_f16(svfloat16_t, svuint8_t, uint64_t);
8846__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16)))
8847svint16_t svluti4_lane_s16(svint16_t, svuint8_t, uint64_t);
8848__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16_x2)))
8849svuint16_t svluti4_lane_u16_x2(svuint16x2_t, svuint8_t, uint64_t);
8850__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16_x2)))
8851svfloat16_t svluti4_lane_f16_x2(svfloat16x2_t, svuint8_t, uint64_t);
8852__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16_x2)))
8853svint16_t svluti4_lane_s16_x2(svint16x2_t, svuint8_t, uint64_t);
8854__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u8)))
8855svuint8_t svluti2_lane(svuint8_t, svuint8_t, uint64_t);
8856__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s8)))
8857svint8_t svluti2_lane(svint8_t, svuint8_t, uint64_t);
8858__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u16)))
8859svuint16_t svluti2_lane(svuint16_t, svuint8_t, uint64_t);
8860__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_f16)))
8861svfloat16_t svluti2_lane(svfloat16_t, svuint8_t, uint64_t);
8862__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s16)))
8863svint16_t svluti2_lane(svint16_t, svuint8_t, uint64_t);
8864__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u8)))
8865svuint8_t svluti4_lane(svuint8_t, svuint8_t, uint64_t);
8866__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s8)))
8867svint8_t svluti4_lane(svint8_t, svuint8_t, uint64_t);
8868__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16)))
8869svuint16_t svluti4_lane(svuint16_t, svuint8_t, uint64_t);
8870__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16)))
8871svfloat16_t svluti4_lane(svfloat16_t, svuint8_t, uint64_t);
8872__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16)))
8873svint16_t svluti4_lane(svint16_t, svuint8_t, uint64_t);
8874__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16_x2)))
8875svuint16_t svluti4_lane(svuint16x2_t, svuint8_t, uint64_t);
8876__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16_x2)))
8877svfloat16_t svluti4_lane(svfloat16x2_t, svuint8_t, uint64_t);
8878__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16_x2)))
8879svint16_t svluti4_lane(svint16x2_t, svuint8_t, uint64_t);
8880__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_u8)))
8881svuint8_t svaesd_u8(svuint8_t, svuint8_t);
8882__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_u8)))
8883svuint8_t svaese_u8(svuint8_t, svuint8_t);
8884__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesimc_u8)))
8885svuint8_t svaesimc_u8(svuint8_t);
8886__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesmc_u8)))
8887svuint8_t svaesmc_u8(svuint8_t);
8888__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_n_u64)))
8889svuint64_t svpmullb_pair_n_u64(svuint64_t, uint64_t);
8890__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_u64)))
8891svuint64_t svpmullb_pair_u64(svuint64_t, svuint64_t);
8892__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_n_u64)))
8893svuint64_t svpmullt_pair_n_u64(svuint64_t, uint64_t);
8894__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_u64)))
8895svuint64_t svpmullt_pair_u64(svuint64_t, svuint64_t);
8896__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_u8)))
8897svuint8_t svaesd(svuint8_t, svuint8_t);
8898__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_u8)))
8899svuint8_t svaese(svuint8_t, svuint8_t);
8900__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesimc_u8)))
8901svuint8_t svaesimc(svuint8_t);
8902__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesmc_u8)))
8903svuint8_t svaesmc(svuint8_t);
8904__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_n_u64)))
8905svuint64_t svpmullb_pair(svuint64_t, uint64_t);
8906__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_u64)))
8907svuint64_t svpmullb_pair(svuint64_t, svuint64_t);
8908__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_n_u64)))
8909svuint64_t svpmullt_pair(svuint64_t, uint64_t);
8910__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_u64)))
8911svuint64_t svpmullt_pair(svuint64_t, svuint64_t);
79518912__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_m)))
79528913svbfloat16_t svadd_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
79538914__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_x)))
......@@ -8180,54 +9141,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_x)))
81809141svbfloat16_t svsub_x(svbool_t, svbfloat16_t, svbfloat16_t);
81819142__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_z)))
81829143svbfloat16_t svsub_z(svbool_t, svbfloat16_t, svbfloat16_t);
8183__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbl2_bf16)))
8184svbfloat16_t svtbl2_bf16(svbfloat16x2_t, svuint16_t);
8185__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbx_bf16)))
8186svbfloat16_t svtbx_bf16(svbfloat16_t, svbfloat16_t, svuint16_t);
8187__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilerw_bf16)))
8188svbool_t svwhilerw_bf16(bfloat16_t const *, bfloat16_t const *);
8189__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilewr_bf16)))
8190svbool_t svwhilewr_bf16(bfloat16_t const *, bfloat16_t const *);
8191__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbl2_bf16)))
8192svbfloat16_t svtbl2(svbfloat16x2_t, svuint16_t);
8193__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbx_bf16)))
8194svbfloat16_t svtbx(svbfloat16_t, svbfloat16_t, svuint16_t);
8195__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilerw_bf16)))
8196svbool_t svwhilerw(bfloat16_t const *, bfloat16_t const *);
8197__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilewr_bf16)))
8198svbool_t svwhilewr(bfloat16_t const *, bfloat16_t const *);
8199__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_u8)))
8200svuint8_t svaesd_u8(svuint8_t, svuint8_t);
8201__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_u8)))
8202svuint8_t svaese_u8(svuint8_t, svuint8_t);
8203__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesimc_u8)))
8204svuint8_t svaesimc_u8(svuint8_t);
8205__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesmc_u8)))
8206svuint8_t svaesmc_u8(svuint8_t);
8207__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_n_u64)))
8208svuint64_t svpmullb_pair_n_u64(svuint64_t, uint64_t);
8209__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_u64)))
8210svuint64_t svpmullb_pair_u64(svuint64_t, svuint64_t);
8211__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_n_u64)))
8212svuint64_t svpmullt_pair_n_u64(svuint64_t, uint64_t);
8213__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_u64)))
8214svuint64_t svpmullt_pair_u64(svuint64_t, svuint64_t);
8215__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_u8)))
8216svuint8_t svaesd(svuint8_t, svuint8_t);
8217__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_u8)))
8218svuint8_t svaese(svuint8_t, svuint8_t);
8219__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesimc_u8)))
8220svuint8_t svaesimc(svuint8_t);
8221__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesmc_u8)))
8222svuint8_t svaesmc(svuint8_t);
8223__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_n_u64)))
8224svuint64_t svpmullb_pair(svuint64_t, uint64_t);
8225__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_u64)))
8226svuint64_t svpmullb_pair(svuint64_t, svuint64_t);
8227__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_n_u64)))
8228svuint64_t svpmullt_pair(svuint64_t, uint64_t);
8229__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_u64)))
8230svuint64_t svpmullt_pair(svuint64_t, svuint64_t);
82319144__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u8)))
82329145svuint8_t svbdep_n_u8(svuint8_t, uint8_t);
82339146__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u32)))
......@@ -8416,6 +9329,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s32)))
84169329svint32_t svextq_s32(svint32_t, svint32_t, int32_t);
84179330__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s64)))
84189331svint64_t svextq_s64(svint64_t, svint64_t, int32_t);
9332__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_mf8)))
9333svmfloat8_t svextq_mf8(svmfloat8_t, svmfloat8_t, int32_t);
84199334__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s16)))
84209335svint16_t svextq_s16(svint16_t, svint16_t, int32_t);
84219336__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u32)))
......@@ -8831,17 +9746,17 @@ svuint32_t svpmov_u32_z(svbool_t);
88319746__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmov_s32_z)))
88329747svint32_t svpmov_s32_z(svbool_t);
88339748__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_u64)))
8834void svst1dq_u64(svbool_t, uint64_t const *, svuint64_t);
9749void svst1dq_u64(svbool_t, uint64_t *, svuint64_t);
88359750__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_f64)))
8836void svst1dq_f64(svbool_t, float64_t const *, svfloat64_t);
9751void svst1dq_f64(svbool_t, float64_t *, svfloat64_t);
88379752__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_s64)))
8838void svst1dq_s64(svbool_t, int64_t const *, svint64_t);
9753void svst1dq_s64(svbool_t, int64_t *, svint64_t);
88399754__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_u64)))
8840void svst1dq_vnum_u64(svbool_t, uint64_t const *, int64_t, svuint64_t);
9755void svst1dq_vnum_u64(svbool_t, uint64_t *, int64_t, svuint64_t);
88419756__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_f64)))
8842void svst1dq_vnum_f64(svbool_t, float64_t const *, int64_t, svfloat64_t);
9757void svst1dq_vnum_f64(svbool_t, float64_t *, int64_t, svfloat64_t);
88439758__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_s64)))
8844void svst1dq_vnum_s64(svbool_t, int64_t const *, int64_t, svint64_t);
9759void svst1dq_vnum_s64(svbool_t, int64_t *, int64_t, svint64_t);
88459760__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u8)))
88469761void svst1q_scatter_u64base_u8(svbool_t, svuint64_t, svuint8_t);
88479762__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u32)))
......@@ -8910,6 +9825,26 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64ba
89109825void svst1q_scatter_u64base_offset_s64(svbool_t, svuint64_t, int64_t, svint64_t);
89119826__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s16)))
89129827void svst1q_scatter_u64base_offset_s16(svbool_t, svuint64_t, int64_t, svint16_t);
9828__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u32)))
9829void svst1q_scatter_s64index_u32(svbool_t, uint32_t *, svint64_t, svuint32_t);
9830__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u64)))
9831void svst1q_scatter_s64index_u64(svbool_t, uint64_t *, svint64_t, svuint64_t);
9832__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u16)))
9833void svst1q_scatter_s64index_u16(svbool_t, uint16_t *, svint64_t, svuint16_t);
9834__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_bf16)))
9835void svst1q_scatter_s64index_bf16(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
9836__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f64)))
9837void svst1q_scatter_s64index_f64(svbool_t, float64_t *, svint64_t, svfloat64_t);
9838__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f32)))
9839void svst1q_scatter_s64index_f32(svbool_t, float32_t *, svint64_t, svfloat32_t);
9840__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f16)))
9841void svst1q_scatter_s64index_f16(svbool_t, float16_t *, svint64_t, svfloat16_t);
9842__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s32)))
9843void svst1q_scatter_s64index_s32(svbool_t, int32_t *, svint64_t, svint32_t);
9844__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s64)))
9845void svst1q_scatter_s64index_s64(svbool_t, int64_t *, svint64_t, svint64_t);
9846__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s16)))
9847void svst1q_scatter_s64index_s16(svbool_t, int16_t *, svint64_t, svint16_t);
89139848__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u32)))
89149849void svst1q_scatter_u64index_u32(svbool_t, uint32_t *, svuint64_t, svuint32_t);
89159850__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u64)))
......@@ -8930,6 +9865,30 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64in
89309865void svst1q_scatter_u64index_s64(svbool_t, int64_t *, svuint64_t, svint64_t);
89319866__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s16)))
89329867void svst1q_scatter_u64index_s16(svbool_t, int16_t *, svuint64_t, svint16_t);
9868__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u8)))
9869void svst1q_scatter_s64offset_u8(svbool_t, uint8_t *, svint64_t, svuint8_t);
9870__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u32)))
9871void svst1q_scatter_s64offset_u32(svbool_t, uint32_t *, svint64_t, svuint32_t);
9872__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u64)))
9873void svst1q_scatter_s64offset_u64(svbool_t, uint64_t *, svint64_t, svuint64_t);
9874__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u16)))
9875void svst1q_scatter_s64offset_u16(svbool_t, uint16_t *, svint64_t, svuint16_t);
9876__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_bf16)))
9877void svst1q_scatter_s64offset_bf16(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
9878__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s8)))
9879void svst1q_scatter_s64offset_s8(svbool_t, int8_t *, svint64_t, svint8_t);
9880__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f64)))
9881void svst1q_scatter_s64offset_f64(svbool_t, float64_t *, svint64_t, svfloat64_t);
9882__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f32)))
9883void svst1q_scatter_s64offset_f32(svbool_t, float32_t *, svint64_t, svfloat32_t);
9884__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f16)))
9885void svst1q_scatter_s64offset_f16(svbool_t, float16_t *, svint64_t, svfloat16_t);
9886__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s32)))
9887void svst1q_scatter_s64offset_s32(svbool_t, int32_t *, svint64_t, svint32_t);
9888__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s64)))
9889void svst1q_scatter_s64offset_s64(svbool_t, int64_t *, svint64_t, svint64_t);
9890__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s16)))
9891void svst1q_scatter_s64offset_s16(svbool_t, int16_t *, svint64_t, svint16_t);
89339892__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u8)))
89349893void svst1q_scatter_u64offset_u8(svbool_t, uint8_t *, svuint64_t, svuint8_t);
89359894__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u32)))
......@@ -8955,17 +9914,17 @@ void svst1q_scatter_u64offset_s64(svbool_t, int64_t *, svuint64_t, svint64_t);
89559914__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s16)))
89569915void svst1q_scatter_u64offset_s16(svbool_t, int16_t *, svuint64_t, svint16_t);
89579916__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_u32)))
8958void svst1wq_u32(svbool_t, uint32_t const *, svuint32_t);
9917void svst1wq_u32(svbool_t, uint32_t *, svuint32_t);
89599918__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_f32)))
8960void svst1wq_f32(svbool_t, float32_t const *, svfloat32_t);
9919void svst1wq_f32(svbool_t, float32_t *, svfloat32_t);
89619920__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_s32)))
8962void svst1wq_s32(svbool_t, int32_t const *, svint32_t);
9921void svst1wq_s32(svbool_t, int32_t *, svint32_t);
89639922__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_u32)))
8964void svst1wq_vnum_u32(svbool_t, uint32_t const *, int64_t, svuint32_t);
9923void svst1wq_vnum_u32(svbool_t, uint32_t *, int64_t, svuint32_t);
89659924__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_f32)))
8966void svst1wq_vnum_f32(svbool_t, float32_t const *, int64_t, svfloat32_t);
9925void svst1wq_vnum_f32(svbool_t, float32_t *, int64_t, svfloat32_t);
89679926__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_s32)))
8968void svst1wq_vnum_s32(svbool_t, int32_t const *, int64_t, svint32_t);
9927void svst1wq_vnum_s32(svbool_t, int32_t *, int64_t, svint32_t);
89699928__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst2q_u8)))
89709929void svst2q_u8(svbool_t, uint8_t const *, svuint8x2_t);
89719930__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst2q_u32)))
......@@ -9132,6 +10091,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s32)))
913210091svint32_t svtblq_s32(svint32_t, svuint32_t);
913310092__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s64)))
913410093svint64_t svtblq_s64(svint64_t, svuint64_t);
10094__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_mf8)))
10095svmfloat8_t svtblq_mf8(svmfloat8_t, svuint8_t);
913510096__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s16)))
913610097svint16_t svtblq_s16(svint16_t, svuint16_t);
913710098__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_u8)))
......@@ -9156,6 +10117,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s32)))
915610117svint32_t svtbxq_s32(svint32_t, svint32_t, svuint32_t);
915710118__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s64)))
915810119svint64_t svtbxq_s64(svint64_t, svint64_t, svuint64_t);
10120__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_mf8)))
10121svmfloat8_t svtbxq_mf8(svmfloat8_t, svmfloat8_t, svuint8_t);
915910122__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s16)))
916010123svint16_t svtbxq_s16(svint16_t, svint16_t, svuint16_t);
916110124__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_u8)))
......@@ -9180,6 +10143,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s32)))
918010143svint32_t svuzpq1_s32(svint32_t, svint32_t);
918110144__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s64)))
918210145svint64_t svuzpq1_s64(svint64_t, svint64_t);
10146__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_mf8)))
10147svmfloat8_t svuzpq1_mf8(svmfloat8_t, svmfloat8_t);
918310148__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s16)))
918410149svint16_t svuzpq1_s16(svint16_t, svint16_t);
918510150__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_u8)))
......@@ -9204,6 +10169,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s32)))
920410169svint32_t svuzpq2_s32(svint32_t, svint32_t);
920510170__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s64)))
920610171svint64_t svuzpq2_s64(svint64_t, svint64_t);
10172__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_mf8)))
10173svmfloat8_t svuzpq2_mf8(svmfloat8_t, svmfloat8_t);
920710174__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s16)))
920810175svint16_t svuzpq2_s16(svint16_t, svint16_t);
920910176__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_u8)))
......@@ -9228,6 +10195,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s32)))
922810195svint32_t svzipq1_s32(svint32_t, svint32_t);
922910196__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s64)))
923010197svint64_t svzipq1_s64(svint64_t, svint64_t);
10198__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_mf8)))
10199svmfloat8_t svzipq1_mf8(svmfloat8_t, svmfloat8_t);
923110200__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s16)))
923210201svint16_t svzipq1_s16(svint16_t, svint16_t);
923310202__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_u8)))
......@@ -9252,6 +10221,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s32)))
925210221svint32_t svzipq2_s32(svint32_t, svint32_t);
925310222__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s64)))
925410223svint64_t svzipq2_s64(svint64_t, svint64_t);
10224__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_mf8)))
10225svmfloat8_t svzipq2_mf8(svmfloat8_t, svmfloat8_t);
925510226__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s16)))
925610227svint16_t svzipq2_s16(svint16_t, svint16_t);
925710228__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddqv_u8)))
......@@ -9330,6 +10301,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s32)))
933010301svint32_t svextq(svint32_t, svint32_t, int32_t);
933110302__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s64)))
933210303svint64_t svextq(svint64_t, svint64_t, int32_t);
10304__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_mf8)))
10305svmfloat8_t svextq(svmfloat8_t, svmfloat8_t, int32_t);
933310306__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svextq_s16)))
933410307svint16_t svextq(svint16_t, svint16_t, int32_t);
933510308__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u32)))
......@@ -9729,17 +10702,17 @@ svuint32_t svpmov_lane_m(svuint32_t, svbool_t, uint64_t);
972910702__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmov_lane_s32_m)))
973010703svint32_t svpmov_lane_m(svint32_t, svbool_t, uint64_t);
973110704__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_u64)))
9732void svst1dq(svbool_t, uint64_t const *, svuint64_t);
10705void svst1dq(svbool_t, uint64_t *, svuint64_t);
973310706__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_f64)))
9734void svst1dq(svbool_t, float64_t const *, svfloat64_t);
10707void svst1dq(svbool_t, float64_t *, svfloat64_t);
973510708__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_s64)))
9736void svst1dq(svbool_t, int64_t const *, svint64_t);
10709void svst1dq(svbool_t, int64_t *, svint64_t);
973710710__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_u64)))
9738void svst1dq_vnum(svbool_t, uint64_t const *, int64_t, svuint64_t);
10711void svst1dq_vnum(svbool_t, uint64_t *, int64_t, svuint64_t);
973910712__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_f64)))
9740void svst1dq_vnum(svbool_t, float64_t const *, int64_t, svfloat64_t);
10713void svst1dq_vnum(svbool_t, float64_t *, int64_t, svfloat64_t);
974110714__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_s64)))
9742void svst1dq_vnum(svbool_t, int64_t const *, int64_t, svint64_t);
10715void svst1dq_vnum(svbool_t, int64_t *, int64_t, svint64_t);
974310716__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u8)))
974410717void svst1q_scatter(svbool_t, svuint64_t, svuint8_t);
974510718__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u32)))
......@@ -9808,6 +10781,26 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64b
980810781void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svint64_t);
980910782__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s16)))
981010783void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svint16_t);
10784__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u32)))
10785void svst1q_scatter_index(svbool_t, uint32_t *, svint64_t, svuint32_t);
10786__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u64)))
10787void svst1q_scatter_index(svbool_t, uint64_t *, svint64_t, svuint64_t);
10788__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u16)))
10789void svst1q_scatter_index(svbool_t, uint16_t *, svint64_t, svuint16_t);
10790__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_bf16)))
10791void svst1q_scatter_index(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
10792__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f64)))
10793void svst1q_scatter_index(svbool_t, float64_t *, svint64_t, svfloat64_t);
10794__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f32)))
10795void svst1q_scatter_index(svbool_t, float32_t *, svint64_t, svfloat32_t);
10796__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f16)))
10797void svst1q_scatter_index(svbool_t, float16_t *, svint64_t, svfloat16_t);
10798__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s32)))
10799void svst1q_scatter_index(svbool_t, int32_t *, svint64_t, svint32_t);
10800__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s64)))
10801void svst1q_scatter_index(svbool_t, int64_t *, svint64_t, svint64_t);
10802__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s16)))
10803void svst1q_scatter_index(svbool_t, int16_t *, svint64_t, svint16_t);
981110804__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u32)))
981210805void svst1q_scatter_index(svbool_t, uint32_t *, svuint64_t, svuint32_t);
981310806__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u64)))
......@@ -9828,6 +10821,30 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64i
982810821void svst1q_scatter_index(svbool_t, int64_t *, svuint64_t, svint64_t);
982910822__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s16)))
983010823void svst1q_scatter_index(svbool_t, int16_t *, svuint64_t, svint16_t);
10824__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u8)))
10825void svst1q_scatter_offset(svbool_t, uint8_t *, svint64_t, svuint8_t);
10826__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u32)))
10827void svst1q_scatter_offset(svbool_t, uint32_t *, svint64_t, svuint32_t);
10828__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u64)))
10829void svst1q_scatter_offset(svbool_t, uint64_t *, svint64_t, svuint64_t);
10830__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u16)))
10831void svst1q_scatter_offset(svbool_t, uint16_t *, svint64_t, svuint16_t);
10832__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_bf16)))
10833void svst1q_scatter_offset(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
10834__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s8)))
10835void svst1q_scatter_offset(svbool_t, int8_t *, svint64_t, svint8_t);
10836__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f64)))
10837void svst1q_scatter_offset(svbool_t, float64_t *, svint64_t, svfloat64_t);
10838__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f32)))
10839void svst1q_scatter_offset(svbool_t, float32_t *, svint64_t, svfloat32_t);
10840__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f16)))
10841void svst1q_scatter_offset(svbool_t, float16_t *, svint64_t, svfloat16_t);
10842__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s32)))
10843void svst1q_scatter_offset(svbool_t, int32_t *, svint64_t, svint32_t);
10844__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s64)))
10845void svst1q_scatter_offset(svbool_t, int64_t *, svint64_t, svint64_t);
10846__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s16)))
10847void svst1q_scatter_offset(svbool_t, int16_t *, svint64_t, svint16_t);
983110848__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u8)))
983210849void svst1q_scatter_offset(svbool_t, uint8_t *, svuint64_t, svuint8_t);
983310850__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u32)))
......@@ -9853,17 +10870,17 @@ void svst1q_scatter_offset(svbool_t, int64_t *, svuint64_t, svint64_t);
985310870__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s16)))
985410871void svst1q_scatter_offset(svbool_t, int16_t *, svuint64_t, svint16_t);
985510872__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_u32)))
9856void svst1wq(svbool_t, uint32_t const *, svuint32_t);
10873void svst1wq(svbool_t, uint32_t *, svuint32_t);
985710874__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_f32)))
9858void svst1wq(svbool_t, float32_t const *, svfloat32_t);
10875void svst1wq(svbool_t, float32_t *, svfloat32_t);
985910876__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_s32)))
9860void svst1wq(svbool_t, int32_t const *, svint32_t);
10877void svst1wq(svbool_t, int32_t *, svint32_t);
986110878__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_u32)))
9862void svst1wq_vnum(svbool_t, uint32_t const *, int64_t, svuint32_t);
10879void svst1wq_vnum(svbool_t, uint32_t *, int64_t, svuint32_t);
986310880__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_f32)))
9864void svst1wq_vnum(svbool_t, float32_t const *, int64_t, svfloat32_t);
10881void svst1wq_vnum(svbool_t, float32_t *, int64_t, svfloat32_t);
986510882__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_s32)))
9866void svst1wq_vnum(svbool_t, int32_t const *, int64_t, svint32_t);
10883void svst1wq_vnum(svbool_t, int32_t *, int64_t, svint32_t);
986710884__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst2q_u8)))
986810885void svst2q(svbool_t, uint8_t const *, svuint8x2_t);
986910886__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst2q_u32)))
......@@ -10030,6 +11047,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s32)))
1003011047svint32_t svtblq(svint32_t, svuint32_t);
1003111048__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s64)))
1003211049svint64_t svtblq(svint64_t, svuint64_t);
11050__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_mf8)))
11051svmfloat8_t svtblq(svmfloat8_t, svuint8_t);
1003311052__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtblq_s16)))
1003411053svint16_t svtblq(svint16_t, svuint16_t);
1003511054__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_u8)))
......@@ -10054,6 +11073,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s32)))
1005411073svint32_t svtbxq(svint32_t, svint32_t, svuint32_t);
1005511074__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s64)))
1005611075svint64_t svtbxq(svint64_t, svint64_t, svuint64_t);
11076__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_mf8)))
11077svmfloat8_t svtbxq(svmfloat8_t, svmfloat8_t, svuint8_t);
1005711078__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtbxq_s16)))
1005811079svint16_t svtbxq(svint16_t, svint16_t, svuint16_t);
1005911080__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_u8)))
......@@ -10078,6 +11099,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s32)))
1007811099svint32_t svuzpq1(svint32_t, svint32_t);
1007911100__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s64)))
1008011101svint64_t svuzpq1(svint64_t, svint64_t);
11102__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_mf8)))
11103svmfloat8_t svuzpq1(svmfloat8_t, svmfloat8_t);
1008111104__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq1_s16)))
1008211105svint16_t svuzpq1(svint16_t, svint16_t);
1008311106__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_u8)))
......@@ -10102,6 +11125,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s32)))
1010211125svint32_t svuzpq2(svint32_t, svint32_t);
1010311126__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s64)))
1010411127svint64_t svuzpq2(svint64_t, svint64_t);
11128__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_mf8)))
11129svmfloat8_t svuzpq2(svmfloat8_t, svmfloat8_t);
1010511130__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzpq2_s16)))
1010611131svint16_t svuzpq2(svint16_t, svint16_t);
1010711132__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_u8)))
......@@ -10126,6 +11151,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s32)))
1012611151svint32_t svzipq1(svint32_t, svint32_t);
1012711152__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s64)))
1012811153svint64_t svzipq1(svint64_t, svint64_t);
11154__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_mf8)))
11155svmfloat8_t svzipq1(svmfloat8_t, svmfloat8_t);
1012911156__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq1_s16)))
1013011157svint16_t svzipq1(svint16_t, svint16_t);
1013111158__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_u8)))
......@@ -10150,6 +11177,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s32)))
1015011177svint32_t svzipq2(svint32_t, svint32_t);
1015111178__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s64)))
1015211179svint64_t svzipq2(svint64_t, svint64_t);
11180__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_mf8)))
11181svmfloat8_t svzipq2(svmfloat8_t, svmfloat8_t);
1015311182__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s16)))
1015411183svint16_t svzipq2(svint16_t, svint16_t);
1015511184__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_bf16)))
......@@ -11522,6 +12551,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_u8)))
1152212551svuint8_t svdup_laneq_u8(svuint8_t, uint64_t);
1152312552__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_s8)))
1152412553svint8_t svdup_laneq_s8(svint8_t, uint64_t);
12554__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_mf8)))
12555svmfloat8_t svdup_laneq_mf8(svmfloat8_t, uint64_t);
1152512556__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_u64)))
1152612557svuint64_t svdup_laneq_u64(svuint64_t, uint64_t);
1152712558__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_f64)))
......@@ -11544,6 +12575,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_u8)))
1154412575svuint8_t svdup_laneq(svuint8_t, uint64_t);
1154512576__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_s8)))
1154612577svint8_t svdup_laneq(svint8_t, uint64_t);
12578__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_mf8)))
12579svmfloat8_t svdup_laneq(svmfloat8_t, uint64_t);
1154712580__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_u64)))
1154812581svuint64_t svdup_laneq(svuint64_t, uint64_t);
1154912582__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_laneq_f64)))
......@@ -18424,12 +19457,6 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_s64_z)))
1842419457svint64_t svadd_s64_z(svbool_t, svint64_t, svint64_t);
1842519458__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_s16_z)))
1842619459svint16_t svadd_s16_z(svbool_t, svint16_t, svint16_t);
18427__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f64)))
18428float64_t svadda_f64(svbool_t, float64_t, svfloat64_t);
18429__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f32)))
18430float32_t svadda_f32(svbool_t, float32_t, svfloat32_t);
18431__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f16)))
18432float16_t svadda_f16(svbool_t, float16_t, svfloat16_t);
1843319460__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddv_s8)))
1843419461int64_t svaddv_s8(svbool_t, svint8_t);
1843519462__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddv_s32)))
......@@ -24810,12 +25837,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_s64_z)))
2481025837svint64_t svadd_z(svbool_t, svint64_t, svint64_t);
2481125838__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_s16_z)))
2481225839svint16_t svadd_z(svbool_t, svint16_t, svint16_t);
24813__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f64)))
24814float64_t svadda(svbool_t, float64_t, svfloat64_t);
24815__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f32)))
24816float32_t svadda(svbool_t, float32_t, svfloat32_t);
24817__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f16)))
24818float16_t svadda(svbool_t, float16_t, svfloat16_t);
2481925840__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddv_s8)))
2482025841int64_t svaddv(svbool_t, svint8_t);
2482125842__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaddv_s32)))
lib/include/arm_vector_types.h+87
......@@ -17,9 +17,62 @@
1717typedef float float32_t;
1818typedef __fp16 float16_t;
1919#if defined(__aarch64__) || defined(__arm64ec__)
20typedef __mfp8 mfloat8_t;
2021typedef double float64_t;
2122#endif
2223
24
25typedef uint64_t fpm_t;
26
27enum __ARM_FPM_FORMAT { __ARM_FPM_E5M2, __ARM_FPM_E4M3 };
28
29enum __ARM_FPM_OVERFLOW { __ARM_FPM_INFNAN, __ARM_FPM_SATURATE };
30
31static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
32__arm_fpm_init(void) {
33 return 0;
34}
35
36static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
37__arm_set_fpm_src1_format(fpm_t __fpm, enum __ARM_FPM_FORMAT __format) {
38 return (__fpm & ~7ull) | (fpm_t)__format;
39}
40
41static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
42__arm_set_fpm_src2_format(fpm_t __fpm, enum __ARM_FPM_FORMAT __format) {
43 return (__fpm & ~0x38ull) | ((fpm_t)__format << 3u);
44}
45
46static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
47__arm_set_fpm_dst_format(fpm_t __fpm, enum __ARM_FPM_FORMAT __format) {
48 return (__fpm & ~0x1c0ull) | ((fpm_t)__format << 6u);
49}
50
51static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
52__arm_set_fpm_overflow_mul(fpm_t __fpm, enum __ARM_FPM_OVERFLOW __behaviour) {
53 return (__fpm & ~0x4000ull) | ((fpm_t)__behaviour << 14u);
54}
55
56static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
57__arm_set_fpm_overflow_cvt(fpm_t __fpm, enum __ARM_FPM_OVERFLOW __behaviour) {
58 return (__fpm & ~0x8000ull) | ((fpm_t)__behaviour << 15u);
59}
60
61static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
62__arm_set_fpm_lscale(fpm_t __fpm, uint64_t __scale) {
63 return (__fpm & ~0x7f0000ull) | (__scale << 16u);
64}
65
66static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
67__arm_set_fpm_nscale(fpm_t __fpm, int64_t __scale) {
68 return (__fpm & ~0xff000000ull) | (((fpm_t)__scale & 0xffu) << 24u);
69}
70
71static __inline__ fpm_t __attribute__((__always_inline__, __nodebug__))
72__arm_set_fpm_lscale2(fpm_t __fpm, uint64_t __scale) {
73 return (uint32_t)__fpm | (__scale << 32u);
74}
75
2376typedef __attribute__((neon_vector_type(8))) int8_t int8x8_t;
2477typedef __attribute__((neon_vector_type(16))) int8_t int8x16_t;
2578typedef __attribute__((neon_vector_type(4))) int16_t int16x4_t;
......@@ -36,6 +89,10 @@ typedef __attribute__((neon_vector_type(2))) uint32_t uint32x2_t;
3689typedef __attribute__((neon_vector_type(4))) uint32_t uint32x4_t;
3790typedef __attribute__((neon_vector_type(1))) uint64_t uint64x1_t;
3891typedef __attribute__((neon_vector_type(2))) uint64_t uint64x2_t;
92#if defined(__aarch64__) || defined(__arm64ec__)
93typedef __attribute__((neon_vector_type(8))) mfloat8_t mfloat8x8_t;
94typedef __attribute__((neon_vector_type(16))) mfloat8_t mfloat8x16_t;
95#endif
3996typedef __attribute__((neon_vector_type(4))) float16_t float16x4_t;
4097typedef __attribute__((neon_vector_type(8))) float16_t float16x8_t;
4198typedef __attribute__((neon_vector_type(2))) float32_t float32x2_t;
......@@ -109,6 +166,16 @@ typedef struct uint64x2x2_t {
109166 uint64x2_t val[2];
110167} uint64x2x2_t;
111168
169#if defined(__aarch64__) || defined(__arm64ec__)
170typedef struct mfloat8x8x2_t {
171 mfloat8x8_t val[2];
172} mfloat8x8x2_t;
173
174typedef struct mfloat8x16x2_t {
175 mfloat8x16_t val[2];
176} mfloat8x16x2_t;
177
178#endif
112179typedef struct float16x4x2_t {
113180 float16x4_t val[2];
114181} float16x4x2_t;
......@@ -199,6 +266,16 @@ typedef struct uint64x2x3_t {
199266 uint64x2_t val[3];
200267} uint64x2x3_t;
201268
269#if defined(__aarch64__) || defined(__arm64ec__)
270typedef struct mfloat8x8x3_t {
271 mfloat8x8_t val[3];
272} mfloat8x8x3_t;
273
274typedef struct mfloat8x16x3_t {
275 mfloat8x16_t val[3];
276} mfloat8x16x3_t;
277
278#endif
202279typedef struct float16x4x3_t {
203280 float16x4_t val[3];
204281} float16x4x3_t;
......@@ -289,6 +366,16 @@ typedef struct uint64x2x4_t {
289366 uint64x2_t val[4];
290367} uint64x2x4_t;
291368
369#if defined(__aarch64__) || defined(__arm64ec__)
370typedef struct mfloat8x8x4_t {
371 mfloat8x8_t val[4];
372} mfloat8x8x4_t;
373
374typedef struct mfloat8x16x4_t {
375 mfloat8x16_t val[4];
376} mfloat8x16x4_t;
377
378#endif
292379typedef struct float16x4x4_t {
293380 float16x4_t val[4];
294381} float16x4x4_t;
lib/include/avx10_2_512bf16intrin.h created+561
......@@ -0,0 +1,561 @@
1/*===----------- avx10_2_512bf16intrin.h - AVX10-BF16 intrinsics ---------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2_512bf16intrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifdef __SSE2__
15
16#ifndef __AVX10_2_512BF16INTRIN_H
17#define __AVX10_2_512BF16INTRIN_H
18
19/* Define the default attributes for the functions in this file. */
20typedef __bf16 __m512bh_u __attribute__((__vector_size__(64), __aligned__(1)));
21
22/* Define the default attributes for the functions in this file. */
23#define __DEFAULT_FN_ATTRS512 \
24 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-512"), \
25 __min_vector_width__(512)))
26
27static __inline __m512bh __DEFAULT_FN_ATTRS512 _mm512_setzero_pbh(void) {
28 return __builtin_bit_cast(__m512bh, _mm512_setzero_ps());
29}
30
31static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_undefined_pbh(void) {
32 return (__m512bh)__builtin_ia32_undef512();
33}
34
35static __inline __m512bh __DEFAULT_FN_ATTRS512 _mm512_set1_pbh(__bf16 bf) {
36 return (__m512bh)(__v32bf){bf, bf, bf, bf, bf, bf, bf, bf, bf, bf, bf,
37 bf, bf, bf, bf, bf, bf, bf, bf, bf, bf, bf,
38 bf, bf, bf, bf, bf, bf, bf, bf, bf, bf};
39}
40
41static __inline __m512bh __DEFAULT_FN_ATTRS512 _mm512_set_pbh(
42 __bf16 bf1, __bf16 bf2, __bf16 bf3, __bf16 bf4, __bf16 bf5, __bf16 bf6,
43 __bf16 bf7, __bf16 bf8, __bf16 bf9, __bf16 bf10, __bf16 bf11, __bf16 bf12,
44 __bf16 bf13, __bf16 bf14, __bf16 bf15, __bf16 bf16, __bf16 bf17,
45 __bf16 bf18, __bf16 bf19, __bf16 bf20, __bf16 bf21, __bf16 bf22,
46 __bf16 bf23, __bf16 bf24, __bf16 bf25, __bf16 bf26, __bf16 bf27,
47 __bf16 bf28, __bf16 bf29, __bf16 bf30, __bf16 bf31, __bf16 bf32) {
48 return (__m512bh)(__v32bf){bf32, bf31, bf30, bf29, bf28, bf27, bf26, bf25,
49 bf24, bf23, bf22, bf21, bf20, bf19, bf18, bf17,
50 bf16, bf15, bf14, bf13, bf12, bf11, bf10, bf9,
51 bf8, bf7, bf6, bf5, bf4, bf3, bf2, bf1};
52}
53
54#define _mm512_setr_pbh(bf1, bf2, bf3, bf4, bf5, bf6, bf7, bf8, bf9, bf10, \
55 bf11, bf12, bf13, bf14, bf15, bf16, bf17, bf18, bf19, \
56 bf20, bf21, bf22, bf23, bf24, bf25, bf26, bf27, bf28, \
57 bf29, bf30, bf31, bf32) \
58 _mm512_set_pbh((bf32), (bf31), (bf30), (bf29), (bf28), (bf27), (bf26), \
59 (bf25), (bf24), (bf23), (bf22), (bf21), (bf20), (bf19), \
60 (bf18), (bf17), (bf16), (bf15), (bf14), (bf13), (bf12), \
61 (bf11), (bf10), (bf9), (bf8), (bf7), (bf6), (bf5), (bf4), \
62 (bf3), (bf2), (bf1))
63
64static __inline__ __m512 __DEFAULT_FN_ATTRS512
65_mm512_castbf16_ps(__m512bh __a) {
66 return (__m512)__a;
67}
68
69static __inline__ __m512d __DEFAULT_FN_ATTRS512
70_mm512_castbf16_pd(__m512bh __a) {
71 return (__m512d)__a;
72}
73
74static __inline__ __m512i __DEFAULT_FN_ATTRS512
75_mm512_castbf16_si512(__m512bh __a) {
76 return (__m512i)__a;
77}
78
79static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_castps_pbh(__m512 __a) {
80 return (__m512bh)__a;
81}
82
83static __inline__ __m512bh __DEFAULT_FN_ATTRS512
84_mm512_castpd_pbh(__m512d __a) {
85 return (__m512bh)__a;
86}
87
88static __inline__ __m512bh __DEFAULT_FN_ATTRS512
89_mm512_castsi512_pbh(__m512i __a) {
90 return (__m512bh)__a;
91}
92
93static __inline__ __m128bh __DEFAULT_FN_ATTRS512
94_mm512_castbf16512_pbh128(__m512bh __a) {
95 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7);
96}
97
98static __inline__ __m256bh __DEFAULT_FN_ATTRS512
99_mm512_castbf16512_pbh256(__m512bh __a) {
100 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
101 12, 13, 14, 15);
102}
103
104static __inline__ __m512bh __DEFAULT_FN_ATTRS512
105_mm512_castbf16128_pbh512(__m128bh __a) {
106 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, -1, -1, -1,
107 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
108 -1, -1, -1, -1, -1, -1, -1, -1, -1);
109}
110
111static __inline__ __m512bh __DEFAULT_FN_ATTRS512
112_mm512_castbf16256_pbh512(__m256bh __a) {
113 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
114 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1,
115 -1, -1, -1, -1, -1, -1, -1, -1);
116}
117
118static __inline__ __m512bh __DEFAULT_FN_ATTRS512
119_mm512_zextbf16128_pbh512(__m128bh __a) {
120 return __builtin_shufflevector(
121 __a, (__v8bf)_mm_setzero_pbh(), 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
122 13, 14, 15, 8, 9, 10, 11, 12, 13, 14, 15, 8, 9, 10, 11, 12, 13, 14, 15);
123}
124
125static __inline__ __m512bh __DEFAULT_FN_ATTRS512
126_mm512_zextbf16256_pbh512(__m256bh __a) {
127 return __builtin_shufflevector(__a, (__v16bf)_mm256_setzero_pbh(), 0, 1, 2, 3,
128 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
129 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
130 29, 30, 31);
131}
132
133static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_abs_pbh(__m512bh __A) {
134 return (__m512bh)_mm512_and_epi32(_mm512_set1_epi32(0x7FFF7FFF),
135 (__m512i)__A);
136}
137
138static __inline__ __m512bh __DEFAULT_FN_ATTRS512
139_mm512_load_pbh(void const *__p) {
140 return *(const __m512bh *)__p;
141}
142
143static __inline__ __m512bh __DEFAULT_FN_ATTRS512
144_mm512_loadu_pbh(void const *__p) {
145 struct __loadu_pbh {
146 __m512bh_u __v;
147 } __attribute__((__packed__, __may_alias__));
148 return ((const struct __loadu_pbh *)__p)->__v;
149}
150
151static __inline__ void __DEFAULT_FN_ATTRS512 _mm512_store_pbh(void *__P,
152 __m512bh __A) {
153 *(__m512bh *)__P = __A;
154}
155
156static __inline__ void __DEFAULT_FN_ATTRS512 _mm512_storeu_pbh(void *__P,
157 __m512bh __A) {
158 struct __storeu_pbh {
159 __m512bh_u __v;
160 } __attribute__((__packed__, __may_alias__));
161 ((struct __storeu_pbh *)__P)->__v = __A;
162}
163
164static __inline__ __m512bh __DEFAULT_FN_ATTRS512
165_mm512_mask_blend_pbh(__mmask32 __U, __m512bh __A, __m512bh __W) {
166 return (__m512bh)__builtin_ia32_selectpbf_512((__mmask32)__U, (__v32bf)__W,
167 (__v32bf)__A);
168}
169
170static __inline__ __m512bh __DEFAULT_FN_ATTRS512
171_mm512_permutex2var_pbh(__m512bh __A, __m512i __I, __m512bh __B) {
172 return (__m512bh)__builtin_ia32_vpermi2varhi512((__v32hi)__A, (__v32hi)__I,
173 (__v32hi)__B);
174}
175
176static __inline__ __m512bh __DEFAULT_FN_ATTRS512
177_mm512_permutexvar_pbh(__m512i __A, __m512bh __B) {
178 return (__m512bh)__builtin_ia32_permvarhi512((__v32hi)__B, (__v32hi)__A);
179}
180
181static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_add_pbh(__m512bh __A,
182 __m512bh __B) {
183 return (__m512bh)((__v32bf)__A + (__v32bf)__B);
184}
185
186static __inline__ __m512bh __DEFAULT_FN_ATTRS512
187_mm512_mask_add_pbh(__m512bh __W, __mmask32 __U, __m512bh __A, __m512bh __B) {
188 return (__m512bh)__builtin_ia32_selectpbf_512(
189 (__mmask32)__U, (__v32bf)_mm512_add_pbh(__A, __B), (__v32bf)__W);
190}
191
192static __inline__ __m512bh __DEFAULT_FN_ATTRS512
193_mm512_maskz_add_pbh(__mmask32 __U, __m512bh __A, __m512bh __B) {
194 return (__m512bh)__builtin_ia32_selectpbf_512(
195 (__mmask32)__U, (__v32bf)_mm512_add_pbh(__A, __B),
196 (__v32bf)_mm512_setzero_pbh());
197}
198
199static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_sub_pbh(__m512bh __A,
200 __m512bh __B) {
201 return (__m512bh)((__v32bf)__A - (__v32bf)__B);
202}
203
204static __inline__ __m512bh __DEFAULT_FN_ATTRS512
205_mm512_mask_sub_pbh(__m512bh __W, __mmask32 __U, __m512bh __A, __m512bh __B) {
206 return (__m512bh)__builtin_ia32_selectpbf_512(
207 (__mmask32)__U, (__v32bf)_mm512_sub_pbh(__A, __B), (__v32bf)__W);
208}
209
210static __inline__ __m512bh __DEFAULT_FN_ATTRS512
211_mm512_maskz_sub_pbh(__mmask32 __U, __m512bh __A, __m512bh __B) {
212 return (__m512bh)__builtin_ia32_selectpbf_512(
213 (__mmask32)__U, (__v32bf)_mm512_sub_pbh(__A, __B),
214 (__v32bf)_mm512_setzero_pbh());
215}
216
217static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mul_pbh(__m512bh __A,
218 __m512bh __B) {
219 return (__m512bh)((__v32bf)__A * (__v32bf)__B);
220}
221
222static __inline__ __m512bh __DEFAULT_FN_ATTRS512
223_mm512_mask_mul_pbh(__m512bh __W, __mmask32 __U, __m512bh __A, __m512bh __B) {
224 return (__m512bh)__builtin_ia32_selectpbf_512(
225 (__mmask32)__U, (__v32bf)_mm512_mul_pbh(__A, __B), (__v32bf)__W);
226}
227
228static __inline__ __m512bh __DEFAULT_FN_ATTRS512
229_mm512_maskz_mul_pbh(__mmask32 __U, __m512bh __A, __m512bh __B) {
230 return (__m512bh)__builtin_ia32_selectpbf_512(
231 (__mmask32)__U, (__v32bf)_mm512_mul_pbh(__A, __B),
232 (__v32bf)_mm512_setzero_pbh());
233}
234
235static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_div_pbh(__m512bh __A,
236 __m512bh __B) {
237 return (__m512bh)((__v32bf)__A / (__v32bf)__B);
238}
239
240static __inline__ __m512bh __DEFAULT_FN_ATTRS512
241_mm512_mask_div_pbh(__m512bh __W, __mmask32 __U, __m512bh __A, __m512bh __B) {
242 return (__m512bh)__builtin_ia32_selectpbf_512(
243 (__mmask32)__U, (__v32bf)_mm512_div_pbh(__A, __B), (__v32bf)__W);
244}
245
246static __inline__ __m512bh __DEFAULT_FN_ATTRS512
247_mm512_maskz_div_pbh(__mmask32 __U, __m512bh __A, __m512bh __B) {
248 return (__m512bh)__builtin_ia32_selectpbf_512(
249 (__mmask32)__U, (__v32bf)_mm512_div_pbh(__A, __B),
250 (__v32bf)_mm512_setzero_pbh());
251}
252
253static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_max_pbh(__m512bh __A,
254 __m512bh __B) {
255 return (__m512bh)__builtin_ia32_vmaxbf16512((__v32bf)__A, (__v32bf)__B);
256}
257
258static __inline__ __m512bh __DEFAULT_FN_ATTRS512
259_mm512_mask_max_pbh(__m512bh __W, __mmask32 __U, __m512bh __A, __m512bh __B) {
260 return (__m512bh)__builtin_ia32_selectpbf_512(
261 (__mmask32)__U, (__v32bf)_mm512_max_pbh(__A, __B), (__v32bf)__W);
262}
263
264static __inline__ __m512bh __DEFAULT_FN_ATTRS512
265_mm512_maskz_max_pbh(__mmask32 __U, __m512bh __A, __m512bh __B) {
266 return (__m512bh)__builtin_ia32_selectpbf_512(
267 (__mmask32)__U, (__v32bf)_mm512_max_pbh(__A, __B),
268 (__v32bf)_mm512_setzero_pbh());
269}
270
271static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_min_pbh(__m512bh __A,
272 __m512bh __B) {
273 return (__m512bh)__builtin_ia32_vminbf16512((__v32bf)__A, (__v32bf)__B);
274}
275
276static __inline__ __m512bh __DEFAULT_FN_ATTRS512
277_mm512_mask_min_pbh(__m512bh __W, __mmask32 __U, __m512bh __A, __m512bh __B) {
278 return (__m512bh)__builtin_ia32_selectpbf_512(
279 (__mmask32)__U, (__v32bf)_mm512_min_pbh(__A, __B), (__v32bf)__W);
280}
281
282static __inline__ __m512bh __DEFAULT_FN_ATTRS512
283_mm512_maskz_min_pbh(__mmask32 __U, __m512bh __A, __m512bh __B) {
284 return (__m512bh)__builtin_ia32_selectpbf_512(
285 (__mmask32)__U, (__v32bf)_mm512_min_pbh(__A, __B),
286 (__v32bf)_mm512_setzero_pbh());
287}
288
289#define _mm512_cmp_pbh_mask(__A, __B, __P) \
290 ((__mmask32)__builtin_ia32_vcmpbf16512_mask((__v32bf)(__m512bh)(__A), \
291 (__v32bf)(__m512bh)(__B), \
292 (int)(__P), (__mmask32) - 1))
293
294#define _mm512_mask_cmp_pbh_mask(__U, __A, __B, __P) \
295 ((__mmask32)__builtin_ia32_vcmpbf16512_mask((__v32bf)(__m512bh)(__A), \
296 (__v32bf)(__m512bh)(__B), \
297 (int)(__P), (__mmask32)(__U)))
298
299#define _mm512_mask_fpclass_pbh_mask(__U, __A, imm) \
300 ((__mmask32)__builtin_ia32_vfpclassbf16512_mask( \
301 (__v32bf)(__m512bh)(__A), (int)(imm), (__mmask32)(__U)))
302
303#define _mm512_fpclass_pbh_mask(__A, imm) \
304 ((__mmask32)__builtin_ia32_vfpclassbf16512_mask( \
305 (__v32bf)(__m512bh)(__A), (int)(imm), (__mmask32) - 1))
306
307static __inline__ __m512bh __DEFAULT_FN_ATTRS512
308_mm512_scalef_pbh(__m512bh __A, __m512bh __B) {
309 return (__m512bh)__builtin_ia32_vscalefbf16512_mask(
310 (__v32bf)__A, (__v32bf)__B, (__v32bf)_mm512_undefined_pbh(),
311 (__mmask32)-1);
312}
313
314static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask_scalef_pbh(
315 __m512bh __W, __mmask32 __U, __m512bh __A, __m512bh __B) {
316 return (__m512bh)__builtin_ia32_vscalefbf16512_mask(
317 (__v32bf)__A, (__v32bf)__B, (__v32bf)__W, (__mmask32)__U);
318}
319
320static __inline__ __m512bh __DEFAULT_FN_ATTRS512
321_mm512_maskz_scalef_pbh(__mmask32 __U, __m512bh __A, __m512bh __B) {
322 return (__m512bh)__builtin_ia32_vscalefbf16512_mask(
323 (__v32bf)__A, (__v32bf)__B, (__v32bf)_mm512_setzero_pbh(),
324 (__mmask32)__U);
325}
326
327static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_rcp_pbh(__m512bh __A) {
328 return (__m512bh)__builtin_ia32_vrcpbf16512_mask(
329 (__v32bf)__A, (__v32bf)_mm512_undefined_pbh(), (__mmask32)-1);
330}
331
332static __inline__ __m512bh __DEFAULT_FN_ATTRS512
333_mm512_mask_rcp_pbh(__m512bh __W, __mmask32 __U, __m512bh __A) {
334 return (__m512bh)__builtin_ia32_vrcpbf16512_mask((__v32bf)__A, (__v32bf)__W,
335 (__mmask32)__U);
336}
337
338static __inline__ __m512bh __DEFAULT_FN_ATTRS512
339_mm512_maskz_rcp_pbh(__mmask32 __U, __m512bh __A) {
340 return (__m512bh)__builtin_ia32_vrcpbf16512_mask(
341 (__v32bf)__A, (__v32bf)_mm512_setzero_pbh(), (__mmask32)__U);
342}
343
344static __inline__ __m512bh __DEFAULT_FN_ATTRS512
345_mm512_getexp_pbh(__m512bh __A) {
346 return (__m512bh)__builtin_ia32_vgetexpbf16512_mask(
347 (__v32bf)__A, (__v32bf)_mm512_undefined_pbh(), (__mmask32)-1);
348}
349
350static __inline__ __m512bh __DEFAULT_FN_ATTRS512
351_mm512_mask_getexp_pbh(__m512bh __W, __mmask32 __U, __m512bh __A) {
352 return (__m512bh)__builtin_ia32_vgetexpbf16512_mask(
353 (__v32bf)__A, (__v32bf)__W, (__mmask32)__U);
354}
355
356static __inline__ __m512bh __DEFAULT_FN_ATTRS512
357_mm512_maskz_getexp_pbh(__mmask32 __U, __m512bh __A) {
358 return (__m512bh)__builtin_ia32_vgetexpbf16512_mask(
359 (__v32bf)__A, (__v32bf)_mm512_setzero_pbh(), (__mmask32)__U);
360}
361
362static __inline__ __m512bh __DEFAULT_FN_ATTRS512
363_mm512_rsqrt_pbh(__m512bh __A) {
364 return (__m512bh)__builtin_ia32_vrsqrtbf16512_mask(
365 (__v32bf)__A, (__v32bf)_mm512_undefined_pbh(), (__mmask32)-1);
366}
367
368static __inline__ __m512bh __DEFAULT_FN_ATTRS512
369_mm512_mask_rsqrt_pbh(__m512bh __W, __mmask32 __U, __m512bh __A) {
370 return (__m512bh)__builtin_ia32_vrsqrtbf16512_mask((__v32bf)__A, (__v32bf)__W,
371 (__mmask32)__U);
372}
373
374static __inline__ __m512bh __DEFAULT_FN_ATTRS512
375_mm512_maskz_rsqrt_pbh(__mmask32 __U, __m512bh __A) {
376 return (__m512bh)__builtin_ia32_vrsqrtbf16512_mask(
377 (__v32bf)__A, (__v32bf)_mm512_setzero_pbh(), (__mmask32)__U);
378}
379
380#define _mm512_reduce_pbh(__A, imm) \
381 ((__m512bh)__builtin_ia32_vreducebf16512_mask( \
382 (__v32bf)(__m512bh)(__A), (int)(imm), (__v32bf)_mm512_undefined_pbh(), \
383 (__mmask32) - 1))
384
385#define _mm512_mask_reduce_pbh(__W, __U, __A, imm) \
386 ((__m512bh)__builtin_ia32_vreducebf16512_mask( \
387 (__v32bf)(__m512bh)(__A), (int)(imm), (__v32bf)(__m512bh)(__W), \
388 (__mmask32)(__U)))
389
390#define _mm512_maskz_reduce_pbh(__U, __A, imm) \
391 ((__m512bh)__builtin_ia32_vreducebf16512_mask( \
392 (__v32bf)(__m512bh)(__A), (int)(imm), (__v32bf)_mm512_setzero_pbh(), \
393 (__mmask32)(__U)))
394
395#define _mm512_roundscale_pbh(__A, imm) \
396 ((__m512bh)__builtin_ia32_vrndscalebf16_mask( \
397 (__v32bf)(__m512bh)(__A), (int)(imm), (__v32bf)_mm512_setzero_pbh(), \
398 (__mmask32) - 1))
399
400#define _mm512_mask_roundscale_pbh(__W, __U, __A, imm) \
401 ((__m512bh)__builtin_ia32_vrndscalebf16_mask( \
402 (__v32bf)(__m512bh)(__A), (int)(imm), (__v32bf)(__m512bh)(__W), \
403 (__mmask32)(__U)))
404
405#define _mm512_maskz_roundscale_pbh(__U, __A, imm) \
406 ((__m512bh)__builtin_ia32_vrndscalebf16_mask( \
407 (__v32bf)(__m512bh)(__A), (int)(imm), (__v32bf)_mm512_setzero_pbh(), \
408 (__mmask32)(__U)))
409
410#define _mm512_getmant_pbh(__A, __B, __C) \
411 ((__m512bh)__builtin_ia32_vgetmantbf16512_mask( \
412 (__v32bf)(__m512bh)(__A), (int)(((__C) << 2) | (__B)), \
413 (__v32bf)_mm512_undefined_pbh(), (__mmask32) - 1))
414
415#define _mm512_mask_getmant_pbh(__W, __U, __A, __B, __C) \
416 ((__m512bh)__builtin_ia32_vgetmantbf16512_mask( \
417 (__v32bf)(__m512bh)(__A), (int)(((__C) << 2) | (__B)), \
418 (__v32bf)(__m512bh)(__W), (__mmask32)(__U)))
419
420#define _mm512_maskz_getmant_pbh(__U, __A, __B, __C) \
421 ((__m512bh)__builtin_ia32_vgetmantbf16512_mask( \
422 (__v32bf)(__m512bh)(__A), (int)(((__C) << 2) | (__B)), \
423 (__v32bf)_mm512_setzero_pbh(), (__mmask32)(__U)))
424
425static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_sqrt_pbh(__m512bh __A) {
426 return (__m512bh)__builtin_ia32_vsqrtbf16512((__v32bf)__A);
427}
428
429static __inline__ __m512bh __DEFAULT_FN_ATTRS512
430_mm512_mask_sqrt_pbh(__m512bh __W, __mmask32 __U, __m512bh __A) {
431 return (__m512bh)__builtin_ia32_selectpbf_512(
432 (__mmask32)__U, (__v32bf)_mm512_sqrt_pbh(__A), (__v32bf)__W);
433}
434
435static __inline__ __m512bh __DEFAULT_FN_ATTRS512
436_mm512_maskz_sqrt_pbh(__mmask32 __U, __m512bh __A) {
437 return (__m512bh)__builtin_ia32_selectpbf_512((__mmask32)__U,
438 (__v32bf)_mm512_sqrt_pbh(__A),
439 (__v32bf)_mm512_setzero_pbh());
440}
441
442static __inline__ __m512bh __DEFAULT_FN_ATTRS512
443_mm512_fmadd_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
444 return (__m512bh)__builtin_ia32_vfmaddnepbh512((__v32bf)__A, (__v32bf)__B,
445 (__v32bf)__C);
446}
447
448static __inline__ __m512bh __DEFAULT_FN_ATTRS512
449_mm512_mask_fmadd_pbh(__m512bh __A, __mmask32 __U, __m512bh __B, __m512bh __C) {
450 return (__m512bh)__builtin_ia32_selectpbf_512(
451 (__mmask32)__U,
452 _mm512_fmadd_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C), (__v32bf)__A);
453}
454
455static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask3_fmadd_pbh(
456 __m512bh __A, __m512bh __B, __m512bh __C, __mmask32 __U) {
457 return (__m512bh)__builtin_ia32_selectpbf_512(
458 (__mmask32)__U,
459 _mm512_fmadd_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C), (__v32bf)__C);
460}
461
462static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fmadd_pbh(
463 __mmask32 __U, __m512bh __A, __m512bh __B, __m512bh __C) {
464 return (__m512bh)__builtin_ia32_selectpbf_512(
465 (__mmask32)__U,
466 _mm512_fmadd_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
467 (__v32bf)_mm512_setzero_pbh());
468}
469
470static __inline__ __m512bh __DEFAULT_FN_ATTRS512
471_mm512_fmsub_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
472 return (__m512bh)__builtin_ia32_vfmaddnepbh512((__v32bf)__A, (__v32bf)__B,
473 -(__v32bf)__C);
474}
475
476static __inline__ __m512bh __DEFAULT_FN_ATTRS512
477_mm512_mask_fmsub_pbh(__m512bh __A, __mmask32 __U, __m512bh __B, __m512bh __C) {
478 return (__m512bh)__builtin_ia32_selectpbf_512(
479 (__mmask32)__U,
480 _mm512_fmsub_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C), (__v32bf)__A);
481}
482
483static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask3_fmsub_pbh(
484 __m512bh __A, __m512bh __B, __m512bh __C, __mmask32 __U) {
485 return (__m512bh)__builtin_ia32_selectpbf_512(
486 (__mmask32)__U,
487 _mm512_fmsub_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C), (__v32bf)__C);
488}
489
490static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fmsub_pbh(
491 __mmask32 __U, __m512bh __A, __m512bh __B, __m512bh __C) {
492 return (__m512bh)__builtin_ia32_selectpbf_512(
493 (__mmask32)__U,
494 _mm512_fmsub_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
495 (__v32bf)_mm512_setzero_pbh());
496}
497
498static __inline__ __m512bh __DEFAULT_FN_ATTRS512
499_mm512_fnmadd_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
500 return (__m512bh)__builtin_ia32_vfmaddnepbh512((__v32bf)__A, -(__v32bf)__B,
501 (__v32bf)__C);
502}
503
504static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask_fnmadd_pbh(
505 __m512bh __A, __mmask32 __U, __m512bh __B, __m512bh __C) {
506 return (__m512bh)__builtin_ia32_selectpbf_512(
507 (__mmask32)__U,
508 _mm512_fnmadd_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
509 (__v32bf)__A);
510}
511
512static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask3_fnmadd_pbh(
513 __m512bh __A, __m512bh __B, __m512bh __C, __mmask32 __U) {
514 return (__m512bh)__builtin_ia32_selectpbf_512(
515 (__mmask32)__U,
516 _mm512_fnmadd_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
517 (__v32bf)__C);
518}
519
520static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fnmadd_pbh(
521 __mmask32 __U, __m512bh __A, __m512bh __B, __m512bh __C) {
522 return (__m512bh)__builtin_ia32_selectpbf_512(
523 (__mmask32)__U,
524 _mm512_fnmadd_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
525 (__v32bf)_mm512_setzero_pbh());
526}
527
528static __inline__ __m512bh __DEFAULT_FN_ATTRS512
529_mm512_fnmsub_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
530 return (__m512bh)__builtin_ia32_vfmaddnepbh512((__v32bf)__A, -(__v32bf)__B,
531 -(__v32bf)__C);
532}
533
534static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask_fnmsub_pbh(
535 __m512bh __A, __mmask32 __U, __m512bh __B, __m512bh __C) {
536 return (__m512bh)__builtin_ia32_selectpbf_512(
537 (__mmask32)__U,
538 _mm512_fnmsub_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
539 (__v32bf)__A);
540}
541
542static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask3_fnmsub_pbh(
543 __m512bh __A, __m512bh __B, __m512bh __C, __mmask32 __U) {
544 return (__m512bh)__builtin_ia32_selectpbf_512(
545 (__mmask32)__U,
546 _mm512_fnmsub_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
547 (__v32bf)__C);
548}
549
550static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fnmsub_pbh(
551 __mmask32 __U, __m512bh __A, __m512bh __B, __m512bh __C) {
552 return (__m512bh)__builtin_ia32_selectpbf_512(
553 (__mmask32)__U,
554 _mm512_fnmsub_pbh((__v32bf)__A, (__v32bf)__B, (__v32bf)__C),
555 (__v32bf)_mm512_setzero_pbh());
556}
557
558#undef __DEFAULT_FN_ATTRS512
559
560#endif
561#endif
lib/include/avx10_2_512convertintrin.h created+320
......@@ -0,0 +1,320 @@
1/*===--------- avx10_2_512convertintrin.h - AVX10_2_512CONVERT -------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2_512convertintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifdef __SSE2__
15
16#ifndef __AVX10_2_512CONVERTINTRIN_H
17#define __AVX10_2_512CONVERTINTRIN_H
18
19/* Define the default attributes for the functions in this file. */
20#define __DEFAULT_FN_ATTRS512 \
21 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-512"), \
22 __min_vector_width__(512)))
23
24static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_cvtx2ps_ph(__m512 __A,
25 __m512 __B) {
26 return (__m512h)__builtin_ia32_vcvt2ps2phx512_mask(
27 (__v16sf)__A, (__v16sf)__B, (__v32hf)_mm512_setzero_ph(), (__mmask32)(-1),
28 _MM_FROUND_CUR_DIRECTION);
29}
30
31static __inline__ __m512h __DEFAULT_FN_ATTRS512
32_mm512_mask_cvtx2ps_ph(__m512h __W, __mmask32 __U, __m512 __A, __m512 __B) {
33 return (__m512h)__builtin_ia32_vcvt2ps2phx512_mask(
34 (__v16sf)__A, (__v16sf)__B, (__v32hf)__W, (__mmask32)__U,
35 _MM_FROUND_CUR_DIRECTION);
36}
37
38static __inline__ __m512h __DEFAULT_FN_ATTRS512
39_mm512_maskz_cvtx2ps_ph(__mmask32 __U, __m512 __A, __m512 __B) {
40 return (__m512h)__builtin_ia32_vcvt2ps2phx512_mask(
41 (__v16sf)__A, (__v16sf)__B, (__v32hf)_mm512_setzero_ph(), (__mmask32)__U,
42 _MM_FROUND_CUR_DIRECTION);
43}
44
45#define _mm512_cvtx_round2ps_ph(A, B, R) \
46 ((__m512h)__builtin_ia32_vcvt2ps2phx512_mask( \
47 (__v16sf)(A), (__v16sf)(B), (__v32hf)_mm512_undefined_ph(), \
48 (__mmask32)(-1), (const int)(R)))
49
50#define _mm512_mask_cvtx_round2ps_ph(W, U, A, B, R) \
51 ((__m512h)__builtin_ia32_vcvt2ps2phx512_mask((__v16sf)(A), (__v16sf)(B), \
52 (__v32hf)(W), (__mmask32)(U), \
53 (const int)(R)))
54
55#define _mm512_maskz_cvtx_round2ps_ph(U, A, B, R) \
56 ((__m512h)__builtin_ia32_vcvt2ps2phx512_mask( \
57 (__v16sf)(A), (__v16sf)(B), (__v32hf)_mm512_setzero_ph(), \
58 (__mmask32)(U), (const int)(R)))
59
60static __inline__ __m256i __DEFAULT_FN_ATTRS512
61_mm512_cvtbiasph_bf8(__m512i __A, __m512h __B) {
62 return (__m256i)__builtin_ia32_vcvtbiasph2bf8_512_mask(
63 (__v64qi)__A, (__v32hf)__B, (__v32qi)_mm256_undefined_si256(),
64 (__mmask32)-1);
65}
66
67static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_mask_cvtbiasph_bf8(
68 __m256i __W, __mmask32 __U, __m512i __A, __m512h __B) {
69 return (__m256i)__builtin_ia32_vcvtbiasph2bf8_512_mask(
70 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)__W, (__mmask32)__U);
71}
72
73static __inline__ __m256i __DEFAULT_FN_ATTRS512
74_mm512_maskz_cvtbiasph_bf8(__mmask32 __U, __m512i __A, __m512h __B) {
75 return (__m256i)__builtin_ia32_vcvtbiasph2bf8_512_mask(
76 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)_mm256_setzero_si256(),
77 (__mmask32)__U);
78}
79
80static __inline__ __m256i __DEFAULT_FN_ATTRS512
81_mm512_cvtbiassph_bf8(__m512i __A, __m512h __B) {
82 return (__m256i)__builtin_ia32_vcvtbiasph2bf8s_512_mask(
83 (__v64qi)__A, (__v32hf)__B, (__v32qi)_mm256_undefined_si256(),
84 (__mmask32)-1);
85}
86
87static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_mask_cvtbiassph_bf8(
88 __m256i __W, __mmask32 __U, __m512i __A, __m512h __B) {
89 return (__m256i)__builtin_ia32_vcvtbiasph2bf8s_512_mask(
90 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)__W, (__mmask32)__U);
91}
92
93static __inline__ __m256i __DEFAULT_FN_ATTRS512
94_mm512_maskz_cvtbiassph_bf8(__mmask32 __U, __m512i __A, __m512h __B) {
95 return (__m256i)__builtin_ia32_vcvtbiasph2bf8s_512_mask(
96 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)_mm256_setzero_si256(),
97 (__mmask32)__U);
98}
99
100static __inline__ __m256i __DEFAULT_FN_ATTRS512
101_mm512_cvtbiasph_hf8(__m512i __A, __m512h __B) {
102 return (__m256i)__builtin_ia32_vcvtbiasph2hf8_512_mask(
103 (__v64qi)__A, (__v32hf)__B, (__v32qi)_mm256_undefined_si256(),
104 (__mmask32)-1);
105}
106
107static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_mask_cvtbiasph_hf8(
108 __m256i __W, __mmask32 __U, __m512i __A, __m512h __B) {
109 return (__m256i)__builtin_ia32_vcvtbiasph2hf8_512_mask(
110 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)__W, (__mmask32)__U);
111}
112
113static __inline__ __m256i __DEFAULT_FN_ATTRS512
114_mm512_maskz_cvtbiasph_hf8(__mmask32 __U, __m512i __A, __m512h __B) {
115 return (__m256i)__builtin_ia32_vcvtbiasph2hf8_512_mask(
116 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)_mm256_setzero_si256(),
117 (__mmask32)__U);
118}
119
120static __inline__ __m256i __DEFAULT_FN_ATTRS512
121_mm512_cvtbiassph_hf8(__m512i __A, __m512h __B) {
122 return (__m256i)__builtin_ia32_vcvtbiasph2hf8s_512_mask(
123 (__v64qi)__A, (__v32hf)__B, (__v32qi)_mm256_undefined_si256(),
124 (__mmask32)-1);
125}
126
127static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_mask_cvtbiassph_hf8(
128 __m256i __W, __mmask32 __U, __m512i __A, __m512h __B) {
129 return (__m256i)__builtin_ia32_vcvtbiasph2hf8s_512_mask(
130 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)__W, (__mmask32)__U);
131}
132
133static __inline__ __m256i __DEFAULT_FN_ATTRS512
134_mm512_maskz_cvtbiassph_hf8(__mmask32 __U, __m512i __A, __m512h __B) {
135 return (__m256i)__builtin_ia32_vcvtbiasph2hf8s_512_mask(
136 (__v64qi)__A, (__v32hf)__B, (__v32qi)(__m256i)_mm256_setzero_si256(),
137 (__mmask32)__U);
138}
139
140static __inline__ __m512i __DEFAULT_FN_ATTRS512 _mm512_cvt2ph_bf8(__m512h __A,
141 __m512h __B) {
142 return (__m512i)__builtin_ia32_vcvt2ph2bf8_512((__v32hf)(__A),
143 (__v32hf)(__B));
144}
145
146static __inline__ __m512i __DEFAULT_FN_ATTRS512
147_mm512_mask_cvt2ph_bf8(__m512i __W, __mmask64 __U, __m512h __A, __m512h __B) {
148 return (__m512i)__builtin_ia32_selectb_512(
149 (__mmask64)__U, (__v64qi)_mm512_cvt2ph_bf8(__A, __B), (__v64qi)__W);
150}
151
152static __inline__ __m512i __DEFAULT_FN_ATTRS512
153_mm512_maskz_cvt2ph_bf8(__mmask64 __U, __m512h __A, __m512h __B) {
154 return (__m512i)__builtin_ia32_selectb_512(
155 (__mmask64)__U, (__v64qi)_mm512_cvt2ph_bf8(__A, __B),
156 (__v64qi)(__m512i)_mm512_setzero_si512());
157}
158
159static __inline__ __m512i __DEFAULT_FN_ATTRS512
160_mm512_cvts2ph_bf8(__m512h __A, __m512h __B) {
161 return (__m512i)__builtin_ia32_vcvt2ph2bf8s_512((__v32hf)(__A),
162 (__v32hf)(__B));
163}
164
165static __inline__ __m512i __DEFAULT_FN_ATTRS512
166_mm512_mask_cvts2ph_bf8(__m512i __W, __mmask64 __U, __m512h __A, __m512h __B) {
167 return (__m512i)__builtin_ia32_selectb_512(
168 (__mmask64)__U, (__v64qi)_mm512_cvts2ph_bf8(__A, __B), (__v64qi)__W);
169}
170
171static __inline__ __m512i __DEFAULT_FN_ATTRS512
172_mm512_maskz_cvts2ph_bf8(__mmask64 __U, __m512h __A, __m512h __B) {
173 return (__m512i)__builtin_ia32_selectb_512(
174 (__mmask64)__U, (__v64qi)_mm512_cvts2ph_bf8(__A, __B),
175 (__v64qi)(__m512i)_mm512_setzero_si512());
176}
177
178static __inline__ __m512i __DEFAULT_FN_ATTRS512 _mm512_cvt2ph_hf8(__m512h __A,
179 __m512h __B) {
180 return (__m512i)__builtin_ia32_vcvt2ph2hf8_512((__v32hf)(__A),
181 (__v32hf)(__B));
182}
183
184static __inline__ __m512i __DEFAULT_FN_ATTRS512
185_mm512_mask_cvt2ph_hf8(__m512i __W, __mmask64 __U, __m512h __A, __m512h __B) {
186 return (__m512i)__builtin_ia32_selectb_512(
187 (__mmask64)__U, (__v64qi)_mm512_cvt2ph_hf8(__A, __B), (__v64qi)__W);
188}
189
190static __inline__ __m512i __DEFAULT_FN_ATTRS512
191_mm512_maskz_cvt2ph_hf8(__mmask64 __U, __m512h __A, __m512h __B) {
192 return (__m512i)__builtin_ia32_selectb_512(
193 (__mmask64)__U, (__v64qi)_mm512_cvt2ph_hf8(__A, __B),
194 (__v64qi)(__m512i)_mm512_setzero_si512());
195}
196
197static __inline__ __m512i __DEFAULT_FN_ATTRS512
198_mm512_cvts2ph_hf8(__m512h __A, __m512h __B) {
199 return (__m512i)__builtin_ia32_vcvt2ph2hf8s_512((__v32hf)(__A),
200 (__v32hf)(__B));
201}
202
203static __inline__ __m512i __DEFAULT_FN_ATTRS512
204_mm512_mask_cvts2ph_hf8(__m512i __W, __mmask64 __U, __m512h __A, __m512h __B) {
205 return (__m512i)__builtin_ia32_selectb_512(
206 (__mmask64)__U, (__v64qi)_mm512_cvts2ph_hf8(__A, __B), (__v64qi)__W);
207}
208
209static __inline__ __m512i __DEFAULT_FN_ATTRS512
210_mm512_maskz_cvts2ph_hf8(__mmask64 __U, __m512h __A, __m512h __B) {
211 return (__m512i)__builtin_ia32_selectb_512(
212 (__mmask64)__U, (__v64qi)_mm512_cvts2ph_hf8(__A, __B),
213 (__v64qi)(__m512i)_mm512_setzero_si512());
214}
215
216static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_cvthf8(__m256i __A) {
217 return (__m512h)__builtin_ia32_vcvthf8_2ph512_mask(
218 (__v32qi)__A, (__v32hf)(__m512h)_mm512_undefined_ph(), (__mmask32)-1);
219}
220
221static __inline__ __m512h __DEFAULT_FN_ATTRS512
222_mm512_mask_cvthf8(__m512h __W, __mmask32 __U, __m256i __A) {
223 return (__m512h)__builtin_ia32_vcvthf8_2ph512_mask(
224 (__v32qi)__A, (__v32hf)(__m512h)__W, (__mmask32)__U);
225}
226
227static __inline__ __m512h __DEFAULT_FN_ATTRS512
228_mm512_maskz_cvthf8(__mmask32 __U, __m256i __A) {
229 return (__m512h)__builtin_ia32_vcvthf8_2ph512_mask(
230 (__v32qi)__A, (__v32hf)(__m512h)_mm512_setzero_ph(), (__mmask32)__U);
231}
232
233static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_cvtph_bf8(__m512h __A) {
234 return (__m256i)__builtin_ia32_vcvtph2bf8_512_mask(
235 (__v32hf)__A, (__v32qi)(__m256i)_mm256_undefined_si256(), (__mmask32)-1);
236}
237
238static __inline__ __m256i __DEFAULT_FN_ATTRS512
239_mm512_mask_cvtph_bf8(__m256i __W, __mmask32 __U, __m512h __A) {
240 return (__m256i)__builtin_ia32_vcvtph2bf8_512_mask(
241 (__v32hf)__A, (__v32qi)(__m256i)__W, (__mmask32)__U);
242}
243
244static __inline__ __m256i __DEFAULT_FN_ATTRS512
245_mm512_maskz_cvtph_bf8(__mmask32 __U, __m512h __A) {
246 return (__m256i)__builtin_ia32_vcvtph2bf8_512_mask(
247 (__v32hf)__A, (__v32qi)(__m256i)_mm256_setzero_si256(), (__mmask32)__U);
248}
249
250static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_cvtsph_bf8(__m512h __A) {
251 return (__m256i)__builtin_ia32_vcvtph2bf8s_512_mask(
252 (__v32hf)__A, (__v32qi)(__m256i)_mm256_undefined_si256(), (__mmask32)-1);
253}
254
255static __inline__ __m256i __DEFAULT_FN_ATTRS512
256_mm512_mask_cvtsph_bf8(__m256i __W, __mmask32 __U, __m512h __A) {
257 return (__m256i)__builtin_ia32_vcvtph2bf8s_512_mask(
258 (__v32hf)__A, (__v32qi)(__m256i)__W, (__mmask32)__U);
259}
260
261static __inline__ __m256i __DEFAULT_FN_ATTRS512
262_mm512_maskz_cvtsph_bf8(__mmask32 __U, __m512h __A) {
263 return (__m256i)__builtin_ia32_vcvtph2bf8s_512_mask(
264 (__v32hf)__A, (__v32qi)(__m256i)_mm256_setzero_si256(), (__mmask32)__U);
265}
266
267static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_cvtph_hf8(__m512h __A) {
268 return (__m256i)__builtin_ia32_vcvtph2hf8_512_mask(
269 (__v32hf)__A, (__v32qi)(__m256i)_mm256_undefined_si256(), (__mmask32)-1);
270}
271
272static __inline__ __m256i __DEFAULT_FN_ATTRS512
273_mm512_mask_cvtph_hf8(__m256i __W, __mmask32 __U, __m512h __A) {
274 return (__m256i)__builtin_ia32_vcvtph2hf8_512_mask(
275 (__v32hf)__A, (__v32qi)(__m256i)__W, (__mmask32)__U);
276}
277
278static __inline__ __m256i __DEFAULT_FN_ATTRS512
279_mm512_maskz_cvtph_hf8(__mmask32 __U, __m512h __A) {
280 return (__m256i)__builtin_ia32_vcvtph2hf8_512_mask(
281 (__v32hf)__A, (__v32qi)(__m256i)_mm256_setzero_si256(), (__mmask32)__U);
282}
283
284static __inline__ __m256i __DEFAULT_FN_ATTRS512 _mm512_cvtsph_hf8(__m512h __A) {
285 return (__m256i)__builtin_ia32_vcvtph2hf8s_512_mask(
286 (__v32hf)__A, (__v32qi)(__m256i)_mm256_undefined_si256(), (__mmask32)-1);
287}
288
289static __inline__ __m256i __DEFAULT_FN_ATTRS512
290_mm512_mask_cvtsph_hf8(__m256i __W, __mmask32 __U, __m512h __A) {
291 return (__m256i)__builtin_ia32_vcvtph2hf8s_512_mask(
292 (__v32hf)__A, (__v32qi)(__m256i)__W, (__mmask32)__U);
293}
294
295static __inline__ __m256i __DEFAULT_FN_ATTRS512
296_mm512_maskz_cvtsph_hf8(__mmask32 __U, __m512h __A) {
297 return (__m256i)__builtin_ia32_vcvtph2hf8s_512_mask(
298 (__v32hf)__A, (__v32qi)(__m256i)_mm256_setzero_si256(), (__mmask32)__U);
299}
300
301static __inline __m512h __DEFAULT_FN_ATTRS512 _mm512_cvtbf8_ph(__m256i __A) {
302 return _mm512_castsi512_ph(_mm512_slli_epi16(_mm512_cvtepi8_epi16(__A), 8));
303}
304
305static __inline __m512h __DEFAULT_FN_ATTRS512
306_mm512_mask_cvtbf8_ph(__m512h __S, __mmask32 __U, __m256i __A) {
307 return _mm512_castsi512_ph(
308 _mm512_mask_slli_epi16((__m512i)__S, __U, _mm512_cvtepi8_epi16(__A), 8));
309}
310
311static __inline __m512h __DEFAULT_FN_ATTRS512
312_mm512_maskz_cvtbf8_ph(__mmask32 __U, __m256i __A) {
313 return _mm512_castsi512_ph(
314 _mm512_slli_epi16(_mm512_maskz_cvtepi8_epi16(__U, __A), 8));
315}
316
317#undef __DEFAULT_FN_ATTRS512
318
319#endif // __AVX10_2_512CONVERTINTRIN_H
320#endif // __SSE2__
lib/include/avx10_2_512minmaxintrin.h created+127
......@@ -0,0 +1,127 @@
1/*===---- avx10_2_512minmaxintrin.h - AVX10_2_512MINMAX intrinsics ---------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2_512minmaxintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifndef __AVX10_2_512MINMAXINTRIN_H
15#define __AVX10_2_512MINMAXINTRIN_H
16
17#define _mm512_minmax_pbh(A, B, C) \
18 ((__m512bh)__builtin_ia32_vminmaxbf16512((__v32bf)(__m512bh)(A), \
19 (__v32bf)(__m512bh)(A), (int)(C)))
20
21#define _mm512_mask_minmax_pbh(W, U, A, B, C) \
22 ((__m512bh)__builtin_ia32_selectpbf_512( \
23 (__mmask32)(U), \
24 (__v32bf)_mm512_minmax_pbh((__v32bf)(__m512bh)(A), \
25 (__v32bf)(__m512bh)(B), (int)(C)), \
26 (__v32bf)(__m512bh)(W)))
27
28#define _mm512_maskz_minmax_pbh(U, A, B, C) \
29 ((__m512bh)__builtin_ia32_selectpbf_512( \
30 (__mmask32)(U), \
31 (__v32bf)_mm512_minmax_pbh((__v32bf)(__m512bh)(A), \
32 (__v32bf)(__m512bh)(B), (int)(C)), \
33 (__v32bf) __builtin_bit_cast(__m512bh, _mm512_setzero_ps())))
34
35#define _mm512_minmax_pd(A, B, C) \
36 ((__m512d)__builtin_ia32_vminmaxpd512_round_mask( \
37 (__v8df)(__m512d)(A), (__v8df)(__m512d)(B), (int)(C), \
38 (__v8df)_mm512_undefined_pd(), (__mmask8)-1, \
39 _MM_FROUND_CUR_DIRECTION))
40
41#define _mm512_mask_minmax_pd(W, U, A, B, C) \
42 ((__m512d)__builtin_ia32_vminmaxpd512_round_mask( \
43 (__v8df)(__m512d)(A), (__v8df)(__m512d)(B), (int)(C), \
44 (__v8df)(__m512d)(W), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
45
46#define _mm512_maskz_minmax_pd(U, A, B, C) \
47 ((__m512d)__builtin_ia32_vminmaxpd512_round_mask( \
48 (__v8df)(__m512d)(A), (__v8df)(__m512d)(B), (int)(C), \
49 (__v8df)_mm512_setzero_pd(), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
50
51#define _mm512_minmax_round_pd(A, B, C, R) \
52 ((__m512d)__builtin_ia32_vminmaxpd512_round_mask( \
53 (__v8df)(__m512d)(A), (__v8df)(__m512d)(B), (int)(C), \
54 (__v8df)_mm512_undefined_pd(), (__mmask8)-1, (int)(R)))
55
56#define _mm512_mask_minmax_round_pd(W, U, A, B, C, R) \
57 ((__m512d)__builtin_ia32_vminmaxpd512_round_mask( \
58 (__v8df)(__m512d)(A), (__v8df)(__m512d)(B), (int)(C), \
59 (__v8df)(__m512d)(W), (__mmask8)(U), (int)(R)))
60
61#define _mm512_maskz_minmax_round_pd(U, A, B, C, R) \
62 ((__m512d)__builtin_ia32_vminmaxpd512_round_mask( \
63 (__v8df)(__m512d)(A), (__v8df)(__m512d)(B), (int)(C), \
64 (__v8df)_mm512_setzero_pd(), (__mmask8)(U), (int)(R)))
65
66#define _mm512_minmax_ph(A, B, C) \
67 ((__m512h)__builtin_ia32_vminmaxph512_round_mask( \
68 (__v32hf)(__m512h)(A), (__v32hf)(__m512h)(B), (int)(C), \
69 (__v32hf)_mm512_undefined_ph(), (__mmask32)-1, \
70 _MM_FROUND_CUR_DIRECTION))
71
72#define _mm512_mask_minmax_ph(W, U, A, B, C) \
73 ((__m512h)__builtin_ia32_vminmaxph512_round_mask( \
74 (__v32hf)(__m512h)(A), (__v32hf)(__m512h)(B), (int)(C), \
75 (__v32hf)(__m512h)(W), (__mmask32)(U), _MM_FROUND_CUR_DIRECTION))
76
77#define _mm512_maskz_minmax_ph(U, A, B, C) \
78 ((__m512h)__builtin_ia32_vminmaxph512_round_mask( \
79 (__v32hf)(__m512h)(A), (__v32hf)(__m512h)(B), (int)(C), \
80 (__v32hf)_mm512_setzero_ph(), (__mmask32)(U), _MM_FROUND_CUR_DIRECTION))
81
82#define _mm512_minmax_round_ph(A, B, C, R) \
83 ((__m512h)__builtin_ia32_vminmaxph512_round_mask( \
84 (__v32hf)(__m512h)(A), (__v32hf)(__m512h)(B), (int)(C), \
85 (__v32hf)_mm512_undefined_ph(), (__mmask32)-1, (int)(R)))
86
87#define _mm512_mask_minmax_round_ph(W, U, A, B, C, R) \
88 ((__m512h)__builtin_ia32_vminmaxph512_round_mask( \
89 (__v32hf)(__m512h)(A), (__v32hf)(__m512h)(B), (int)(C), \
90 (__v32hf)(__m512h)(W), (__mmask32)(U), (int)(R)))
91
92#define _mm512_maskz_minmax_round_ph(U, A, B, C, R) \
93 ((__m512h)__builtin_ia32_vminmaxph512_round_mask( \
94 (__v32hf)(__m512h)(A), (__v32hf)(__m512h)(B), (int)(C), \
95 (__v32hf)_mm512_setzero_ph(), (__mmask32)(U), (int)(R)))
96
97#define _mm512_minmax_ps(A, B, C) \
98 ((__m512)__builtin_ia32_vminmaxps512_round_mask( \
99 (__v16sf)(__m512)(A), (__v16sf)(__m512)(B), (int)(C), \
100 (__v16sf)_mm512_undefined_ps(), (__mmask16)-1, \
101 _MM_FROUND_CUR_DIRECTION))
102
103#define _mm512_mask_minmax_ps(W, U, A, B, C) \
104 ((__m512)__builtin_ia32_vminmaxps512_round_mask( \
105 (__v16sf)(__m512)(A), (__v16sf)(__m512)(B), (int)(C), (__v16sf)(W), \
106 (__mmask16)(U), _MM_FROUND_CUR_DIRECTION))
107
108#define _mm512_maskz_minmax_ps(U, A, B, C) \
109 ((__m512)__builtin_ia32_vminmaxps512_round_mask( \
110 (__v16sf)(__m512)(A), (__v16sf)(__m512)(B), (int)(C), \
111 (__v16sf)_mm512_setzero_ps(), (__mmask16)(U), _MM_FROUND_CUR_DIRECTION))
112
113#define _mm512_minmax_round_ps(A, B, C, R) \
114 ((__m512)__builtin_ia32_vminmaxps512_round_mask( \
115 (__v16sf)(__m512)(A), (__v16sf)(__m512)(B), (int)(C), \
116 (__v16sf)_mm512_undefined_ps(), (__mmask16)-1, (int)(R)))
117
118#define _mm512_mask_minmax_round_ps(W, U, A, B, C, R) \
119 ((__m512)__builtin_ia32_vminmaxps512_round_mask( \
120 (__v16sf)(__m512)(A), (__v16sf)(__m512)(B), (int)(C), (__v16sf)(W), \
121 (__mmask16)(U), (int)(R)))
122
123#define _mm512_maskz_minmax_round_ps(U, A, B, C, R) \
124 ((__m512)__builtin_ia32_vminmaxps512_round_mask( \
125 (__v16sf)(__m512)(A), (__v16sf)(__m512)(B), (int)(C), \
126 (__v16sf)_mm512_setzero_ps(), (__mmask16)(U), (int)(R)))
127#endif // __AVX10_2_512MINMAXINTRIN_H
lib/include/avx10_2_512niintrin.h created+314
......@@ -0,0 +1,314 @@
1/*===---- avx10_2_512niintrin.h - AVX10.2-512 new instruction intrinsics ---===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2_512niintrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifdef __SSE2__
15
16#ifndef __AVX10_2_512NIINTRIN_H
17#define __AVX10_2_512NIINTRIN_H
18
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-512"), \
21 __min_vector_width__(512)))
22
23/* VNNI FP16 */
24static __inline__ __m512 __DEFAULT_FN_ATTRS _mm512_dpph_ps(__m512 __W,
25 __m512h __A,
26 __m512h __B) {
27 return (__m512)__builtin_ia32_vdpphps512((__v16sf)__W, (__v32hf)__A,
28 (__v32hf)__B);
29}
30
31static __inline__ __m512 __DEFAULT_FN_ATTRS _mm512_mask_dpph_ps(__m512 __W,
32 __mmask16 __U,
33 __m512h __A,
34 __m512h __B) {
35 return (__m512)__builtin_ia32_selectps_512(
36 (__mmask16)__U, (__v16sf)_mm512_dpph_ps(__W, __A, __B), (__v16sf)__W);
37}
38
39static __inline__ __m512 __DEFAULT_FN_ATTRS _mm512_maskz_dpph_ps(__mmask16 __U,
40 __m512 __W,
41 __m512h __A,
42 __m512h __B) {
43 return (__m512)__builtin_ia32_selectps_512(
44 (__mmask16)__U, (__v16sf)_mm512_dpph_ps(__W, __A, __B),
45 (__v16sf)_mm512_setzero_ps());
46}
47
48/* VMPSADBW */
49#define _mm512_mpsadbw_epu8(A, B, imm) \
50 ((__m512i)__builtin_ia32_mpsadbw512((__v64qi)(__m512i)(A), \
51 (__v64qi)(__m512i)(B), (int)(imm)))
52
53#define _mm512_mask_mpsadbw_epu8(W, U, A, B, imm) \
54 ((__m512i)__builtin_ia32_selectw_512( \
55 (__mmask32)(U), (__v32hi)_mm512_mpsadbw_epu8((A), (B), (imm)), \
56 (__v32hi)(__m512i)(W)))
57
58#define _mm512_maskz_mpsadbw_epu8(U, A, B, imm) \
59 ((__m512i)__builtin_ia32_selectw_512( \
60 (__mmask32)(U), (__v32hi)_mm512_mpsadbw_epu8((A), (B), (imm)), \
61 (__v32hi)_mm512_setzero_si512()))
62
63/* VNNI INT8 */
64static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbssd_epi32(__m512i __W,
65 __m512i __A,
66 __m512i __B) {
67 return (__m512i)__builtin_ia32_vpdpbssd512((__v16si)__W, (__v16si)__A,
68 (__v16si)__B);
69}
70
71static __inline__ __m512i __DEFAULT_FN_ATTRS
72_mm512_mask_dpbssd_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
73 return (__m512i)__builtin_ia32_selectd_512(
74 __U, (__v16si)_mm512_dpbssd_epi32(__W, __A, __B), (__v16si)__W);
75}
76
77static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbssd_epi32(
78 __mmask16 __U, __m512i __W, __m512i __A, __m512i __B) {
79 return (__m512i)__builtin_ia32_selectd_512(
80 __U, (__v16si)_mm512_dpbssd_epi32(__W, __A, __B),
81 (__v16si)_mm512_setzero_si512());
82}
83
84static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbssds_epi32(__m512i __W,
85 __m512i __A,
86 __m512i __B) {
87 return (__m512i)__builtin_ia32_vpdpbssds512((__v16si)__W, (__v16si)__A,
88 (__v16si)__B);
89}
90
91static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpbssds_epi32(
92 __m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
93 return (__m512i)__builtin_ia32_selectd_512(
94 __U, (__v16si)_mm512_dpbssds_epi32(__W, __A, __B), (__v16si)__W);
95}
96
97static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbssds_epi32(
98 __mmask16 __U, __m512i __W, __m512i __A, __m512i __B) {
99 return (__m512i)__builtin_ia32_selectd_512(
100 __U, (__v16si)_mm512_dpbssds_epi32(__W, __A, __B),
101 (__v16si)_mm512_setzero_si512());
102}
103
104static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbsud_epi32(__m512i __W,
105 __m512i __A,
106 __m512i __B) {
107 return (__m512i)__builtin_ia32_vpdpbsud512((__v16si)__W, (__v16si)__A,
108 (__v16si)__B);
109}
110
111static __inline__ __m512i __DEFAULT_FN_ATTRS
112_mm512_mask_dpbsud_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
113 return (__m512i)__builtin_ia32_selectd_512(
114 __U, (__v16si)_mm512_dpbsud_epi32(__W, __A, __B), (__v16si)__W);
115}
116
117static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbsud_epi32(
118 __mmask16 __U, __m512i __W, __m512i __A, __m512i __B) {
119 return (__m512i)__builtin_ia32_selectd_512(
120 __U, (__v16si)_mm512_dpbsud_epi32(__W, __A, __B),
121 (__v16si)_mm512_setzero_si512());
122}
123
124static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbsuds_epi32(__m512i __W,
125 __m512i __A,
126 __m512i __B) {
127 return (__m512i)__builtin_ia32_vpdpbsuds512((__v16si)__W, (__v16si)__A,
128 (__v16si)__B);
129}
130
131static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpbsuds_epi32(
132 __m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
133 return (__m512i)__builtin_ia32_selectd_512(
134 __U, (__v16si)_mm512_dpbsuds_epi32(__W, __A, __B), (__v16si)__W);
135}
136
137static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbsuds_epi32(
138 __mmask16 __U, __m512i __W, __m512i __A, __m512i __B) {
139 return (__m512i)__builtin_ia32_selectd_512(
140 __U, (__v16si)_mm512_dpbsuds_epi32(__W, __A, __B),
141 (__v16si)_mm512_setzero_si512());
142}
143
144static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbuud_epi32(__m512i __W,
145 __m512i __A,
146 __m512i __B) {
147 return (__m512i)__builtin_ia32_vpdpbuud512((__v16si)__W, (__v16si)__A,
148 (__v16si)__B);
149}
150
151static __inline__ __m512i __DEFAULT_FN_ATTRS
152_mm512_mask_dpbuud_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
153 return (__m512i)__builtin_ia32_selectd_512(
154 __U, (__v16si)_mm512_dpbuud_epi32(__W, __A, __B), (__v16si)__W);
155}
156
157static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbuud_epi32(
158 __mmask16 __U, __m512i __W, __m512i __A, __m512i __B) {
159 return (__m512i)__builtin_ia32_selectd_512(
160 __U, (__v16si)_mm512_dpbuud_epi32(__W, __A, __B),
161 (__v16si)_mm512_setzero_si512());
162}
163
164static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbuuds_epi32(__m512i __W,
165 __m512i __A,
166 __m512i __B) {
167 return (__m512i)__builtin_ia32_vpdpbuuds512((__v16si)__W, (__v16si)__A,
168 (__v16si)__B);
169}
170
171static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpbuuds_epi32(
172 __m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
173 return (__m512i)__builtin_ia32_selectd_512(
174 __U, (__v16si)_mm512_dpbuuds_epi32(__W, __A, __B), (__v16si)__W);
175}
176
177static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbuuds_epi32(
178 __mmask16 __U, __m512i __W, __m512i __A, __m512i __B) {
179 return (__m512i)__builtin_ia32_selectd_512(
180 __U, (__v16si)_mm512_dpbuuds_epi32(__W, __A, __B),
181 (__v16si)_mm512_setzero_si512());
182}
183
184/* VNNI INT16 */
185static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwsud_epi32(__m512i __A,
186 __m512i __B,
187 __m512i __C) {
188 return (__m512i)__builtin_ia32_vpdpwsud512((__v16si)__A, (__v16si)__B,
189 (__v16si)__C);
190}
191
192static __inline__ __m512i __DEFAULT_FN_ATTRS
193_mm512_mask_dpwsud_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
194 return (__m512i)__builtin_ia32_selectd_512(
195 (__mmask16)__U, (__v16si)_mm512_dpwsud_epi32(__A, __B, __C),
196 (__v16si)__A);
197}
198
199static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwsud_epi32(
200 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
201 return (__m512i)__builtin_ia32_selectd_512(
202 (__mmask16)__U, (__v16si)_mm512_dpwsud_epi32(__A, __B, __C),
203 (__v16si)_mm512_setzero_si512());
204}
205
206static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwsuds_epi32(__m512i __A,
207 __m512i __B,
208 __m512i __C) {
209 return (__m512i)__builtin_ia32_vpdpwsuds512((__v16si)__A, (__v16si)__B,
210 (__v16si)__C);
211}
212
213static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwsuds_epi32(
214 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
215 return (__m512i)__builtin_ia32_selectd_512(
216 (__mmask16)__U, (__v16si)_mm512_dpwsuds_epi32(__A, __B, __C),
217 (__v16si)__A);
218}
219
220static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwsuds_epi32(
221 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
222 return (__m512i)__builtin_ia32_selectd_512(
223 (__mmask16)__U, (__v16si)_mm512_dpwsuds_epi32(__A, __B, __C),
224 (__v16si)_mm512_setzero_si512());
225}
226
227static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwusd_epi32(__m512i __A,
228 __m512i __B,
229 __m512i __C) {
230 return (__m512i)__builtin_ia32_vpdpwusd512((__v16si)__A, (__v16si)__B,
231 (__v16si)__C);
232}
233
234static __inline__ __m512i __DEFAULT_FN_ATTRS
235_mm512_mask_dpwusd_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
236 return (__m512i)__builtin_ia32_selectd_512(
237 (__mmask16)__U, (__v16si)_mm512_dpwusd_epi32(__A, __B, __C),
238 (__v16si)__A);
239}
240
241static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwusd_epi32(
242 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
243 return (__m512i)__builtin_ia32_selectd_512(
244 (__mmask16)__U, (__v16si)_mm512_dpwusd_epi32(__A, __B, __C),
245 (__v16si)_mm512_setzero_si512());
246}
247
248static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwusds_epi32(__m512i __A,
249 __m512i __B,
250 __m512i __C) {
251 return (__m512i)__builtin_ia32_vpdpwusds512((__v16si)__A, (__v16si)__B,
252 (__v16si)__C);
253}
254
255static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwusds_epi32(
256 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
257 return (__m512i)__builtin_ia32_selectd_512(
258 (__mmask16)__U, (__v16si)_mm512_dpwusds_epi32(__A, __B, __C),
259 (__v16si)__A);
260}
261
262static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwusds_epi32(
263 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
264 return (__m512i)__builtin_ia32_selectd_512(
265 (__mmask16)__U, (__v16si)_mm512_dpwusds_epi32(__A, __B, __C),
266 (__v16si)_mm512_setzero_si512());
267}
268
269static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwuud_epi32(__m512i __A,
270 __m512i __B,
271 __m512i __C) {
272 return (__m512i)__builtin_ia32_vpdpwuud512((__v16si)__A, (__v16si)__B,
273 (__v16si)__C);
274}
275
276static __inline__ __m512i __DEFAULT_FN_ATTRS
277_mm512_mask_dpwuud_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
278 return (__m512i)__builtin_ia32_selectd_512(
279 (__mmask16)__U, (__v16si)_mm512_dpwuud_epi32(__A, __B, __C),
280 (__v16si)__A);
281}
282
283static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwuud_epi32(
284 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
285 return (__m512i)__builtin_ia32_selectd_512(
286 (__mmask16)__U, (__v16si)_mm512_dpwuud_epi32(__A, __B, __C),
287 (__v16si)_mm512_setzero_si512());
288}
289
290static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwuuds_epi32(__m512i __A,
291 __m512i __B,
292 __m512i __C) {
293 return (__m512i)__builtin_ia32_vpdpwuuds512((__v16si)__A, (__v16si)__B,
294 (__v16si)__C);
295}
296
297static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwuuds_epi32(
298 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
299 return (__m512i)__builtin_ia32_selectd_512(
300 (__mmask16)__U, (__v16si)_mm512_dpwuuds_epi32(__A, __B, __C),
301 (__v16si)__A);
302}
303
304static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwuuds_epi32(
305 __m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
306 return (__m512i)__builtin_ia32_selectd_512(
307 (__mmask16)__U, (__v16si)_mm512_dpwuuds_epi32(__A, __B, __C),
308 (__v16si)_mm512_setzero_si512());
309}
310
311#undef __DEFAULT_FN_ATTRS
312
313#endif /* __SSE2__ */
314#endif /* __AVX10_2_512NIINTRIN_H */
lib/include/avx10_2_512satcvtdsintrin.h created+303
......@@ -0,0 +1,303 @@
1/*===----- avx10_2_512satcvtdsintrin.h - AVX10_2_512SATCVTDS intrinsics ----===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2_512satcvtdsintrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifndef __AVX10_2_512SATCVTDSINTRIN_H
15#define __AVX10_2_512SATCVTDSINTRIN_H
16
17/* Define the default attributes for the functions in this file. */
18#define __DEFAULT_FN_ATTRS \
19 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-512"), \
20 __min_vector_width__(512)))
21
22// 512 bit : Double -> Int
23static __inline__ __m256i __DEFAULT_FN_ATTRS _mm512_cvttspd_epi32(__m512d __A) {
24 return ((__m256i)__builtin_ia32_vcvttpd2dqs512_round_mask(
25 (__v8df)__A, (__v8si)_mm256_undefined_si256(), (__mmask8)-1,
26 _MM_FROUND_CUR_DIRECTION));
27}
28
29static __inline__ __m256i __DEFAULT_FN_ATTRS
30_mm512_mask_cvttspd_epi32(__m256i __W, __mmask8 __U, __m512d __A) {
31 return ((__m256i)__builtin_ia32_vcvttpd2dqs512_round_mask(
32 (__v8df)__A, (__v8si)__W, __U, _MM_FROUND_CUR_DIRECTION));
33}
34
35static __inline__ __m256i __DEFAULT_FN_ATTRS
36_mm512_maskz_cvttspd_epi32(__mmask8 __U, __m512d __A) {
37 return ((__m256i)__builtin_ia32_vcvttpd2dqs512_round_mask(
38 (__v8df)__A, (__v8si)_mm256_setzero_si256(), __U,
39 _MM_FROUND_CUR_DIRECTION));
40}
41
42#define _mm512_cvtts_roundpd_epi32(__A, __R) \
43 ((__m256i)__builtin_ia32_vcvttpd2dqs512_round_mask( \
44 (__v8df)(__m512d)(__A), (__v8si)_mm256_undefined_si256(), \
45 (__mmask8) - 1, (const int)(__R)))
46
47#define _mm512_mask_cvtts_roundpd_epi32(__W, __U, __A, __R) \
48 ((__m256i)__builtin_ia32_vcvttpd2dqs512_round_mask( \
49 (__v8df)(__m512d)(__A), (__v8si)(__m256i)(__W), (__mmask8)(__U), \
50 (const int)(__R)))
51
52#define _mm512_maskz_cvtts_roundpd_epi32(__U, __A, __R) \
53 ((__m256i)__builtin_ia32_vcvttpd2dqs512_round_mask( \
54 (__v8df)(__m512d)(__A), (__v8si)_mm256_setzero_si256(), (__mmask8)(__U), \
55 (const int)(__R)))
56
57// 512 bit : Double -> uInt
58static __inline__ __m256i __DEFAULT_FN_ATTRS _mm512_cvttspd_epu32(__m512d __A) {
59 return ((__m256i)__builtin_ia32_vcvttpd2udqs512_round_mask(
60 (__v8df)__A, (__v8si)_mm256_undefined_si256(), (__mmask8)-1,
61 _MM_FROUND_CUR_DIRECTION));
62}
63
64static __inline__ __m256i __DEFAULT_FN_ATTRS
65_mm512_mask_cvttspd_epu32(__m256i __W, __mmask8 __U, __m512d __A) {
66 return ((__m256i)__builtin_ia32_vcvttpd2udqs512_round_mask(
67 (__v8df)__A, (__v8si)__W, __U, _MM_FROUND_CUR_DIRECTION));
68}
69
70static __inline__ __m256i __DEFAULT_FN_ATTRS
71_mm512_maskz_cvttspd_epu32(__mmask8 __U, __m512d __A) {
72 return ((__m256i)__builtin_ia32_vcvttpd2udqs512_round_mask(
73 (__v8df)__A, (__v8si)_mm256_setzero_si256(), __U,
74 _MM_FROUND_CUR_DIRECTION));
75}
76
77#define _mm512_cvtts_roundpd_epu32(__A, __R) \
78 ((__m256i)__builtin_ia32_vcvttpd2udqs512_round_mask( \
79 (__v8df)(__m512d)(__A), (__v8si)_mm256_undefined_si256(), \
80 (__mmask8) - 1, (const int)(__R)))
81
82#define _mm512_mask_cvtts_roundpd_epu32(__W, __U, __A, __R) \
83 ((__m256i)__builtin_ia32_vcvttpd2udqs512_round_mask( \
84 (__v8df)(__m512d)(__A), (__v8si)(__m256i)(__W), (__mmask8)(__U), \
85 (const int)(__R)))
86
87#define _mm512_maskz_cvtts_roundpd_epu32(__U, __A, __R) \
88 ((__m256i)__builtin_ia32_vcvttpd2udqs512_round_mask( \
89 (__v8df)(__m512d)(__A), (__v8si)_mm256_setzero_si256(), (__mmask8)(__U), \
90 (const int)(__R)))
91
92// 512 bit : Double -> Long
93
94static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_cvttspd_epi64(__m512d __A) {
95 return ((__m512i)__builtin_ia32_vcvttpd2qqs512_round_mask(
96 (__v8df)__A, (__v8di)_mm512_undefined_epi32(), (__mmask8)-1,
97 _MM_FROUND_CUR_DIRECTION));
98}
99static __inline__ __m512i __DEFAULT_FN_ATTRS
100_mm512_mask_cvttspd_epi64(__m512i __W, __mmask8 __U, __m512d __A) {
101 return ((__m512i)__builtin_ia32_vcvttpd2qqs512_round_mask(
102 (__v8df)__A, (__v8di)__W, __U, _MM_FROUND_CUR_DIRECTION));
103}
104static __inline__ __m512i __DEFAULT_FN_ATTRS
105_mm512_maskz_cvttspd_epi64(__mmask8 __U, __m512d __A) {
106 return ((__m512i)__builtin_ia32_vcvttpd2qqs512_round_mask(
107 (__v8df)__A, (__v8di)_mm512_setzero_si512(), __U,
108 _MM_FROUND_CUR_DIRECTION));
109}
110
111#define _mm512_cvtts_roundpd_epi64(__A, __R) \
112 ((__m512i)__builtin_ia32_vcvttpd2qqs512_round_mask( \
113 (__v8df)(__m512d)(__A), (__v8di)_mm512_undefined_epi32(), \
114 (__mmask8) - 1, (const int)(__R)))
115
116#define _mm512_mask_cvtts_roundpd_epi64(__W, __U, __A, __R) \
117 ((__m512i)__builtin_ia32_vcvttpd2qqs512_round_mask( \
118 (__v8df)(__m512d)(__A), (__v8di)(__m512i)(__W), (__mmask8)(__U), \
119 (const int)(__R)))
120
121#define _mm512_maskz_cvtts_roundpd_epi64(__U, __A, __R) \
122 ((__m512i)__builtin_ia32_vcvttpd2qqs512_round_mask( \
123 (__v8df)(__m512d)(__A), (__v8di)_mm512_setzero_si512(), (__mmask8)(__U), \
124 (const int)(__R)))
125
126// 512 bit : Double -> ULong
127
128static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_cvttspd_epu64(__m512d __A) {
129 return ((__m512i)__builtin_ia32_vcvttpd2uqqs512_round_mask(
130 (__v8df)__A, (__v8di)_mm512_undefined_epi32(), (__mmask8)-1,
131 _MM_FROUND_CUR_DIRECTION));
132}
133
134static __inline__ __m512i __DEFAULT_FN_ATTRS
135_mm512_mask_cvttspd_epu64(__m512i __W, __mmask8 __U, __m512d __A) {
136 return ((__m512i)__builtin_ia32_vcvttpd2uqqs512_round_mask(
137 (__v8df)__A, (__v8di)__W, __U, _MM_FROUND_CUR_DIRECTION));
138}
139
140static __inline__ __m512i __DEFAULT_FN_ATTRS
141_mm512_maskz_cvttspd_epu64(__mmask8 __U, __m512d __A) {
142 return ((__m512i)__builtin_ia32_vcvttpd2uqqs512_round_mask(
143 (__v8df)__A, (__v8di)_mm512_setzero_si512(), __U,
144 _MM_FROUND_CUR_DIRECTION));
145}
146
147#define _mm512_cvtts_roundpd_epu64(__A, __R) \
148 ((__m512i)__builtin_ia32_vcvttpd2uqqs512_round_mask( \
149 (__v8df)(__m512d)(__A), (__v8di)_mm512_undefined_epi32(), \
150 (__mmask8) - 1, (const int)(__R)))
151
152#define _mm512_mask_cvtts_roundpd_epu64(__W, __U, __A, __R) \
153 ((__m512i)__builtin_ia32_vcvttpd2uqqs512_round_mask( \
154 (__v8df)(__m512d)(__A), (__v8di)(__m512i)(__W), (__mmask8)(__U), \
155 (const int)(__R)))
156
157#define _mm512_maskz_cvtts_roundpd_epu64(__U, __A, __R) \
158 ((__m512i)__builtin_ia32_vcvttpd2uqqs512_round_mask( \
159 (__v8df)(__m512d)(__A), (__v8di)_mm512_setzero_si512(), (__mmask8)(__U), \
160 (const int)(__R)))
161
162// 512 bit: Float -> int
163static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_cvttsps_epi32(__m512 __A) {
164 return ((__m512i)__builtin_ia32_vcvttps2dqs512_round_mask(
165 (__v16sf)(__A), (__v16si)_mm512_undefined_epi32(), (__mmask16)-1,
166 _MM_FROUND_CUR_DIRECTION));
167}
168
169static __inline__ __m512i __DEFAULT_FN_ATTRS
170_mm512_mask_cvttsps_epi32(__m512i __W, __mmask16 __U, __m512 __A) {
171 return ((__m512i)__builtin_ia32_vcvttps2dqs512_round_mask(
172 (__v16sf)(__A), (__v16si)(__W), __U, _MM_FROUND_CUR_DIRECTION));
173}
174
175static __inline__ __m512i __DEFAULT_FN_ATTRS
176_mm512_maskz_cvttsps_epi32(__mmask16 __U, __m512 __A) {
177 return ((__m512i)__builtin_ia32_vcvttps2dqs512_round_mask(
178 (__v16sf)(__A), (__v16si)_mm512_setzero_si512(), __U,
179 _MM_FROUND_CUR_DIRECTION));
180}
181
182#define _mm512_cvtts_roundps_epi32(__A, __R) \
183 ((__m512i)__builtin_ia32_vcvttps2dqs512_round_mask( \
184 (__v16sf)(__m512)(__A), (__v16si)_mm512_undefined_epi32(), \
185 (__mmask16) - 1, (const int)(__R)))
186
187#define _mm512_mask_cvtts_roundps_epi32(__W, __U, __A, __R) \
188 ((__m512i)__builtin_ia32_vcvttps2dqs512_round_mask( \
189 (__v16sf)(__m512)(__A), (__v16si)(__m512i)(__W), (__mmask16)(__U), \
190 (const int)(__R)))
191
192#define _mm512_maskz_cvtts_roundps_epi32(__U, __A, __R) \
193 ((__m512i)__builtin_ia32_vcvttps2dqs512_round_mask( \
194 (__v16sf)(__m512)(__A), (__v16si)_mm512_setzero_si512(), \
195 (__mmask16)(__U), (const int)(__R)))
196
197// 512 bit: Float -> uint
198static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_cvttsps_epu32(__m512 __A) {
199 return ((__m512i)__builtin_ia32_vcvttps2udqs512_round_mask(
200 (__v16sf)(__A), (__v16si)_mm512_undefined_epi32(), (__mmask16)-1,
201 _MM_FROUND_CUR_DIRECTION));
202}
203
204static __inline__ __m512i __DEFAULT_FN_ATTRS
205_mm512_mask_cvttsps_epu32(__m512i __W, __mmask16 __U, __m512 __A) {
206 return ((__m512i)__builtin_ia32_vcvttps2udqs512_round_mask(
207 (__v16sf)(__A), (__v16si)(__W), __U, _MM_FROUND_CUR_DIRECTION));
208}
209
210static __inline__ __m512i __DEFAULT_FN_ATTRS
211_mm512_maskz_cvttsps_epu32(__mmask16 __U, __m512 __A) {
212 return ((__m512i)__builtin_ia32_vcvttps2udqs512_round_mask(
213 (__v16sf)(__A), (__v16si)_mm512_setzero_si512(), __U,
214 _MM_FROUND_CUR_DIRECTION));
215}
216
217#define _mm512_cvtts_roundps_epu32(__A, __R) \
218 ((__m512i)__builtin_ia32_vcvttps2udqs512_round_mask( \
219 (__v16sf)(__m512)(__A), (__v16si)_mm512_undefined_epi32(), \
220 (__mmask16) - 1, (const int)(__R)))
221
222#define _mm512_mask_cvtts_roundps_epu32(__W, __U, __A, __R) \
223 ((__m512i)__builtin_ia32_vcvttps2udqs512_round_mask( \
224 (__v16sf)(__m512)(__A), (__v16si)(__m512i)(__W), (__mmask16)(__U), \
225 (const int)(__R)))
226
227#define _mm512_maskz_cvtts_roundps_epu32(__U, __A, __R) \
228 ((__m512i)__builtin_ia32_vcvttps2udqs512_round_mask( \
229 (__v16sf)(__m512)(__A), (__v16si)_mm512_setzero_si512(), \
230 (__mmask16)(__U), (const int)(__R)))
231
232// 512 bit : float -> long
233static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_cvttsps_epi64(__m256 __A) {
234 return ((__m512i)__builtin_ia32_vcvttps2qqs512_round_mask(
235 (__v8sf)__A, (__v8di)_mm512_undefined_epi32(), (__mmask8)-1,
236 _MM_FROUND_CUR_DIRECTION));
237}
238
239static __inline__ __m512i __DEFAULT_FN_ATTRS
240_mm512_mask_cvttsps_epi64(__m512i __W, __mmask8 __U, __m256 __A) {
241 return ((__m512i)__builtin_ia32_vcvttps2qqs512_round_mask(
242 (__v8sf)__A, (__v8di)__W, __U, _MM_FROUND_CUR_DIRECTION));
243}
244
245static __inline__ __m512i __DEFAULT_FN_ATTRS
246_mm512_maskz_cvttsps_epi64(__mmask8 __U, __m256 __A) {
247 return ((__m512i)__builtin_ia32_vcvttps2qqs512_round_mask(
248 (__v8sf)__A, (__v8di)_mm512_setzero_si512(), __U,
249 _MM_FROUND_CUR_DIRECTION));
250}
251
252#define _mm512_cvtts_roundps_epi64(__A, __R) \
253 ((__m512i)__builtin_ia32_vcvttps2qqs512_round_mask( \
254 (__v8sf)(__m256)(__A), (__v8di)_mm512_undefined_epi32(), (__mmask8) - 1, \
255 (const int)(__R)))
256
257#define _mm512_mask_cvtts_roundps_epi64(__W, __U, __A, __R) \
258 ((__m512i)__builtin_ia32_vcvttps2qqs512_round_mask( \
259 (__v8sf)(__m256)(__A), (__v8di)(__m512i)(__W), (__mmask8)(__U), \
260 (const int)(__R)))
261
262#define _mm512_maskz_cvtts_roundps_epi64(__U, __A, __R) \
263 ((__m512i)__builtin_ia32_vcvttps2qqs512_round_mask( \
264 (__v8sf)(__m256)(__A), (__v8di)_mm512_setzero_si512(), (__mmask8)(__U), \
265 (const int)(__R)))
266
267// 512 bit : float -> ulong
268static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_cvttsps_epu64(__m256 __A) {
269 return ((__m512i)__builtin_ia32_vcvttps2uqqs512_round_mask(
270 (__v8sf)__A, (__v8di)_mm512_undefined_epi32(), (__mmask8)-1,
271 _MM_FROUND_CUR_DIRECTION));
272}
273
274static __inline__ __m512i __DEFAULT_FN_ATTRS
275_mm512_mask_cvttsps_epu64(__m512i __W, __mmask8 __U, __m256 __A) {
276 return ((__m512i)__builtin_ia32_vcvttps2uqqs512_round_mask(
277 (__v8sf)__A, (__v8di)__W, __U, _MM_FROUND_CUR_DIRECTION));
278}
279
280static __inline__ __m512i __DEFAULT_FN_ATTRS
281_mm512_maskz_cvttsps_epu64(__mmask8 __U, __m256 __A) {
282 return ((__m512i)__builtin_ia32_vcvttps2uqqs512_round_mask(
283 (__v8sf)__A, (__v8di)_mm512_setzero_si512(), __U,
284 _MM_FROUND_CUR_DIRECTION));
285}
286
287#define _mm512_cvtts_roundps_epu64(__A, __R) \
288 ((__m512i)__builtin_ia32_vcvttps2uqqs512_round_mask( \
289 (__v8sf)(__m256)(__A), (__v8di)_mm512_undefined_epi32(), (__mmask8) - 1, \
290 (const int)(__R)))
291
292#define _mm512_mask_cvtts_roundps_epu64(__W, __U, __A, __R) \
293 ((__m512i)__builtin_ia32_vcvttps2uqqs512_round_mask( \
294 (__v8sf)(__m256)(__A), (__v8di)(__m512i)(__W), (__mmask8)(__U), \
295 (const int)(__R)))
296
297#define _mm512_maskz_cvtts_roundps_epu64(__U, __A, __R) \
298 ((__m512i)__builtin_ia32_vcvttps2uqqs512_round_mask( \
299 (__v8sf)(__m256)(__A), (__v8di)_mm512_setzero_si512(), (__mmask8)(__U), \
300 (const int)(__R)))
301
302#undef __DEFAULT_FN_ATTRS
303#endif // __AVX10_2_512SATCVTDSINTRIN_H
lib/include/avx10_2_512satcvtintrin.h created+301
......@@ -0,0 +1,301 @@
1/*===------ avx10_2_512satcvtintrin.h - AVX10_2_512SATCVT intrinsics -------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2_512satcvtintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifndef __AVX10_2_512SATCVTINTRIN_H
15#define __AVX10_2_512SATCVTINTRIN_H
16
17#define _mm512_ipcvtbf16_epi8(A) \
18 ((__m512i)__builtin_ia32_vcvtbf162ibs512((__v32bf)(__m512bh)(A)))
19
20#define _mm512_mask_ipcvtbf16_epi8(W, U, A) \
21 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
22 (__v32hi)_mm512_ipcvtbf16_epi8(A), \
23 (__v32hi)(__m512i)(W)))
24
25#define _mm512_maskz_ipcvtbf16_epi8(U, A) \
26 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
27 (__v32hi)_mm512_ipcvtbf16_epi8(A), \
28 (__v32hi)_mm512_setzero_si512()))
29
30#define _mm512_ipcvtbf16_epu8(A) \
31 ((__m512i)__builtin_ia32_vcvtbf162iubs512((__v32bf)(__m512bh)(A)))
32
33#define _mm512_mask_ipcvtbf16_epu8(W, U, A) \
34 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
35 (__v32hi)_mm512_ipcvtbf16_epu8(A), \
36 (__v32hi)(__m512i)(W)))
37
38#define _mm512_maskz_ipcvtbf16_epu8(U, A) \
39 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
40 (__v32hi)_mm512_ipcvtbf16_epu8(A), \
41 (__v32hi)_mm512_setzero_si512()))
42
43#define _mm512_ipcvttbf16_epi8(A) \
44 ((__m512i)__builtin_ia32_vcvttbf162ibs512((__v32bf)(__m512bh)(A)))
45
46#define _mm512_mask_ipcvttbf16_epi8(W, U, A) \
47 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
48 (__v32hi)_mm512_ipcvttbf16_epi8(A), \
49 (__v32hi)(__m512i)(W)))
50
51#define _mm512_maskz_ipcvttbf16_epi8(U, A) \
52 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
53 (__v32hi)_mm512_ipcvttbf16_epi8(A), \
54 (__v32hi)_mm512_setzero_si512()))
55
56#define _mm512_ipcvttbf16_epu8(A) \
57 ((__m512i)__builtin_ia32_vcvttbf162iubs512((__v32bf)(__m512bh)(A)))
58
59#define _mm512_mask_ipcvttbf16_epu8(W, U, A) \
60 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
61 (__v32hi)_mm512_ipcvttbf16_epu8(A), \
62 (__v32hi)(__m512i)(W)))
63
64#define _mm512_maskz_ipcvttbf16_epu8(U, A) \
65 ((__m512i)__builtin_ia32_selectw_512((__mmask32)(U), \
66 (__v32hi)_mm512_ipcvttbf16_epu8(A), \
67 (__v32hi)_mm512_setzero_si512()))
68
69#define _mm512_ipcvtph_epi8(A) \
70 ((__m512i)__builtin_ia32_vcvtph2ibs512_mask( \
71 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)-1, \
72 _MM_FROUND_CUR_DIRECTION))
73
74#define _mm512_mask_ipcvtph_epi8(W, U, A) \
75 ((__m512i)__builtin_ia32_vcvtph2ibs512_mask((__v32hf)(__m512h)(A), \
76 (__v32hu)(W), (__mmask32)(U), \
77 _MM_FROUND_CUR_DIRECTION))
78
79#define _mm512_maskz_ipcvtph_epi8(U, A) \
80 ((__m512i)__builtin_ia32_vcvtph2ibs512_mask( \
81 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)(U), \
82 _MM_FROUND_CUR_DIRECTION))
83
84#define _mm512_ipcvt_roundph_epi8(A, R) \
85 ((__m512i)__builtin_ia32_vcvtph2ibs512_mask((__v32hf)(__m512h)(A), \
86 (__v32hu)_mm512_setzero_si512(), \
87 (__mmask32)-1, (const int)R))
88
89#define _mm512_mask_ipcvt_roundph_epi8(W, U, A, R) \
90 ((__m512i)__builtin_ia32_vcvtph2ibs512_mask( \
91 (__v32hf)(__m512h)(A), (__v32hu)(W), (__mmask32)(U), (const int)R))
92
93#define _mm512_maskz_ipcvt_roundph_epi8(U, A, R) \
94 ((__m512i)__builtin_ia32_vcvtph2ibs512_mask((__v32hf)(__m512h)(A), \
95 (__v32hu)_mm512_setzero_si512(), \
96 (__mmask32)(U), (const int)R))
97
98#define _mm512_ipcvtph_epu8(A) \
99 ((__m512i)__builtin_ia32_vcvtph2iubs512_mask( \
100 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)-1, \
101 _MM_FROUND_CUR_DIRECTION))
102
103#define _mm512_mask_ipcvtph_epu8(W, U, A) \
104 ((__m512i)__builtin_ia32_vcvtph2iubs512_mask((__v32hf)(__m512h)(A), \
105 (__v32hu)(W), (__mmask32)(U), \
106 _MM_FROUND_CUR_DIRECTION))
107
108#define _mm512_maskz_ipcvtph_epu8(U, A) \
109 ((__m512i)__builtin_ia32_vcvtph2iubs512_mask( \
110 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)(U), \
111 _MM_FROUND_CUR_DIRECTION))
112
113#define _mm512_ipcvt_roundph_epu8(A, R) \
114 ((__m512i)__builtin_ia32_vcvtph2iubs512_mask( \
115 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)-1, \
116 (const int)R))
117
118#define _mm512_mask_ipcvt_roundph_epu8(W, U, A, R) \
119 ((__m512i)__builtin_ia32_vcvtph2iubs512_mask( \
120 (__v32hf)(__m512h)(A), (__v32hu)(W), (__mmask32)(U), (const int)R))
121
122#define _mm512_maskz_ipcvt_roundph_epu8(U, A, R) \
123 ((__m512i)__builtin_ia32_vcvtph2iubs512_mask( \
124 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)(U), \
125 (const int)R))
126
127#define _mm512_ipcvtps_epi8(A) \
128 ((__m512i)__builtin_ia32_vcvtps2ibs512_mask( \
129 (__v16sf)(__m512)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)-1, \
130 _MM_FROUND_CUR_DIRECTION))
131
132#define _mm512_mask_ipcvtps_epi8(W, U, A) \
133 ((__m512i)__builtin_ia32_vcvtps2ibs512_mask((__v16sf)(__m512)(A), \
134 (__v16su)(W), (__mmask16)(U), \
135 _MM_FROUND_CUR_DIRECTION))
136
137#define _mm512_maskz_ipcvtps_epi8(U, A) \
138 ((__m512i)__builtin_ia32_vcvtps2ibs512_mask( \
139 (__v16sf)(__m512)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)(U), \
140 _MM_FROUND_CUR_DIRECTION))
141
142#define _mm512_ipcvt_roundps_epi8(A, R) \
143 ((__m512i)__builtin_ia32_vcvtps2ibs512_mask((__v16sf)(__m512)(A), \
144 (__v16su)_mm512_setzero_si512(), \
145 (__mmask16)-1, (const int)R))
146
147#define _mm512_mask_ipcvt_roundps_epi8(W, U, A, R) \
148 ((__m512i)__builtin_ia32_vcvtps2ibs512_mask( \
149 (__v16sf)(__m512)(A), (__v16su)(W), (__mmask16)(U), (const int)R))
150
151#define _mm512_maskz_ipcvt_roundps_epi8(U, A, R) \
152 ((__m512i)__builtin_ia32_vcvtps2ibs512_mask((__v16sf)(__m512)(A), \
153 (__v16su)_mm512_setzero_si512(), \
154 (__mmask16)(U), (const int)R))
155
156#define _mm512_ipcvtps_epu8(A) \
157 ((__m512i)__builtin_ia32_vcvtps2iubs512_mask( \
158 (__v16sf)(__m512)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)-1, \
159 _MM_FROUND_CUR_DIRECTION))
160
161#define _mm512_mask_ipcvtps_epu8(W, U, A) \
162 ((__m512i)__builtin_ia32_vcvtps2iubs512_mask((__v16sf)(__m512)(A), \
163 (__v16su)(W), (__mmask16)(U), \
164 _MM_FROUND_CUR_DIRECTION))
165
166#define _mm512_maskz_ipcvtps_epu8(U, A) \
167 ((__m512i)__builtin_ia32_vcvtps2iubs512_mask( \
168 (__v16sf)(__m512)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)(U), \
169 _MM_FROUND_CUR_DIRECTION))
170
171#define _mm512_ipcvt_roundps_epu8(A, R) \
172 ((__m512i)__builtin_ia32_vcvtps2iubs512_mask( \
173 (__v16sf)(__m512)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)-1, \
174 (const int)R))
175
176#define _mm512_mask_ipcvt_roundps_epu8(W, U, A, R) \
177 ((__m512i)__builtin_ia32_vcvtps2iubs512_mask( \
178 (__v16sf)(__m512)(A), (__v16su)(W), (__mmask16)(U), (const int)R))
179
180#define _mm512_maskz_ipcvt_roundps_epu8(U, A, R) \
181 ((__m512i)__builtin_ia32_vcvtps2iubs512_mask( \
182 (__v16sf)(__m512)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)(U), \
183 (const int)R))
184
185#define _mm512_ipcvttph_epi8(A) \
186 ((__m512i)__builtin_ia32_vcvttph2ibs512_mask( \
187 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)-1, \
188 _MM_FROUND_CUR_DIRECTION))
189
190#define _mm512_mask_ipcvttph_epi8(W, U, A) \
191 ((__m512i)__builtin_ia32_vcvttph2ibs512_mask((__v32hf)(__m512h)(A), \
192 (__v32hu)(W), (__mmask32)(U), \
193 _MM_FROUND_CUR_DIRECTION))
194
195#define _mm512_maskz_ipcvttph_epi8(U, A) \
196 ((__m512i)__builtin_ia32_vcvttph2ibs512_mask( \
197 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)(U), \
198 _MM_FROUND_CUR_DIRECTION))
199
200#define _mm512_ipcvtt_roundph_epi8(A, S) \
201 ((__m512i)__builtin_ia32_vcvttph2ibs512_mask( \
202 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)-1, \
203 S))
204
205#define _mm512_mask_ipcvtt_roundph_epi8(W, U, A, S) \
206 ((__m512i)__builtin_ia32_vcvttph2ibs512_mask( \
207 (__v32hf)(__m512h)(A), (__v32hu)(W), (__mmask32)(U), S))
208
209#define _mm512_maskz_ipcvtt_roundph_epi8(U, A, S) \
210 ((__m512i)__builtin_ia32_vcvttph2ibs512_mask( \
211 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)(U), \
212 S))
213
214#define _mm512_ipcvttph_epu8(A) \
215 ((__m512i)__builtin_ia32_vcvttph2iubs512_mask( \
216 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)-1, \
217 _MM_FROUND_CUR_DIRECTION))
218
219#define _mm512_mask_ipcvttph_epu8(W, U, A) \
220 ((__m512i)__builtin_ia32_vcvttph2iubs512_mask((__v32hf)(__m512h)(A), \
221 (__v32hu)(W), (__mmask32)(U), \
222 _MM_FROUND_CUR_DIRECTION))
223
224#define _mm512_maskz_ipcvttph_epu8(U, A) \
225 ((__m512i)__builtin_ia32_vcvttph2iubs512_mask( \
226 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)(U), \
227 _MM_FROUND_CUR_DIRECTION))
228
229#define _mm512_ipcvtt_roundph_epu8(A, S) \
230 ((__m512i)__builtin_ia32_vcvttph2iubs512_mask( \
231 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)-1, \
232 S))
233
234#define _mm512_mask_ipcvtt_roundph_epu8(W, U, A, S) \
235 ((__m512i)__builtin_ia32_vcvttph2iubs512_mask( \
236 (__v32hf)(__m512h)(A), (__v32hu)(W), (__mmask32)(U), S))
237
238#define _mm512_maskz_ipcvtt_roundph_epu8(U, A, S) \
239 ((__m512i)__builtin_ia32_vcvttph2iubs512_mask( \
240 (__v32hf)(__m512h)(A), (__v32hu)_mm512_setzero_si512(), (__mmask32)(U), \
241 S))
242
243#define _mm512_ipcvttps_epi8(A) \
244 ((__m512i)__builtin_ia32_vcvttps2ibs512_mask( \
245 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)-1, \
246 _MM_FROUND_CUR_DIRECTION))
247
248#define _mm512_mask_ipcvttps_epi8(W, U, A) \
249 ((__m512i)__builtin_ia32_vcvttps2ibs512_mask((__v16sf)(__m512h)(A), \
250 (__v16su)(W), (__mmask16)(U), \
251 _MM_FROUND_CUR_DIRECTION))
252
253#define _mm512_maskz_ipcvttps_epi8(U, A) \
254 ((__m512i)__builtin_ia32_vcvttps2ibs512_mask( \
255 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)(U), \
256 _MM_FROUND_CUR_DIRECTION))
257
258#define _mm512_ipcvtt_roundps_epi8(A, S) \
259 ((__m512i)__builtin_ia32_vcvttps2ibs512_mask( \
260 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)-1, \
261 S))
262
263#define _mm512_mask_ipcvtt_roundps_epi8(W, U, A, S) \
264 ((__m512i)__builtin_ia32_vcvttps2ibs512_mask( \
265 (__v16sf)(__m512h)(A), (__v16su)(W), (__mmask16)(U), S))
266
267#define _mm512_maskz_ipcvtt_roundps_epi8(U, A, S) \
268 ((__m512i)__builtin_ia32_vcvttps2ibs512_mask( \
269 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)(U), \
270 S))
271
272#define _mm512_ipcvttps_epu8(A) \
273 ((__m512i)__builtin_ia32_vcvttps2iubs512_mask( \
274 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)-1, \
275 _MM_FROUND_CUR_DIRECTION))
276
277#define _mm512_mask_ipcvttps_epu8(W, U, A) \
278 ((__m512i)__builtin_ia32_vcvttps2iubs512_mask((__v16sf)(__m512h)(A), \
279 (__v16su)(W), (__mmask16)(U), \
280 _MM_FROUND_CUR_DIRECTION))
281
282#define _mm512_maskz_ipcvttps_epu8(U, A) \
283 ((__m512i)__builtin_ia32_vcvttps2iubs512_mask( \
284 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)(U), \
285 _MM_FROUND_CUR_DIRECTION))
286
287#define _mm512_ipcvtt_roundps_epu8(A, S) \
288 ((__m512i)__builtin_ia32_vcvttps2iubs512_mask( \
289 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)-1, \
290 S))
291
292#define _mm512_mask_ipcvtt_roundps_epu8(W, U, A, S) \
293 ((__m512i)__builtin_ia32_vcvttps2iubs512_mask( \
294 (__v16sf)(__m512h)(A), (__v16su)(W), (__mmask16)(U), S))
295
296#define _mm512_maskz_ipcvtt_roundps_epu8(U, A, S) \
297 ((__m512i)__builtin_ia32_vcvttps2iubs512_mask( \
298 (__v16sf)(__m512h)(A), (__v16su)_mm512_setzero_si512(), (__mmask16)(U), \
299 S))
300
301#endif // __AVX10_2_512SATCVTINTRIN_H
lib/include/avx10_2bf16intrin.h created+1085
......@@ -0,0 +1,1085 @@
1/*===-------------- avx10_2bf16intrin.h - AVX10-BF16 intrinsics ------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2bf16intrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifdef __SSE2__
15
16#ifndef __AVX10_2BF16INTRIN_H
17#define __AVX10_2BF16INTRIN_H
18
19typedef __bf16 __m128bh_u __attribute__((__vector_size__(16), __aligned__(1)));
20typedef __bf16 __m256bh_u __attribute__((__vector_size__(32), __aligned__(1)));
21
22/* Define the default attributes for the functions in this file. */
23#define __DEFAULT_FN_ATTRS256 \
24 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
25 __min_vector_width__(256)))
26#define __DEFAULT_FN_ATTRS128 \
27 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
28 __min_vector_width__(128)))
29
30static __inline __m256bh __DEFAULT_FN_ATTRS256 _mm256_setzero_pbh(void) {
31 return __builtin_bit_cast(__m256bh, _mm256_setzero_ps());
32}
33
34static __inline __m128bh __DEFAULT_FN_ATTRS128 _mm_setzero_pbh(void) {
35 return __builtin_bit_cast(__m128bh, _mm_setzero_ps());
36}
37
38static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_castbf16_ps(__m128bh __a) {
39 return (__m128)__a;
40}
41
42static __inline__ __m256 __DEFAULT_FN_ATTRS256
43_mm256_castbf16_ps(__m256bh __a) {
44 return (__m256)__a;
45}
46
47static __inline__ __m256d __DEFAULT_FN_ATTRS256
48_mm256_castbf16_pd(__m256bh __a) {
49 return (__m256d)__a;
50}
51
52static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_castbf16_pd(__m128bh __a) {
53 return (__m128d)__a;
54}
55
56static __inline__ __m128i __DEFAULT_FN_ATTRS128
57_mm_castbf16_si128(__m128bh __a) {
58 return (__m128i)__a;
59}
60
61static __inline__ __m256i __DEFAULT_FN_ATTRS256
62_mm256_castbf16_si256(__m256bh __a) {
63 return (__m256i)__a;
64}
65
66static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_castps_pbh(__m128 __a) {
67 return (__m128bh)__a;
68}
69
70static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_castps_pbh(__m256 __a) {
71 return (__m256bh)__a;
72}
73
74static __inline__ __bf16 __DEFAULT_FN_ATTRS128 _mm_cvtsbh_bf16(__m128bh __a) {
75 return __a[0];
76}
77
78static __inline__ __bf16 __DEFAULT_FN_ATTRS256
79_mm256_cvtsbh_bf16(__m256bh __a) {
80 return __a[0];
81}
82
83static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_castpd_pbh(__m128d __a) {
84 return (__m128bh)__a;
85}
86
87static __inline__ __m256bh __DEFAULT_FN_ATTRS256
88_mm256_castpd_pbh(__m256d __a) {
89 return (__m256bh)__a;
90}
91
92static __inline__ __m128bh __DEFAULT_FN_ATTRS128
93_mm_castsi128_pbh(__m128i __a) {
94 return (__m128bh)__a;
95}
96
97static __inline__ __m256bh __DEFAULT_FN_ATTRS256
98_mm256_castsi256_pbh(__m256i __a) {
99 return (__m256bh)__a;
100}
101
102static __inline__ __m128bh __DEFAULT_FN_ATTRS256
103_mm256_castbf16256_pbh128(__m256bh __a) {
104 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7);
105}
106
107static __inline__ __m256bh __DEFAULT_FN_ATTRS256
108_mm256_castbf16128_pbh256(__m128bh __a) {
109 return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, -1, -1, -1,
110 -1, -1, -1, -1, -1);
111}
112
113static __inline__ __m256bh __DEFAULT_FN_ATTRS256
114_mm256_zextbf16128_pbh256(__m128bh __a) {
115 return __builtin_shufflevector(__a, (__v8bf)_mm_setzero_pbh(), 0, 1, 2, 3, 4,
116 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
117}
118
119static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_undefined_pbh(void) {
120 return (__m256bh)__builtin_ia32_undef256();
121}
122
123static __inline__ __m128bh __DEFAULT_FN_ATTRS128
124_mm_load_sbh(void const *__dp) {
125 __m128bh src = (__v8bf)_mm_setzero_pbh();
126 return (__m128bh)__builtin_ia32_loadsbf16128_mask((const __v8bf *)__dp, src,
127 1);
128}
129
130static __inline__ __m128bh __DEFAULT_FN_ATTRS128
131_mm_mask_load_sbh(__m128bh __W, __mmask8 __U, const void *__A) {
132 __m128bh src = (__v8bf)__builtin_shufflevector(
133 (__v8bf)__W, (__v8bf)_mm_setzero_pbh(), 0, 8, 8, 8, 8, 8, 8, 8);
134
135 return (__m128bh)__builtin_ia32_loadsbf16128_mask((const __v8bf *)__A, src,
136 __U & 1);
137}
138
139static __inline__ __m128bh __DEFAULT_FN_ATTRS128
140_mm_maskz_load_sbh(__mmask8 __U, const void *__A) {
141 return (__m128bh)__builtin_ia32_loadsbf16128_mask(
142 (const __v8bf *)__A, (__v8bf)_mm_setzero_pbh(), __U & 1);
143}
144
145static __inline__ __m256bh __DEFAULT_FN_ATTRS256
146_mm256_load_pbh(void const *__p) {
147 return *(const __m256bh *)__p;
148}
149
150static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_load_pbh(void const *__p) {
151 return *(const __m128bh *)__p;
152}
153
154static __inline__ __m256bh __DEFAULT_FN_ATTRS256
155_mm256_loadu_pbh(void const *__p) {
156 struct __loadu_pbh {
157 __m256bh_u __v;
158 } __attribute__((__packed__, __may_alias__));
159 return ((const struct __loadu_pbh *)__p)->__v;
160}
161
162static __inline__ __m128bh __DEFAULT_FN_ATTRS128
163_mm_loadu_pbh(void const *__p) {
164 struct __loadu_pbh {
165 __m128bh_u __v;
166 } __attribute__((__packed__, __may_alias__));
167 return ((const struct __loadu_pbh *)__p)->__v;
168}
169
170static __inline__ void __DEFAULT_FN_ATTRS128 _mm_store_sbh(void *__dp,
171 __m128bh __a) {
172 struct __mm_store_sbh_struct {
173 __bf16 __u;
174 } __attribute__((__packed__, __may_alias__));
175 ((struct __mm_store_sbh_struct *)__dp)->__u = __a[0];
176}
177
178static __inline__ void __DEFAULT_FN_ATTRS128 _mm_mask_store_sbh(void *__W,
179 __mmask8 __U,
180 __m128bh __A) {
181 __builtin_ia32_storesbf16128_mask((__v8bf *)__W, __A, __U & 1);
182}
183
184static __inline__ void __DEFAULT_FN_ATTRS256 _mm256_store_pbh(void *__P,
185 __m256bh __A) {
186 *(__m256bh *)__P = __A;
187}
188
189static __inline__ void __DEFAULT_FN_ATTRS128 _mm_store_pbh(void *__P,
190 __m128bh __A) {
191 *(__m128bh *)__P = __A;
192}
193
194static __inline__ void __DEFAULT_FN_ATTRS256 _mm256_storeu_pbh(void *__P,
195 __m256bh __A) {
196 struct __storeu_pbh {
197 __m256bh_u __v;
198 } __attribute__((__packed__, __may_alias__));
199 ((struct __storeu_pbh *)__P)->__v = __A;
200}
201
202static __inline__ void __DEFAULT_FN_ATTRS128 _mm_storeu_pbh(void *__P,
203 __m128bh __A) {
204 struct __storeu_pbh {
205 __m128bh_u __v;
206 } __attribute__((__packed__, __may_alias__));
207 ((struct __storeu_pbh *)__P)->__v = __A;
208}
209
210static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_move_sbh(__m128bh __a,
211 __m128bh __b) {
212 __a[0] = __b[0];
213 return __a;
214}
215
216static __inline__ __m128bh __DEFAULT_FN_ATTRS128
217_mm_mask_move_sbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
218 return __builtin_ia32_selectsbf_128(__U, _mm_move_sbh(__A, __B), __W);
219}
220
221static __inline__ __m128bh __DEFAULT_FN_ATTRS128
222_mm_maskz_move_sbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
223 return __builtin_ia32_selectsbf_128(__U, _mm_move_sbh(__A, __B),
224 _mm_setzero_pbh());
225}
226
227static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_undefined_pbh(void) {
228 return (__m128bh)__builtin_ia32_undef128();
229}
230
231static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_set_sbh(__bf16 bf) {
232 return (__v8bf)__builtin_shufflevector(
233 (__v8bf){bf, bf, bf, bf, bf, bf, bf, bf}, (__v8bf)_mm_setzero_pbh(), 0, 8,
234 8, 8, 8, 8, 8, 8);
235}
236
237static __inline __m128bh __DEFAULT_FN_ATTRS128 _mm_set1_pbh(__bf16 bf) {
238 return (__m128bh)(__v8bf){bf, bf, bf, bf, bf, bf, bf, bf};
239}
240
241static __inline __m256bh __DEFAULT_FN_ATTRS256 _mm256_set1_pbh(__bf16 bf) {
242 return (__m256bh)(__v16bf){bf, bf, bf, bf, bf, bf, bf, bf,
243 bf, bf, bf, bf, bf, bf, bf, bf};
244}
245
246static __inline __m128bh __DEFAULT_FN_ATTRS128
247_mm_set_pbh(__bf16 bf1, __bf16 bf2, __bf16 bf3, __bf16 bf4, __bf16 bf5,
248 __bf16 bf6, __bf16 bf7, __bf16 bf8) {
249 return (__m128bh)(__v8bf){bf1, bf2, bf3, bf4, bf5, bf6, bf7, bf8};
250}
251
252static __inline __m256bh __DEFAULT_FN_ATTRS256 _mm256_set_pbh(
253 __bf16 bf1, __bf16 bf2, __bf16 bf3, __bf16 bf4, __bf16 bf5, __bf16 bf6,
254 __bf16 bf7, __bf16 bf8, __bf16 bf9, __bf16 bf10, __bf16 bf11, __bf16 bf12,
255 __bf16 bf13, __bf16 bf14, __bf16 bf15, __bf16 bf16) {
256 return (__m256bh)(__v16bf){bf1, bf2, bf3, bf4, bf5, bf6, bf7, bf8,
257 bf9, bf10, bf11, bf12, bf13, bf14, bf15, bf16};
258}
259
260#define _mm_setr_pbh(bf1, bf2, bf3, bf4, bf5, bf6, bf7, bf8) \
261 _mm_set_pbh((bf8), (bf7), (bf6), (bf5), (bf4), (bf3), (bf2), (bf1))
262
263#define _mm256_setr_pbh(bf1, bf2, bf3, bf4, bf5, bf6, bf7, bf8, bf9, bf10, \
264 bf11, bf12, bf13, bf14, bf15, bf16) \
265 _mm256_set_pbh((bf16), (bf15), (bf14), (bf13), (bf12), (bf11), (bf10), \
266 (bf9), (bf8), (bf7), (bf6), (bf5), (bf4), (bf3), (bf2), \
267 (bf1))
268
269static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_abs_pbh(__m256bh __A) {
270 return (__m256bh)_mm256_and_epi32(_mm256_set1_epi32(0x7FFF7FFF),
271 (__m256i)__A);
272}
273
274static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_abs_pbh(__m128bh __A) {
275 return (__m128bh)_mm_and_epi32(_mm_set1_epi32(0x7FFF7FFF), (__m128i)__A);
276}
277
278static __inline__ __m128bh __DEFAULT_FN_ATTRS128
279_mm_mask_blend_pbh(__mmask8 __U, __m128bh __A, __m128bh __W) {
280 return (__m128bh)__builtin_ia32_selectpbf_128((__mmask8)__U, (__v8bf)__W,
281 (__v8bf)__A);
282}
283
284static __inline__ __m256bh __DEFAULT_FN_ATTRS256
285_mm256_mask_blend_pbh(__mmask16 __U, __m256bh __A, __m256bh __W) {
286 return (__m256bh)__builtin_ia32_selectpbf_256((__mmask16)__U, (__v16bf)__W,
287 (__v16bf)__A);
288}
289
290static __inline__ __m128bh __DEFAULT_FN_ATTRS128
291_mm_permutex2var_pbh(__m128bh __A, __m128i __I, __m128bh __B) {
292 return (__m128bh)__builtin_ia32_vpermi2varhi128((__v8hi)__A, (__v8hi)__I,
293 (__v8hi)__B);
294}
295
296static __inline__ __m256bh __DEFAULT_FN_ATTRS256
297_mm256_permutex2var_pbh(__m256bh __A, __m256i __I, __m256bh __B) {
298 return (__m256bh)__builtin_ia32_vpermi2varhi256((__v16hi)__A, (__v16hi)__I,
299 (__v16hi)__B);
300}
301
302static __inline__ __m128bh __DEFAULT_FN_ATTRS128
303_mm_permutexvar_pbh(__m128i __A, __m128bh __B) {
304 return (__m128bh)__builtin_ia32_permvarhi128((__v8hi)__B, (__v8hi)__A);
305}
306
307static __inline__ __m256bh __DEFAULT_FN_ATTRS256
308_mm256_permutexvar_pbh(__m256i __A, __m256bh __B) {
309 return (__m256bh)__builtin_ia32_permvarhi256((__v16hi)__B, (__v16hi)__A);
310}
311
312static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_add_pbh(__m256bh __A,
313 __m256bh __B) {
314 return (__m256bh)((__v16bf)__A + (__v16bf)__B);
315}
316
317static __inline__ __m256bh __DEFAULT_FN_ATTRS256
318_mm256_mask_add_pbh(__m256bh __W, __mmask16 __U, __m256bh __A, __m256bh __B) {
319 return (__m256bh)__builtin_ia32_selectpbf_256(
320 (__mmask16)__U, (__v16bf)_mm256_add_pbh(__A, __B), (__v16bf)__W);
321}
322
323static __inline__ __m256bh __DEFAULT_FN_ATTRS256
324_mm256_maskz_add_pbh(__mmask16 __U, __m256bh __A, __m256bh __B) {
325 return (__m256bh)__builtin_ia32_selectpbf_256(
326 (__mmask16)__U, (__v16bf)_mm256_add_pbh(__A, __B),
327 (__v16bf)_mm256_setzero_pbh());
328}
329
330static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_add_pbh(__m128bh __A,
331 __m128bh __B) {
332 return (__m128bh)((__v8bf)__A + (__v8bf)__B);
333}
334
335static __inline__ __m128bh __DEFAULT_FN_ATTRS128
336_mm_mask_add_pbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
337 return (__m128bh)__builtin_ia32_selectpbf_128(
338 (__mmask8)__U, (__v8bf)_mm_add_pbh(__A, __B), (__v8bf)__W);
339}
340
341static __inline__ __m128bh __DEFAULT_FN_ATTRS128
342_mm_maskz_add_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
343 return (__m128bh)__builtin_ia32_selectpbf_128(
344 (__mmask8)__U, (__v8bf)_mm_add_pbh(__A, __B), (__v8bf)_mm_setzero_pbh());
345}
346
347static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_sub_pbh(__m256bh __A,
348 __m256bh __B) {
349 return (__m256bh)((__v16bf)__A - (__v16bf)__B);
350}
351
352static __inline__ __m256bh __DEFAULT_FN_ATTRS256
353_mm256_mask_sub_pbh(__m256bh __W, __mmask16 __U, __m256bh __A, __m256bh __B) {
354 return (__m256bh)__builtin_ia32_selectpbf_256(
355 (__mmask16)__U, (__v16bf)_mm256_sub_pbh(__A, __B), (__v16bf)__W);
356}
357
358static __inline__ __m256bh __DEFAULT_FN_ATTRS256
359_mm256_maskz_sub_pbh(__mmask16 __U, __m256bh __A, __m256bh __B) {
360 return (__m256bh)__builtin_ia32_selectpbf_256(
361 (__mmask16)__U, (__v16bf)_mm256_sub_pbh(__A, __B),
362 (__v16bf)_mm256_setzero_pbh());
363}
364
365static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_sub_pbh(__m128bh __A,
366 __m128bh __B) {
367 return (__m128bh)((__v8bf)__A - (__v8bf)__B);
368}
369
370static __inline__ __m128bh __DEFAULT_FN_ATTRS128
371_mm_mask_sub_pbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
372 return (__m128bh)__builtin_ia32_selectpbf_128(
373 (__mmask8)__U, (__v8bf)_mm_sub_pbh(__A, __B), (__v8bf)__W);
374}
375
376static __inline__ __m128bh __DEFAULT_FN_ATTRS128
377_mm_maskz_sub_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
378 return (__m128bh)__builtin_ia32_selectpbf_128(
379 (__mmask8)__U, (__v8bf)_mm_sub_pbh(__A, __B), (__v8bf)_mm_setzero_pbh());
380}
381
382static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mul_pbh(__m256bh __A,
383 __m256bh __B) {
384 return (__m256bh)((__v16bf)__A * (__v16bf)__B);
385}
386
387static __inline__ __m256bh __DEFAULT_FN_ATTRS256
388_mm256_mask_mul_pbh(__m256bh __W, __mmask16 __U, __m256bh __A, __m256bh __B) {
389 return (__m256bh)__builtin_ia32_selectpbf_256(
390 (__mmask16)__U, (__v16bf)_mm256_mul_pbh(__A, __B), (__v16bf)__W);
391}
392
393static __inline__ __m256bh __DEFAULT_FN_ATTRS256
394_mm256_maskz_mul_pbh(__mmask16 __U, __m256bh __A, __m256bh __B) {
395 return (__m256bh)__builtin_ia32_selectpbf_256(
396 (__mmask16)__U, (__v16bf)_mm256_mul_pbh(__A, __B),
397 (__v16bf)_mm256_setzero_pbh());
398}
399
400static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_mul_pbh(__m128bh __A,
401 __m128bh __B) {
402 return (__m128bh)((__v8bf)__A * (__v8bf)__B);
403}
404
405static __inline__ __m128bh __DEFAULT_FN_ATTRS128
406_mm_mask_mul_pbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
407 return (__m128bh)__builtin_ia32_selectpbf_128(
408 (__mmask8)__U, (__v8bf)_mm_mul_pbh(__A, __B), (__v8bf)__W);
409}
410
411static __inline__ __m128bh __DEFAULT_FN_ATTRS128
412_mm_maskz_mul_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
413 return (__m128bh)__builtin_ia32_selectpbf_128(
414 (__mmask8)__U, (__v8bf)_mm_mul_pbh(__A, __B), (__v8bf)_mm_setzero_pbh());
415}
416
417static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_div_pbh(__m256bh __A,
418 __m256bh __B) {
419 return (__m256bh)((__v16bf)__A / (__v16bf)__B);
420}
421
422static __inline__ __m256bh __DEFAULT_FN_ATTRS256
423_mm256_mask_div_pbh(__m256bh __W, __mmask16 __U, __m256bh __A, __m256bh __B) {
424 return (__m256bh)__builtin_ia32_selectpbf_256(
425 (__mmask16)__U, (__v16bf)_mm256_div_pbh(__A, __B), (__v16bf)__W);
426}
427
428static __inline__ __m256bh __DEFAULT_FN_ATTRS256
429_mm256_maskz_div_pbh(__mmask16 __U, __m256bh __A, __m256bh __B) {
430 return (__m256bh)__builtin_ia32_selectpbf_256(
431 (__mmask16)__U, (__v16bf)_mm256_div_pbh(__A, __B),
432 (__v16bf)_mm256_setzero_pbh());
433}
434
435static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_div_pbh(__m128bh __A,
436 __m128bh __B) {
437 return (__m128bh)((__v8bf)__A / (__v8bf)__B);
438}
439
440static __inline__ __m128bh __DEFAULT_FN_ATTRS128
441_mm_mask_div_pbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
442 return (__m128bh)__builtin_ia32_selectpbf_128(
443 (__mmask8)__U, (__v8bf)_mm_div_pbh(__A, __B), (__v8bf)__W);
444}
445
446static __inline__ __m128bh __DEFAULT_FN_ATTRS128
447_mm_maskz_div_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
448 return (__m128bh)__builtin_ia32_selectpbf_128(
449 (__mmask8)__U, (__v8bf)_mm_div_pbh(__A, __B), (__v8bf)_mm_setzero_pbh());
450}
451
452static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_max_pbh(__m256bh __A,
453 __m256bh __B) {
454 return (__m256bh)__builtin_ia32_vmaxbf16256((__v16bf)__A, (__v16bf)__B);
455}
456
457static __inline__ __m256bh __DEFAULT_FN_ATTRS256
458_mm256_mask_max_pbh(__m256bh __W, __mmask16 __U, __m256bh __A, __m256bh __B) {
459 return (__m256bh)__builtin_ia32_selectpbf_256(
460 (__mmask16)__U, (__v16bf)_mm256_max_pbh(__A, __B), (__v16bf)__W);
461}
462
463static __inline__ __m256bh __DEFAULT_FN_ATTRS256
464_mm256_maskz_max_pbh(__mmask16 __U, __m256bh __A, __m256bh __B) {
465 return (__m256bh)__builtin_ia32_selectpbf_256(
466 (__mmask16)__U, (__v16bf)_mm256_max_pbh(__A, __B),
467 (__v16bf)_mm256_setzero_pbh());
468}
469
470static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_max_pbh(__m128bh __A,
471 __m128bh __B) {
472 return (__m128bh)__builtin_ia32_vmaxbf16128((__v8bf)__A, (__v8bf)__B);
473}
474
475static __inline__ __m128bh __DEFAULT_FN_ATTRS128
476_mm_mask_max_pbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
477 return (__m128bh)__builtin_ia32_selectpbf_128(
478 (__mmask8)__U, (__v8bf)_mm_max_pbh(__A, __B), (__v8bf)__W);
479}
480
481static __inline__ __m128bh __DEFAULT_FN_ATTRS128
482_mm_maskz_max_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
483 return (__m128bh)__builtin_ia32_selectpbf_128(
484 (__mmask8)__U, (__v8bf)_mm_max_pbh(__A, __B), (__v8bf)_mm_setzero_pbh());
485}
486
487static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_min_pbh(__m256bh __A,
488 __m256bh __B) {
489 return (__m256bh)__builtin_ia32_vminbf16256((__v16bf)__A, (__v16bf)__B);
490}
491
492static __inline__ __m256bh __DEFAULT_FN_ATTRS256
493_mm256_mask_min_pbh(__m256bh __W, __mmask16 __U, __m256bh __A, __m256bh __B) {
494 return (__m256bh)__builtin_ia32_selectpbf_256(
495 (__mmask16)__U, (__v16bf)_mm256_min_pbh(__A, __B), (__v16bf)__W);
496}
497
498static __inline__ __m256bh __DEFAULT_FN_ATTRS256
499_mm256_maskz_min_pbh(__mmask16 __U, __m256bh __A, __m256bh __B) {
500 return (__m256bh)__builtin_ia32_selectpbf_256(
501 (__mmask16)__U, (__v16bf)_mm256_min_pbh(__A, __B),
502 (__v16bf)_mm256_setzero_pbh());
503}
504
505static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_min_pbh(__m128bh __A,
506 __m128bh __B) {
507 return (__m128bh)__builtin_ia32_vminbf16128((__v8bf)__A, (__v8bf)__B);
508}
509
510static __inline__ __m128bh __DEFAULT_FN_ATTRS128
511_mm_mask_min_pbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
512 return (__m128bh)__builtin_ia32_selectpbf_128(
513 (__mmask8)__U, (__v8bf)_mm_min_pbh(__A, __B), (__v8bf)__W);
514}
515
516static __inline__ __m128bh __DEFAULT_FN_ATTRS128
517_mm_maskz_min_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
518 return (__m128bh)__builtin_ia32_selectpbf_128(
519 (__mmask8)__U, (__v8bf)_mm_min_pbh(__A, __B), (__v8bf)_mm_setzero_pbh());
520}
521
522static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comieq_sbh(__m128bh A,
523 __m128bh B) {
524 return __builtin_ia32_vcomisbf16eq((__v8bf)A, (__v8bf)B);
525}
526
527static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comilt_sbh(__m128bh A,
528 __m128bh B) {
529 return __builtin_ia32_vcomisbf16lt((__v8bf)A, (__v8bf)B);
530}
531
532static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comile_sbh(__m128bh A,
533 __m128bh B) {
534 return __builtin_ia32_vcomisbf16le((__v8bf)A, (__v8bf)B);
535}
536
537static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comigt_sbh(__m128bh A,
538 __m128bh B) {
539 return __builtin_ia32_vcomisbf16gt((__v8bf)A, (__v8bf)B);
540}
541
542static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comige_sbh(__m128bh A,
543 __m128bh B) {
544 return __builtin_ia32_vcomisbf16ge((__v8bf)A, (__v8bf)B);
545}
546
547static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comineq_sbh(__m128bh A,
548 __m128bh B) {
549 return __builtin_ia32_vcomisbf16neq((__v8bf)A, (__v8bf)B);
550}
551
552#define _mm256_cmp_pbh_mask(__A, __B, __P) \
553 ((__mmask16)__builtin_ia32_vcmpbf16256_mask((__v16bf)(__m256bh)(__A), \
554 (__v16bf)(__m256bh)(__B), \
555 (int)(__P), (__mmask16) - 1))
556
557#define _mm256_mask_cmp_pbh_mask(__U, __A, __B, __P) \
558 ((__mmask16)__builtin_ia32_vcmpbf16256_mask((__v16bf)(__m256bh)(__A), \
559 (__v16bf)(__m256bh)(__B), \
560 (int)(__P), (__mmask16)(__U)))
561
562#define _mm_cmp_pbh_mask(__A, __B, __P) \
563 ((__mmask8)__builtin_ia32_vcmpbf16128_mask((__v8bf)(__m128bh)(__A), \
564 (__v8bf)(__m128bh)(__B), \
565 (int)(__P), (__mmask8) - 1))
566
567#define _mm_mask_cmp_pbh_mask(__U, __A, __B, __P) \
568 ((__mmask8)__builtin_ia32_vcmpbf16128_mask((__v8bf)(__m128bh)(__A), \
569 (__v8bf)(__m128bh)(__B), \
570 (int)(__P), (__mmask8)(__U)))
571
572#define _mm256_mask_fpclass_pbh_mask(__U, __A, imm) \
573 ((__mmask16)__builtin_ia32_vfpclassbf16256_mask( \
574 (__v16bf)(__m256bh)(__A), (int)(imm), (__mmask16)(__U)))
575
576#define _mm256_fpclass_pbh_mask(__A, imm) \
577 ((__mmask16)__builtin_ia32_vfpclassbf16256_mask( \
578 (__v16bf)(__m256bh)(__A), (int)(imm), (__mmask16) - 1))
579
580#define _mm_mask_fpclass_pbh_mask(__U, __A, imm) \
581 ((__mmask8)__builtin_ia32_vfpclassbf16128_mask((__v8bf)(__m128bh)(__A), \
582 (int)(imm), (__mmask8)(__U)))
583
584#define _mm_fpclass_pbh_mask(__A, imm) \
585 ((__mmask8)__builtin_ia32_vfpclassbf16128_mask((__v8bf)(__m128bh)(__A), \
586 (int)(imm), (__mmask8) - 1))
587
588static __inline__ __m256bh __DEFAULT_FN_ATTRS256
589_mm256_scalef_pbh(__m256bh __A, __m256bh __B) {
590 return (__m256bh)__builtin_ia32_vscalefbf16256_mask(
591 (__v16bf)__A, (__v16bf)__B, (__v16bf)_mm256_undefined_pbh(),
592 (__mmask16)-1);
593}
594
595static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask_scalef_pbh(
596 __m256bh __W, __mmask16 __U, __m256bh __A, __m256bh __B) {
597 return (__m256bh)__builtin_ia32_vscalefbf16256_mask(
598 (__v16bf)__A, (__v16bf)__B, (__v16bf)__W, (__mmask16)__U);
599}
600
601static __inline__ __m256bh __DEFAULT_FN_ATTRS256
602_mm256_maskz_scalef_pbh(__mmask16 __U, __m256bh __A, __m256bh __B) {
603 return (__m256bh)__builtin_ia32_vscalefbf16256_mask(
604 (__v16bf)__A, (__v16bf)__B, (__v16bf)_mm256_setzero_pbh(),
605 (__mmask16)__U);
606}
607
608static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_scalef_pbh(__m128bh __A,
609 __m128bh __B) {
610 return (__m128bh)__builtin_ia32_vscalefbf16128_mask(
611 (__v8bf)__A, (__v8bf)__B, (__v8bf)_mm_undefined_pbh(), (__mmask8)-1);
612}
613
614static __inline__ __m128bh __DEFAULT_FN_ATTRS128
615_mm_mask_scalef_pbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
616 return (__m128bh)__builtin_ia32_vscalefbf16128_mask(
617 (__v8bf)__A, (__v8bf)__B, (__v8bf)__W, (__mmask8)__U);
618}
619
620static __inline__ __m128bh __DEFAULT_FN_ATTRS128
621_mm_maskz_scalef_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
622 return (__m128bh)__builtin_ia32_vscalefbf16128_mask(
623 (__v8bf)__A, (__v8bf)__B, (__v8bf)_mm_setzero_pbh(), (__mmask8)__U);
624}
625
626static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_rcp_pbh(__m256bh __A) {
627 return (__m256bh)__builtin_ia32_vrcpbf16256_mask(
628 (__v16bf)__A, (__v16bf)_mm256_undefined_pbh(), (__mmask16)-1);
629}
630
631static __inline__ __m256bh __DEFAULT_FN_ATTRS256
632_mm256_mask_rcp_pbh(__m256bh __W, __mmask16 __U, __m256bh __A) {
633 return (__m256bh)__builtin_ia32_vrcpbf16256_mask((__v16bf)__A, (__v16bf)__W,
634 (__mmask16)__U);
635}
636
637static __inline__ __m256bh __DEFAULT_FN_ATTRS256
638_mm256_maskz_rcp_pbh(__mmask16 __U, __m256bh __A) {
639 return (__m256bh)__builtin_ia32_vrcpbf16256_mask(
640 (__v16bf)__A, (__v16bf)_mm256_setzero_pbh(), (__mmask16)__U);
641}
642
643static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_rcp_pbh(__m128bh __A) {
644 return (__m128bh)__builtin_ia32_vrcpbf16128_mask(
645 (__v8bf)__A, (__v8bf)_mm_undefined_pbh(), (__mmask8)-1);
646}
647
648static __inline__ __m128bh __DEFAULT_FN_ATTRS128
649_mm_mask_rcp_pbh(__m128bh __W, __mmask8 __U, __m128bh __A) {
650 return (__m128bh)__builtin_ia32_vrcpbf16128_mask((__v8bf)__A, (__v8bf)__W,
651 (__mmask8)__U);
652}
653
654static __inline__ __m128bh __DEFAULT_FN_ATTRS128
655_mm_maskz_rcp_pbh(__mmask8 __U, __m128bh __A) {
656 return (__m128bh)__builtin_ia32_vrcpbf16128_mask(
657 (__v8bf)__A, (__v8bf)_mm_setzero_pbh(), (__mmask8)__U);
658}
659
660static __inline__ __m256bh __DEFAULT_FN_ATTRS256
661_mm256_getexp_pbh(__m256bh __A) {
662 return (__m256bh)__builtin_ia32_vgetexpbf16256_mask(
663 (__v16bf)__A, (__v16bf)_mm256_undefined_pbh(), (__mmask16)-1);
664}
665
666static __inline__ __m256bh __DEFAULT_FN_ATTRS256
667_mm256_mask_getexp_pbh(__m256bh __W, __mmask16 __U, __m256bh __A) {
668 return (__m256bh)__builtin_ia32_vgetexpbf16256_mask(
669 (__v16bf)__A, (__v16bf)__W, (__mmask16)__U);
670}
671
672static __inline__ __m256bh __DEFAULT_FN_ATTRS256
673_mm256_maskz_getexp_pbh(__mmask16 __U, __m256bh __A) {
674 return (__m256bh)__builtin_ia32_vgetexpbf16256_mask(
675 (__v16bf)__A, (__v16bf)_mm256_setzero_pbh(), (__mmask16)__U);
676}
677
678static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_getexp_pbh(__m128bh __A) {
679 return (__m128bh)__builtin_ia32_vgetexpbf16128_mask(
680 (__v8bf)__A, (__v8bf)_mm_undefined_pbh(), (__mmask8)-1);
681}
682
683static __inline__ __m128bh __DEFAULT_FN_ATTRS128
684_mm_mask_getexp_pbh(__m128bh __W, __mmask8 __U, __m128bh __A) {
685 return (__m128bh)__builtin_ia32_vgetexpbf16128_mask((__v8bf)__A, (__v8bf)__W,
686 (__mmask8)__U);
687}
688
689static __inline__ __m128bh __DEFAULT_FN_ATTRS128
690_mm_maskz_getexp_pbh(__mmask8 __U, __m128bh __A) {
691 return (__m128bh)__builtin_ia32_vgetexpbf16128_mask(
692 (__v8bf)__A, (__v8bf)_mm_setzero_pbh(), (__mmask8)__U);
693}
694
695static __inline__ __m256bh __DEFAULT_FN_ATTRS256
696_mm256_rsqrt_pbh(__m256bh __A) {
697 return (__m256bh)__builtin_ia32_vrsqrtbf16256_mask(
698 (__v16bf)__A, (__v16bf)_mm256_undefined_pbh(), (__mmask16)-1);
699}
700
701static __inline__ __m256bh __DEFAULT_FN_ATTRS256
702_mm256_mask_rsqrt_pbh(__m256bh __W, __mmask16 __U, __m256bh __A) {
703 return (__m256bh)__builtin_ia32_vrsqrtbf16256_mask((__v16bf)__A, (__v16bf)__W,
704 (__mmask16)__U);
705}
706
707static __inline__ __m256bh __DEFAULT_FN_ATTRS256
708_mm256_maskz_rsqrt_pbh(__mmask16 __U, __m256bh __A) {
709 return (__m256bh)__builtin_ia32_vrsqrtbf16256_mask(
710 (__v16bf)__A, (__v16bf)_mm256_setzero_pbh(), (__mmask16)__U);
711}
712
713static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_rsqrt_pbh(__m128bh __A) {
714 return (__m128bh)__builtin_ia32_vrsqrtbf16128_mask(
715 (__v8bf)__A, (__v8bf)_mm_undefined_pbh(), (__mmask8)-1);
716}
717
718static __inline__ __m128bh __DEFAULT_FN_ATTRS128
719_mm_mask_rsqrt_pbh(__m128bh __W, __mmask8 __U, __m128bh __A) {
720 return (__m128bh)__builtin_ia32_vrsqrtbf16128_mask((__v8bf)__A, (__v8bf)__W,
721 (__mmask8)__U);
722}
723
724static __inline__ __m128bh __DEFAULT_FN_ATTRS128
725_mm_maskz_rsqrt_pbh(__mmask8 __U, __m128bh __A) {
726 return (__m128bh)__builtin_ia32_vrsqrtbf16128_mask(
727 (__v8bf)__A, (__v8bf)_mm_setzero_pbh(), (__mmask8)__U);
728}
729
730#define _mm256_reduce_pbh(__A, imm) \
731 ((__m256bh)__builtin_ia32_vreducebf16256_mask( \
732 (__v16bf)(__m256bh)(__A), (int)(imm), (__v16bf)_mm256_undefined_pbh(), \
733 (__mmask16) - 1))
734
735#define _mm256_mask_reduce_pbh(__W, __U, __A, imm) \
736 ((__m256bh)__builtin_ia32_vreducebf16256_mask( \
737 (__v16bf)(__m256bh)(__A), (int)(imm), (__v16bf)(__m256bh)(__W), \
738 (__mmask16)(__U)))
739
740#define _mm256_maskz_reduce_pbh(__U, __A, imm) \
741 ((__m256bh)__builtin_ia32_vreducebf16256_mask( \
742 (__v16bf)(__m256bh)(__A), (int)(imm), (__v16bf)_mm256_setzero_pbh(), \
743 (__mmask16)(__U)))
744
745#define _mm_reduce_pbh(__A, imm) \
746 ((__m128bh)__builtin_ia32_vreducebf16128_mask( \
747 (__v8bf)(__m128bh)(__A), (int)(imm), (__v8bf)_mm_undefined_pbh(), \
748 (__mmask8) - 1))
749
750#define _mm_mask_reduce_pbh(__W, __U, __A, imm) \
751 ((__m128bh)__builtin_ia32_vreducebf16128_mask( \
752 (__v8bf)(__m128bh)(__A), (int)(imm), (__v8bf)(__m128bh)(__W), \
753 (__mmask8)(__U)))
754
755#define _mm_maskz_reduce_pbh(__U, __A, imm) \
756 ((__m128bh)__builtin_ia32_vreducebf16128_mask( \
757 (__v8bf)(__m128bh)(__A), (int)(imm), (__v8bf)_mm_setzero_pbh(), \
758 (__mmask8)(__U)))
759
760#define _mm256_roundscale_pbh(__A, imm) \
761 ((__m256bh)__builtin_ia32_vrndscalebf16_256_mask( \
762 (__v16bf)(__m256bh)(__A), (int)(imm), (__v16bf)_mm256_setzero_pbh(), \
763 (__mmask16) - 1))
764
765#define _mm256_mask_roundscale_pbh(__W, __U, __A, imm) \
766 ((__m256bh)__builtin_ia32_vrndscalebf16_256_mask( \
767 (__v16bf)(__m256bh)(__A), (int)(imm), (__v16bf)(__m256bh)(__W), \
768 (__mmask16)(__U)))
769
770#define _mm256_maskz_roundscale_pbh(__U, __A, imm) \
771 ((__m256bh)__builtin_ia32_vrndscalebf16_256_mask( \
772 (__v16bf)(__m256bh)(__A), (int)(imm), (__v16bf)_mm256_setzero_pbh(), \
773 (__mmask16)(__U)))
774
775#define _mm_roundscale_pbh(__A, imm) \
776 ((__m128bh)__builtin_ia32_vrndscalebf16_128_mask( \
777 (__v8bf)(__m128bh)(__A), (int)(imm), (__v8bf)_mm_setzero_pbh(), \
778 (__mmask8) - 1))
779
780#define _mm_mask_roundscale_pbh(__W, __U, __A, imm) \
781 ((__m128bh)__builtin_ia32_vrndscalebf16_128_mask( \
782 (__v8bf)(__m128bh)(__A), (int)(imm), (__v8bf)(__m128bh)(__W), \
783 (__mmask8)(__U)))
784
785#define _mm_maskz_roundscale_pbh(__U, __A, imm) \
786 ((__m128bh)__builtin_ia32_vrndscalebf16_128_mask( \
787 (__v8bf)(__m128bh)(__A), (int)(imm), (__v8bf)_mm_setzero_pbh(), \
788 (__mmask8)(__U)))
789
790#define _mm256_getmant_pbh(__A, __B, __C) \
791 ((__m256bh)__builtin_ia32_vgetmantbf16256_mask( \
792 (__v16bf)(__m256bh)(__A), (int)(((__C) << 2) | (__B)), \
793 (__v16bf)_mm256_undefined_pbh(), (__mmask16) - 1))
794
795#define _mm256_mask_getmant_pbh(__W, __U, __A, __B, __C) \
796 ((__m256bh)__builtin_ia32_vgetmantbf16256_mask( \
797 (__v16bf)(__m256bh)(__A), (int)(((__C) << 2) | (__B)), \
798 (__v16bf)(__m256bh)(__W), (__mmask16)(__U)))
799
800#define _mm256_maskz_getmant_pbh(__U, __A, __B, __C) \
801 ((__m256bh)__builtin_ia32_vgetmantbf16256_mask( \
802 (__v16bf)(__m256bh)(__A), (int)(((__C) << 2) | (__B)), \
803 (__v16bf)_mm256_setzero_pbh(), (__mmask16)(__U)))
804
805#define _mm_getmant_pbh(__A, __B, __C) \
806 ((__m128bh)__builtin_ia32_vgetmantbf16128_mask( \
807 (__v8bf)(__m128bh)(__A), (int)(((__C) << 2) | (__B)), \
808 (__v8bf)_mm_undefined_pbh(), (__mmask8) - 1))
809
810#define _mm_mask_getmant_pbh(__W, __U, __A, __B, __C) \
811 ((__m128bh)__builtin_ia32_vgetmantbf16128_mask( \
812 (__v8bf)(__m128bh)(__A), (int)(((__C) << 2) | (__B)), \
813 (__v8bf)(__m128bh)(__W), (__mmask8)(__U)))
814
815#define _mm_maskz_getmant_pbh(__U, __A, __B, __C) \
816 ((__m128bh)__builtin_ia32_vgetmantbf16128_mask( \
817 (__v8bf)(__m128bh)(__A), (int)(((__C) << 2) | (__B)), \
818 (__v8bf)_mm_setzero_pbh(), (__mmask8)(__U)))
819
820static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_sqrt_pbh(__m256bh __A) {
821 return (__m256bh)__builtin_ia32_vsqrtbf16256((__v16bf)__A);
822}
823
824static __inline__ __m256bh __DEFAULT_FN_ATTRS256
825_mm256_mask_sqrt_pbh(__m256bh __W, __mmask16 __U, __m256bh __A) {
826 return (__m256bh)__builtin_ia32_selectpbf_256(
827 (__mmask16)__U, (__v16bf)_mm256_sqrt_pbh(__A), (__v16bf)__W);
828}
829
830static __inline__ __m256bh __DEFAULT_FN_ATTRS256
831_mm256_maskz_sqrt_pbh(__mmask16 __U, __m256bh __A) {
832 return (__m256bh)__builtin_ia32_selectpbf_256((__mmask16)__U,
833 (__v16bf)_mm256_sqrt_pbh(__A),
834 (__v16bf)_mm256_setzero_pbh());
835}
836
837static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_sqrt_pbh(__m128bh __A) {
838 return (__m128bh)__builtin_ia32_vsqrtbf16((__v8bf)__A);
839}
840
841static __inline__ __m128bh __DEFAULT_FN_ATTRS128
842_mm_mask_sqrt_pbh(__m128bh __W, __mmask8 __U, __m128bh __A) {
843 return (__m128bh)__builtin_ia32_selectpbf_128(
844 (__mmask8)__U, (__v8bf)_mm_sqrt_pbh(__A), (__v8bf)__W);
845}
846
847static __inline__ __m128bh __DEFAULT_FN_ATTRS128
848_mm_maskz_sqrt_pbh(__mmask8 __U, __m128bh __A) {
849 return (__m128bh)__builtin_ia32_selectpbf_128(
850 (__mmask8)__U, (__v8bf)_mm_sqrt_pbh(__A), (__v8bf)_mm_setzero_pbh());
851}
852
853static __inline__ __m256bh __DEFAULT_FN_ATTRS256
854_mm256_fmadd_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
855 return (__m256bh)__builtin_ia32_vfmaddnepbh256((__v16bf)__A, (__v16bf)__B,
856 (__v16bf)__C);
857}
858
859static __inline__ __m256bh __DEFAULT_FN_ATTRS256
860_mm256_mask_fmadd_pbh(__m256bh __A, __mmask16 __U, __m256bh __B, __m256bh __C) {
861 return (__m256bh)__builtin_ia32_selectpbf_256(
862 (__mmask16)__U,
863 _mm256_fmadd_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C), (__v16bf)__A);
864}
865
866static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask3_fmadd_pbh(
867 __m256bh __A, __m256bh __B, __m256bh __C, __mmask16 __U) {
868 return (__m256bh)__builtin_ia32_selectpbf_256(
869 (__mmask16)__U,
870 _mm256_fmadd_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C), (__v16bf)__C);
871}
872
873static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fmadd_pbh(
874 __mmask16 __U, __m256bh __A, __m256bh __B, __m256bh __C) {
875 return (__m256bh)__builtin_ia32_selectpbf_256(
876 (__mmask16)__U,
877 _mm256_fmadd_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
878 (__v16bf)_mm256_setzero_pbh());
879}
880
881static __inline__ __m256bh __DEFAULT_FN_ATTRS256
882_mm256_fmsub_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
883 return (__m256bh)__builtin_ia32_vfmaddnepbh256((__v16bf)__A, (__v16bf)__B,
884 -(__v16bf)__C);
885}
886
887static __inline__ __m256bh __DEFAULT_FN_ATTRS256
888_mm256_mask_fmsub_pbh(__m256bh __A, __mmask16 __U, __m256bh __B, __m256bh __C) {
889 return (__m256bh)__builtin_ia32_selectpbf_256(
890 (__mmask16)__U,
891 _mm256_fmsub_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C), (__v16bf)__A);
892}
893
894static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask3_fmsub_pbh(
895 __m256bh __A, __m256bh __B, __m256bh __C, __mmask16 __U) {
896 return (__m256bh)__builtin_ia32_selectpbf_256(
897 (__mmask16)__U,
898 _mm256_fmsub_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C), (__v16bf)__C);
899}
900
901static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fmsub_pbh(
902 __mmask16 __U, __m256bh __A, __m256bh __B, __m256bh __C) {
903 return (__m256bh)__builtin_ia32_selectpbf_256(
904 (__mmask16)__U,
905 _mm256_fmsub_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
906 (__v16bf)_mm256_setzero_pbh());
907}
908
909static __inline__ __m256bh __DEFAULT_FN_ATTRS256
910_mm256_fnmadd_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
911 return (__m256bh)__builtin_ia32_vfmaddnepbh256((__v16bf)__A, -(__v16bf)__B,
912 (__v16bf)__C);
913}
914
915static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask_fnmadd_pbh(
916 __m256bh __A, __mmask16 __U, __m256bh __B, __m256bh __C) {
917 return (__m256bh)__builtin_ia32_selectpbf_256(
918 (__mmask16)__U,
919 _mm256_fnmadd_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
920 (__v16bf)__A);
921}
922
923static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask3_fnmadd_pbh(
924 __m256bh __A, __m256bh __B, __m256bh __C, __mmask16 __U) {
925 return (__m256bh)__builtin_ia32_selectpbf_256(
926 (__mmask16)__U,
927 _mm256_fnmadd_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
928 (__v16bf)__C);
929}
930
931static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fnmadd_pbh(
932 __mmask16 __U, __m256bh __A, __m256bh __B, __m256bh __C) {
933 return (__m256bh)__builtin_ia32_selectpbf_256(
934 (__mmask16)__U,
935 _mm256_fnmadd_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
936 (__v16bf)_mm256_setzero_pbh());
937}
938
939static __inline__ __m256bh __DEFAULT_FN_ATTRS256
940_mm256_fnmsub_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
941 return (__m256bh)__builtin_ia32_vfmaddnepbh256((__v16bf)__A, -(__v16bf)__B,
942 -(__v16bf)__C);
943}
944
945static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask_fnmsub_pbh(
946 __m256bh __A, __mmask16 __U, __m256bh __B, __m256bh __C) {
947 return (__m256bh)__builtin_ia32_selectpbf_256(
948 (__mmask16)__U,
949 _mm256_fnmsub_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
950 (__v16bf)__A);
951}
952
953static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask3_fnmsub_pbh(
954 __m256bh __A, __m256bh __B, __m256bh __C, __mmask16 __U) {
955 return (__m256bh)__builtin_ia32_selectpbf_256(
956 (__mmask16)__U,
957 _mm256_fnmsub_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
958 (__v16bf)__C);
959}
960
961static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fnmsub_pbh(
962 __mmask16 __U, __m256bh __A, __m256bh __B, __m256bh __C) {
963 return (__m256bh)__builtin_ia32_selectpbf_256(
964 (__mmask16)__U,
965 _mm256_fnmsub_pbh((__v16bf)__A, (__v16bf)__B, (__v16bf)__C),
966 (__v16bf)_mm256_setzero_pbh());
967}
968
969static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fmadd_pbh(__m128bh __A,
970 __m128bh __B,
971 __m128bh __C) {
972 return (__m128bh)__builtin_ia32_vfmaddnepbh128((__v8bf)__A, (__v8bf)__B,
973 (__v8bf)__C);
974}
975
976static __inline__ __m128bh __DEFAULT_FN_ATTRS128
977_mm_mask_fmadd_pbh(__m128bh __A, __mmask8 __U, __m128bh __B, __m128bh __C) {
978 return (__m128bh)__builtin_ia32_selectpbf_128(
979 (__mmask8)__U, _mm_fmadd_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
980 (__v8bf)__A);
981}
982
983static __inline__ __m128bh __DEFAULT_FN_ATTRS128
984_mm_mask3_fmadd_pbh(__m128bh __A, __m128bh __B, __m128bh __C, __mmask8 __U) {
985 return (__m128bh)__builtin_ia32_selectpbf_128(
986 (__mmask8)__U, _mm_fmadd_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
987 (__v8bf)__C);
988}
989
990static __inline__ __m128bh __DEFAULT_FN_ATTRS128
991_mm_maskz_fmadd_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
992 return (__m128bh)__builtin_ia32_selectpbf_128(
993 (__mmask8)__U, _mm_fmadd_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
994 (__v8bf)_mm_setzero_pbh());
995}
996
997static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fmsub_pbh(__m128bh __A,
998 __m128bh __B,
999 __m128bh __C) {
1000 return (__m128bh)__builtin_ia32_vfmaddnepbh128((__v8bf)__A, (__v8bf)__B,
1001 -(__v8bf)__C);
1002}
1003
1004static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1005_mm_mask_fmsub_pbh(__m128bh __A, __mmask8 __U, __m128bh __B, __m128bh __C) {
1006 return (__m128bh)__builtin_ia32_selectpbf_128(
1007 (__mmask8)__U, _mm_fmsub_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1008 (__v8bf)__A);
1009}
1010
1011static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1012_mm_mask3_fmsub_pbh(__m128bh __A, __m128bh __B, __m128bh __C, __mmask8 __U) {
1013 return (__m128bh)__builtin_ia32_selectpbf_128(
1014 (__mmask8)__U, _mm_fmsub_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1015 (__v8bf)__C);
1016}
1017
1018static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1019_mm_maskz_fmsub_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
1020 return (__m128bh)__builtin_ia32_selectpbf_128(
1021 (__mmask8)__U, _mm_fmsub_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1022 (__v8bf)_mm_setzero_pbh());
1023}
1024
1025static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fnmadd_pbh(__m128bh __A,
1026 __m128bh __B,
1027 __m128bh __C) {
1028 return (__m128bh)__builtin_ia32_vfmaddnepbh128((__v8bf)__A, -(__v8bf)__B,
1029 (__v8bf)__C);
1030}
1031
1032static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1033_mm_mask_fnmadd_pbh(__m128bh __A, __mmask8 __U, __m128bh __B, __m128bh __C) {
1034 return (__m128bh)__builtin_ia32_selectpbf_128(
1035 (__mmask8)__U, _mm_fnmadd_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1036 (__v8bf)__A);
1037}
1038
1039static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1040_mm_mask3_fnmadd_pbh(__m128bh __A, __m128bh __B, __m128bh __C, __mmask8 __U) {
1041 return (__m128bh)__builtin_ia32_selectpbf_128(
1042 (__mmask8)__U, _mm_fnmadd_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1043 (__v8bf)__C);
1044}
1045
1046static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1047_mm_maskz_fnmadd_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
1048 return (__m128bh)__builtin_ia32_selectpbf_128(
1049 (__mmask8)__U, _mm_fnmadd_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1050 (__v8bf)_mm_setzero_pbh());
1051}
1052
1053static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fnmsub_pbh(__m128bh __A,
1054 __m128bh __B,
1055 __m128bh __C) {
1056 return (__m128bh)__builtin_ia32_vfmaddnepbh128((__v8bf)__A, -(__v8bf)__B,
1057 -(__v8bf)__C);
1058}
1059
1060static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1061_mm_mask_fnmsub_pbh(__m128bh __A, __mmask8 __U, __m128bh __B, __m128bh __C) {
1062 return (__m128bh)__builtin_ia32_selectpbf_128(
1063 (__mmask8)__U, _mm_fnmsub_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1064 (__v8bf)__A);
1065}
1066
1067static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1068_mm_mask3_fnmsub_pbh(__m128bh __A, __m128bh __B, __m128bh __C, __mmask8 __U) {
1069 return (__m128bh)__builtin_ia32_selectpbf_128(
1070 (__mmask8)__U, _mm_fnmsub_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1071 (__v8bf)__C);
1072}
1073
1074static __inline__ __m128bh __DEFAULT_FN_ATTRS128
1075_mm_maskz_fnmsub_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
1076 return (__m128bh)__builtin_ia32_selectpbf_128(
1077 (__mmask8)__U, _mm_fnmsub_pbh((__v8bf)__A, (__v8bf)__B, (__v8bf)__C),
1078 (__v8bf)_mm_setzero_pbh());
1079}
1080
1081#undef __DEFAULT_FN_ATTRS128
1082#undef __DEFAULT_FN_ATTRS256
1083
1084#endif
1085#endif
lib/include/avx10_2convertintrin.h created+590
......@@ -0,0 +1,590 @@
1/*===--------------- avx10_2convertintrin.h - AVX10_2CONVERT ---------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2convertintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifdef __SSE2__
15
16#ifndef __AVX10_2CONVERTINTRIN_H
17#define __AVX10_2CONVERTINTRIN_H
18
19/* Define the default attributes for the functions in this file. */
20#define __DEFAULT_FN_ATTRS128 \
21 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
22 __min_vector_width__(128)))
23#define __DEFAULT_FN_ATTRS256 \
24 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
25 __min_vector_width__(256)))
26
27static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_cvtx2ps_ph(__m128 __A,
28 __m128 __B) {
29 return (__m128h)__builtin_ia32_vcvt2ps2phx128_mask(
30 (__v4sf)__A, (__v4sf)__B, (__v8hf)_mm_setzero_ph(), (__mmask8)(-1));
31}
32
33static __inline__ __m128h __DEFAULT_FN_ATTRS128
34_mm_mask_cvtx2ps_ph(__m128h __W, __mmask8 __U, __m128 __A, __m128 __B) {
35 return (__m128h)__builtin_ia32_vcvt2ps2phx128_mask(
36 (__v4sf)__A, (__v4sf)__B, (__v8hf)__W, (__mmask8)__U);
37}
38
39static __inline__ __m128h __DEFAULT_FN_ATTRS128
40_mm_maskz_cvtx2ps_ph(__mmask8 __U, __m128 __A, __m128 __B) {
41 return (__m128h)__builtin_ia32_vcvt2ps2phx128_mask(
42 (__v4sf)__A, (__v4sf)__B, (__v8hf)_mm_setzero_ph(), (__mmask8)__U);
43}
44
45static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_cvtx2ps_ph(__m256 __A,
46 __m256 __B) {
47 return (__m256h)__builtin_ia32_vcvt2ps2phx256_mask(
48 (__v8sf)__A, (__v8sf)__B, (__v16hf)_mm256_setzero_ph(), (__mmask16)(-1),
49 _MM_FROUND_CUR_DIRECTION);
50}
51
52static __inline__ __m256h __DEFAULT_FN_ATTRS256
53_mm256_mask_cvtx2ps_ph(__m256h __W, __mmask16 __U, __m256 __A, __m256 __B) {
54 return (__m256h)__builtin_ia32_vcvt2ps2phx256_mask(
55 (__v8sf)__A, (__v8sf)__B, (__v16hf)__W, (__mmask16)__U,
56 _MM_FROUND_CUR_DIRECTION);
57}
58
59static __inline__ __m256h __DEFAULT_FN_ATTRS256
60_mm256_maskz_cvtx2ps_ph(__mmask16 __U, __m256 __A, __m256 __B) {
61 return (__m256h)__builtin_ia32_vcvt2ps2phx256_mask(
62 (__v8sf)__A, (__v8sf)__B, (__v16hf)_mm256_setzero_ph(), (__mmask16)__U,
63 _MM_FROUND_CUR_DIRECTION);
64}
65
66#define _mm256_cvtx_round2ps_ph(A, B, R) \
67 ((__m256h)__builtin_ia32_vcvt2ps2phx256_mask( \
68 (__v8sf)(A), (__v8sf)(B), (__v16hf)_mm256_undefined_ph(), \
69 (__mmask16)(-1), (const int)(R)))
70
71#define _mm256_mask_cvtx_round2ps_ph(W, U, A, B, R) \
72 ((__m256h)__builtin_ia32_vcvt2ps2phx256_mask( \
73 (__v8sf)(A), (__v8sf)(B), (__v16hf)(W), (__mmask16)(U), (const int)(R)))
74
75#define _mm256_maskz_cvtx_round2ps_ph(U, A, B, R) \
76 ((__m256h)__builtin_ia32_vcvt2ps2phx256_mask( \
77 (__v8sf)(A), (__v8sf)(B), (__v16hf)(_mm256_setzero_ph()), \
78 (__mmask16)(U), (const int)(R)))
79
80static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvtbiasph_bf8(__m128i __A,
81 __m128h __B) {
82 return (__m128i)__builtin_ia32_vcvtbiasph2bf8_128_mask(
83 (__v16qi)__A, (__v8hf)__B, (__v16qi)_mm_undefined_si128(), (__mmask8)-1);
84}
85
86static __inline__ __m128i __DEFAULT_FN_ATTRS128
87_mm_mask_cvtbiasph_bf8(__m128i __W, __mmask8 __U, __m128i __A, __m128h __B) {
88 return (__m128i)__builtin_ia32_vcvtbiasph2bf8_128_mask(
89 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)__W, (__mmask8)__U);
90}
91
92static __inline__ __m128i __DEFAULT_FN_ATTRS128
93_mm_maskz_cvtbiasph_bf8(__mmask8 __U, __m128i __A, __m128h __B) {
94 return (__m128i)__builtin_ia32_vcvtbiasph2bf8_128_mask(
95 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
96 (__mmask8)__U);
97}
98
99static __inline__ __m128i __DEFAULT_FN_ATTRS256
100_mm256_cvtbiasph_bf8(__m256i __A, __m256h __B) {
101 return (__m128i)__builtin_ia32_vcvtbiasph2bf8_256_mask(
102 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_undefined_si128(),
103 (__mmask16)-1);
104}
105
106static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_mask_cvtbiasph_bf8(
107 __m128i __W, __mmask16 __U, __m256i __A, __m256h __B) {
108 return (__m128i)__builtin_ia32_vcvtbiasph2bf8_256_mask(
109 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)__W, (__mmask16)__U);
110}
111
112static __inline__ __m128i __DEFAULT_FN_ATTRS256
113_mm256_maskz_cvtbiasph_bf8(__mmask16 __U, __m256i __A, __m256h __B) {
114 return (__m128i)__builtin_ia32_vcvtbiasph2bf8_256_mask(
115 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
116 (__mmask16)__U);
117}
118
119static __inline__ __m128i __DEFAULT_FN_ATTRS128
120_mm_cvtbiassph_bf8(__m128i __A, __m128h __B) {
121 return (__m128i)__builtin_ia32_vcvtbiasph2bf8s_128_mask(
122 (__v16qi)__A, (__v8hf)__B, (__v16qi)_mm_undefined_si128(), (__mmask8)-1);
123}
124
125static __inline__ __m128i __DEFAULT_FN_ATTRS128
126_mm_mask_cvtbiassph_bf8(__m128i __W, __mmask8 __U, __m128i __A, __m128h __B) {
127 return (__m128i)__builtin_ia32_vcvtbiasph2bf8s_128_mask(
128 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)__W, (__mmask8)__U);
129}
130
131static __inline__ __m128i __DEFAULT_FN_ATTRS128
132_mm_maskz_cvtbiassph_bf8(__mmask8 __U, __m128i __A, __m128h __B) {
133 return (__m128i)__builtin_ia32_vcvtbiasph2bf8s_128_mask(
134 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
135 (__mmask8)__U);
136}
137
138static __inline__ __m128i __DEFAULT_FN_ATTRS256
139_mm256_cvtbiassph_bf8(__m256i __A, __m256h __B) {
140 return (__m128i)__builtin_ia32_vcvtbiasph2bf8s_256_mask(
141 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_undefined_si128(),
142 (__mmask16)-1);
143}
144
145static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_mask_cvtbiassph_bf8(
146 __m128i __W, __mmask16 __U, __m256i __A, __m256h __B) {
147 return (__m128i)__builtin_ia32_vcvtbiasph2bf8s_256_mask(
148 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)__W, (__mmask16)__U);
149}
150
151static __inline__ __m128i __DEFAULT_FN_ATTRS256
152_mm256_maskz_cvtbiassph_bf8(__mmask16 __U, __m256i __A, __m256h __B) {
153 return (__m128i)__builtin_ia32_vcvtbiasph2bf8s_256_mask(
154 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
155 (__mmask16)__U);
156}
157
158static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvtbiasph_hf8(__m128i __A,
159 __m128h __B) {
160 return (__m128i)__builtin_ia32_vcvtbiasph2hf8_128_mask(
161 (__v16qi)__A, (__v8hf)__B, (__v16qi)_mm_undefined_si128(), (__mmask8)-1);
162}
163
164static __inline__ __m128i __DEFAULT_FN_ATTRS128
165_mm_mask_cvtbiasph_hf8(__m128i __W, __mmask8 __U, __m128i __A, __m128h __B) {
166 return (__m128i)__builtin_ia32_vcvtbiasph2hf8_128_mask(
167 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)__W, (__mmask8)__U);
168}
169
170static __inline__ __m128i __DEFAULT_FN_ATTRS128
171_mm_maskz_cvtbiasph_hf8(__mmask8 __U, __m128i __A, __m128h __B) {
172 return (__m128i)__builtin_ia32_vcvtbiasph2hf8_128_mask(
173 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
174 (__mmask8)__U);
175}
176
177static __inline__ __m128i __DEFAULT_FN_ATTRS256
178_mm256_cvtbiasph_hf8(__m256i __A, __m256h __B) {
179 return (__m128i)__builtin_ia32_vcvtbiasph2hf8_256_mask(
180 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_undefined_si128(),
181 (__mmask16)-1);
182}
183
184static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_mask_cvtbiasph_hf8(
185 __m128i __W, __mmask16 __U, __m256i __A, __m256h __B) {
186 return (__m128i)__builtin_ia32_vcvtbiasph2hf8_256_mask(
187 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)__W, (__mmask16)__U);
188}
189
190static __inline__ __m128i __DEFAULT_FN_ATTRS256
191_mm256_maskz_cvtbiasph_hf8(__mmask16 __U, __m256i __A, __m256h __B) {
192 return (__m128i)__builtin_ia32_vcvtbiasph2hf8_256_mask(
193 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
194 (__mmask16)__U);
195}
196
197static __inline__ __m128i __DEFAULT_FN_ATTRS128
198_mm_cvtbiassph_hf8(__m128i __A, __m128h __B) {
199 return (__m128i)__builtin_ia32_vcvtbiasph2hf8s_128_mask(
200 (__v16qi)__A, (__v8hf)__B, (__v16qi)_mm_undefined_si128(), (__mmask8)-1);
201}
202
203static __inline__ __m128i __DEFAULT_FN_ATTRS128
204_mm_mask_cvtbiassph_hf8(__m128i __W, __mmask8 __U, __m128i __A, __m128h __B) {
205 return (__m128i)__builtin_ia32_vcvtbiasph2hf8s_128_mask(
206 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)__W, (__mmask8)__U);
207}
208
209static __inline__ __m128i __DEFAULT_FN_ATTRS128
210_mm_maskz_cvtbiassph_hf8(__mmask8 __U, __m128i __A, __m128h __B) {
211 return (__m128i)__builtin_ia32_vcvtbiasph2hf8s_128_mask(
212 (__v16qi)__A, (__v8hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
213 (__mmask8)__U);
214}
215
216static __inline__ __m128i __DEFAULT_FN_ATTRS256
217_mm256_cvtbiassph_hf8(__m256i __A, __m256h __B) {
218 return (__m128i)__builtin_ia32_vcvtbiasph2hf8s_256_mask(
219 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_undefined_si128(),
220 (__mmask16)-1);
221}
222
223static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_mask_cvtbiassph_hf8(
224 __m128i __W, __mmask16 __U, __m256i __A, __m256h __B) {
225 return (__m128i)__builtin_ia32_vcvtbiasph2hf8s_256_mask(
226 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)__W, (__mmask16)__U);
227}
228
229static __inline__ __m128i __DEFAULT_FN_ATTRS256
230_mm256_maskz_cvtbiassph_hf8(__mmask16 __U, __m256i __A, __m256h __B) {
231 return (__m128i)__builtin_ia32_vcvtbiasph2hf8s_256_mask(
232 (__v32qi)__A, (__v16hf)__B, (__v16qi)(__m128i)_mm_setzero_si128(),
233 (__mmask16)__U);
234}
235
236static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvt2ph_bf8(__m128h __A,
237 __m128h __B) {
238 return (__m128i)__builtin_ia32_vcvt2ph2bf8_128((__v8hf)(__A), (__v8hf)(__B));
239}
240
241static __inline__ __m128i __DEFAULT_FN_ATTRS128
242_mm_mask_cvt2ph_bf8(__m128i __W, __mmask16 __U, __m128h __A, __m128h __B) {
243 return (__m128i)__builtin_ia32_selectb_128(
244 (__mmask16)__U, (__v16qi)_mm_cvt2ph_bf8(__A, __B), (__v16qi)__W);
245}
246
247static __inline__ __m128i __DEFAULT_FN_ATTRS128
248_mm_maskz_cvt2ph_bf8(__mmask16 __U, __m128h __A, __m128h __B) {
249 return (__m128i)__builtin_ia32_selectb_128(
250 (__mmask16)__U, (__v16qi)_mm_cvt2ph_bf8(__A, __B),
251 (__v16qi)(__m128i)_mm_setzero_si128());
252}
253
254static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_cvt2ph_bf8(__m256h __A,
255 __m256h __B) {
256 return (__m256i)__builtin_ia32_vcvt2ph2bf8_256((__v16hf)(__A),
257 (__v16hf)(__B));
258}
259
260static __inline__ __m256i __DEFAULT_FN_ATTRS256
261_mm256_mask_cvt2ph_bf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
262 return (__m256i)__builtin_ia32_selectb_256(
263 (__mmask16)__U, (__v32qi)_mm256_cvt2ph_bf8(__A, __B), (__v32qi)__W);
264}
265
266static __inline__ __m256i __DEFAULT_FN_ATTRS256
267_mm256_maskz_cvt2ph_bf8(__mmask32 __U, __m256h __A, __m256h __B) {
268 return (__m256i)__builtin_ia32_selectb_256(
269 (__mmask16)__U, (__v32qi)_mm256_cvt2ph_bf8(__A, __B),
270 (__v32qi)(__m256i)_mm256_setzero_si256());
271}
272
273static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvts2ph_bf8(__m128h __A,
274 __m128h __B) {
275 return (__m128i)__builtin_ia32_vcvt2ph2bf8s_128((__v8hf)(__A), (__v8hf)(__B));
276}
277
278static __inline__ __m128i __DEFAULT_FN_ATTRS128
279_mm_mask_cvts2ph_bf8(__m128i __W, __mmask16 __U, __m128h __A, __m128h __B) {
280 return (__m128i)__builtin_ia32_selectb_128(
281 (__mmask16)__U, (__v16qi)_mm_cvts2ph_bf8(__A, __B), (__v16qi)__W);
282}
283
284static __inline__ __m128i __DEFAULT_FN_ATTRS128
285_mm_maskz_cvts2ph_bf8(__mmask16 __U, __m128h __A, __m128h __B) {
286 return (__m128i)__builtin_ia32_selectb_128(
287 (__mmask16)__U, (__v16qi)_mm_cvts2ph_bf8(__A, __B),
288 (__v16qi)(__m128i)_mm_setzero_si128());
289}
290
291static __inline__ __m256i __DEFAULT_FN_ATTRS256
292_mm256_cvts2ph_bf8(__m256h __A, __m256h __B) {
293 return (__m256i)__builtin_ia32_vcvt2ph2bf8s_256((__v16hf)(__A),
294 (__v16hf)(__B));
295}
296
297static __inline__ __m256i __DEFAULT_FN_ATTRS256
298_mm256_mask_cvts2ph_bf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
299 return (__m256i)__builtin_ia32_selectb_256(
300 (__mmask16)__U, (__v32qi)_mm256_cvts2ph_bf8(__A, __B), (__v32qi)__W);
301}
302
303static __inline__ __m256i __DEFAULT_FN_ATTRS256
304_mm256_maskz_cvts2ph_bf8(__mmask32 __U, __m256h __A, __m256h __B) {
305 return (__m256i)__builtin_ia32_selectb_256(
306 (__mmask16)__U, (__v32qi)_mm256_cvts2ph_bf8(__A, __B),
307 (__v32qi)(__m256i)_mm256_setzero_si256());
308}
309
310static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvt2ph_hf8(__m128h __A,
311 __m128h __B) {
312 return (__m128i)__builtin_ia32_vcvt2ph2hf8_128((__v8hf)(__A), (__v8hf)(__B));
313}
314
315static __inline__ __m128i __DEFAULT_FN_ATTRS128
316_mm_mask_cvt2ph_hf8(__m128i __W, __mmask16 __U, __m128h __A, __m128h __B) {
317 return (__m128i)__builtin_ia32_selectb_128(
318 (__mmask16)__U, (__v16qi)_mm_cvt2ph_hf8(__A, __B), (__v16qi)__W);
319}
320
321static __inline__ __m128i __DEFAULT_FN_ATTRS128
322_mm_maskz_cvt2ph_hf8(__mmask16 __U, __m128h __A, __m128h __B) {
323 return (__m128i)__builtin_ia32_selectb_128(
324 (__mmask16)__U, (__v16qi)_mm_cvt2ph_hf8(__A, __B),
325 (__v16qi)(__m128i)_mm_setzero_si128());
326}
327
328static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_cvt2ph_hf8(__m256h __A,
329 __m256h __B) {
330 return (__m256i)__builtin_ia32_vcvt2ph2hf8_256((__v16hf)(__A),
331 (__v16hf)(__B));
332}
333
334static __inline__ __m256i __DEFAULT_FN_ATTRS256
335_mm256_mask_cvt2ph_hf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
336 return (__m256i)__builtin_ia32_selectb_256(
337 (__mmask16)__U, (__v32qi)_mm256_cvt2ph_hf8(__A, __B), (__v32qi)__W);
338}
339
340static __inline__ __m256i __DEFAULT_FN_ATTRS256
341_mm256_maskz_cvt2ph_hf8(__mmask32 __U, __m256h __A, __m256h __B) {
342 return (__m256i)__builtin_ia32_selectb_256(
343 (__mmask16)__U, (__v32qi)_mm256_cvt2ph_hf8(__A, __B),
344 (__v32qi)(__m256i)_mm256_setzero_si256());
345}
346
347static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvts2ph_hf8(__m128h __A,
348 __m128h __B) {
349 return (__m128i)__builtin_ia32_vcvt2ph2hf8s_128((__v8hf)(__A), (__v8hf)(__B));
350}
351
352static __inline__ __m128i __DEFAULT_FN_ATTRS128
353_mm_mask_cvts2ph_hf8(__m128i __W, __mmask16 __U, __m128h __A, __m128h __B) {
354 return (__m128i)__builtin_ia32_selectb_128(
355 (__mmask16)__U, (__v16qi)_mm_cvts2ph_hf8(__A, __B), (__v16qi)__W);
356}
357
358static __inline__ __m128i __DEFAULT_FN_ATTRS128
359_mm_maskz_cvts2ph_hf8(__mmask16 __U, __m128h __A, __m128h __B) {
360 return (__m128i)__builtin_ia32_selectb_128(
361 (__mmask16)__U, (__v16qi)_mm_cvts2ph_hf8(__A, __B),
362 (__v16qi)(__m128i)_mm_setzero_si128());
363}
364
365static __inline__ __m256i __DEFAULT_FN_ATTRS256
366_mm256_cvts2ph_hf8(__m256h __A, __m256h __B) {
367 return (__m256i)__builtin_ia32_vcvt2ph2hf8s_256((__v16hf)(__A),
368 (__v16hf)(__B));
369}
370
371static __inline__ __m256i __DEFAULT_FN_ATTRS256
372_mm256_mask_cvts2ph_hf8(__m256i __W, __mmask32 __U, __m256h __A, __m256h __B) {
373 return (__m256i)__builtin_ia32_selectb_256(
374 (__mmask16)__U, (__v32qi)_mm256_cvts2ph_hf8(__A, __B), (__v32qi)__W);
375}
376
377static __inline__ __m256i __DEFAULT_FN_ATTRS256
378_mm256_maskz_cvts2ph_hf8(__mmask32 __U, __m256h __A, __m256h __B) {
379 return (__m256i)__builtin_ia32_selectb_256(
380 (__mmask16)__U, (__v32qi)_mm256_cvts2ph_hf8(__A, __B),
381 (__v32qi)(__m256i)_mm256_setzero_si256());
382}
383
384static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_cvthf8(__m128i __A) {
385 return (__m128h)__builtin_ia32_vcvthf8_2ph128_mask(
386 (__v16qi)__A, (__v8hf)(__m128h)_mm_undefined_ph(), (__mmask8)-1);
387}
388
389static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_cvthf8(__m128h __W,
390 __mmask8 __U,
391 __m128i __A) {
392 return (__m128h)__builtin_ia32_vcvthf8_2ph128_mask(
393 (__v16qi)__A, (__v8hf)(__m128h)__W, (__mmask8)__U);
394}
395
396static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_cvthf8(__mmask8 __U,
397 __m128i __A) {
398 return (__m128h)__builtin_ia32_vcvthf8_2ph128_mask(
399 (__v16qi)__A, (__v8hf)(__m128h)_mm_setzero_ph(), (__mmask8)__U);
400}
401
402static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_cvthf8(__m128i __A) {
403 return (__m256h)__builtin_ia32_vcvthf8_2ph256_mask(
404 (__v16qi)__A, (__v16hf)(__m256h)_mm256_undefined_ph(), (__mmask16)-1);
405}
406
407static __inline__ __m256h __DEFAULT_FN_ATTRS256
408_mm256_mask_cvthf8(__m256h __W, __mmask16 __U, __m128i __A) {
409 return (__m256h)__builtin_ia32_vcvthf8_2ph256_mask(
410 (__v16qi)__A, (__v16hf)(__m256h)__W, (__mmask16)__U);
411}
412
413static __inline__ __m256h __DEFAULT_FN_ATTRS256
414_mm256_maskz_cvthf8(__mmask16 __U, __m128i __A) {
415 return (__m256h)__builtin_ia32_vcvthf8_2ph256_mask(
416 (__v16qi)__A, (__v16hf)(__m256h)_mm256_setzero_ph(), (__mmask16)__U);
417}
418
419static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvtph_bf8(__m128h __A) {
420 return (__m128i)__builtin_ia32_vcvtph2bf8_128_mask(
421 (__v8hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask8)-1);
422}
423
424static __inline__ __m128i __DEFAULT_FN_ATTRS128
425_mm_mask_cvtph_bf8(__m128i __W, __mmask8 __U, __m128h __A) {
426 return (__m128i)__builtin_ia32_vcvtph2bf8_128_mask(
427 (__v8hf)__A, (__v16qi)(__m128i)__W, (__mmask8)__U);
428}
429
430static __inline__ __m128i __DEFAULT_FN_ATTRS128
431_mm_maskz_cvtph_bf8(__mmask8 __U, __m128h __A) {
432 return (__m128i)__builtin_ia32_vcvtph2bf8_128_mask(
433 (__v8hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask8)__U);
434}
435
436static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_cvtph_bf8(__m256h __A) {
437 return (__m128i)__builtin_ia32_vcvtph2bf8_256_mask(
438 (__v16hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask16)-1);
439}
440
441static __inline__ __m128i __DEFAULT_FN_ATTRS256
442_mm256_mask_cvtph_bf8(__m128i __W, __mmask16 __U, __m256h __A) {
443 return (__m128i)__builtin_ia32_vcvtph2bf8_256_mask(
444 (__v16hf)__A, (__v16qi)(__m128i)__W, (__mmask16)__U);
445}
446
447static __inline__ __m128i __DEFAULT_FN_ATTRS256
448_mm256_maskz_cvtph_bf8(__mmask16 __U, __m256h __A) {
449 return (__m128i)__builtin_ia32_vcvtph2bf8_256_mask(
450 (__v16hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask16)__U);
451}
452
453static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvtsph_bf8(__m128h __A) {
454 return (__m128i)__builtin_ia32_vcvtph2bf8s_128_mask(
455 (__v8hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask8)-1);
456}
457
458static __inline__ __m128i __DEFAULT_FN_ATTRS128
459_mm_mask_cvtsph_bf8(__m128i __W, __mmask8 __U, __m128h __A) {
460 return (__m128i)__builtin_ia32_vcvtph2bf8s_128_mask(
461 (__v8hf)__A, (__v16qi)(__m128i)__W, (__mmask8)__U);
462}
463
464static __inline__ __m128i __DEFAULT_FN_ATTRS128
465_mm_maskz_cvtsph_bf8(__mmask8 __U, __m128h __A) {
466 return (__m128i)__builtin_ia32_vcvtph2bf8s_128_mask(
467 (__v8hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask8)__U);
468}
469
470static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_cvtsph_bf8(__m256h __A) {
471 return (__m128i)__builtin_ia32_vcvtph2bf8s_256_mask(
472 (__v16hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask16)-1);
473}
474
475static __inline__ __m128i __DEFAULT_FN_ATTRS256
476_mm256_mask_cvtsph_bf8(__m128i __W, __mmask16 __U, __m256h __A) {
477 return (__m128i)__builtin_ia32_vcvtph2bf8s_256_mask(
478 (__v16hf)__A, (__v16qi)(__m128i)__W, (__mmask16)__U);
479}
480
481static __inline__ __m128i __DEFAULT_FN_ATTRS256
482_mm256_maskz_cvtsph_bf8(__mmask16 __U, __m256h __A) {
483 return (__m128i)__builtin_ia32_vcvtph2bf8s_256_mask(
484 (__v16hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask16)__U);
485}
486
487static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvtph_hf8(__m128h __A) {
488 return (__m128i)__builtin_ia32_vcvtph2hf8_128_mask(
489 (__v8hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask8)-1);
490}
491
492static __inline__ __m128i __DEFAULT_FN_ATTRS128
493_mm_mask_cvtph_hf8(__m128i __W, __mmask8 __U, __m128h __A) {
494 return (__m128i)__builtin_ia32_vcvtph2hf8_128_mask(
495 (__v8hf)__A, (__v16qi)(__m128i)__W, (__mmask8)__U);
496}
497
498static __inline__ __m128i __DEFAULT_FN_ATTRS128
499_mm_maskz_cvtph_hf8(__mmask8 __U, __m128h __A) {
500 return (__m128i)__builtin_ia32_vcvtph2hf8_128_mask(
501 (__v8hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask8)__U);
502}
503
504static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_cvtph_hf8(__m256h __A) {
505 return (__m128i)__builtin_ia32_vcvtph2hf8_256_mask(
506 (__v16hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask16)-1);
507}
508
509static __inline__ __m128i __DEFAULT_FN_ATTRS256
510_mm256_mask_cvtph_hf8(__m128i __W, __mmask16 __U, __m256h __A) {
511 return (__m128i)__builtin_ia32_vcvtph2hf8_256_mask(
512 (__v16hf)__A, (__v16qi)(__m128i)__W, (__mmask16)__U);
513}
514
515static __inline__ __m128i __DEFAULT_FN_ATTRS256
516_mm256_maskz_cvtph_hf8(__mmask16 __U, __m256h __A) {
517 return (__m128i)__builtin_ia32_vcvtph2hf8_256_mask(
518 (__v16hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask16)__U);
519}
520
521static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvtsph_hf8(__m128h __A) {
522 return (__m128i)__builtin_ia32_vcvtph2hf8s_128_mask(
523 (__v8hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask8)-1);
524}
525
526static __inline__ __m128i __DEFAULT_FN_ATTRS128
527_mm_mask_cvtsph_hf8(__m128i __W, __mmask8 __U, __m128h __A) {
528 return (__m128i)__builtin_ia32_vcvtph2hf8s_128_mask(
529 (__v8hf)__A, (__v16qi)(__m128i)__W, (__mmask8)__U);
530}
531
532static __inline__ __m128i __DEFAULT_FN_ATTRS128
533_mm_maskz_cvtsph_hf8(__mmask8 __U, __m128h __A) {
534 return (__m128i)__builtin_ia32_vcvtph2hf8s_128_mask(
535 (__v8hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask8)__U);
536}
537
538static __inline__ __m128i __DEFAULT_FN_ATTRS256 _mm256_cvtsph_hf8(__m256h __A) {
539 return (__m128i)__builtin_ia32_vcvtph2hf8s_256_mask(
540 (__v16hf)__A, (__v16qi)(__m128i)_mm_undefined_si128(), (__mmask16)-1);
541}
542
543static __inline__ __m128i __DEFAULT_FN_ATTRS256
544_mm256_mask_cvtsph_hf8(__m128i __W, __mmask16 __U, __m256h __A) {
545 return (__m128i)__builtin_ia32_vcvtph2hf8s_256_mask(
546 (__v16hf)__A, (__v16qi)(__m128i)__W, (__mmask16)__U);
547}
548
549static __inline__ __m128i __DEFAULT_FN_ATTRS256
550_mm256_maskz_cvtsph_hf8(__mmask16 __U, __m256h __A) {
551 return (__m128i)__builtin_ia32_vcvtph2hf8s_256_mask(
552 (__v16hf)__A, (__v16qi)(__m128i)_mm_setzero_si128(), (__mmask16)__U);
553}
554
555static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_cvtbf8_ph(__m128i __A) {
556 return _mm_castsi128_ph(_mm_slli_epi16(_mm_cvtepi8_epi16(__A), 8));
557}
558
559static __inline__ __m128h __DEFAULT_FN_ATTRS128
560_mm_mask_cvtbf8_ph(__m128h __S, __mmask8 __U, __m128i __A) {
561 return _mm_castsi128_ph(
562 _mm_mask_slli_epi16((__m128i)__S, __U, _mm_cvtepi8_epi16(__A), 8));
563}
564
565static __inline__ __m128h __DEFAULT_FN_ATTRS128
566_mm_maskz_cvtbf8_ph(__mmask8 __U, __m128i __A) {
567 return _mm_castsi128_ph(_mm_slli_epi16(_mm_maskz_cvtepi8_epi16(__U, __A), 8));
568}
569
570static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_cvtbf8_ph(__m128i __A) {
571 return _mm256_castsi256_ph(_mm256_slli_epi16(_mm256_cvtepi8_epi16(__A), 8));
572}
573
574static __inline__ __m256h __DEFAULT_FN_ATTRS256
575_mm256_mask_cvtbf8_ph(__m256h __S, __mmask16 __U, __m128i __A) {
576 return _mm256_castsi256_ph(
577 _mm256_mask_slli_epi16((__m256i)__S, __U, _mm256_cvtepi8_epi16(__A), 8));
578}
579
580static __inline__ __m256h __DEFAULT_FN_ATTRS256
581_mm256_maskz_cvtbf8_ph(__mmask16 __U, __m128i __A) {
582 return _mm256_castsi256_ph(
583 _mm256_slli_epi16(_mm256_maskz_cvtepi8_epi16(__U, __A), 8));
584}
585
586#undef __DEFAULT_FN_ATTRS128
587#undef __DEFAULT_FN_ATTRS256
588
589#endif // __AVX10_2CONVERTINTRIN_H
590#endif // __SSE2__
lib/include/avx10_2copyintrin.h created+66
......@@ -0,0 +1,66 @@
1/*===---- avx10_2copyintrin.h - AVX10.2 Copy intrinsics -------------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2copyintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifndef __AVX10_2COPYINTRIN_H
15#define __AVX10_2COPYINTRIN_H
16
17/* Define the default attributes for the functions in this file. */
18#define __DEFAULT_FN_ATTRS128 \
19 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
20 __min_vector_width__(128)))
21
22/// Constructs a 128-bit integer vector, setting the lower 32 bits to the
23/// lower 32 bits of the parameter \a __A; the upper bits are zeoroed.
24///
25/// \code{.operation}
26/// result[31:0] := __A[31:0]
27/// result[MAX:32] := 0
28/// \endcode
29///
30/// \headerfile <immintrin.h>
31///
32/// This intrinsic corresponds to the <c> VMOVD </c> instruction.
33///
34/// \param __A
35/// A 128-bit integer vector.
36/// \returns A 128-bit integer vector. The lower 32 bits are copied from the
37/// parameter \a __A; the upper bits are zeroed.
38static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_move_epi32(__m128i __A) {
39 return (__m128i)__builtin_shufflevector(
40 (__v4si)__A, (__v4si)_mm_setzero_si128(), 0, 4, 4, 4);
41}
42
43/// Constructs a 128-bit integer vector, setting the lower 16 bits to the
44/// lower 16 bits of the parameter \a __A; the upper bits are zeoroed.
45///
46/// \code{.operation}
47/// result[15:0] := __A[15:0]
48/// result[MAX:16] := 0
49/// \endcode
50///
51/// \headerfile <immintrin.h>
52///
53/// This intrinsic corresponds to the <c> VMOVW </c> instruction.
54///
55/// \param __A
56/// A 128-bit integer vector.
57/// \returns A 128-bit integer vector. The lower 16 bits are copied from the
58/// parameter \a __A; the upper bits are zeroed.
59static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_move_epi16(__m128i __A) {
60 return (__m128i)__builtin_shufflevector(
61 (__v8hi)__A, (__v8hi)_mm_setzero_si128(), 0, 8, 8, 8, 8, 8, 8, 8);
62}
63
64#undef __DEFAULT_FN_ATTRS128
65
66#endif // __AVX10_2COPYINTRIN_H
lib/include/avx10_2minmaxintrin.h created+277
......@@ -0,0 +1,277 @@
1/*===-------- avx10_2minmaxintrin.h - AVX10_2MINMAX intrinsics -------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2minmaxintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifndef __AVX10_2MINMAXINTRIN_H
15#define __AVX10_2MINMAXINTRIN_H
16
17#define _mm_minmax_pbh(A, B, C) \
18 ((__m128bh)__builtin_ia32_vminmaxbf16128((__m128bh)(__v8bf)(A), \
19 (__m128bh)(__v8bf)(B), (int)(C)))
20
21#define _mm_mask_minmax_pbh(W, U, A, B, C) \
22 ((__m128bh)__builtin_ia32_selectpbf_128( \
23 (__mmask8)(U), \
24 (__v8bf)_mm_minmax_pbh((__m128bh)(__v8bf)(A), (__m128bh)(__v8bf)(B), \
25 (int)(C)), \
26 (__v8bf)(W)))
27
28#define _mm_maskz_minmax_pbh(U, A, B, C) \
29 ((__m128bh)__builtin_ia32_selectpbf_128( \
30 (__mmask8)(U), \
31 (__v8bf)_mm_minmax_pbh((__m128bh)(__v8bf)(A), (__m128bh)(__v8bf)(B), \
32 (int)(C)), \
33 (__v8bf) __builtin_bit_cast(__m128bh, _mm_setzero_ps())))
34
35#define _mm256_minmax_pbh(A, B, C) \
36 ((__m256bh)__builtin_ia32_vminmaxbf16256((__m256bh)(__v16bf)(A), \
37 (__m256bh)(__v16bf)(B), (int)(C)))
38
39#define _mm256_mask_minmax_pbh(W, U, A, B, C) \
40 ((__m256bh)__builtin_ia32_selectpbf_256( \
41 (__mmask16)(U), \
42 (__v16bf)_mm256_minmax_pbh((__m256bh)(__v16bf)(A), \
43 (__m256bh)(__v16bf)(B), (int)(C)), \
44 (__v16bf)(W)))
45
46#define _mm256_maskz_minmax_pbh(U, A, B, C) \
47 ((__m256bh)__builtin_ia32_selectpbf_256( \
48 (__mmask16)(U), \
49 (__v16bf)_mm256_minmax_pbh((__m256bh)(__v16bf)(A), \
50 (__m256bh)(__v16bf)(B), (int)(C)), \
51 (__v16bf) __builtin_bit_cast(__m256bh, _mm256_setzero_ps())))
52
53#define _mm_minmax_pd(A, B, C) \
54 ((__m128d)__builtin_ia32_vminmaxpd128_mask( \
55 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
56 (__v2df)_mm_setzero_pd(), (__mmask8)-1))
57
58#define _mm_mask_minmax_pd(W, U, A, B, C) \
59 ((__m128d)__builtin_ia32_vminmaxpd128_mask( \
60 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
61 (__v2df)(__m128d)(W), (__mmask8)(U)))
62
63#define _mm_maskz_minmax_pd(U, A, B, C) \
64 ((__m128d)__builtin_ia32_vminmaxpd128_mask( \
65 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
66 (__v2df)_mm_setzero_pd(), (__mmask8)(U)))
67
68#define _mm256_minmax_pd(A, B, C) \
69 ((__m256d)__builtin_ia32_vminmaxpd256_round_mask( \
70 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
71 (__v4df)_mm256_setzero_pd(), (__mmask8)-1, _MM_FROUND_NO_EXC))
72
73#define _mm256_mask_minmax_pd(W, U, A, B, C) \
74 ((__m256d)__builtin_ia32_vminmaxpd256_round_mask( \
75 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
76 (__v4df)(__m256d)(W), (__mmask8)(U), _MM_FROUND_NO_EXC))
77
78#define _mm256_maskz_minmax_pd(U, A, B, C) \
79 ((__m256d)__builtin_ia32_vminmaxpd256_round_mask( \
80 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
81 (__v4df)_mm256_setzero_pd(), (__mmask8)(U), _MM_FROUND_NO_EXC))
82
83#define _mm256_minmax_round_pd(A, B, C, R) \
84 ((__m256d)__builtin_ia32_vminmaxpd256_round_mask( \
85 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
86 (__v4df)_mm256_undefined_pd(), (__mmask8)-1, (int)(R)))
87
88#define _mm256_mask_minmax_round_pd(W, U, A, B, C, R) \
89 ((__m256d)__builtin_ia32_vminmaxpd256_round_mask( \
90 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
91 (__v4df)(__m256d)(W), (__mmask8)(U), (int)(R)))
92
93#define _mm256_maskz_minmax_round_pd(U, A, B, C, R) \
94 ((__m256d)__builtin_ia32_vminmaxpd256_round_mask( \
95 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
96 (__v4df)_mm256_setzero_pd(), (__mmask8)(U), (int)(R)))
97
98#define _mm_minmax_ph(A, B, C) \
99 ((__m128h)__builtin_ia32_vminmaxph128_mask( \
100 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
101 (__v8hf)_mm_setzero_ph(), (__mmask8)-1))
102
103#define _mm_mask_minmax_ph(W, U, A, B, C) \
104 ((__m128h)__builtin_ia32_vminmaxph128_mask( \
105 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
106 (__v8hf)(__m128h)(W), (__mmask16)-1))
107
108#define _mm_maskz_minmax_ph(U, A, B, C) \
109 ((__m128h)__builtin_ia32_vminmaxph128_mask( \
110 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
111 (__v8hf)_mm_setzero_ph(), (__mmask8)(U)))
112
113#define _mm256_minmax_ph(A, B, C) \
114 ((__m256h)__builtin_ia32_vminmaxph256_round_mask( \
115 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (int)(C), \
116 (__v16hf)_mm256_setzero_ph(), (__mmask16)-1, _MM_FROUND_NO_EXC))
117
118#define _mm256_mask_minmax_ph(W, U, A, B, C) \
119 ((__m256h)__builtin_ia32_vminmaxph256_round_mask( \
120 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (int)(C), \
121 (__v16hf)(__m256h)(W), (__mmask16)(U), _MM_FROUND_NO_EXC))
122
123#define _mm256_maskz_minmax_ph(U, A, B, C) \
124 ((__m256h)__builtin_ia32_vminmaxph256_round_mask( \
125 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (int)(C), \
126 (__v16hf)_mm256_setzero_ph(), (__mmask16)(U), _MM_FROUND_NO_EXC))
127
128#define _mm256_minmax_round_ph(A, B, C, R) \
129 ((__m256h)__builtin_ia32_vminmaxph256_round_mask( \
130 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (int)(C), \
131 (__v16hf)_mm256_undefined_ph(), (__mmask16)-1, (int)(R)))
132
133#define _mm256_mask_minmax_round_ph(W, U, A, B, C, R) \
134 ((__m256h)__builtin_ia32_vminmaxph256_round_mask( \
135 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (C), \
136 (__v16hf)(__m256h)(W), (__mmask16)(U), (int)(R)))
137
138#define _mm256_maskz_minmax_round_ph(U, A, B, C, R) \
139 ((__m256h)__builtin_ia32_vminmaxph256_round_mask( \
140 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (int)(C), \
141 (__v16hf)_mm256_setzero_ph(), (__mmask16)(U), (int)(R)))
142
143#define _mm_minmax_ps(A, B, C) \
144 ((__m128)__builtin_ia32_vminmaxps128_mask( \
145 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), \
146 (__v4sf)_mm_setzero_ps(), (__mmask8)-1))
147
148#define _mm_mask_minmax_ps(W, U, A, B, C) \
149 ((__m128)__builtin_ia32_vminmaxps128_mask( \
150 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), (__v4sf)(__m128)(W), \
151 (__mmask8)(U)))
152
153#define _mm_maskz_minmax_ps(U, A, B, C) \
154 ((__m128)__builtin_ia32_vminmaxps128_mask( \
155 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), \
156 (__v4sf)_mm_setzero_ps(), (__mmask8)(U)))
157
158#define _mm256_minmax_ps(A, B, C) \
159 ((__m256)__builtin_ia32_vminmaxps256_round_mask( \
160 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), \
161 (__v8sf)_mm256_setzero_ps(), (__mmask8)-1, _MM_FROUND_NO_EXC))
162
163#define _mm256_mask_minmax_ps(W, U, A, B, C) \
164 ((__m256)__builtin_ia32_vminmaxps256_round_mask( \
165 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), (__v8sf)(__m256)(W), \
166 (__mmask8)(U), _MM_FROUND_NO_EXC))
167
168#define _mm256_maskz_minmax_ps(U, A, B, C) \
169 ((__m256)__builtin_ia32_vminmaxps256_round_mask( \
170 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), \
171 (__v8sf)_mm256_setzero_ps(), (__mmask8)(U), _MM_FROUND_NO_EXC))
172
173#define _mm256_minmax_round_ps(A, B, C, R) \
174 ((__m256)__builtin_ia32_vminmaxps256_round_mask( \
175 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), \
176 (__v8sf)_mm256_undefined_ps(), (__mmask8)-1, (int)(R)))
177
178#define _mm256_mask_minmax_round_ps(W, U, A, B, C, R) \
179 ((__m256)__builtin_ia32_vminmaxps256_round_mask( \
180 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), (__v8sf)(__m256)(W), \
181 (__mmask8)(U), (int)(R)))
182
183#define _mm256_maskz_minmax_round_ps(U, A, B, C, R) \
184 ((__m256)__builtin_ia32_vminmaxps256_round_mask( \
185 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), \
186 (__v8sf)_mm256_setzero_ps(), (__mmask8)(U), (int)(R)))
187
188#define _mm_minmax_sd(A, B, C) \
189 ((__m128d)__builtin_ia32_vminmaxsd_round_mask( \
190 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
191 (__v2df)_mm_undefined_pd(), (__mmask8)-1, _MM_FROUND_CUR_DIRECTION))
192
193#define _mm_mask_minmax_sd(W, U, A, B, C) \
194 ((__m128d)__builtin_ia32_vminmaxsd_round_mask( \
195 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
196 (__v2df)(__m128d)(W), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
197
198#define _mm_maskz_minmax_sd(U, A, B, C) \
199 ((__m128d)__builtin_ia32_vminmaxsd_round_mask( \
200 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
201 (__v2df)_mm_setzero_pd(), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
202
203#define _mm_minmax_round_sd(A, B, C, R) \
204 ((__m128d)__builtin_ia32_vminmaxsd_round_mask( \
205 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
206 (__v2df)_mm_undefined_pd(), (__mmask8)-1, (int)(R)))
207
208#define _mm_mask_minmax_round_sd(W, U, A, B, C, R) \
209 ((__m128d)__builtin_ia32_vminmaxsd_round_mask( \
210 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
211 (__v2df)(__m128d)(W), (__mmask8)(U), (int)(R)))
212
213#define _mm_maskz_minmax_round_sd(U, A, B, C, R) \
214 ((__m128d)__builtin_ia32_vminmaxsd_round_mask( \
215 (__v2df)(__m128d)(A), (__v2df)(__m128d)(B), (int)(C), \
216 (__v2df)_mm_setzero_pd(), (__mmask8)(U), (int)(R)))
217
218#define _mm_minmax_sh(A, B, C) \
219 ((__m128h)__builtin_ia32_vminmaxsh_round_mask( \
220 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
221 (__v8hf)_mm_undefined_ph(), (__mmask8)-1, _MM_FROUND_CUR_DIRECTION))
222
223#define _mm_mask_minmax_sh(W, U, A, B, C) \
224 ((__m128h)__builtin_ia32_vminmaxsh_round_mask( \
225 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
226 (__v8hf)(__m128h)(W), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
227
228#define _mm_maskz_minmax_sh(U, A, B, C) \
229 ((__m128h)__builtin_ia32_vminmaxsh_round_mask( \
230 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
231 (__v8hf)_mm_setzero_ph(), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
232
233#define _mm_minmax_round_sh(A, B, C, R) \
234 ((__m128h)__builtin_ia32_vminmaxsh_round_mask( \
235 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
236 (__v8hf)_mm_undefined_ph(), (__mmask8)-1, (int)(R)))
237
238#define _mm_mask_minmax_round_sh(W, U, A, B, C, R) \
239 ((__m128h)__builtin_ia32_vminmaxsh_round_mask( \
240 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
241 (__v8hf)(__m128h)(W), (__mmask8)(U), (int)(R)))
242
243#define _mm_maskz_minmax_round_sh(U, A, B, C, R) \
244 ((__m128h)__builtin_ia32_vminmaxsh_round_mask( \
245 (__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (int)(C), \
246 (__v8hf)_mm_setzero_ph(), (__mmask8)(U), (int)(R)))
247
248#define _mm_minmax_ss(A, B, C) \
249 ((__m128)__builtin_ia32_vminmaxss_round_mask( \
250 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), \
251 (__v4sf)_mm_undefined_ps(), (__mmask8)-1, _MM_FROUND_CUR_DIRECTION))
252
253#define _mm_mask_minmax_ss(W, U, A, B, C) \
254 ((__m128)__builtin_ia32_vminmaxss_round_mask( \
255 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), (__v4sf)(W), \
256 (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
257
258#define _mm_maskz_minmax_ss(U, A, B, C) \
259 ((__m128)__builtin_ia32_vminmaxss_round_mask( \
260 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), \
261 (__v4sf)_mm_setzero_ps(), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION))
262
263#define _mm_minmax_round_ss(A, B, C, R) \
264 ((__m128)__builtin_ia32_vminmaxss_round_mask( \
265 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), \
266 (__v4sf)_mm_undefined_ps(), (__mmask8)-1, (int)(R)))
267
268#define _mm_mask_minmax_round_ss(W, U, A, B, C, R) \
269 ((__m128)__builtin_ia32_vminmaxss_round_mask( \
270 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), (__v4sf)(W), \
271 (__mmask8)(U), (int)(R)))
272
273#define _mm_maskz_minmax_round_ss(U, A, B, C, R) \
274 ((__m128)__builtin_ia32_vminmaxss_round_mask( \
275 (__v4sf)(__m128)(A), (__v4sf)(__m128)(B), (int)(C), \
276 (__v4sf)_mm_setzero_ps(), (__mmask8)(U), (int)(R)))
277#endif // __AVX10_2MINMAXINTRIN_H
lib/include/avx10_2niintrin.h created+2075
......@@ -0,0 +1,2075 @@
1/*===---- avx10_2niintrin.h - AVX10.2 new instruction intrinsics -----------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error "Never use <avx10_2niintrin.h> directly; include <immintrin.h> instead."
11#endif
12
13#ifdef __SSE2__
14
15#ifndef __AVX10_2NIINTRIN_H
16#define __AVX10_2NIINTRIN_H
17
18#define __DEFAULT_FN_ATTRS128 \
19 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
20 __min_vector_width__(128)))
21#define __DEFAULT_FN_ATTRS256 \
22 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
23 __min_vector_width__(256)))
24
25/* VNNI FP16 */
26static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_dpph_ps(__m128 __W,
27 __m128h __A,
28 __m128h __B) {
29 return (__m128)__builtin_ia32_vdpphps128((__v4sf)__W, (__v8hf)__A,
30 (__v8hf)__B);
31}
32
33static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_mask_dpph_ps(__m128 __W,
34 __mmask8 __U,
35 __m128h __A,
36 __m128h __B) {
37 return (__m128)__builtin_ia32_selectps_128(
38 (__mmask8)__U, (__v4sf)_mm_dpph_ps(__W, __A, __B), (__v4sf)__W);
39}
40
41static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_maskz_dpph_ps(__mmask8 __U,
42 __m128 __W,
43 __m128h __A,
44 __m128h __B) {
45 return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
46 (__v4sf)_mm_dpph_ps(__W, __A, __B),
47 (__v4sf)_mm_setzero_ps());
48}
49
50static __inline__ __m256 __DEFAULT_FN_ATTRS256 _mm256_dpph_ps(__m256 __W,
51 __m256h __A,
52 __m256h __B) {
53 return (__m256)__builtin_ia32_vdpphps256((__v8sf)__W, (__v16hf)__A,
54 (__v16hf)__B);
55}
56
57static __inline__ __m256 __DEFAULT_FN_ATTRS256
58_mm256_mask_dpph_ps(__m256 __W, __mmask8 __U, __m256h __A, __m256h __B) {
59 return (__m256)__builtin_ia32_selectps_256(
60 (__mmask8)__U, (__v8sf)_mm256_dpph_ps(__W, __A, __B), (__v8sf)__W);
61}
62
63static __inline__ __m256 __DEFAULT_FN_ATTRS256
64_mm256_maskz_dpph_ps(__mmask8 __U, __m256 __W, __m256h __A, __m256h __B) {
65 return (__m256)__builtin_ia32_selectps_256(
66 (__mmask8)__U, (__v8sf)_mm256_dpph_ps(__W, __A, __B),
67 (__v8sf)_mm256_setzero_ps());
68}
69
70/* VMPSADBW */
71#define _mm_mask_mpsadbw_epu8(W, U, A, B, imm) \
72 ((__m128i)__builtin_ia32_selectw_128( \
73 (__mmask8)(U), (__v8hi)_mm_mpsadbw_epu8((A), (B), (imm)), \
74 (__v8hi)(__m128i)(W)))
75
76#define _mm_maskz_mpsadbw_epu8(U, A, B, imm) \
77 ((__m128i)__builtin_ia32_selectw_128( \
78 (__mmask8)(U), (__v8hi)_mm_mpsadbw_epu8((A), (B), (imm)), \
79 (__v8hi)_mm_setzero_si128()))
80
81#define _mm256_mask_mpsadbw_epu8(W, U, A, B, imm) \
82 ((__m256i)__builtin_ia32_selectw_256( \
83 (__mmask16)(U), (__v16hi)_mm256_mpsadbw_epu8((A), (B), (imm)), \
84 (__v16hi)(__m256i)(W)))
85
86#define _mm256_maskz_mpsadbw_epu8(U, A, B, imm) \
87 ((__m256i)__builtin_ia32_selectw_256( \
88 (__mmask16)(U), (__v16hi)_mm256_mpsadbw_epu8((A), (B), (imm)), \
89 (__v16hi)_mm256_setzero_si256()))
90
91/* VNNI INT8 */
92static __inline__ __m128i __DEFAULT_FN_ATTRS128
93_mm_mask_dpbssd_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
94 return (__m128i)__builtin_ia32_selectd_128(
95 __U, (__v4si)_mm_dpbssd_epi32(__W, __A, __B), (__v4si)__W);
96}
97
98static __inline__ __m128i __DEFAULT_FN_ATTRS128
99_mm_maskz_dpbssd_epi32(__mmask8 __U, __m128i __W, __m128i __A, __m128i __B) {
100 return (__m128i)__builtin_ia32_selectd_128(
101 __U, (__v4si)_mm_dpbssd_epi32(__W, __A, __B),
102 (__v4si)_mm_setzero_si128());
103}
104
105static __inline__ __m256i __DEFAULT_FN_ATTRS256
106_mm256_mask_dpbssd_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
107 return (__m256i)__builtin_ia32_selectd_256(
108 __U, (__v8si)_mm256_dpbssd_epi32(__W, __A, __B), (__v8si)__W);
109}
110
111static __inline__ __m256i __DEFAULT_FN_ATTRS256
112_mm256_maskz_dpbssd_epi32(__mmask8 __U, __m256i __W, __m256i __A, __m256i __B) {
113 return (__m256i)__builtin_ia32_selectd_256(
114 __U, (__v8si)_mm256_dpbssd_epi32(__W, __A, __B),
115 (__v8si)_mm256_setzero_si256());
116}
117
118static __inline__ __m128i __DEFAULT_FN_ATTRS128
119_mm_mask_dpbssds_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
120 return (__m128i)__builtin_ia32_selectd_128(
121 __U, (__v4si)_mm_dpbssds_epi32(__W, __A, __B), (__v4si)__W);
122}
123
124static __inline__ __m128i __DEFAULT_FN_ATTRS128
125_mm_maskz_dpbssds_epi32(__mmask8 __U, __m128i __W, __m128i __A, __m128i __B) {
126 return (__m128i)__builtin_ia32_selectd_128(
127 __U, (__v4si)_mm_dpbssds_epi32(__W, __A, __B),
128 (__v4si)_mm_setzero_si128());
129}
130
131static __inline__ __m256i __DEFAULT_FN_ATTRS256
132_mm256_mask_dpbssds_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
133 return (__m256i)__builtin_ia32_selectd_256(
134 __U, (__v8si)_mm256_dpbssds_epi32(__W, __A, __B), (__v8si)__W);
135}
136
137static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpbssds_epi32(
138 __mmask8 __U, __m256i __W, __m256i __A, __m256i __B) {
139 return (__m256i)__builtin_ia32_selectd_256(
140 __U, (__v8si)_mm256_dpbssds_epi32(__W, __A, __B),
141 (__v8si)_mm256_setzero_si256());
142}
143
144static __inline__ __m128i __DEFAULT_FN_ATTRS128
145_mm_mask_dpbsud_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
146 return (__m128i)__builtin_ia32_selectd_128(
147 __U, (__v4si)_mm_dpbsud_epi32(__W, __A, __B), (__v4si)__W);
148}
149
150static __inline__ __m128i __DEFAULT_FN_ATTRS128
151_mm_maskz_dpbsud_epi32(__mmask8 __U, __m128i __W, __m128i __A, __m128i __B) {
152 return (__m128i)__builtin_ia32_selectd_128(
153 __U, (__v4si)_mm_dpbsud_epi32(__W, __A, __B),
154 (__v4si)_mm_setzero_si128());
155}
156
157static __inline__ __m256i __DEFAULT_FN_ATTRS256
158_mm256_mask_dpbsud_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
159 return (__m256i)__builtin_ia32_selectd_256(
160 __U, (__v8si)_mm256_dpbsud_epi32(__W, __A, __B), (__v8si)__W);
161}
162
163static __inline__ __m256i __DEFAULT_FN_ATTRS256
164_mm256_maskz_dpbsud_epi32(__mmask8 __U, __m256i __W, __m256i __A, __m256i __B) {
165 return (__m256i)__builtin_ia32_selectd_256(
166 __U, (__v8si)_mm256_dpbsud_epi32(__W, __A, __B),
167 (__v8si)_mm256_setzero_si256());
168}
169
170static __inline__ __m128i __DEFAULT_FN_ATTRS128
171_mm_mask_dpbsuds_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
172 return (__m128i)__builtin_ia32_selectd_128(
173 __U, (__v4si)_mm_dpbsuds_epi32(__W, __A, __B), (__v4si)__W);
174}
175
176static __inline__ __m128i __DEFAULT_FN_ATTRS128
177_mm_maskz_dpbsuds_epi32(__mmask8 __U, __m128i __W, __m128i __A, __m128i __B) {
178 return (__m128i)__builtin_ia32_selectd_128(
179 __U, (__v4si)_mm_dpbsuds_epi32(__W, __A, __B),
180 (__v4si)_mm_setzero_si128());
181}
182
183static __inline__ __m256i __DEFAULT_FN_ATTRS256
184_mm256_mask_dpbsuds_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
185 return (__m256i)__builtin_ia32_selectd_256(
186 __U, (__v8si)_mm256_dpbsuds_epi32(__W, __A, __B), (__v8si)__W);
187}
188
189static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpbsuds_epi32(
190 __mmask8 __U, __m256i __W, __m256i __A, __m256i __B) {
191 return (__m256i)__builtin_ia32_selectd_256(
192 __U, (__v8si)_mm256_dpbsuds_epi32(__W, __A, __B),
193 (__v8si)_mm256_setzero_si256());
194}
195
196static __inline__ __m128i __DEFAULT_FN_ATTRS128
197_mm_mask_dpbuud_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
198 return (__m128i)__builtin_ia32_selectd_128(
199 __U, (__v4si)_mm_dpbuud_epi32(__W, __A, __B), (__v4si)__W);
200}
201
202static __inline__ __m128i __DEFAULT_FN_ATTRS128
203_mm_maskz_dpbuud_epi32(__mmask8 __U, __m128i __W, __m128i __A, __m128i __B) {
204 return (__m128i)__builtin_ia32_selectd_128(
205 __U, (__v4si)_mm_dpbuud_epi32(__W, __A, __B),
206 (__v4si)_mm_setzero_si128());
207}
208
209static __inline__ __m256i __DEFAULT_FN_ATTRS256
210_mm256_mask_dpbuud_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
211 return (__m256i)__builtin_ia32_selectd_256(
212 __U, (__v8si)_mm256_dpbuud_epi32(__W, __A, __B), (__v8si)__W);
213}
214
215static __inline__ __m256i __DEFAULT_FN_ATTRS256
216_mm256_maskz_dpbuud_epi32(__mmask8 __U, __m256i __W, __m256i __A, __m256i __B) {
217 return (__m256i)__builtin_ia32_selectd_256(
218 __U, (__v8si)_mm256_dpbuud_epi32(__W, __A, __B),
219 (__v8si)_mm256_setzero_si256());
220}
221
222static __inline__ __m128i __DEFAULT_FN_ATTRS128
223_mm_mask_dpbuuds_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
224 return (__m128i)__builtin_ia32_selectd_128(
225 __U, (__v4si)_mm_dpbuuds_epi32(__W, __A, __B), (__v4si)__W);
226}
227
228static __inline__ __m128i __DEFAULT_FN_ATTRS128
229_mm_maskz_dpbuuds_epi32(__mmask8 __U, __m128i __W, __m128i __A, __m128i __B) {
230 return (__m128i)__builtin_ia32_selectd_128(
231 __U, (__v4si)_mm_dpbuuds_epi32(__W, __A, __B),
232 (__v4si)_mm_setzero_si128());
233}
234
235static __inline__ __m256i __DEFAULT_FN_ATTRS256
236_mm256_mask_dpbuuds_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
237 return (__m256i)__builtin_ia32_selectd_256(
238 __U, (__v8si)_mm256_dpbuuds_epi32(__W, __A, __B), (__v8si)__W);
239}
240
241static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpbuuds_epi32(
242 __mmask8 __U, __m256i __W, __m256i __A, __m256i __B) {
243 return (__m256i)__builtin_ia32_selectd_256(
244 __U, (__v8si)_mm256_dpbuuds_epi32(__W, __A, __B),
245 (__v8si)_mm256_setzero_si256());
246}
247
248/* VNNI INT16 */
249static __inline__ __m128i __DEFAULT_FN_ATTRS128
250_mm_mask_dpwsud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
251 return (__m128i)__builtin_ia32_selectd_128(
252 (__mmask8)__U, (__v4si)_mm_dpwsud_epi32(__A, __B, __C), (__v4si)__A);
253}
254
255static __inline__ __m128i __DEFAULT_FN_ATTRS128
256_mm_maskz_dpwsud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
257 return (__m128i)__builtin_ia32_selectd_128(
258 (__mmask8)__U, (__v4si)_mm_dpwsud_epi32(__A, __B, __C),
259 (__v4si)_mm_setzero_si128());
260}
261
262static __inline__ __m256i __DEFAULT_FN_ATTRS256
263_mm256_mask_dpwsud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
264 return (__m256i)__builtin_ia32_selectd_256(
265 (__mmask8)__U, (__v8si)_mm256_dpwsud_epi32(__A, __B, __C), (__v8si)__A);
266}
267
268static __inline__ __m256i __DEFAULT_FN_ATTRS256
269_mm256_maskz_dpwsud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
270 return (__m256i)__builtin_ia32_selectd_256(
271 (__mmask8)__U, (__v8si)_mm256_dpwsud_epi32(__A, __B, __C),
272 (__v8si)_mm256_setzero_si256());
273}
274
275static __inline__ __m128i __DEFAULT_FN_ATTRS128
276_mm_mask_dpwsuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
277 return (__m128i)__builtin_ia32_selectd_128(
278 (__mmask8)__U, (__v4si)_mm_dpwsuds_epi32(__A, __B, __C), (__v4si)__A);
279}
280
281static __inline__ __m128i __DEFAULT_FN_ATTRS128
282_mm_maskz_dpwsuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
283 return (__m128i)__builtin_ia32_selectd_128(
284 (__mmask8)__U, (__v4si)_mm_dpwsuds_epi32(__A, __B, __C),
285 (__v4si)_mm_setzero_si128());
286}
287
288static __inline__ __m256i __DEFAULT_FN_ATTRS256
289_mm256_mask_dpwsuds_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
290 return (__m256i)__builtin_ia32_selectd_256(
291 (__mmask8)__U, (__v8si)_mm256_dpwsuds_epi32(__A, __B, __C), (__v8si)__A);
292}
293
294static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpwsuds_epi32(
295 __m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
296 return (__m256i)__builtin_ia32_selectd_256(
297 (__mmask8)__U, (__v8si)_mm256_dpwsuds_epi32(__A, __B, __C),
298 (__v8si)_mm256_setzero_si256());
299}
300
301static __inline__ __m128i __DEFAULT_FN_ATTRS128
302_mm_mask_dpwusd_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
303 return (__m128i)__builtin_ia32_selectd_128(
304 (__mmask8)__U, (__v4si)_mm_dpwusd_epi32(__A, __B, __C), (__v4si)__A);
305}
306
307static __inline__ __m128i __DEFAULT_FN_ATTRS128
308_mm_maskz_dpwusd_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
309 return (__m128i)__builtin_ia32_selectd_128(
310 (__mmask8)__U, (__v4si)_mm_dpwusd_epi32(__A, __B, __C),
311 (__v4si)_mm_setzero_si128());
312}
313
314static __inline__ __m256i __DEFAULT_FN_ATTRS256
315_mm256_mask_dpwusd_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
316 return (__m256i)__builtin_ia32_selectd_256(
317 (__mmask8)__U, (__v8si)_mm256_dpwusd_epi32(__A, __B, __C), (__v8si)__A);
318}
319
320static __inline__ __m256i __DEFAULT_FN_ATTRS256
321_mm256_maskz_dpwusd_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
322 return (__m256i)__builtin_ia32_selectd_256(
323 (__mmask8)__U, (__v8si)_mm256_dpwusd_epi32(__A, __B, __C),
324 (__v8si)_mm256_setzero_si256());
325}
326
327static __inline__ __m128i __DEFAULT_FN_ATTRS128
328_mm_mask_dpwusds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
329 return (__m128i)__builtin_ia32_selectd_128(
330 (__mmask8)__U, (__v4si)_mm_dpwusds_epi32(__A, __B, __C), (__v4si)__A);
331}
332
333static __inline__ __m128i __DEFAULT_FN_ATTRS128
334_mm_maskz_dpwusds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
335 return (__m128i)__builtin_ia32_selectd_128(
336 (__mmask8)__U, (__v4si)_mm_dpwusds_epi32(__A, __B, __C),
337 (__v4si)_mm_setzero_si128());
338}
339
340static __inline__ __m256i __DEFAULT_FN_ATTRS256
341_mm256_mask_dpwusds_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
342 return (__m256i)__builtin_ia32_selectd_256(
343 (__mmask8)__U, (__v8si)_mm256_dpwusds_epi32(__A, __B, __C), (__v8si)__A);
344}
345
346static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpwusds_epi32(
347 __m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
348 return (__m256i)__builtin_ia32_selectd_256(
349 (__mmask8)__U, (__v8si)_mm256_dpwusds_epi32(__A, __B, __C),
350 (__v8si)_mm256_setzero_si256());
351}
352
353static __inline__ __m128i __DEFAULT_FN_ATTRS128
354_mm_mask_dpwuud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
355 return (__m128i)__builtin_ia32_selectd_128(
356 (__mmask8)__U, (__v4si)_mm_dpwuud_epi32(__A, __B, __C), (__v4si)__A);
357}
358
359static __inline__ __m128i __DEFAULT_FN_ATTRS128
360_mm_maskz_dpwuud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
361 return (__m128i)__builtin_ia32_selectd_128(
362 (__mmask8)__U, (__v4si)_mm_dpwuud_epi32(__A, __B, __C),
363 (__v4si)_mm_setzero_si128());
364}
365
366static __inline__ __m256i __DEFAULT_FN_ATTRS256
367_mm256_mask_dpwuud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
368 return (__m256i)__builtin_ia32_selectd_256(
369 (__mmask8)__U, (__v8si)_mm256_dpwuud_epi32(__A, __B, __C), (__v8si)__A);
370}
371
372static __inline__ __m256i __DEFAULT_FN_ATTRS256
373_mm256_maskz_dpwuud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
374 return (__m256i)__builtin_ia32_selectd_256(
375 (__mmask8)__U, (__v8si)_mm256_dpwuud_epi32(__A, __B, __C),
376 (__v8si)_mm256_setzero_si256());
377}
378
379static __inline__ __m128i __DEFAULT_FN_ATTRS128
380_mm_mask_dpwuuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
381 return (__m128i)__builtin_ia32_selectd_128(
382 (__mmask8)__U, (__v4si)_mm_dpwuuds_epi32(__A, __B, __C), (__v4si)__A);
383}
384
385static __inline__ __m128i __DEFAULT_FN_ATTRS128
386_mm_maskz_dpwuuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
387 return (__m128i)__builtin_ia32_selectd_128(
388 (__mmask8)__U, (__v4si)_mm_dpwuuds_epi32(__A, __B, __C),
389 (__v4si)_mm_setzero_si128());
390}
391
392static __inline__ __m256i __DEFAULT_FN_ATTRS256
393_mm256_mask_dpwuuds_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
394 return (__m256i)__builtin_ia32_selectd_256(
395 (__mmask8)__U, (__v8si)_mm256_dpwuuds_epi32(__A, __B, __C), (__v8si)__A);
396}
397
398static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpwuuds_epi32(
399 __m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
400 return (__m256i)__builtin_ia32_selectd_256(
401 (__mmask8)__U, (__v8si)_mm256_dpwuuds_epi32(__A, __B, __C),
402 (__v8si)_mm256_setzero_si256());
403}
404
405/* YMM Rounding */
406#define _mm256_add_round_pd(A, B, R) \
407 ((__m256d)__builtin_ia32_vaddpd256_round((__v4df)(__m256d)(A), \
408 (__v4df)(__m256d)(B), (int)(R)))
409
410#define _mm256_mask_add_round_pd(W, U, A, B, R) \
411 ((__m256d)__builtin_ia32_selectpd_256( \
412 (__mmask8)(U), (__v4df)_mm256_add_round_pd((A), (B), (R)), \
413 (__v4df)(__m256d)(W)))
414
415#define _mm256_maskz_add_round_pd(U, A, B, R) \
416 ((__m256d)__builtin_ia32_selectpd_256( \
417 (__mmask8)(U), (__v4df)_mm256_add_round_pd((A), (B), (R)), \
418 (__v4df)_mm256_setzero_pd()))
419
420#define _mm256_add_round_ph(A, B, R) \
421 ((__m256h)__builtin_ia32_vaddph256_round((__v16hf)(__m256h)(A), \
422 (__v16hf)(__m256h)(B), (int)(R)))
423
424#define _mm256_mask_add_round_ph(W, U, A, B, R) \
425 ((__m256h)__builtin_ia32_selectph_256( \
426 (__mmask16)(U), (__v16hf)_mm256_add_round_ph((A), (B), (R)), \
427 (__v16hf)(__m256h)(W)))
428
429#define _mm256_maskz_add_round_ph(U, A, B, R) \
430 ((__m256h)__builtin_ia32_selectph_256( \
431 (__mmask16)(U), (__v16hf)_mm256_add_round_ph((A), (B), (R)), \
432 (__v16hf)_mm256_setzero_ph()))
433
434#define _mm256_add_round_ps(A, B, R) \
435 ((__m256)__builtin_ia32_vaddps256_round((__v8sf)(__m256)(A), \
436 (__v8sf)(__m256)(B), (int)(R)))
437
438#define _mm256_mask_add_round_ps(W, U, A, B, R) \
439 ((__m256)__builtin_ia32_selectps_256( \
440 (__mmask8)(U), (__v8sf)_mm256_add_round_ps((A), (B), (R)), \
441 (__v8sf)(__m256)(W)))
442
443#define _mm256_maskz_add_round_ps(U, A, B, R) \
444 ((__m256)__builtin_ia32_selectps_256( \
445 (__mmask8)(U), (__v8sf)_mm256_add_round_ps((A), (B), (R)), \
446 (__v8sf)_mm256_setzero_ps()))
447
448#define _mm256_cmp_round_pd_mask(A, B, P, R) \
449 ((__mmask8)__builtin_ia32_vcmppd256_round_mask( \
450 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(P), (__mmask8)-1, \
451 (int)(R)))
452
453#define _mm256_mask_cmp_round_pd_mask(U, A, B, P, R) \
454 ((__mmask8)__builtin_ia32_vcmppd256_round_mask( \
455 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(P), (__mmask8)(U), \
456 (int)(R)))
457
458#define _mm256_cmp_round_ph_mask(A, B, P, R) \
459 ((__mmask16)__builtin_ia32_vcmpph256_round_mask( \
460 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (int)(P), (__mmask16)-1, \
461 (int)(R)))
462
463#define _mm256_mask_cmp_round_ph_mask(U, A, B, P, R) \
464 ((__mmask16)__builtin_ia32_vcmpph256_round_mask( \
465 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (int)(P), (__mmask16)(U), \
466 (int)(R)))
467
468#define _mm256_cmp_round_ps_mask(A, B, P, R) \
469 ((__mmask8)__builtin_ia32_vcmpps256_round_mask( \
470 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(P), (__mmask8)-1, \
471 (int)(R)))
472
473#define _mm256_mask_cmp_round_ps_mask(U, A, B, P, R) \
474 ((__mmask8)__builtin_ia32_vcmpps256_round_mask( \
475 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(P), (__mmask8)(U), \
476 (int)(R)))
477
478#define _mm256_cvt_roundepi32_ph(A, R) \
479 ((__m128h)__builtin_ia32_vcvtdq2ph256_round_mask( \
480 (__v8si)(A), (__v8hf)_mm_undefined_ph(), (__mmask8)(-1), (int)(R)))
481
482#define _mm256_mask_cvt_roundepi32_ph(W, U, A, R) \
483 ((__m128h)__builtin_ia32_vcvtdq2ph256_round_mask((__v8si)(A), (__v8hf)(W), \
484 (__mmask8)(U), (int)(R)))
485
486#define _mm256_maskz_cvt_roundepi32_ph(U, A, R) \
487 ((__m128h)__builtin_ia32_vcvtdq2ph256_round_mask( \
488 (__v8si)(A), (__v8hf)_mm_setzero_ph(), (__mmask8)(U), (int)(R)))
489
490#define _mm256_cvt_roundepi32_ps(A, R) \
491 ((__m256)__builtin_ia32_vcvtdq2ps256_round_mask((__v8si)(__m256i)(A), \
492 (__v8sf)_mm256_setzero_ps(), \
493 (__mmask8)-1, (int)(R)))
494
495#define _mm256_mask_cvt_roundepi32_ps(W, U, A, R) \
496 ((__m256)__builtin_ia32_vcvtdq2ps256_round_mask( \
497 (__v8si)(__m256i)(A), (__v8sf)(__m256)(W), (__mmask8)(U), (int)(R)))
498
499#define _mm256_maskz_cvt_roundepi32_ps(U, A, R) \
500 ((__m256)__builtin_ia32_vcvtdq2ps256_round_mask((__v8si)(__m256i)(A), \
501 (__v8sf)_mm256_setzero_ps(), \
502 (__mmask8)(U), (int)(R)))
503
504#define _mm256_cvt_roundpd_epi32(A, R) \
505 ((__m128i)__builtin_ia32_vcvtpd2dq256_round_mask( \
506 (__v4df)(__m256d)(A), (__v4si)_mm_setzero_si128(), (__mmask8)-1, \
507 (int)(R)))
508
509#define _mm256_mask_cvt_roundpd_epi32(W, U, A, R) \
510 ((__m128i)__builtin_ia32_vcvtpd2dq256_round_mask( \
511 (__v4df)(__m256d)(A), (__v4si)(__m128i)(W), (__mmask8)(U), (int)(R)))
512
513#define _mm256_maskz_cvt_roundpd_epi32(U, A, R) \
514 ((__m128i)__builtin_ia32_vcvtpd2dq256_round_mask( \
515 (__v4df)(__m256d)(A), (__v4si)_mm_setzero_si128(), (__mmask8)(U), \
516 (int)(R)))
517
518#define _mm256_cvt_roundpd_ph(A, R) \
519 ((__m128h)__builtin_ia32_vcvtpd2ph256_round_mask( \
520 (__v4df)(A), (__v8hf)_mm_undefined_ph(), (__mmask8)(-1), (int)(R)))
521
522#define _mm256_mask_cvt_roundpd_ph(W, U, A, R) \
523 ((__m128h)__builtin_ia32_vcvtpd2ph256_round_mask((__v4df)(A), (__v8hf)(W), \
524 (__mmask8)(U), (int)(R)))
525
526#define _mm256_maskz_cvt_roundpd_ph(U, A, R) \
527 ((__m128h)__builtin_ia32_vcvtpd2ph256_round_mask( \
528 (__v4df)(A), (__v8hf)_mm_setzero_ph(), (__mmask8)(U), (int)(R)))
529
530#define _mm256_cvt_roundpd_ps(A, R) \
531 ((__m128)__builtin_ia32_vcvtpd2ps256_round_mask( \
532 (__v4df)(__m256d)(A), (__v4sf)_mm_setzero_ps(), (__mmask8)-1, (int)(R)))
533
534#define _mm256_mask_cvt_roundpd_ps(W, U, A, R) \
535 ((__m128)__builtin_ia32_vcvtpd2ps256_round_mask( \
536 (__v4df)(__m256d)(A), (__v4sf)(__m128)(W), (__mmask8)(U), (int)(R)))
537
538#define _mm256_maskz_cvt_roundpd_ps(U, A, R) \
539 ((__m128)__builtin_ia32_vcvtpd2ps256_round_mask((__v4df)(__m256d)(A), \
540 (__v4sf)_mm_setzero_ps(), \
541 (__mmask8)(U), (int)(R)))
542
543#define _mm256_cvt_roundpd_epi64(A, R) \
544 ((__m256i)__builtin_ia32_vcvtpd2qq256_round_mask( \
545 (__v4df)(__m256d)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)-1, \
546 (int)(R)))
547
548#define _mm256_mask_cvt_roundpd_epi64(W, U, A, R) \
549 ((__m256i)__builtin_ia32_vcvtpd2qq256_round_mask( \
550 (__v4df)(__m256d)(A), (__v4di)(__m256i)(W), (__mmask8)(U), (int)(R)))
551
552#define _mm256_maskz_cvt_roundpd_epi64(U, A, R) \
553 ((__m256i)__builtin_ia32_vcvtpd2qq256_round_mask( \
554 (__v4df)(__m256d)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)(U), \
555 (int)(R)))
556
557#define _mm256_cvt_roundpd_epu32(A, R) \
558 ((__m128i)__builtin_ia32_vcvtpd2udq256_round_mask( \
559 (__v4df)(__m256d)(A), (__v4su)_mm_setzero_si128(), (__mmask8)-1, \
560 (int)(R)))
561
562#define _mm256_mask_cvt_roundpd_epu32(W, U, A, R) \
563 ((__m128i)__builtin_ia32_vcvtpd2udq256_round_mask( \
564 (__v4df)(__m256d)(A), (__v4su)(__m128i)(W), (__mmask8)(U), (int)(R)))
565
566#define _mm256_maskz_cvt_roundpd_epu32(U, A, R) \
567 ((__m128i)__builtin_ia32_vcvtpd2udq256_round_mask( \
568 (__v4df)(__m256d)(A), (__v4su)_mm_setzero_si128(), (__mmask8)(U), \
569 (int)(R)))
570
571#define _mm256_cvt_roundpd_epu64(A, R) \
572 ((__m256i)__builtin_ia32_vcvtpd2uqq256_round_mask( \
573 (__v4df)(__m256d)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)-1, \
574 (int)(R)))
575
576#define _mm256_mask_cvt_roundpd_epu64(W, U, A, R) \
577 ((__m256i)__builtin_ia32_vcvtpd2uqq256_round_mask( \
578 (__v4df)(__m256d)(A), (__v4du)(__m256i)(W), (__mmask8)(U), (int)(R)))
579
580#define _mm256_maskz_cvt_roundpd_epu64(U, A, R) \
581 ((__m256i)__builtin_ia32_vcvtpd2uqq256_round_mask( \
582 (__v4df)(__m256d)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)(U), \
583 (int)(R)))
584
585#define _mm256_cvt_roundph_epi32(A, R) \
586 ((__m256i)__builtin_ia32_vcvtph2dq256_round_mask( \
587 (__v8hf)(A), (__v8si)_mm256_undefined_si256(), (__mmask8)(-1), \
588 (int)(R)))
589
590#define _mm256_mask_cvt_roundph_epi32(W, U, A, R) \
591 ((__m256i)__builtin_ia32_vcvtph2dq256_round_mask((__v8hf)(A), (__v8si)(W), \
592 (__mmask8)(U), (int)(R)))
593
594#define _mm256_maskz_cvt_roundph_epi32(U, A, R) \
595 ((__m256i)__builtin_ia32_vcvtph2dq256_round_mask( \
596 (__v8hf)(A), (__v8si)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
597
598#define _mm256_cvt_roundph_pd(A, R) \
599 ((__m256d)__builtin_ia32_vcvtph2pd256_round_mask( \
600 (__v8hf)(A), (__v4df)_mm256_undefined_pd(), (__mmask8)(-1), (int)(R)))
601
602#define _mm256_mask_cvt_roundph_pd(W, U, A, R) \
603 ((__m256d)__builtin_ia32_vcvtph2pd256_round_mask((__v8hf)(A), (__v4df)(W), \
604 (__mmask8)(U), (int)(R)))
605
606#define _mm256_maskz_cvt_roundph_pd(U, A, R) \
607 ((__m256d)__builtin_ia32_vcvtph2pd256_round_mask( \
608 (__v8hf)(A), (__v4df)_mm256_setzero_pd(), (__mmask8)(U), (int)(R)))
609
610#define _mm256_cvtx_roundph_ps(A, R) \
611 ((__m256)__builtin_ia32_vcvtph2psx256_round_mask( \
612 (__v8hf)(A), (__v8sf)_mm256_undefined_ps(), (__mmask8)(-1), (int)(R)))
613
614#define _mm256_mask_cvtx_roundph_ps(W, U, A, R) \
615 ((__m256)__builtin_ia32_vcvtph2psx256_round_mask((__v8hf)(A), (__v8sf)(W), \
616 (__mmask8)(U), (int)(R)))
617
618#define _mm256_maskz_cvtx_roundph_ps(U, A, R) \
619 ((__m256)__builtin_ia32_vcvtph2psx256_round_mask( \
620 (__v8hf)(A), (__v8sf)_mm256_setzero_ps(), (__mmask8)(U), (int)(R)))
621
622#define _mm256_cvt_roundph_epi64(A, R) \
623 ((__m256i)__builtin_ia32_vcvtph2qq256_round_mask( \
624 (__v8hf)(A), (__v4di)_mm256_undefined_si256(), (__mmask8)(-1), \
625 (int)(R)))
626
627#define _mm256_mask_cvt_roundph_epi64(W, U, A, R) \
628 ((__m256i)__builtin_ia32_vcvtph2qq256_round_mask((__v8hf)(A), (__v4di)(W), \
629 (__mmask8)(U), (int)(R)))
630
631#define _mm256_maskz_cvt_roundph_epi64(U, A, R) \
632 ((__m256i)__builtin_ia32_vcvtph2qq256_round_mask( \
633 (__v8hf)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
634
635#define _mm256_cvt_roundph_epu32(A, R) \
636 ((__m256i)__builtin_ia32_vcvtph2udq256_round_mask( \
637 (__v8hf)(A), (__v8su)_mm256_undefined_si256(), (__mmask8)(-1), \
638 (int)(R)))
639
640#define _mm256_mask_cvt_roundph_epu32(W, U, A, R) \
641 ((__m256i)__builtin_ia32_vcvtph2udq256_round_mask((__v8hf)(A), (__v8su)(W), \
642 (__mmask8)(U), (int)(R)))
643
644#define _mm256_maskz_cvt_roundph_epu32(U, A, R) \
645 ((__m256i)__builtin_ia32_vcvtph2udq256_round_mask( \
646 (__v8hf)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
647
648#define _mm256_cvt_roundph_epu64(A, R) \
649 ((__m256i)__builtin_ia32_vcvtph2uqq256_round_mask( \
650 (__v8hf)(A), (__v4du)_mm256_undefined_si256(), (__mmask8)(-1), \
651 (int)(R)))
652
653#define _mm256_mask_cvt_roundph_epu64(W, U, A, R) \
654 ((__m256i)__builtin_ia32_vcvtph2uqq256_round_mask((__v8hf)(A), (__v4du)(W), \
655 (__mmask8)(U), (int)(R)))
656
657#define _mm256_maskz_cvt_roundph_epu64(U, A, R) \
658 ((__m256i)__builtin_ia32_vcvtph2uqq256_round_mask( \
659 (__v8hf)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
660
661#define _mm256_cvt_roundph_epu16(A, R) \
662 ((__m256i)__builtin_ia32_vcvtph2uw256_round_mask( \
663 (__v16hf)(A), (__v16hu)_mm256_undefined_si256(), (__mmask16)(-1), \
664 (int)(R)))
665
666#define _mm256_mask_cvt_roundph_epu16(W, U, A, R) \
667 ((__m256i)__builtin_ia32_vcvtph2uw256_round_mask((__v16hf)(A), (__v16hu)(W), \
668 (__mmask16)(U), (int)(R)))
669
670#define _mm256_maskz_cvt_roundph_epu16(U, A, R) \
671 ((__m256i)__builtin_ia32_vcvtph2uw256_round_mask( \
672 (__v16hf)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)(U), \
673 (int)(R)))
674
675#define _mm256_cvt_roundph_epi16(A, R) \
676 ((__m256i)__builtin_ia32_vcvtph2w256_round_mask( \
677 (__v16hf)(A), (__v16hi)_mm256_undefined_si256(), (__mmask16)(-1), \
678 (int)(R)))
679
680#define _mm256_mask_cvt_roundph_epi16(W, U, A, R) \
681 ((__m256i)__builtin_ia32_vcvtph2w256_round_mask((__v16hf)(A), (__v16hi)(W), \
682 (__mmask16)(U), (int)(R)))
683
684#define _mm256_maskz_cvt_roundph_epi16(U, A, R) \
685 ((__m256i)__builtin_ia32_vcvtph2w256_round_mask( \
686 (__v16hf)(A), (__v16hi)_mm256_setzero_si256(), (__mmask16)(U), \
687 (int)(R)))
688
689#define _mm256_cvt_roundps_epi32(A, R) \
690 ((__m256i)__builtin_ia32_vcvtps2dq256_round_mask( \
691 (__v8sf)(__m256)(A), (__v8si)_mm256_setzero_si256(), (__mmask8)-1, \
692 (int)(R)))
693
694#define _mm256_mask_cvt_roundps_epi32(W, U, A, R) \
695 ((__m256i)__builtin_ia32_vcvtps2dq256_round_mask( \
696 (__v8sf)(__m256)(A), (__v8si)(__m256i)(W), (__mmask8)(U), (int)(R)))
697
698#define _mm256_maskz_cvt_roundps_epi32(U, A, R) \
699 ((__m256i)__builtin_ia32_vcvtps2dq256_round_mask( \
700 (__v8sf)(__m256)(A), (__v8si)_mm256_setzero_si256(), (__mmask8)(U), \
701 (int)(R)))
702
703#define _mm256_cvt_roundps_pd(A, R) \
704 ((__m256d)__builtin_ia32_vcvtps2pd256_round_mask( \
705 (__v4sf)(__m128)(A), (__v4df)_mm256_undefined_pd(), (__mmask8)-1, \
706 (int)(R)))
707
708#define _mm256_mask_cvt_roundps_pd(W, U, A, R) \
709 ((__m256d)__builtin_ia32_vcvtps2pd256_round_mask( \
710 (__v4sf)(__m128)(A), (__v4df)(__m256d)(W), (__mmask8)(U), (int)(R)))
711
712#define _mm256_maskz_cvt_roundps_pd(U, A, R) \
713 ((__m256d)__builtin_ia32_vcvtps2pd256_round_mask( \
714 (__v4sf)(__m128)(A), (__v4df)_mm256_setzero_pd(), (__mmask8)(U), \
715 (int)(R)))
716
717#define _mm256_cvt_roundps_ph(A, I) \
718 ((__m128i)__builtin_ia32_vcvtps2ph256_mask((__v8sf)(__m256)(A), (int)(I), \
719 (__v8hi)_mm_undefined_si128(), \
720 (__mmask8)-1))
721
722/* FIXME: We may use these way in future.
723#define _mm256_cvt_roundps_ph(A, I) \
724 ((__m128i)__builtin_ia32_vcvtps2ph256_round_mask( \
725 (__v8sf)(__m256)(A), (int)(I), (__v8hi)_mm_undefined_si128(), \
726 (__mmask8)-1))
727#define _mm256_mask_cvt_roundps_ph(U, W, A, I) \
728 ((__m128i)__builtin_ia32_vcvtps2ph256_round_mask( \
729 (__v8sf)(__m256)(A), (int)(I), (__v8hi)(__m128i)(U), (__mmask8)(W)))
730#define _mm256_maskz_cvt_roundps_ph(W, A, I) \
731 ((__m128i)__builtin_ia32_vcvtps2ph256_round_mask( \
732 (__v8sf)(__m256)(A), (int)(I), (__v8hi)_mm_setzero_si128(), \
733 (__mmask8)(W))) */
734
735#define _mm256_cvtx_roundps_ph(A, R) \
736 ((__m128h)__builtin_ia32_vcvtps2phx256_round_mask( \
737 (__v8sf)(A), (__v8hf)_mm_undefined_ph(), (__mmask8)(-1), (int)(R)))
738
739#define _mm256_mask_cvtx_roundps_ph(W, U, A, R) \
740 ((__m128h)__builtin_ia32_vcvtps2phx256_round_mask((__v8sf)(A), (__v8hf)(W), \
741 (__mmask8)(U), (int)(R)))
742
743#define _mm256_maskz_cvtx_roundps_ph(U, A, R) \
744 ((__m128h)__builtin_ia32_vcvtps2phx256_round_mask( \
745 (__v8sf)(A), (__v8hf)_mm_setzero_ph(), (__mmask8)(U), (int)(R)))
746
747#define _mm256_cvt_roundps_epi64(A, R) \
748 ((__m256i)__builtin_ia32_vcvtps2qq256_round_mask( \
749 (__v4sf)(__m128)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)-1, \
750 (int)(R)))
751
752#define _mm256_mask_cvt_roundps_epi64(W, U, A, R) \
753 ((__m256i)__builtin_ia32_vcvtps2qq256_round_mask( \
754 (__v4sf)(__m128)(A), (__v4di)(__m256i)(W), (__mmask8)(U), (int)(R)))
755
756#define _mm256_maskz_cvt_roundps_epi64(U, A, R) \
757 ((__m256i)__builtin_ia32_vcvtps2qq256_round_mask( \
758 (__v4sf)(__m128)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)(U), \
759 (int)(R)))
760
761#define _mm256_cvt_roundps_epu32(A, R) \
762 ((__m256i)__builtin_ia32_vcvtps2udq256_round_mask( \
763 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)-1, \
764 (int)(R)))
765
766#define _mm256_mask_cvt_roundps_epu32(W, U, A, R) \
767 ((__m256i)__builtin_ia32_vcvtps2udq256_round_mask( \
768 (__v8sf)(__m256)(A), (__v8su)(__m256i)(W), (__mmask8)(U), (int)(R)))
769
770#define _mm256_maskz_cvt_roundps_epu32(U, A, R) \
771 ((__m256i)__builtin_ia32_vcvtps2udq256_round_mask( \
772 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)(U), \
773 (int)(R)))
774
775#define _mm256_cvt_roundps_epu64(A, R) \
776 ((__m256i)__builtin_ia32_vcvtps2uqq256_round_mask( \
777 (__v4sf)(__m128)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)-1, \
778 (int)(R)))
779
780#define _mm256_mask_cvt_roundps_epu64(W, U, A, R) \
781 ((__m256i)__builtin_ia32_vcvtps2uqq256_round_mask( \
782 (__v4sf)(__m128)(A), (__v4du)(__m256i)(W), (__mmask8)(U), (int)(R)))
783
784#define _mm256_maskz_cvt_roundps_epu64(U, A, R) \
785 ((__m256i)__builtin_ia32_vcvtps2uqq256_round_mask( \
786 (__v4sf)(__m128)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)(U), \
787 (int)(R)))
788
789#define _mm256_cvt_roundepi64_pd(A, R) \
790 ((__m256d)__builtin_ia32_vcvtqq2pd256_round_mask( \
791 (__v4di)(__m256i)(A), (__v4df)_mm256_setzero_pd(), (__mmask8)-1, \
792 (int)(R)))
793
794#define _mm256_mask_cvt_roundepi64_pd(W, U, A, R) \
795 ((__m256d)__builtin_ia32_vcvtqq2pd256_round_mask( \
796 (__v4di)(__m256i)(A), (__v4df)(__m256d)(W), (__mmask8)(U), (int)(R)))
797
798#define _mm256_maskz_cvt_roundepi64_pd(U, A, R) \
799 ((__m256d)__builtin_ia32_vcvtqq2pd256_round_mask( \
800 (__v4di)(__m256i)(A), (__v4df)_mm256_setzero_pd(), (__mmask8)(U), \
801 (int)(R)))
802
803#define _mm256_cvt_roundepi64_ph(A, R) \
804 ((__m128h)__builtin_ia32_vcvtqq2ph256_round_mask( \
805 (__v4di)(A), (__v8hf)_mm_undefined_ph(), (__mmask8)(-1), (int)(R)))
806
807#define _mm256_mask_cvt_roundepi64_ph(W, U, A, R) \
808 ((__m128h)__builtin_ia32_vcvtqq2ph256_round_mask((__v4di)(A), (__v8hf)(W), \
809 (__mmask8)(U), (int)(R)))
810
811#define _mm256_maskz_cvt_roundepi64_ph(U, A, R) \
812 ((__m128h)__builtin_ia32_vcvtqq2ph256_round_mask( \
813 (__v4di)(A), (__v8hf)_mm_setzero_ph(), (__mmask8)(U), (int)(R)))
814
815#define _mm256_cvt_roundepi64_ps(A, R) \
816 ((__m128)__builtin_ia32_vcvtqq2ps256_round_mask( \
817 (__v4di)(__m256i)(A), (__v4sf)_mm_setzero_ps(), (__mmask8)-1, (int)(R)))
818
819#define _mm256_mask_cvt_roundepi64_ps(W, U, A, R) \
820 ((__m128)__builtin_ia32_vcvtqq2ps256_round_mask( \
821 (__v4di)(__m256i)(A), (__v4sf)(__m128)(W), (__mmask8)(U), (int)(R)))
822
823#define _mm256_maskz_cvt_roundepi64_ps(U, A, R) \
824 ((__m128)__builtin_ia32_vcvtqq2ps256_round_mask((__v4di)(__m256i)(A), \
825 (__v4sf)_mm_setzero_ps(), \
826 (__mmask8)(U), (int)(R)))
827
828#define _mm256_cvtt_roundpd_epi32(A, R) \
829 ((__m128i)__builtin_ia32_vcvttpd2dq256_round_mask( \
830 (__v4df)(__m256d)(A), (__v4si)_mm_setzero_si128(), (__mmask8)-1, \
831 (int)(R)))
832
833#define _mm256_mask_cvtt_roundpd_epi32(W, U, A, R) \
834 ((__m128i)__builtin_ia32_vcvttpd2dq256_round_mask( \
835 (__v4df)(__m256d)(A), (__v4si)(__m128i)(W), (__mmask8)(U), (int)(R)))
836
837#define _mm256_maskz_cvtt_roundpd_epi32(U, A, R) \
838 ((__m128i)__builtin_ia32_vcvttpd2dq256_round_mask( \
839 (__v4df)(__m256d)(A), (__v4si)_mm_setzero_si128(), (__mmask8)(U), \
840 (int)(R)))
841
842#define _mm256_cvtt_roundpd_epi64(A, R) \
843 ((__m256i)__builtin_ia32_vcvttpd2qq256_round_mask( \
844 (__v4df)(__m256d)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)-1, \
845 (int)(R)))
846
847#define _mm256_mask_cvtt_roundpd_epi64(W, U, A, R) \
848 ((__m256i)__builtin_ia32_vcvttpd2qq256_round_mask( \
849 (__v4df)(__m256d)(A), (__v4di)(__m256i)(W), (__mmask8)(U), (int)(R)))
850
851#define _mm256_maskz_cvtt_roundpd_epi64(U, A, R) \
852 ((__m256i)__builtin_ia32_vcvttpd2qq256_round_mask( \
853 (__v4df)(__m256d)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)(U), \
854 (int)(R)))
855
856#define _mm256_cvtt_roundpd_epu32(A, R) \
857 ((__m128i)__builtin_ia32_vcvttpd2udq256_round_mask( \
858 (__v4df)(__m256d)(A), (__v4su)_mm_setzero_si128(), (__mmask8)-1, \
859 (int)(R)))
860
861#define _mm256_mask_cvtt_roundpd_epu32(W, U, A, R) \
862 ((__m128i)__builtin_ia32_vcvttpd2udq256_round_mask( \
863 (__v4df)(__m256d)(A), (__v4su)(__m128i)(W), (__mmask8)(U), (int)(R)))
864
865#define _mm256_maskz_cvtt_roundpd_epu32(U, A, R) \
866 ((__m128i)__builtin_ia32_vcvttpd2udq256_round_mask( \
867 (__v4df)(__m256d)(A), (__v4su)_mm_setzero_si128(), (__mmask8)(U), \
868 (int)(R)))
869
870#define _mm256_cvtt_roundpd_epu64(A, R) \
871 ((__m256i)__builtin_ia32_vcvttpd2uqq256_round_mask( \
872 (__v4df)(__m256d)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)-1, \
873 (int)(R)))
874
875#define _mm256_mask_cvtt_roundpd_epu64(W, U, A, R) \
876 ((__m256i)__builtin_ia32_vcvttpd2uqq256_round_mask( \
877 (__v4df)(__m256d)(A), (__v4du)(__m256i)(W), (__mmask8)(U), (int)(R)))
878
879#define _mm256_maskz_cvtt_roundpd_epu64(U, A, R) \
880 ((__m256i)__builtin_ia32_vcvttpd2uqq256_round_mask( \
881 (__v4df)(__m256d)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)(U), \
882 (int)(R)))
883
884#define _mm256_cvtt_roundph_epi32(A, R) \
885 ((__m256i)__builtin_ia32_vcvttph2dq256_round_mask( \
886 (__v8hf)(A), (__v8si)_mm256_undefined_si256(), (__mmask8)(-1), \
887 (int)(R)))
888
889#define _mm256_mask_cvtt_roundph_epi32(W, U, A, R) \
890 ((__m256i)__builtin_ia32_vcvttph2dq256_round_mask((__v8hf)(A), (__v8si)(W), \
891 (__mmask8)(U), (int)(R)))
892
893#define _mm256_maskz_cvtt_roundph_epi32(U, A, R) \
894 ((__m256i)__builtin_ia32_vcvttph2dq256_round_mask( \
895 (__v8hf)(A), (__v8si)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
896
897#define _mm256_cvtt_roundph_epi64(A, R) \
898 ((__m256i)__builtin_ia32_vcvttph2qq256_round_mask( \
899 (__v8hf)(A), (__v4di)_mm256_undefined_si256(), (__mmask8)(-1), \
900 (int)(R)))
901
902#define _mm256_mask_cvtt_roundph_epi64(W, U, A, R) \
903 ((__m256i)__builtin_ia32_vcvttph2qq256_round_mask((__v8hf)(A), (__v4di)(W), \
904 (__mmask8)(U), (int)(R)))
905
906#define _mm256_maskz_cvtt_roundph_epi64(U, A, R) \
907 ((__m256i)__builtin_ia32_vcvttph2qq256_round_mask( \
908 (__v8hf)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
909
910#define _mm256_cvtt_roundph_epu32(A, R) \
911 ((__m256i)__builtin_ia32_vcvttph2udq256_round_mask( \
912 (__v8hf)(A), (__v8su)_mm256_undefined_si256(), (__mmask8)(-1), \
913 (int)(R)))
914
915#define _mm256_mask_cvtt_roundph_epu32(W, U, A, R) \
916 ((__m256i)__builtin_ia32_vcvttph2udq256_round_mask((__v8hf)(A), (__v8su)(W), \
917 (__mmask8)(U), (int)(R)))
918
919#define _mm256_maskz_cvtt_roundph_epu32(U, A, R) \
920 ((__m256i)__builtin_ia32_vcvttph2udq256_round_mask( \
921 (__v8hf)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
922
923#define _mm256_cvtt_roundph_epu64(A, R) \
924 ((__m256i)__builtin_ia32_vcvttph2uqq256_round_mask( \
925 (__v8hf)(A), (__v4du)_mm256_undefined_si256(), (__mmask8)(-1), \
926 (int)(R)))
927
928#define _mm256_mask_cvtt_roundph_epu64(W, U, A, R) \
929 ((__m256i)__builtin_ia32_vcvttph2uqq256_round_mask((__v8hf)(A), (__v4du)(W), \
930 (__mmask8)(U), (int)(R)))
931
932#define _mm256_maskz_cvtt_roundph_epu64(U, A, R) \
933 ((__m256i)__builtin_ia32_vcvttph2uqq256_round_mask( \
934 (__v8hf)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)(U), (int)(R)))
935
936#define _mm256_cvtt_roundph_epu16(A, R) \
937 ((__m256i)__builtin_ia32_vcvttph2uw256_round_mask( \
938 (__v16hf)(A), (__v16hu)_mm256_undefined_si256(), (__mmask16)(-1), \
939 (int)(R)))
940
941#define _mm256_mask_cvtt_roundph_epu16(W, U, A, R) \
942 ((__m256i)__builtin_ia32_vcvttph2uw256_round_mask( \
943 (__v16hf)(A), (__v16hu)(W), (__mmask16)(U), (int)(R)))
944
945#define _mm256_maskz_cvtt_roundph_epu16(U, A, R) \
946 ((__m256i)__builtin_ia32_vcvttph2uw256_round_mask( \
947 (__v16hf)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)(U), \
948 (int)(R)))
949
950#define _mm256_cvtt_roundph_epi16(A, R) \
951 ((__m256i)__builtin_ia32_vcvttph2w256_round_mask( \
952 (__v16hf)(A), (__v16hi)_mm256_undefined_si256(), (__mmask16)(-1), \
953 (int)(R)))
954
955#define _mm256_mask_cvtt_roundph_epi16(W, U, A, R) \
956 ((__m256i)__builtin_ia32_vcvttph2w256_round_mask((__v16hf)(A), (__v16hi)(W), \
957 (__mmask16)(U), (int)(R)))
958
959#define _mm256_maskz_cvtt_roundph_epi16(U, A, R) \
960 ((__m256i)__builtin_ia32_vcvttph2w256_round_mask( \
961 (__v16hf)(A), (__v16hi)_mm256_setzero_si256(), (__mmask16)(U), \
962 (int)(R)))
963
964#define _mm256_cvtt_roundps_epi32(A, R) \
965 ((__m256i)__builtin_ia32_vcvttps2dq256_round_mask( \
966 (__v8sf)(__m256)(A), (__v8si)_mm256_setzero_si256(), (__mmask8)-1, \
967 (int)(R)))
968
969#define _mm256_mask_cvtt_roundps_epi32(W, U, A, R) \
970 ((__m256i)__builtin_ia32_vcvttps2dq256_round_mask( \
971 (__v8sf)(__m256)(A), (__v8si)(__m256i)(W), (__mmask8)(U), (int)(R)))
972
973#define _mm256_maskz_cvtt_roundps_epi32(U, A, R) \
974 ((__m256i)__builtin_ia32_vcvttps2dq256_round_mask( \
975 (__v8sf)(__m256)(A), (__v8si)_mm256_setzero_si256(), (__mmask8)(U), \
976 (int)(R)))
977
978#define _mm256_cvtt_roundps_epi64(A, R) \
979 ((__m256i)__builtin_ia32_vcvttps2qq256_round_mask( \
980 (__v4sf)(__m128)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)-1, \
981 (int)(R)))
982
983#define _mm256_mask_cvtt_roundps_epi64(W, U, A, R) \
984 ((__m256i)__builtin_ia32_vcvttps2qq256_round_mask( \
985 (__v4sf)(__m128)(A), (__v4di)(__m256i)(W), (__mmask8)(U), (int)(R)))
986
987#define _mm256_maskz_cvtt_roundps_epi64(U, A, R) \
988 ((__m256i)__builtin_ia32_vcvttps2qq256_round_mask( \
989 (__v4sf)(__m128)(A), (__v4di)_mm256_setzero_si256(), (__mmask8)(U), \
990 (int)(R)))
991
992#define _mm256_cvtt_roundps_epu32(A, R) \
993 ((__m256i)__builtin_ia32_vcvttps2udq256_round_mask( \
994 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)-1, \
995 (int)(R)))
996
997#define _mm256_mask_cvtt_roundps_epu32(W, U, A, R) \
998 ((__m256i)__builtin_ia32_vcvttps2udq256_round_mask( \
999 (__v8sf)(__m256)(A), (__v8su)(__m256i)(W), (__mmask8)(U), (int)(R)))
1000
1001#define _mm256_maskz_cvtt_roundps_epu32(U, A, R) \
1002 ((__m256i)__builtin_ia32_vcvttps2udq256_round_mask( \
1003 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)(U), \
1004 (int)(R)))
1005
1006#define _mm256_cvtt_roundps_epu64(A, R) \
1007 ((__m256i)__builtin_ia32_vcvttps2uqq256_round_mask( \
1008 (__v4sf)(__m128)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)-1, \
1009 (int)(R)))
1010
1011#define _mm256_mask_cvtt_roundps_epu64(W, U, A, R) \
1012 ((__m256i)__builtin_ia32_vcvttps2uqq256_round_mask( \
1013 (__v4sf)(__m128)(A), (__v4du)(__m256i)(W), (__mmask8)(U), (int)(R)))
1014
1015#define _mm256_maskz_cvtt_roundps_epu64(U, A, R) \
1016 ((__m256i)__builtin_ia32_vcvttps2uqq256_round_mask( \
1017 (__v4sf)(__m128)(A), (__v4du)_mm256_setzero_si256(), (__mmask8)(U), \
1018 (int)(R)))
1019
1020#define _mm256_cvt_roundepu32_ph(A, R) \
1021 ((__m128h)__builtin_ia32_vcvtudq2ph256_round_mask( \
1022 (__v8su)(A), (__v8hf)_mm_undefined_ph(), (__mmask8)(-1), (int)(R)))
1023
1024#define _mm256_mask_cvt_roundepu32_ph(W, U, A, R) \
1025 ((__m128h)__builtin_ia32_vcvtudq2ph256_round_mask((__v8su)(A), (__v8hf)(W), \
1026 (__mmask8)(U), (int)(R)))
1027
1028#define _mm256_maskz_cvt_roundepu32_ph(U, A, R) \
1029 ((__m128h)__builtin_ia32_vcvtudq2ph256_round_mask( \
1030 (__v8su)(A), (__v8hf)_mm_setzero_ph(), (__mmask8)(U), (int)(R)))
1031
1032#define _mm256_cvt_roundepu32_ps(A, R) \
1033 ((__m256)__builtin_ia32_vcvtudq2ps256_round_mask( \
1034 (__v8su)(__m256i)(A), (__v8sf)_mm256_setzero_ps(), (__mmask8)-1, \
1035 (int)(R)))
1036
1037#define _mm256_mask_cvt_roundepu32_ps(W, U, A, R) \
1038 ((__m256)__builtin_ia32_vcvtudq2ps256_round_mask( \
1039 (__v8su)(__m256i)(A), (__v8sf)(__m256)(W), (__mmask8)(U), (int)(R)))
1040
1041#define _mm256_maskz_cvt_roundepu32_ps(U, A, R) \
1042 ((__m256)__builtin_ia32_vcvtudq2ps256_round_mask( \
1043 (__v8su)(__m256i)(A), (__v8sf)_mm256_setzero_ps(), (__mmask8)(U), \
1044 (int)(R)))
1045
1046#define _mm256_cvt_roundepu64_pd(A, R) \
1047 ((__m256d)__builtin_ia32_vcvtuqq2pd256_round_mask( \
1048 (__v4du)(__m256i)(A), (__v4df)_mm256_setzero_pd(), (__mmask8)-1, \
1049 (int)(R)))
1050
1051#define _mm256_mask_cvt_roundepu64_pd(W, U, A, R) \
1052 ((__m256d)__builtin_ia32_vcvtuqq2pd256_round_mask( \
1053 (__v4du)(__m256i)(A), (__v4df)(__m256d)(W), (__mmask8)(U), (int)(R)))
1054
1055#define _mm256_maskz_cvt_roundepu64_pd(U, A, R) \
1056 ((__m256d)__builtin_ia32_vcvtuqq2pd256_round_mask( \
1057 (__v4du)(__m256i)(A), (__v4df)_mm256_setzero_pd(), (__mmask8)(U), \
1058 (int)(R)))
1059
1060#define _mm256_cvt_roundepu64_ph(A, R) \
1061 ((__m128h)__builtin_ia32_vcvtuqq2ph256_round_mask( \
1062 (__v4du)(A), (__v8hf)_mm_undefined_ph(), (__mmask8)(-1), (int)(R)))
1063
1064#define _mm256_mask_cvt_roundepu64_ph(W, U, A, R) \
1065 ((__m128h)__builtin_ia32_vcvtuqq2ph256_round_mask((__v4du)(A), (__v8hf)(W), \
1066 (__mmask8)(U), (int)(R)))
1067
1068#define _mm256_maskz_cvt_roundepu64_ph(U, A, R) \
1069 ((__m128h)__builtin_ia32_vcvtuqq2ph256_round_mask( \
1070 (__v4du)(A), (__v8hf)_mm_setzero_ph(), (__mmask8)(U), (int)(R)))
1071
1072#define _mm256_cvt_roundepu64_ps(A, R) \
1073 ((__m128)__builtin_ia32_vcvtuqq2ps256_round_mask( \
1074 (__v4du)(__m256i)(A), (__v4sf)_mm_setzero_ps(), (__mmask8)-1, (int)(R)))
1075
1076#define _mm256_mask_cvt_roundepu64_ps(W, U, A, R) \
1077 ((__m128)__builtin_ia32_vcvtuqq2ps256_round_mask( \
1078 (__v4du)(__m256i)(A), (__v4sf)(__m128)(W), (__mmask8)(U), (int)(R)))
1079
1080#define _mm256_maskz_cvt_roundepu64_ps(U, A, R) \
1081 ((__m128)__builtin_ia32_vcvtuqq2ps256_round_mask((__v4du)(__m256i)(A), \
1082 (__v4sf)_mm_setzero_ps(), \
1083 (__mmask8)(U), (int)(R)))
1084
1085#define _mm256_cvt_roundepu16_ph(A, R) \
1086 ((__m256h)__builtin_ia32_vcvtuw2ph256_round_mask( \
1087 (__v16hu)(A), (__v16hf)_mm256_undefined_ph(), (__mmask16)(-1), \
1088 (int)(R)))
1089
1090#define _mm256_mask_cvt_roundepu16_ph(W, U, A, R) \
1091 ((__m256h)__builtin_ia32_vcvtuw2ph256_round_mask((__v16hu)(A), (__v16hf)(W), \
1092 (__mmask16)(U), (int)(R)))
1093
1094#define _mm256_maskz_cvt_roundepu16_ph(U, A, R) \
1095 ((__m256h)__builtin_ia32_vcvtuw2ph256_round_mask( \
1096 (__v16hu)(A), (__v16hf)_mm256_setzero_ph(), (__mmask16)(U), (int)(R)))
1097
1098#define _mm256_cvt_roundepi16_ph(A, R) \
1099 ((__m256h)__builtin_ia32_vcvtw2ph256_round_mask( \
1100 (__v16hi)(A), (__v16hf)_mm256_undefined_ph(), (__mmask16)(-1), \
1101 (int)(R)))
1102
1103#define _mm256_mask_cvt_roundepi16_ph(W, U, A, R) \
1104 ((__m256h)__builtin_ia32_vcvtw2ph256_round_mask((__v16hi)(A), (__v16hf)(W), \
1105 (__mmask16)(U), (int)(R)))
1106
1107#define _mm256_maskz_cvt_roundepi16_ph(U, A, R) \
1108 ((__m256h)__builtin_ia32_vcvtw2ph256_round_mask( \
1109 (__v16hi)(A), (__v16hf)_mm256_setzero_ph(), (__mmask16)(U), (int)(R)))
1110
1111#define _mm256_div_round_pd(A, B, R) \
1112 ((__m256d)__builtin_ia32_vdivpd256_round((__v4df)(__m256d)(A), \
1113 (__v4df)(__m256d)(B), (int)(R)))
1114
1115#define _mm256_mask_div_round_pd(W, U, A, B, R) \
1116 ((__m256d)__builtin_ia32_selectpd_256( \
1117 (__mmask8)(U), (__v4df)_mm256_div_round_pd((A), (B), (R)), \
1118 (__v4df)(__m256d)(W)))
1119
1120#define _mm256_maskz_div_round_pd(U, A, B, R) \
1121 ((__m256d)__builtin_ia32_selectpd_256( \
1122 (__mmask8)(U), (__v4df)_mm256_div_round_pd((A), (B), (R)), \
1123 (__v4df)_mm256_setzero_pd()))
1124
1125#define _mm256_div_round_ph(A, B, R) \
1126 ((__m256h)__builtin_ia32_vdivph256_round((__v16hf)(__m256h)(A), \
1127 (__v16hf)(__m256h)(B), (int)(R)))
1128
1129#define _mm256_mask_div_round_ph(W, U, A, B, R) \
1130 ((__m256h)__builtin_ia32_selectph_256( \
1131 (__mmask16)(U), (__v16hf)_mm256_div_round_ph((A), (B), (R)), \
1132 (__v16hf)(__m256h)(W)))
1133
1134#define _mm256_maskz_div_round_ph(U, A, B, R) \
1135 ((__m256h)__builtin_ia32_selectph_256( \
1136 (__mmask16)(U), (__v16hf)_mm256_div_round_ph((A), (B), (R)), \
1137 (__v16hf)_mm256_setzero_ph()))
1138
1139#define _mm256_div_round_ps(A, B, R) \
1140 ((__m256)__builtin_ia32_vdivps256_round((__v8sf)(__m256)(A), \
1141 (__v8sf)(__m256)(B), (int)(R)))
1142
1143#define _mm256_mask_div_round_ps(W, U, A, B, R) \
1144 ((__m256)__builtin_ia32_selectps_256( \
1145 (__mmask8)(U), (__v8sf)_mm256_div_round_ps((A), (B), (R)), \
1146 (__v8sf)(__m256)(W)))
1147
1148#define _mm256_maskz_div_round_ps(U, A, B, R) \
1149 ((__m256)__builtin_ia32_selectps_256( \
1150 (__mmask8)(U), (__v8sf)_mm256_div_round_ps((A), (B), (R)), \
1151 (__v8sf)_mm256_setzero_ps()))
1152
1153#define _mm256_fcmadd_round_pch(A, B, C, R) \
1154 ((__m256h)__builtin_ia32_vfcmaddcph256_round_mask3( \
1155 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1156 (__mmask8)-1, (int)(R)))
1157
1158#define _mm256_mask_fcmadd_round_pch(A, U, B, C, R) \
1159 ((__m256h)__builtin_ia32_vfcmaddcph256_round_mask( \
1160 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1161 (__mmask8)(U), (int)(R)))
1162
1163#define _mm256_mask3_fcmadd_round_pch(A, B, C, U, R) \
1164 ((__m256h)__builtin_ia32_vfcmaddcph256_round_mask3( \
1165 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1166 (__mmask8)(U), (int)(R)))
1167
1168#define _mm256_maskz_fcmadd_round_pch(U, A, B, C, R) \
1169 ((__m256h)__builtin_ia32_vfcmaddcph256_round_maskz( \
1170 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1171 (__mmask8)(U), (int)(R)))
1172
1173#define _mm256_cmul_round_pch(A, B, R) \
1174 ((__m256h)__builtin_ia32_vfcmulcph256_round_mask( \
1175 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), \
1176 (__v8sf)(__m256h)_mm256_undefined_ph(), (__mmask8)-1, (int)(R)))
1177
1178#define _mm256_mask_cmul_round_pch(W, U, A, B, R) \
1179 ((__m256h)__builtin_ia32_vfcmulcph256_round_mask( \
1180 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(W), \
1181 (__mmask8)(U), (int)(R)))
1182
1183#define _mm256_maskz_cmul_round_pch(U, A, B, R) \
1184 ((__m256h)__builtin_ia32_vfcmulcph256_round_mask( \
1185 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), \
1186 (__v8sf)(__m256h)_mm256_setzero_ph(), (__mmask8)(U), (int)(R)))
1187
1188#define _mm256_fixupimm_round_pd(A, B, C, imm, R) \
1189 ((__m256d)__builtin_ia32_vfixupimmpd256_round_mask( \
1190 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4di)(__m256i)(C), \
1191 (int)(imm), (__mmask8)-1, (int)(R)))
1192
1193#define _mm256_mask_fixupimm_round_pd(A, U, B, C, imm, R) \
1194 ((__m256d)__builtin_ia32_vfixupimmpd256_round_mask( \
1195 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4di)(__m256i)(C), \
1196 (int)(imm), (__mmask8)(U), (int)(R)))
1197
1198#define _mm256_maskz_fixupimm_round_pd(U, A, B, C, imm, R) \
1199 ((__m256d)__builtin_ia32_vfixupimmpd256_round_maskz( \
1200 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4di)(__m256i)(C), \
1201 (int)(imm), (__mmask8)(U), (int)(R)))
1202
1203#define _mm256_fixupimm_round_ps(A, B, C, imm, R) \
1204 ((__m256)__builtin_ia32_vfixupimmps256_round_mask( \
1205 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8si)(__m256i)(C), \
1206 (int)(imm), (__mmask8)-1, (int)(R)))
1207
1208#define _mm256_mask_fixupimm_round_ps(A, U, B, C, imm, R) \
1209 ((__m256)__builtin_ia32_vfixupimmps256_round_mask( \
1210 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8si)(__m256i)(C), \
1211 (int)(imm), (__mmask8)(U), (int)(R)))
1212
1213#define _mm256_maskz_fixupimm_round_ps(U, A, B, C, imm, R) \
1214 ((__m256)__builtin_ia32_vfixupimmps256_round_maskz( \
1215 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8si)(__m256i)(C), \
1216 (int)(imm), (__mmask8)(U), (int)(R)))
1217
1218#define _mm256_fmadd_round_pd(A, B, C, R) \
1219 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1220 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1221 (__mmask8)-1, (int)(R)))
1222
1223#define _mm256_mask_fmadd_round_pd(A, U, B, C, R) \
1224 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1225 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1226 (__mmask8)(U), (int)(R)))
1227
1228#define _mm256_mask3_fmadd_round_pd(A, B, C, U, R) \
1229 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask3( \
1230 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1231 (__mmask8)(U), (int)(R)))
1232
1233#define _mm256_maskz_fmadd_round_pd(U, A, B, C, R) \
1234 ((__m256d)__builtin_ia32_vfmaddpd256_round_maskz( \
1235 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1236 (__mmask8)(U), (int)(R)))
1237
1238#define _mm256_fmsub_round_pd(A, B, C, R) \
1239 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1240 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1241 (__mmask8)-1, (int)(R)))
1242
1243#define _mm256_mask_fmsub_round_pd(A, U, B, C, R) \
1244 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1245 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1246 (__mmask8)(U), (int)(R)))
1247
1248#define _mm256_maskz_fmsub_round_pd(U, A, B, C, R) \
1249 ((__m256d)__builtin_ia32_vfmaddpd256_round_maskz( \
1250 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1251 (__mmask8)(U), (int)(R)))
1252
1253#define _mm256_fnmadd_round_pd(A, B, C, R) \
1254 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1255 -(__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1256 (__mmask8)-1, (int)(R)))
1257
1258#define _mm256_mask3_fnmadd_round_pd(A, B, C, U, R) \
1259 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask3( \
1260 -(__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1261 (__mmask8)(U), (int)(R)))
1262
1263#define _mm256_maskz_fnmadd_round_pd(U, A, B, C, R) \
1264 ((__m256d)__builtin_ia32_vfmaddpd256_round_maskz( \
1265 -(__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1266 (__mmask8)(U), (int)(R)))
1267
1268#define _mm256_fnmsub_round_pd(A, B, C, R) \
1269 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1270 -(__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1271 (__mmask8)-1, (int)(R)))
1272
1273#define _mm256_maskz_fnmsub_round_pd(U, A, B, C, R) \
1274 ((__m256d)__builtin_ia32_vfmaddpd256_round_maskz( \
1275 -(__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1276 (__mmask8)(U), (int)(R)))
1277
1278#define _mm256_fmadd_round_ph(A, B, C, R) \
1279 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1280 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1281 (__mmask16)-1, (int)(R)))
1282
1283#define _mm256_mask_fmadd_round_ph(A, U, B, C, R) \
1284 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1285 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1286 (__mmask16)(U), (int)(R)))
1287
1288#define _mm256_mask3_fmadd_round_ph(A, B, C, U, R) \
1289 ((__m256h)__builtin_ia32_vfmaddph256_round_mask3( \
1290 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1291 (__mmask16)(U), (int)(R)))
1292
1293#define _mm256_maskz_fmadd_round_ph(U, A, B, C, R) \
1294 ((__m256h)__builtin_ia32_vfmaddph256_round_maskz( \
1295 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1296 (__mmask16)(U), (int)(R)))
1297
1298#define _mm256_fmsub_round_ph(A, B, C, R) \
1299 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1300 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1301 (__mmask16)-1, (int)(R)))
1302
1303#define _mm256_mask_fmsub_round_ph(A, U, B, C, R) \
1304 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1305 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1306 (__mmask16)(U), (int)(R)))
1307
1308#define _mm256_maskz_fmsub_round_ph(U, A, B, C, R) \
1309 ((__m256h)__builtin_ia32_vfmaddph256_round_maskz( \
1310 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1311 (__mmask16)(U), (int)(R)))
1312
1313#define _mm256_fnmadd_round_ph(A, B, C, R) \
1314 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1315 (__v16hf)(__m256h)(A), -(__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1316 (__mmask16)-1, (int)(R)))
1317
1318#define _mm256_mask3_fnmadd_round_ph(A, B, C, U, R) \
1319 ((__m256h)__builtin_ia32_vfmaddph256_round_mask3( \
1320 -(__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1321 (__mmask16)(U), (int)(R)))
1322
1323#define _mm256_maskz_fnmadd_round_ph(U, A, B, C, R) \
1324 ((__m256h)__builtin_ia32_vfmaddph256_round_maskz( \
1325 -(__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1326 (__mmask16)(U), (int)(R)))
1327
1328#define _mm256_fnmsub_round_ph(A, B, C, R) \
1329 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1330 (__v16hf)(__m256h)(A), -(__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1331 (__mmask16)-1, (int)(R)))
1332
1333#define _mm256_maskz_fnmsub_round_ph(U, A, B, C, R) \
1334 ((__m256h)__builtin_ia32_vfmaddph256_round_maskz( \
1335 -(__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1336 (__mmask16)(U), (int)(R)))
1337
1338#define _mm256_fmadd_round_ps(A, B, C, R) \
1339 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1340 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1341 (__mmask8)-1, (int)(R)))
1342
1343#define _mm256_mask_fmadd_round_ps(A, U, B, C, R) \
1344 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1345 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1346 (__mmask8)(U), (int)(R)))
1347
1348#define _mm256_mask3_fmadd_round_ps(A, B, C, U, R) \
1349 ((__m256)__builtin_ia32_vfmaddps256_round_mask3( \
1350 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1351 (__mmask8)(U), (int)(R)))
1352
1353#define _mm256_maskz_fmadd_round_ps(U, A, B, C, R) \
1354 ((__m256)__builtin_ia32_vfmaddps256_round_maskz( \
1355 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1356 (__mmask8)(U), (int)(R)))
1357
1358#define _mm256_fmsub_round_ps(A, B, C, R) \
1359 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1360 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1361 (__mmask8)-1, (int)(R)))
1362
1363#define _mm256_mask_fmsub_round_ps(A, U, B, C, R) \
1364 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1365 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1366 (__mmask8)(U), (int)(R)))
1367
1368#define _mm256_maskz_fmsub_round_ps(U, A, B, C, R) \
1369 ((__m256)__builtin_ia32_vfmaddps256_round_maskz( \
1370 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1371 (__mmask8)(U), (int)(R)))
1372
1373#define _mm256_fnmadd_round_ps(A, B, C, R) \
1374 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1375 (__v8sf)(__m256)(A), -(__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1376 (__mmask8)-1, (int)(R)))
1377
1378#define _mm256_mask3_fnmadd_round_ps(A, B, C, U, R) \
1379 ((__m256)__builtin_ia32_vfmaddps256_round_mask3( \
1380 -(__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1381 (__mmask8)(U), (int)(R)))
1382
1383#define _mm256_maskz_fnmadd_round_ps(U, A, B, C, R) \
1384 ((__m256)__builtin_ia32_vfmaddps256_round_maskz( \
1385 -(__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1386 (__mmask8)(U), (int)(R)))
1387
1388#define _mm256_fnmsub_round_ps(A, B, C, R) \
1389 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1390 (__v8sf)(__m256)(A), -(__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1391 (__mmask8)-1, (int)(R)))
1392
1393#define _mm256_maskz_fnmsub_round_ps(U, A, B, C, R) \
1394 ((__m256)__builtin_ia32_vfmaddps256_round_maskz( \
1395 -(__v8sf)(__m256)(A), (__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1396 (__mmask8)(U), (int)(R)))
1397
1398#define _mm256_fmadd_round_pch(A, B, C, R) \
1399 ((__m256h)__builtin_ia32_vfmaddcph256_round_mask3( \
1400 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1401 (__mmask8)-1, (int)(R)))
1402
1403#define _mm256_mask_fmadd_round_pch(A, U, B, C, R) \
1404 ((__m256h)__builtin_ia32_vfmaddcph256_round_mask( \
1405 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1406 (__mmask8)(U), (int)(R)))
1407
1408#define _mm256_mask3_fmadd_round_pch(A, B, C, U, R) \
1409 ((__m256h)__builtin_ia32_vfmaddcph256_round_mask3( \
1410 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1411 (__mmask8)(U), (int)(R)))
1412
1413#define _mm256_maskz_fmadd_round_pch(U, A, B, C, R) \
1414 ((__m256h)__builtin_ia32_vfmaddcph256_round_maskz( \
1415 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(C), \
1416 (__mmask8)(U), (int)(R)))
1417
1418#define _mm256_fmaddsub_round_pd(A, B, C, R) \
1419 ((__m256d)__builtin_ia32_vfmaddsubpd256_round_mask( \
1420 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1421 (__mmask8)-1, (int)(R)))
1422
1423#define _mm256_mask_fmaddsub_round_pd(A, U, B, C, R) \
1424 ((__m256d)__builtin_ia32_vfmaddsubpd256_round_mask( \
1425 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1426 (__mmask8)(U), (int)(R)))
1427
1428#define _mm256_mask3_fmaddsub_round_pd(A, B, C, U, R) \
1429 ((__m256d)__builtin_ia32_vfmaddsubpd256_round_mask3( \
1430 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1431 (__mmask8)(U), (int)(R)))
1432
1433#define _mm256_maskz_fmaddsub_round_pd(U, A, B, C, R) \
1434 ((__m256d)__builtin_ia32_vfmaddsubpd256_round_maskz( \
1435 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1436 (__mmask8)(U), (int)(R)))
1437
1438#define _mm256_fmsubadd_round_pd(A, B, C, R) \
1439 ((__m256d)__builtin_ia32_vfmaddsubpd256_round_mask( \
1440 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1441 (__mmask8)-1, (int)(R)))
1442
1443#define _mm256_mask_fmsubadd_round_pd(A, U, B, C, R) \
1444 ((__m256d)__builtin_ia32_vfmaddsubpd256_round_mask( \
1445 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1446 (__mmask8)(U), (int)(R)))
1447
1448#define _mm256_maskz_fmsubadd_round_pd(U, A, B, C, R) \
1449 ((__m256d)__builtin_ia32_vfmaddsubpd256_round_maskz( \
1450 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1451 (__mmask8)(U), (int)(R)))
1452
1453#define _mm256_fmaddsub_round_ph(A, B, C, R) \
1454 ((__m256h)__builtin_ia32_vfmaddsubph256_round_mask( \
1455 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1456 (__mmask16)-1, (int)(R)))
1457
1458#define _mm256_mask_fmaddsub_round_ph(A, U, B, C, R) \
1459 ((__m256h)__builtin_ia32_vfmaddsubph256_round_mask( \
1460 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1461 (__mmask16)(U), (int)(R)))
1462
1463#define _mm256_mask3_fmaddsub_round_ph(A, B, C, U, R) \
1464 ((__m256h)__builtin_ia32_vfmaddsubph256_round_mask3( \
1465 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1466 (__mmask16)(U), (int)(R)))
1467
1468#define _mm256_maskz_fmaddsub_round_ph(U, A, B, C, R) \
1469 ((__m256h)__builtin_ia32_vfmaddsubph256_round_maskz( \
1470 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1471 (__mmask16)(U), (int)(R)))
1472
1473#define _mm256_fmsubadd_round_ph(A, B, C, R) \
1474 ((__m256h)__builtin_ia32_vfmaddsubph256_round_mask( \
1475 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1476 (__mmask16)-1, (int)(R)))
1477
1478#define _mm256_mask_fmsubadd_round_ph(A, U, B, C, R) \
1479 ((__m256h)__builtin_ia32_vfmaddsubph256_round_mask( \
1480 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1481 (__mmask16)(U), (int)(R)))
1482
1483#define _mm256_maskz_fmsubadd_round_ph(U, A, B, C, R) \
1484 ((__m256h)__builtin_ia32_vfmaddsubph256_round_maskz( \
1485 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1486 (__mmask16)(U), (int)(R)))
1487
1488#define _mm256_fmaddsub_round_ps(A, B, C, R) \
1489 ((__m256)__builtin_ia32_vfmaddsubps256_round_mask( \
1490 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1491 (__mmask8)-1, (int)(R)))
1492
1493#define _mm256_mask_fmaddsub_round_ps(A, U, B, C, R) \
1494 ((__m256)__builtin_ia32_vfmaddsubps256_round_mask( \
1495 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1496 (__mmask8)(U), (int)(R)))
1497
1498#define _mm256_mask3_fmaddsub_round_ps(A, B, C, U, R) \
1499 ((__m256)__builtin_ia32_vfmaddsubps256_round_mask3( \
1500 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1501 (__mmask8)(U), (int)(R)))
1502
1503#define _mm256_maskz_fmaddsub_round_ps(U, A, B, C, R) \
1504 ((__m256)__builtin_ia32_vfmaddsubps256_round_maskz( \
1505 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1506 (__mmask8)(U), (int)(R)))
1507
1508#define _mm256_fmsubadd_round_ps(A, B, C, R) \
1509 ((__m256)__builtin_ia32_vfmaddsubps256_round_mask( \
1510 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1511 (__mmask8)-1, (int)(R)))
1512
1513#define _mm256_mask_fmsubadd_round_ps(A, U, B, C, R) \
1514 ((__m256)__builtin_ia32_vfmaddsubps256_round_mask( \
1515 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1516 (__mmask8)(U), (int)(R)))
1517
1518#define _mm256_maskz_fmsubadd_round_ps(U, A, B, C, R) \
1519 ((__m256)__builtin_ia32_vfmaddsubps256_round_maskz( \
1520 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1521 (__mmask8)(U), (int)(R)))
1522#define _mm256_mask3_fmsub_round_pd(A, B, C, U, R) \
1523 ((__m256d)__builtin_ia32_vfmsubpd256_round_mask3( \
1524 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1525 (__mmask8)(U), (int)(R)))
1526
1527#define _mm256_mask3_fmsubadd_round_pd(A, B, C, U, R) \
1528 ((__m256d)__builtin_ia32_vfmsubaddpd256_round_mask3( \
1529 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1530 (__mmask8)(U), (int)(R)))
1531
1532#define _mm256_mask_fnmadd_round_pd(A, U, B, C, R) \
1533 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1534 (__v4df)(__m256d)(A), -(__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1535 (__mmask8)(U), (int)(R)))
1536
1537#define _mm256_mask_fnmsub_round_pd(A, U, B, C, R) \
1538 ((__m256d)__builtin_ia32_vfmaddpd256_round_mask( \
1539 (__v4df)(__m256d)(A), -(__v4df)(__m256d)(B), -(__v4df)(__m256d)(C), \
1540 (__mmask8)(U), (int)(R)))
1541
1542#define _mm256_mask3_fnmsub_round_pd(A, B, C, U, R) \
1543 ((__m256d)__builtin_ia32_vfmsubpd256_round_mask3( \
1544 -(__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(C), \
1545 (__mmask8)(U), (int)(R)))
1546
1547#define _mm256_mask3_fmsub_round_ph(A, B, C, U, R) \
1548 ((__m256h)__builtin_ia32_vfmsubph256_round_mask3( \
1549 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1550 (__mmask16)(U), (int)(R)))
1551
1552#define _mm256_mask3_fmsubadd_round_ph(A, B, C, U, R) \
1553 ((__m256h)__builtin_ia32_vfmsubaddph256_round_mask3( \
1554 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1555 (__mmask16)(U), (int)(R)))
1556
1557#define _mm256_mask_fnmadd_round_ph(A, U, B, C, R) \
1558 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1559 (__v16hf)(__m256h)(A), -(__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1560 (__mmask16)(U), (int)(R)))
1561
1562#define _mm256_mask_fnmsub_round_ph(A, U, B, C, R) \
1563 ((__m256h)__builtin_ia32_vfmaddph256_round_mask( \
1564 (__v16hf)(__m256h)(A), -(__v16hf)(__m256h)(B), -(__v16hf)(__m256h)(C), \
1565 (__mmask16)(U), (int)(R)))
1566
1567#define _mm256_mask3_fnmsub_round_ph(A, B, C, U, R) \
1568 ((__m256h)__builtin_ia32_vfmsubph256_round_mask3( \
1569 -(__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(C), \
1570 (__mmask16)(U), (int)(R)))
1571
1572#define _mm256_mask3_fmsub_round_ps(A, B, C, U, R) \
1573 ((__m256)__builtin_ia32_vfmsubps256_round_mask3( \
1574 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1575 (__mmask8)(U), (int)(R)))
1576
1577#define _mm256_mask3_fmsubadd_round_ps(A, B, C, U, R) \
1578 ((__m256)__builtin_ia32_vfmsubaddps256_round_mask3( \
1579 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1580 (__mmask8)(U), (int)(R)))
1581
1582#define _mm256_mask_fnmadd_round_ps(A, U, B, C, R) \
1583 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1584 (__v8sf)(__m256)(A), -(__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1585 (__mmask8)(U), (int)(R)))
1586
1587#define _mm256_mask_fnmsub_round_ps(A, U, B, C, R) \
1588 ((__m256)__builtin_ia32_vfmaddps256_round_mask( \
1589 (__v8sf)(__m256)(A), -(__v8sf)(__m256)(B), -(__v8sf)(__m256)(C), \
1590 (__mmask8)(U), (int)(R)))
1591
1592#define _mm256_mask3_fnmsub_round_ps(A, B, C, U, R) \
1593 ((__m256)__builtin_ia32_vfmsubps256_round_mask3( \
1594 -(__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(C), \
1595 (__mmask8)(U), (int)(R)))
1596
1597#define _mm256_mul_round_pch(A, B, R) \
1598 ((__m256h)__builtin_ia32_vfmulcph256_round_mask( \
1599 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), \
1600 (__v8sf)(__m256h)_mm256_undefined_ph(), (__mmask8)-1, (int)(R)))
1601
1602#define _mm256_mask_mul_round_pch(W, U, A, B, R) \
1603 ((__m256h)__builtin_ia32_vfmulcph256_round_mask( \
1604 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), (__v8sf)(__m256h)(W), \
1605 (__mmask8)(U), (int)(R)))
1606
1607#define _mm256_maskz_mul_round_pch(U, A, B, R) \
1608 ((__m256h)__builtin_ia32_vfmulcph256_round_mask( \
1609 (__v8sf)(__m256h)(A), (__v8sf)(__m256h)(B), \
1610 (__v8sf)(__m256h)_mm256_setzero_ph(), (__mmask8)(U), (int)(R)))
1611
1612#define _mm256_getexp_round_pd(A, R) \
1613 ((__m256d)__builtin_ia32_vgetexppd256_round_mask( \
1614 (__v4df)(__m256d)(A), (__v4df)_mm256_undefined_pd(), (__mmask8)-1, \
1615 (int)(R)))
1616
1617#define _mm256_mask_getexp_round_pd(W, U, A, R) \
1618 ((__m256d)__builtin_ia32_vgetexppd256_round_mask( \
1619 (__v4df)(__m256d)(A), (__v4df)(__m256d)(W), (__mmask8)(U), (int)(R)))
1620
1621#define _mm256_maskz_getexp_round_pd(U, A, R) \
1622 ((__m256d)__builtin_ia32_vgetexppd256_round_mask( \
1623 (__v4df)(__m256d)(A), (__v4df)_mm256_setzero_pd(), (__mmask8)(U), \
1624 (int)(R)))
1625
1626#define _mm256_getexp_round_ph(A, R) \
1627 ((__m256h)__builtin_ia32_vgetexpph256_round_mask( \
1628 (__v16hf)(__m256h)(A), (__v16hf)_mm256_undefined_ph(), (__mmask16)-1, \
1629 (int)(R)))
1630
1631#define _mm256_mask_getexp_round_ph(W, U, A, R) \
1632 ((__m256h)__builtin_ia32_vgetexpph256_round_mask( \
1633 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(W), (__mmask16)(U), (int)(R)))
1634
1635#define _mm256_maskz_getexp_round_ph(U, A, R) \
1636 ((__m256h)__builtin_ia32_vgetexpph256_round_mask( \
1637 (__v16hf)(__m256h)(A), (__v16hf)_mm256_setzero_ph(), (__mmask16)(U), \
1638 (int)(R)))
1639
1640#define _mm256_getexp_round_ps(A, R) \
1641 ((__m256)__builtin_ia32_vgetexpps256_round_mask( \
1642 (__v8sf)(__m256)(A), (__v8sf)_mm256_undefined_ps(), (__mmask8)-1, \
1643 (int)(R)))
1644
1645#define _mm256_mask_getexp_round_ps(W, U, A, R) \
1646 ((__m256)__builtin_ia32_vgetexpps256_round_mask( \
1647 (__v8sf)(__m256)(A), (__v8sf)(__m256)(W), (__mmask8)(U), (int)(R)))
1648
1649#define _mm256_maskz_getexp_round_ps(U, A, R) \
1650 ((__m256)__builtin_ia32_vgetexpps256_round_mask((__v8sf)(__m256)(A), \
1651 (__v8sf)_mm256_setzero_ps(), \
1652 (__mmask8)(U), (int)(R)))
1653
1654#define _mm256_getmant_round_pd(A, B, C, R) \
1655 ((__m256d)__builtin_ia32_vgetmantpd256_round_mask( \
1656 (__v4df)(__m256d)(A), (int)(((C) << 2) | (B)), \
1657 (__v4df)_mm256_undefined_pd(), (__mmask8)-1, (int)(R)))
1658
1659#define _mm256_mask_getmant_round_pd(W, U, A, B, C, R) \
1660 ((__m256d)__builtin_ia32_vgetmantpd256_round_mask( \
1661 (__v4df)(__m256d)(A), (int)(((C) << 2) | (B)), (__v4df)(__m256d)(W), \
1662 (__mmask8)(U), (int)(R)))
1663
1664#define _mm256_maskz_getmant_round_pd(U, A, B, C, R) \
1665 ((__m256d)__builtin_ia32_vgetmantpd256_round_mask( \
1666 (__v4df)(__m256d)(A), (int)(((C) << 2) | (B)), \
1667 (__v4df)_mm256_setzero_pd(), (__mmask8)(U), (int)(R)))
1668
1669#define _mm256_getmant_round_ph(A, B, C, R) \
1670 ((__m256h)__builtin_ia32_vgetmantph256_round_mask( \
1671 (__v16hf)(__m256h)(A), (int)(((C) << 2) | (B)), \
1672 (__v16hf)_mm256_undefined_ph(), (__mmask16)-1, (int)(R)))
1673
1674#define _mm256_mask_getmant_round_ph(W, U, A, B, C, R) \
1675 ((__m256h)__builtin_ia32_vgetmantph256_round_mask( \
1676 (__v16hf)(__m256h)(A), (int)(((C) << 2) | (B)), (__v16hf)(__m256h)(W), \
1677 (__mmask16)(U), (int)(R)))
1678
1679#define _mm256_maskz_getmant_round_ph(U, A, B, C, R) \
1680 ((__m256h)__builtin_ia32_vgetmantph256_round_mask( \
1681 (__v16hf)(__m256h)(A), (int)(((C) << 2) | (B)), \
1682 (__v16hf)_mm256_setzero_ph(), (__mmask16)(U), (int)(R)))
1683
1684#define _mm256_getmant_round_ps(A, B, C, R) \
1685 ((__m256)__builtin_ia32_vgetmantps256_round_mask( \
1686 (__v8sf)(__m256)(A), (int)(((C) << 2) | (B)), \
1687 (__v8sf)_mm256_undefined_ps(), (__mmask8)-1, (int)(R)))
1688
1689#define _mm256_mask_getmant_round_ps(W, U, A, B, C, R) \
1690 ((__m256)__builtin_ia32_vgetmantps256_round_mask( \
1691 (__v8sf)(__m256)(A), (int)(((C) << 2) | (B)), (__v8sf)(__m256)(W), \
1692 (__mmask8)(U), (int)(R)))
1693
1694#define _mm256_maskz_getmant_round_ps(U, A, B, C, R) \
1695 ((__m256)__builtin_ia32_vgetmantps256_round_mask( \
1696 (__v8sf)(__m256)(A), (int)(((C) << 2) | (B)), \
1697 (__v8sf)_mm256_setzero_ps(), (__mmask8)(U), (int)(R)))
1698
1699#define _mm256_max_round_pd(A, B, R) \
1700 ((__m256d)__builtin_ia32_vmaxpd256_round((__v4df)(__m256d)(A), \
1701 (__v4df)(__m256d)(B), (int)(R)))
1702
1703#define _mm256_mask_max_round_pd(W, U, A, B, R) \
1704 ((__m256d)__builtin_ia32_selectpd_256( \
1705 (__mmask8)(U), (__v4df)_mm256_max_round_pd((A), (B), (R)), \
1706 (__v4df)(__m256d)(W)))
1707
1708#define _mm256_maskz_max_round_pd(U, A, B, R) \
1709 ((__m256d)__builtin_ia32_selectpd_256( \
1710 (__mmask8)(U), (__v4df)_mm256_max_round_pd((A), (B), (R)), \
1711 (__v4df)_mm256_setzero_pd()))
1712
1713#define _mm256_max_round_ph(A, B, R) \
1714 ((__m256h)__builtin_ia32_vmaxph256_round((__v16hf)(__m256h)(A), \
1715 (__v16hf)(__m256h)(B), (int)(R)))
1716
1717#define _mm256_mask_max_round_ph(W, U, A, B, R) \
1718 ((__m256h)__builtin_ia32_selectph_256( \
1719 (__mmask16)(U), (__v16hf)_mm256_max_round_ph((A), (B), (R)), \
1720 (__v16hf)(__m256h)(W)))
1721
1722#define _mm256_maskz_max_round_ph(U, A, B, R) \
1723 ((__m256h)__builtin_ia32_selectph_256( \
1724 (__mmask16)(U), (__v16hf)_mm256_max_round_ph((A), (B), (R)), \
1725 (__v16hf)_mm256_setzero_ph()))
1726
1727#define _mm256_max_round_ps(A, B, R) \
1728 ((__m256)__builtin_ia32_vmaxps256_round((__v8sf)(__m256)(A), \
1729 (__v8sf)(__m256)(B), (int)(R)))
1730
1731#define _mm256_mask_max_round_ps(W, U, A, B, R) \
1732 ((__m256)__builtin_ia32_selectps_256( \
1733 (__mmask8)(U), (__v8sf)_mm256_max_round_ps((A), (B), (R)), \
1734 (__v8sf)(__m256)(W)))
1735
1736#define _mm256_maskz_max_round_ps(U, A, B, R) \
1737 ((__m256)__builtin_ia32_selectps_256( \
1738 (__mmask8)(U), (__v8sf)_mm256_max_round_ps((A), (B), (R)), \
1739 (__v8sf)_mm256_setzero_ps()))
1740
1741#define _mm256_min_round_pd(A, B, R) \
1742 ((__m256d)__builtin_ia32_vminpd256_round((__v4df)(__m256d)(A), \
1743 (__v4df)(__m256d)(B), (int)(R)))
1744
1745#define _mm256_mask_min_round_pd(W, U, A, B, R) \
1746 ((__m256d)__builtin_ia32_selectpd_256( \
1747 (__mmask8)(U), (__v4df)_mm256_min_round_pd((A), (B), (R)), \
1748 (__v4df)(__m256d)(W)))
1749
1750#define _mm256_maskz_min_round_pd(U, A, B, R) \
1751 ((__m256d)__builtin_ia32_selectpd_256( \
1752 (__mmask8)(U), (__v4df)_mm256_min_round_pd((A), (B), (R)), \
1753 (__v4df)_mm256_setzero_pd()))
1754
1755#define _mm256_min_round_ph(A, B, R) \
1756 ((__m256h)__builtin_ia32_vminph256_round((__v16hf)(__m256h)(A), \
1757 (__v16hf)(__m256h)(B), (int)(R)))
1758
1759#define _mm256_mask_min_round_ph(W, U, A, B, R) \
1760 ((__m256h)__builtin_ia32_selectph_256( \
1761 (__mmask16)(U), (__v16hf)_mm256_min_round_ph((A), (B), (R)), \
1762 (__v16hf)(__m256h)(W)))
1763
1764#define _mm256_maskz_min_round_ph(U, A, B, R) \
1765 ((__m256h)__builtin_ia32_selectph_256( \
1766 (__mmask16)(U), (__v16hf)_mm256_min_round_ph((A), (B), (R)), \
1767 (__v16hf)_mm256_setzero_ph()))
1768
1769#define _mm256_min_round_ps(A, B, R) \
1770 ((__m256)__builtin_ia32_vminps256_round((__v8sf)(__m256)(A), \
1771 (__v8sf)(__m256)(B), (int)(R)))
1772
1773#define _mm256_mask_min_round_ps(W, U, A, B, R) \
1774 ((__m256)__builtin_ia32_selectps_256( \
1775 (__mmask8)(U), (__v8sf)_mm256_min_round_ps((A), (B), (R)), \
1776 (__v8sf)(__m256)(W)))
1777
1778#define _mm256_maskz_min_round_ps(U, A, B, R) \
1779 ((__m256)__builtin_ia32_selectps_256( \
1780 (__mmask8)(U), (__v8sf)_mm256_min_round_ps((A), (B), (R)), \
1781 (__v8sf)_mm256_setzero_ps()))
1782
1783#define _mm256_mul_round_pd(A, B, R) \
1784 ((__m256d)__builtin_ia32_vmulpd256_round((__v4df)(__m256d)(A), \
1785 (__v4df)(__m256d)(B), (int)(R)))
1786
1787#define _mm256_mask_mul_round_pd(W, U, A, B, R) \
1788 ((__m256d)__builtin_ia32_selectpd_256( \
1789 (__mmask8)(U), (__v4df)_mm256_mul_round_pd((A), (B), (R)), \
1790 (__v4df)(__m256d)(W)))
1791
1792#define _mm256_maskz_mul_round_pd(U, A, B, R) \
1793 ((__m256d)__builtin_ia32_selectpd_256( \
1794 (__mmask8)(U), (__v4df)_mm256_mul_round_pd((A), (B), (R)), \
1795 (__v4df)_mm256_setzero_pd()))
1796
1797#define _mm256_mul_round_ph(A, B, R) \
1798 ((__m256h)__builtin_ia32_vmulph256_round((__v16hf)(__m256h)(A), \
1799 (__v16hf)(__m256h)(B), (int)(R)))
1800
1801#define _mm256_mask_mul_round_ph(W, U, A, B, R) \
1802 ((__m256h)__builtin_ia32_selectph_256( \
1803 (__mmask16)(U), (__v16hf)_mm256_mul_round_ph((A), (B), (R)), \
1804 (__v16hf)(__m256h)(W)))
1805
1806#define _mm256_maskz_mul_round_ph(U, A, B, R) \
1807 ((__m256h)__builtin_ia32_selectph_256( \
1808 (__mmask16)(U), (__v16hf)_mm256_mul_round_ph((A), (B), (R)), \
1809 (__v16hf)_mm256_setzero_ph()))
1810
1811#define _mm256_mul_round_ps(A, B, R) \
1812 ((__m256)__builtin_ia32_vmulps256_round((__v8sf)(__m256)(A), \
1813 (__v8sf)(__m256)(B), (int)(R)))
1814
1815#define _mm256_mask_mul_round_ps(W, U, A, B, R) \
1816 ((__m256)__builtin_ia32_selectps_256( \
1817 (__mmask8)(U), (__v8sf)_mm256_mul_round_ps((A), (B), (R)), \
1818 (__v8sf)(__m256)(W)))
1819
1820#define _mm256_maskz_mul_round_ps(U, A, B, R) \
1821 ((__m256)__builtin_ia32_selectps_256( \
1822 (__mmask8)(U), (__v8sf)_mm256_mul_round_ps((A), (B), (R)), \
1823 (__v8sf)_mm256_setzero_ps()))
1824
1825#define _mm256_range_round_pd(A, B, C, R) \
1826 ((__m256d)__builtin_ia32_vrangepd256_round_mask( \
1827 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
1828 (__v4df)_mm256_setzero_pd(), (__mmask8)-1, (int)(R)))
1829
1830#define _mm256_mask_range_round_pd(W, U, A, B, C, R) \
1831 ((__m256d)__builtin_ia32_vrangepd256_round_mask( \
1832 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
1833 (__v4df)(__m256d)(W), (__mmask8)(U), (int)(R)))
1834
1835#define _mm256_maskz_range_round_pd(U, A, B, C, R) \
1836 ((__m256d)__builtin_ia32_vrangepd256_round_mask( \
1837 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (int)(C), \
1838 (__v4df)_mm256_setzero_pd(), (__mmask8)(U), (int)(R)))
1839
1840#define _mm256_range_round_ps(A, B, C, R) \
1841 ((__m256)__builtin_ia32_vrangeps256_round_mask( \
1842 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), \
1843 (__v8sf)_mm256_setzero_ps(), (__mmask8)-1, (int)(R)))
1844
1845#define _mm256_mask_range_round_ps(W, U, A, B, C, R) \
1846 ((__m256)__builtin_ia32_vrangeps256_round_mask( \
1847 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), (__v8sf)(__m256)(W), \
1848 (__mmask8)(U), (int)(R)))
1849
1850#define _mm256_maskz_range_round_ps(U, A, B, C, R) \
1851 ((__m256)__builtin_ia32_vrangeps256_round_mask( \
1852 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (int)(C), \
1853 (__v8sf)_mm256_setzero_ps(), (__mmask8)(U), (int)(R)))
1854
1855#define _mm256_reduce_round_pd(A, B, R) \
1856 ((__m256d)__builtin_ia32_vreducepd256_round_mask( \
1857 (__v4df)(__m256d)(A), (int)(B), (__v4df)_mm256_setzero_pd(), \
1858 (__mmask8)-1, (int)(R)))
1859
1860#define _mm256_mask_reduce_round_pd(W, U, A, B, R) \
1861 ((__m256d)__builtin_ia32_vreducepd256_round_mask( \
1862 (__v4df)(__m256d)(A), (int)(B), (__v4df)(__m256d)(W), (__mmask8)(U), \
1863 (int)(R)))
1864
1865#define _mm256_maskz_reduce_round_pd(U, A, B, R) \
1866 ((__m256d)__builtin_ia32_vreducepd256_round_mask( \
1867 (__v4df)(__m256d)(A), (int)(B), (__v4df)_mm256_setzero_pd(), \
1868 (__mmask8)(U), (int)(R)))
1869
1870#define _mm256_mask_reduce_round_ph(W, U, A, imm, R) \
1871 ((__m256h)__builtin_ia32_vreduceph256_round_mask( \
1872 (__v16hf)(__m256h)(A), (int)(imm), (__v16hf)(__m256h)(W), \
1873 (__mmask16)(U), (int)(R)))
1874
1875#define _mm256_maskz_reduce_round_ph(U, A, imm, R) \
1876 ((__m256h)__builtin_ia32_vreduceph256_round_mask( \
1877 (__v16hf)(__m256h)(A), (int)(imm), (__v16hf)_mm256_setzero_ph(), \
1878 (__mmask16)(U), (int)(R)))
1879
1880#define _mm256_reduce_round_ph(A, imm, R) \
1881 ((__m256h)__builtin_ia32_vreduceph256_round_mask( \
1882 (__v16hf)(__m256h)(A), (int)(imm), (__v16hf)_mm256_undefined_ph(), \
1883 (__mmask16)-1, (int)(R)))
1884
1885#define _mm256_reduce_round_ps(A, B, R) \
1886 ((__m256)__builtin_ia32_vreduceps256_round_mask( \
1887 (__v8sf)(__m256)(A), (int)(B), (__v8sf)_mm256_setzero_ps(), \
1888 (__mmask8)-1, (int)(R)))
1889
1890#define _mm256_mask_reduce_round_ps(W, U, A, B, R) \
1891 ((__m256)__builtin_ia32_vreduceps256_round_mask( \
1892 (__v8sf)(__m256)(A), (int)(B), (__v8sf)(__m256)(W), (__mmask8)(U), \
1893 (int)(R)))
1894
1895#define _mm256_maskz_reduce_round_ps(U, A, B, R) \
1896 ((__m256)__builtin_ia32_vreduceps256_round_mask( \
1897 (__v8sf)(__m256)(A), (int)(B), (__v8sf)_mm256_setzero_ps(), \
1898 (__mmask8)(U), (int)(R)))
1899
1900#define _mm256_roundscale_round_pd(A, imm, R) \
1901 ((__m256d)__builtin_ia32_vrndscalepd256_round_mask( \
1902 (__v4df)(__m256d)(A), (int)(imm), (__v4df)_mm256_undefined_pd(), \
1903 (__mmask8)-1, (int)(R)))
1904
1905#define _mm256_mask_roundscale_round_pd(A, B, C, imm, R) \
1906 ((__m256d)__builtin_ia32_vrndscalepd256_round_mask( \
1907 (__v4df)(__m256d)(C), (int)(imm), (__v4df)(__m256d)(A), (__mmask8)(B), \
1908 (int)(R)))
1909
1910#define _mm256_maskz_roundscale_round_pd(A, B, imm, R) \
1911 ((__m256d)__builtin_ia32_vrndscalepd256_round_mask( \
1912 (__v4df)(__m256d)(B), (int)(imm), (__v4df)_mm256_setzero_pd(), \
1913 (__mmask8)(A), (int)(R)))
1914
1915#define _mm256_roundscale_round_ph(A, imm, R) \
1916 ((__m256h)__builtin_ia32_vrndscaleph256_round_mask( \
1917 (__v16hf)(__m256h)(A), (int)(imm), (__v16hf)_mm256_undefined_ph(), \
1918 (__mmask16)-1, (int)(R)))
1919
1920#define _mm256_mask_roundscale_round_ph(A, B, C, imm, R) \
1921 ((__m256h)__builtin_ia32_vrndscaleph256_round_mask( \
1922 (__v16hf)(__m256h)(C), (int)(imm), (__v16hf)(__m256h)(A), \
1923 (__mmask16)(B), (int)(R)))
1924
1925#define _mm256_maskz_roundscale_round_ph(A, B, imm, R) \
1926 ((__m256h)__builtin_ia32_vrndscaleph256_round_mask( \
1927 (__v16hf)(__m256h)(B), (int)(imm), (__v16hf)_mm256_setzero_ph(), \
1928 (__mmask16)(A), (int)(R)))
1929
1930#define _mm256_roundscale_round_ps(A, imm, R) \
1931 ((__m256)__builtin_ia32_vrndscaleps256_round_mask( \
1932 (__v8sf)(__m256)(A), (int)(imm), (__v8sf)_mm256_undefined_ps(), \
1933 (__mmask8)-1, (int)(R)))
1934
1935#define _mm256_mask_roundscale_round_ps(A, B, C, imm, R) \
1936 ((__m256)__builtin_ia32_vrndscaleps256_round_mask( \
1937 (__v8sf)(__m256)(C), (int)(imm), (__v8sf)(__m256)(A), (__mmask8)(B), \
1938 (int)(R)))
1939
1940#define _mm256_maskz_roundscale_round_ps(A, B, imm, R) \
1941 ((__m256)__builtin_ia32_vrndscaleps256_round_mask( \
1942 (__v8sf)(__m256)(B), (int)(imm), (__v8sf)_mm256_setzero_ps(), \
1943 (__mmask8)(A), (int)(R)))
1944
1945#define _mm256_scalef_round_pd(A, B, R) \
1946 ((__m256d)__builtin_ia32_vscalefpd256_round_mask( \
1947 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), \
1948 (__v4df)_mm256_undefined_pd(), (__mmask8)-1, (int)(R)))
1949
1950#define _mm256_mask_scalef_round_pd(W, U, A, B, R) \
1951 ((__m256d)__builtin_ia32_vscalefpd256_round_mask( \
1952 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)(__m256d)(W), \
1953 (__mmask8)(U), (int)(R)))
1954
1955#define _mm256_maskz_scalef_round_pd(U, A, B, R) \
1956 ((__m256d)__builtin_ia32_vscalefpd256_round_mask( \
1957 (__v4df)(__m256d)(A), (__v4df)(__m256d)(B), (__v4df)_mm256_setzero_pd(), \
1958 (__mmask8)(U), (int)(R)))
1959
1960#define _mm256_scalef_round_ph(A, B, R) \
1961 ((__m256h)__builtin_ia32_vscalefph256_round_mask( \
1962 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), \
1963 (__v16hf)_mm256_undefined_ph(), (__mmask16)-1, (int)(R)))
1964
1965#define _mm256_mask_scalef_round_ph(W, U, A, B, R) \
1966 ((__m256h)__builtin_ia32_vscalefph256_round_mask( \
1967 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), (__v16hf)(__m256h)(W), \
1968 (__mmask16)(U), (int)(R)))
1969
1970#define _mm256_maskz_scalef_round_ph(U, A, B, R) \
1971 ((__m256h)__builtin_ia32_vscalefph256_round_mask( \
1972 (__v16hf)(__m256h)(A), (__v16hf)(__m256h)(B), \
1973 (__v16hf)_mm256_setzero_ph(), (__mmask16)(U), (int)(R)))
1974
1975#define _mm256_scalef_round_ps(A, B, R) \
1976 ((__m256)__builtin_ia32_vscalefps256_round_mask( \
1977 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)_mm256_undefined_ps(), \
1978 (__mmask8)-1, (int)(R)))
1979
1980#define _mm256_mask_scalef_round_ps(W, U, A, B, R) \
1981 ((__m256)__builtin_ia32_vscalefps256_round_mask( \
1982 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)(__m256)(W), \
1983 (__mmask8)(U), (int)(R)))
1984
1985#define _mm256_maskz_scalef_round_ps(U, A, B, R) \
1986 ((__m256)__builtin_ia32_vscalefps256_round_mask( \
1987 (__v8sf)(__m256)(A), (__v8sf)(__m256)(B), (__v8sf)_mm256_setzero_ps(), \
1988 (__mmask8)(U), (int)(R)))
1989
1990#define _mm256_sqrt_round_pd(A, R) \
1991 ((__m256d)__builtin_ia32_vsqrtpd256_round((__v4df)(__m256d)(A), (int)(R)))
1992
1993#define _mm256_mask_sqrt_round_pd(W, U, A, R) \
1994 ((__m256d)__builtin_ia32_selectpd_256( \
1995 (__mmask8)(U), (__v4df)_mm256_sqrt_round_pd((A), (R)), \
1996 (__v4df)(__m256d)(W)))
1997
1998#define _mm256_maskz_sqrt_round_pd(U, A, R) \
1999 ((__m256d)__builtin_ia32_selectpd_256( \
2000 (__mmask8)(U), (__v4df)_mm256_sqrt_round_pd((A), (R)), \
2001 (__v4df)_mm256_setzero_pd()))
2002
2003#define _mm256_sqrt_round_ph(A, R) \
2004 ((__m256h)__builtin_ia32_vsqrtph256_round((__v16hf)(__m256h)(A), (int)(R)))
2005
2006#define _mm256_mask_sqrt_round_ph(W, U, A, R) \
2007 ((__m256h)__builtin_ia32_selectph_256( \
2008 (__mmask16)(U), (__v16hf)_mm256_sqrt_round_ph((A), (R)), \
2009 (__v16hf)(__m256h)(W)))
2010
2011#define _mm256_maskz_sqrt_round_ph(U, A, R) \
2012 ((__m256h)__builtin_ia32_selectph_256( \
2013 (__mmask16)(U), (__v16hf)_mm256_sqrt_round_ph((A), (R)), \
2014 (__v16hf)_mm256_setzero_ph()))
2015
2016#define _mm256_sqrt_round_ps(A, R) \
2017 ((__m256)__builtin_ia32_vsqrtps256_round((__v8sf)(__m256)(A), (int)(R)))
2018
2019#define _mm256_mask_sqrt_round_ps(W, U, A, R) \
2020 ((__m256)__builtin_ia32_selectps_256((__mmask8)(U), \
2021 (__v8sf)_mm256_sqrt_round_ps((A), (R)), \
2022 (__v8sf)(__m256)(W)))
2023
2024#define _mm256_maskz_sqrt_round_ps(U, A, R) \
2025 ((__m256)__builtin_ia32_selectps_256((__mmask8)(U), \
2026 (__v8sf)_mm256_sqrt_round_ps((A), (R)), \
2027 (__v8sf)_mm256_setzero_ps()))
2028
2029#define _mm256_sub_round_pd(A, B, R) \
2030 ((__m256d)__builtin_ia32_vsubpd256_round((__v4df)(__m256d)(A), \
2031 (__v4df)(__m256d)(B), (int)(R)))
2032
2033#define _mm256_mask_sub_round_pd(W, U, A, B, R) \
2034 ((__m256d)__builtin_ia32_selectpd_256( \
2035 (__mmask8)(U), (__v4df)_mm256_sub_round_pd((A), (B), (R)), \
2036 (__v4df)(__m256d)(W)))
2037
2038#define _mm256_maskz_sub_round_pd(U, A, B, R) \
2039 ((__m256d)__builtin_ia32_selectpd_256( \
2040 (__mmask8)(U), (__v4df)_mm256_sub_round_pd((A), (B), (R)), \
2041 (__v4df)_mm256_setzero_pd()))
2042
2043#define _mm256_sub_round_ph(A, B, R) \
2044 ((__m256h)__builtin_ia32_vsubph256_round((__v16hf)(__m256h)(A), \
2045 (__v16hf)(__m256h)(B), (int)(R)))
2046
2047#define _mm256_mask_sub_round_ph(W, U, A, B, R) \
2048 ((__m256h)__builtin_ia32_selectph_256( \
2049 (__mmask16)(U), (__v16hf)_mm256_sub_round_ph((A), (B), (R)), \
2050 (__v16hf)(__m256h)(W)))
2051
2052#define _mm256_maskz_sub_round_ph(U, A, B, R) \
2053 ((__m256h)__builtin_ia32_selectph_256( \
2054 (__mmask16)(U), (__v16hf)_mm256_sub_round_ph((A), (B), (R)), \
2055 (__v16hf)_mm256_setzero_ph()))
2056
2057#define _mm256_sub_round_ps(A, B, R) \
2058 ((__m256)__builtin_ia32_vsubps256_round((__v8sf)(__m256)(A), \
2059 (__v8sf)(__m256)(B), (int)(R)))
2060
2061#define _mm256_mask_sub_round_ps(W, U, A, B, R) \
2062 ((__m256)__builtin_ia32_selectps_256( \
2063 (__mmask8)(U), (__v8sf)_mm256_sub_round_ps((A), (B), (R)), \
2064 (__v8sf)(__m256)(W)))
2065
2066#define _mm256_maskz_sub_round_ps(U, A, B, R) \
2067 ((__m256)__builtin_ia32_selectps_256( \
2068 (__mmask8)(U), (__v8sf)_mm256_sub_round_ps((A), (B), (R)), \
2069 (__v8sf)_mm256_setzero_ps()))
2070
2071#undef __DEFAULT_FN_ATTRS256
2072#undef __DEFAULT_FN_ATTRS128
2073
2074#endif /* __AVX10_2NIINTRIN_H */
2075#endif /* __SSE2__ */
lib/include/avx10_2satcvtdsintrin.h created+496
......@@ -0,0 +1,496 @@
1/*===----------- avx10_2satcvtdsintrin.h - AVX512SATCVTDS intrinsics --------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10#ifndef __IMMINTRIN_H
11#error \
12 "Never use <avx10_2satcvtdsintrin.h> directly; include <immintrin.h> instead."
13#endif // __IMMINTRIN_H
14
15#ifndef __AVX10_2SATCVTDSINTRIN_H
16#define __AVX10_2SATCVTDSINTRIN_H
17
18/* Define the default attributes for the functions in this file. */
19#define __DEFAULT_FN_ATTRS256 \
20 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
21 __min_vector_width__(256)))
22
23#define __DEFAULT_FN_ATTRS128 \
24 __attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
25 __min_vector_width__(128)))
26
27#define _mm_cvtts_roundsd_i32(__A, __R) \
28 ((int)__builtin_ia32_vcvttsd2sis32((__v2df)(__m128)(__A), (const int)(__R)))
29
30#define _mm_cvtts_roundsd_si32(__A, __R) \
31 ((int)__builtin_ia32_vcvttsd2sis32((__v2df)(__m128d)(__A), (const int)(__R)))
32
33#define _mm_cvtts_roundsd_u32(__A, __R) \
34 ((unsigned int)__builtin_ia32_vcvttsd2usis32((__v2df)(__m128d)(__A), \
35 (const int)(__R)))
36
37#define _mm_cvtts_roundss_i32(__A, __R) \
38 ((int)__builtin_ia32_vcvttss2sis32((__v4sf)(__m128)(__A), (const int)(__R)))
39
40#define _mm_cvtts_roundss_si32(__A, __R) \
41 ((int)__builtin_ia32_vcvttss2sis32((__v4sf)(__m128)(__A), (const int)(__R)))
42
43#define _mm_cvtts_roundss_u32(__A, __R) \
44 ((unsigned int)__builtin_ia32_vcvttss2usis32((__v4sf)(__m128)(__A), \
45 (const int)(__R)))
46
47#ifdef __x86_64__
48#define _mm_cvtts_roundss_u64(__A, __R) \
49 ((unsigned long long)__builtin_ia32_vcvttss2usis64((__v4sf)(__m128)(__A), \
50 (const int)(__R)))
51
52#define _mm_cvtts_roundsd_u64(__A, __R) \
53 ((unsigned long long)__builtin_ia32_vcvttsd2usis64((__v2df)(__m128d)(__A), \
54 (const int)(__R)))
55
56#define _mm_cvtts_roundss_i64(__A, __R) \
57 ((long long)__builtin_ia32_vcvttss2sis64((__v4sf)(__m128)(__A), \
58 (const int)(__R)))
59
60#define _mm_cvtts_roundss_si64(__A, __R) \
61 ((long long)__builtin_ia32_vcvttss2sis64((__v4sf)(__m128)(__A), \
62 (const int)(__R)))
63
64#define _mm_cvtts_roundsd_si64(__A, __R) \
65 ((long long)__builtin_ia32_vcvttsd2sis64((__v2df)(__m128d)(__A), \
66 (const int)(__R)))
67
68#define _mm_cvtts_roundsd_i64(__A, __R) \
69 ((long long)__builtin_ia32_vcvttsd2sis64((__v2df)(__m128d)(__A), \
70 (const int)(__R)))
71#endif /* __x86_64__ */
72
73// 128 Bit : Double -> int
74static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttspd_epi32(__m128d __A) {
75 return ((__m128i)__builtin_ia32_vcvttpd2dqs128_mask(
76 (__v2df)__A, (__v4si)(__m128i)_mm_undefined_si128(), (__mmask8)(-1)));
77}
78
79static __inline__ __m128i __DEFAULT_FN_ATTRS128
80_mm_mask_cvttspd_epi32(__m128i __W, __mmask8 __U, __m128d __A) {
81 return ((__m128i)__builtin_ia32_vcvttpd2dqs128_mask((__v2df)__A, (__v4si)__W,
82 __U));
83}
84
85static __inline__ __m128i __DEFAULT_FN_ATTRS128
86_mm_maskz_cvttspd_epi32(__mmask16 __U, __m128d __A) {
87 return ((__m128i)__builtin_ia32_vcvttpd2dqs128_mask(
88 (__v2df)__A, (__v4si)(__m128i)_mm_setzero_si128(), __U));
89}
90
91// 256 Bit : Double -> int
92static __inline__ __m128i __DEFAULT_FN_ATTRS256
93_mm256_cvttspd_epi32(__m256d __A) {
94 return ((__m128i)__builtin_ia32_vcvttpd2dqs256_round_mask(
95 (__v4df)__A, (__v4si)_mm_undefined_si128(), (__mmask8)-1,
96 _MM_FROUND_CUR_DIRECTION));
97}
98
99static __inline__ __m128i __DEFAULT_FN_ATTRS256
100_mm256_mask_cvttspd_epi32(__m128i __W, __mmask8 __U, __m256d __A) {
101 return ((__m128i)__builtin_ia32_vcvttpd2dqs256_round_mask(
102 (__v4df)__A, (__v4si)__W, __U, _MM_FROUND_CUR_DIRECTION));
103}
104
105static __inline__ __m128i __DEFAULT_FN_ATTRS256
106_mm256_maskz_cvttspd_epi32(__mmask8 __U, __m256d __A) {
107 return ((__m128i)__builtin_ia32_vcvttpd2dqs256_round_mask(
108 (__v4df)__A, (__v4si)_mm_setzero_si128(), __U, _MM_FROUND_CUR_DIRECTION));
109}
110
111#define _mm256_cvtts_roundpd_epi32(__A, __R) \
112 ((__m128i)__builtin_ia32_vcvttpd2dqs256_round_mask( \
113 (__v4df)(__m256d)__A, (__v4si)(__m128i)_mm_undefined_si128(), \
114 (__mmask8) - 1, (int)(__R)))
115
116#define _mm256_mask_cvtts_roundpd_epi32(__W, __U, __A, __R) \
117 ((__m128i)__builtin_ia32_vcvttpd2dqs256_round_mask( \
118 (__v4df)(__m256d)__A, (__v4si)(__m128i)__W, (__mmask8)__U, (int)(__R)))
119
120#define _mm256_maskz_cvtts_roundpd_epi32(__U, __A, __R) \
121 ((__m128i)__builtin_ia32_vcvttpd2dqs256_round_mask( \
122 (__v4df)(__m256d)__A, (__v4si)(__m128i)_mm_setzero_si128(), \
123 (__mmask8)__U, (int)(__R)))
124
125// 128 Bit : Double -> uint
126static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttspd_epu32(__m128d __A) {
127 return ((__m128i)__builtin_ia32_vcvttpd2udqs128_mask(
128 (__v2df)__A, (__v4si)(__m128i)_mm_undefined_si128(), (__mmask8)(-1)));
129}
130
131static __inline__ __m128i __DEFAULT_FN_ATTRS128
132_mm_mask_cvttspd_epu32(__m128i __W, __mmask8 __U, __m128d __A) {
133 return ((__m128i)__builtin_ia32_vcvttpd2udqs128_mask(
134 (__v2df)__A, (__v4si)(__m128i)__W, (__mmask8)__U));
135}
136
137static __inline__ __m128i __DEFAULT_FN_ATTRS128
138_mm_maskz_cvttspd_epu32(__mmask8 __U, __m128d __A) {
139 return ((__m128i)__builtin_ia32_vcvttpd2udqs128_mask(
140 (__v2df)__A, (__v4si)(__m128i)_mm_setzero_si128(), __U));
141}
142
143// 256 Bit : Double -> uint
144static __inline__ __m128i __DEFAULT_FN_ATTRS256
145_mm256_cvttspd_epu32(__m256d __A) {
146 return ((__m128i)__builtin_ia32_vcvttpd2udqs256_round_mask(
147 (__v4df)__A, (__v4si)_mm_undefined_si128(), (__mmask8)-1,
148 _MM_FROUND_CUR_DIRECTION));
149}
150
151static __inline__ __m128i __DEFAULT_FN_ATTRS256
152_mm256_mask_cvttspd_epu32(__m128i __W, __mmask8 __U, __m256d __A) {
153 return ((__m128i)__builtin_ia32_vcvttpd2udqs256_round_mask(
154 (__v4df)__A, (__v4si)__W, __U, _MM_FROUND_CUR_DIRECTION));
155}
156
157static __inline__ __m128i __DEFAULT_FN_ATTRS256
158_mm256_maskz_cvttspd_epu32(__mmask8 __U, __m256d __A) {
159 return ((__m128i)__builtin_ia32_vcvttpd2udqs256_round_mask(
160 (__v4df)__A, (__v4si)_mm_setzero_si128(), __U, _MM_FROUND_CUR_DIRECTION));
161}
162
163#define _mm256_cvtts_roundpd_epu32(__A, __R) \
164 ((__m128i)__builtin_ia32_vcvttpd2udqs256_round_mask( \
165 (__v4df)(__m256d)__A, (__v4si)(__m128i)_mm_undefined_si128(), \
166 (__mmask8) - 1, (int)(__R)))
167
168#define _mm256_mask_cvtts_roundpd_epu32(__W, __U, __A, __R) \
169 ((__m128i)__builtin_ia32_vcvttpd2udqs256_round_mask( \
170 (__v4df)(__m256d)__A, (__v4si)(__m128i)__W, (__mmask8)__U, (int)(__R)))
171
172#define _mm256_maskz_cvtts_roundpd_epu32(__U, __A, __R) \
173 ((__m128i)__builtin_ia32_vcvttpd2udqs256_round_mask( \
174 (__v4df)(__m256d)__A, (__v4si)(__m128i)_mm_setzero_si128(), \
175 (__mmask8)__U, (int)(__R)))
176
177// 128 Bit : Double -> long
178static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttspd_epi64(__m128d __A) {
179 return ((__m128i)__builtin_ia32_vcvttpd2qqs128_mask(
180 (__v2df)__A, (__v2di)_mm_undefined_si128(), (__mmask8)-1));
181}
182
183static __inline__ __m128i __DEFAULT_FN_ATTRS128
184_mm_mask_cvttspd_epi64(__m128i __W, __mmask8 __U, __m128d __A) {
185 return ((__m128i)__builtin_ia32_vcvttpd2qqs128_mask((__v2df)__A, (__v2di)__W,
186 (__mmask8)__U));
187}
188
189static __inline__ __m128i __DEFAULT_FN_ATTRS128
190_mm_maskz_cvttspd_epi64(__mmask8 __U, __m128d __A) {
191 return ((__m128i)__builtin_ia32_vcvttpd2qqs128_mask(
192 (__v2df)__A, (__v2di)_mm_setzero_si128(), (__mmask8)__U));
193}
194
195// 256 Bit : Double -> long
196static __inline__ __m256i __DEFAULT_FN_ATTRS256
197_mm256_cvttspd_epi64(__m256d __A) {
198 return ((__m256i)__builtin_ia32_vcvttpd2qqs256_round_mask(
199 (__v4df)__A, (__v4di)_mm256_undefined_si256(), (__mmask8)-1,
200 _MM_FROUND_CUR_DIRECTION));
201}
202
203static __inline__ __m256i __DEFAULT_FN_ATTRS256
204_mm256_mask_cvttspd_epi64(__m256i __W, __mmask8 __U, __m256d __A) {
205 return ((__m256i)__builtin_ia32_vcvttpd2qqs256_round_mask(
206 (__v4df)__A, (__v4di)__W, __U, _MM_FROUND_CUR_DIRECTION));
207}
208
209static __inline__ __m256i __DEFAULT_FN_ATTRS256
210_mm256_maskz_cvttspd_epi64(__mmask8 __U, __m256d __A) {
211 return ((__m256i)__builtin_ia32_vcvttpd2qqs256_round_mask(
212 (__v4df)__A, (__v4di)_mm256_setzero_si256(), __U,
213 _MM_FROUND_CUR_DIRECTION));
214}
215
216#define _mm256_cvtts_roundpd_epi64(__A, __R) \
217 ((__m256i)__builtin_ia32_vcvttpd2qqs256_round_mask( \
218 (__v4df)__A, (__v4di)_mm256_undefined_si256(), (__mmask8) - 1, \
219 (int)__R))
220
221#define _mm256_mask_cvtts_roundpd_epi64(__W, __U, __A, __R) \
222 ((__m256i)__builtin_ia32_vcvttpd2qqs256_round_mask((__v4df)__A, (__v4di)__W, \
223 (__mmask8)__U, (int)__R))
224
225#define _mm256_maskz_cvtts_roundpd_epi64(__U, __A, __R) \
226 ((__m256i)__builtin_ia32_vcvttpd2qqs256_round_mask( \
227 (__v4df)__A, (__v4di)_mm256_setzero_si256(), (__mmask8)__U, (int)__R))
228
229// 128 Bit : Double -> ulong
230static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttspd_epu64(__m128d __A) {
231 return ((__m128i)__builtin_ia32_vcvttpd2uqqs128_mask(
232 (__v2df)__A, (__v2di)_mm_undefined_si128(), (__mmask8)-1));
233}
234
235static __inline__ __m128i __DEFAULT_FN_ATTRS128
236_mm_mask_cvttspd_epu64(__m128i __W, __mmask8 __U, __m128d __A) {
237 return ((__m128i)__builtin_ia32_vcvttpd2uqqs128_mask((__v2df)__A, (__v2di)__W,
238 (__mmask8)__U));
239}
240
241static __inline__ __m128i __DEFAULT_FN_ATTRS128
242_mm_maskz_cvttspd_epu64(__mmask8 __U, __m128d __A) {
243 return ((__m128i)__builtin_ia32_vcvttpd2uqqs128_mask(
244 (__v2df)__A, (__v2di)_mm_setzero_si128(), (__mmask8)__U));
245}
246
247// 256 Bit : Double -> ulong
248
249static __inline__ __m256i __DEFAULT_FN_ATTRS256
250_mm256_cvttspd_epu64(__m256d __A) {
251 return ((__m256i)__builtin_ia32_vcvttpd2uqqs256_round_mask(
252 (__v4df)__A, (__v4di)_mm256_undefined_si256(), (__mmask8)-1,
253 _MM_FROUND_CUR_DIRECTION));
254}
255
256static __inline__ __m256i __DEFAULT_FN_ATTRS256
257_mm256_mask_cvttspd_epu64(__m256i __W, __mmask8 __U, __m256d __A) {
258 return ((__m256i)__builtin_ia32_vcvttpd2uqqs256_round_mask(
259 (__v4df)__A, (__v4di)__W, __U, _MM_FROUND_CUR_DIRECTION));
260}
261
262static __inline__ __m256i __DEFAULT_FN_ATTRS256
263_mm256_maskz_cvttspd_epu64(__mmask8 __U, __m256d __A) {
264 return ((__m256i)__builtin_ia32_vcvttpd2uqqs256_round_mask(
265 (__v4df)__A, (__v4di)_mm256_setzero_si256(), __U,
266 _MM_FROUND_CUR_DIRECTION));
267}
268
269#define _mm256_cvtts_roundpd_epu64(__A, __R) \
270 ((__m256i)__builtin_ia32_vcvttpd2uqqs256_round_mask( \
271 (__v4df)__A, (__v4di)_mm256_undefined_si256(), (__mmask8) - 1, \
272 (int)__R))
273
274#define _mm256_mask_cvtts_roundpd_epu64(__W, __U, __A, __R) \
275 ((__m256i)__builtin_ia32_vcvttpd2uqqs256_round_mask( \
276 (__v4df)__A, (__v4di)__W, (__mmask8)__U, (int)__R))
277
278#define _mm256_maskz_cvtts_roundpd_epu64(__U, __A, __R) \
279 ((__m256i)__builtin_ia32_vcvttpd2uqqs256_round_mask( \
280 (__v4df)__A, (__v4di)_mm256_setzero_si256(), (__mmask8)__U, (int)__R))
281
282// 128 Bit : float -> int
283static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttsps_epi32(__m128 __A) {
284 return ((__m128i)__builtin_ia32_vcvttps2dqs128_mask(
285 (__v4sf)__A, (__v4si)(__m128i)_mm_undefined_si128(), (__mmask8)(-1)));
286}
287
288static __inline__ __m128i __DEFAULT_FN_ATTRS128
289_mm_mask_cvttsps_epi32(__m128i __W, __mmask8 __U, __m128 __A) {
290 return ((__m128i)__builtin_ia32_vcvttps2dqs128_mask((__v4sf)__A, (__v4si)__W,
291 (__mmask8)__U));
292}
293
294static __inline__ __m128i __DEFAULT_FN_ATTRS128
295_mm_maskz_cvttsps_epi32(__mmask8 __U, __m128 __A) {
296 return ((__m128i)__builtin_ia32_vcvttps2dqs128_mask(
297 (__v4sf)__A, (__v4si)(__m128i)_mm_setzero_si128(), (__mmask8)__U));
298}
299
300// 256 Bit : float -> int
301static __inline__ __m256i __DEFAULT_FN_ATTRS256
302_mm256_cvttsps_epi32(__m256 __A) {
303 return ((__m256i)__builtin_ia32_vcvttps2dqs256_round_mask(
304 (__v8sf)__A, (__v8si)_mm256_undefined_si256(), (__mmask8)-1,
305 _MM_FROUND_CUR_DIRECTION));
306}
307
308static __inline__ __m256i __DEFAULT_FN_ATTRS256
309_mm256_mask_cvttsps_epi32(__m256i __W, __mmask8 __U, __m256 __A) {
310 return ((__m256i)__builtin_ia32_vcvttps2dqs256_round_mask(
311 (__v8sf)__A, (__v8si)__W, __U, _MM_FROUND_CUR_DIRECTION));
312}
313
314static __inline__ __m256i __DEFAULT_FN_ATTRS256
315_mm256_maskz_cvttsps_epi32(__mmask8 __U, __m256 __A) {
316 return ((__m256i)__builtin_ia32_vcvttps2dqs256_round_mask(
317 (__v8sf)__A, (__v8si)_mm256_setzero_si256(), __U,
318 _MM_FROUND_CUR_DIRECTION));
319}
320
321#define _mm256_cvtts_roundps_epi32(__A, __R) \
322 ((__m256i)__builtin_ia32_vcvttps2dqs256_round_mask( \
323 (__v8sf)(__m256)__A, (__v8si)(__m256i)_mm256_undefined_si256(), \
324 (__mmask8) - 1, (int)(__R)))
325
326#define _mm256_mask_cvtts_roundps_epi32(__W, __U, __A, __R) \
327 ((__m256i)__builtin_ia32_vcvttps2dqs256_round_mask( \
328 (__v8sf)(__m256)__A, (__v8si)(__m256i)__W, (__mmask8)__U, (int)(__R)))
329
330#define _mm256_maskz_cvtts_roundps_epi32(__U, __A, __R) \
331 ((__m256i)__builtin_ia32_vcvttps2dqs256_round_mask( \
332 (__v8sf)(__m256)__A, (__v8si)(__m256i)_mm256_setzero_si256(), \
333 (__mmask8)__U, (int)(__R)))
334
335// 128 Bit : float -> uint
336static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttsps_epu32(__m128 __A) {
337 return ((__m128i)__builtin_ia32_vcvttps2udqs128_mask(
338 (__v4sf)__A, (__v4si)(__m128i)_mm_undefined_si128(), (__mmask8)(-1)));
339}
340
341static __inline__ __m128i __DEFAULT_FN_ATTRS128
342_mm_mask_cvttsps_epu32(__m128i __W, __mmask8 __U, __m128 __A) {
343 return ((__m128i)__builtin_ia32_vcvttps2udqs128_mask((__v4sf)__A, (__v4si)__W,
344 (__mmask8)__U));
345}
346
347static __inline__ __m128i __DEFAULT_FN_ATTRS128
348_mm_maskz_cvttsps_epu32(__mmask8 __U, __m128 __A) {
349 return ((__m128i)__builtin_ia32_vcvttps2udqs128_mask(
350 (__v4sf)__A, (__v4si)_mm_setzero_si128(), (__mmask8)__U));
351}
352
353// 256 Bit : float -> uint
354
355static __inline__ __m256i __DEFAULT_FN_ATTRS256
356_mm256_cvttsps_epu32(__m256 __A) {
357 return ((__m256i)__builtin_ia32_vcvttps2udqs256_round_mask(
358 (__v8sf)__A, (__v8si)_mm256_undefined_si256(), (__mmask8)-1,
359 _MM_FROUND_CUR_DIRECTION));
360}
361
362static __inline__ __m256i __DEFAULT_FN_ATTRS256
363_mm256_mask_cvttsps_epu32(__m256i __W, __mmask8 __U, __m256 __A) {
364 return ((__m256i)__builtin_ia32_vcvttps2udqs256_round_mask(
365 (__v8sf)__A, (__v8si)__W, __U, _MM_FROUND_CUR_DIRECTION));
366}
367
368static __inline__ __m256i __DEFAULT_FN_ATTRS256
369_mm256_maskz_cvttsps_epu32(__mmask8 __U, __m256 __A) {
370 return ((__m256i)__builtin_ia32_vcvttps2udqs256_round_mask(
371 (__v8sf)__A, (__v8si)_mm256_setzero_si256(), __U,
372 _MM_FROUND_CUR_DIRECTION));
373}
374
375#define _mm256_cvtts_roundps_epu32(__A, __R) \
376 ((__m256i)__builtin_ia32_vcvttps2udqs256_round_mask( \
377 (__v8sf)(__m256)__A, (__v8si)(__m256i)_mm256_undefined_si256(), \
378 (__mmask8) - 1, (int)(__R)))
379
380#define _mm256_mask_cvtts_roundps_epu32(__W, __U, __A, __R) \
381 ((__m256i)__builtin_ia32_vcvttps2udqs256_round_mask( \
382 (__v8sf)(__m256)__A, (__v8si)(__m256i)__W, (__mmask8)__U, (int)(__R)))
383
384#define _mm256_maskz_cvtts_roundps_epu32(__U, __A, __R) \
385 ((__m256i)__builtin_ia32_vcvttps2udqs256_round_mask( \
386 (__v8sf)(__m256)__A, (__v8si)(__m256i)_mm256_setzero_si256(), \
387 (__mmask8)__U, (int)(__R)))
388
389// 128 bit : float -> long
390static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttsps_epi64(__m128 __A) {
391 return ((__m128i)__builtin_ia32_vcvttps2qqs128_mask(
392 (__v4sf)__A, (__v2di)_mm_undefined_si128(), (__mmask8)-1));
393}
394
395static __inline__ __m128i __DEFAULT_FN_ATTRS128
396_mm_mask_cvttsps_epi64(__m128i __W, __mmask8 __U, __m128 __A) {
397 return ((__m128i)__builtin_ia32_vcvttps2qqs128_mask(
398 (__v4sf)__A, (__v2di)(__m128i)__W, (__mmask8)__U));
399}
400
401static __inline__ __m128i __DEFAULT_FN_ATTRS128
402_mm_maskz_cvttsps_epi64(__mmask8 __U, __m128 __A) {
403 return ((__m128i)__builtin_ia32_vcvttps2qqs128_mask(
404 (__v4sf)__A, (__v2di)_mm_setzero_si128(), (__mmask8)__U));
405}
406// 256 bit : float -> long
407
408static __inline__ __m256i __DEFAULT_FN_ATTRS256
409_mm256_cvttsps_epi64(__m128 __A) {
410 return ((__m256i)__builtin_ia32_vcvttps2qqs256_round_mask(
411 (__v4sf)__A, (__v4di)_mm256_undefined_si256(), (__mmask8)-1,
412 _MM_FROUND_CUR_DIRECTION));
413}
414static __inline__ __m256i __DEFAULT_FN_ATTRS256
415_mm256_mask_cvttsps_epi64(__m256i __W, __mmask8 __U, __m128 __A) {
416 return ((__m256i)__builtin_ia32_vcvttps2qqs256_round_mask(
417 (__v4sf)__A, (__v4di)__W, __U, _MM_FROUND_CUR_DIRECTION));
418}
419
420static __inline__ __m256i __DEFAULT_FN_ATTRS256
421_mm256_maskz_cvttsps_epi64(__mmask8 __U, __m128 __A) {
422 return ((__m256i)__builtin_ia32_vcvttps2qqs256_round_mask(
423 (__v4sf)__A, (__v4di)_mm256_setzero_si256(), __U,
424 _MM_FROUND_CUR_DIRECTION));
425}
426
427#define _mm256_cvtts_roundps_epi64(__A, __R) \
428 ((__m256i)__builtin_ia32_vcvttps2qqs256_round_mask( \
429 (__v4sf)(__m128)__A, (__v4di)_mm256_undefined_si256(), (__mmask8) - 1, \
430 (int)__R))
431
432#define _mm256_mask_cvtts_roundps_epi64(__W, __U, __A, __R) \
433 ((__m256i)__builtin_ia32_vcvttps2qqs256_round_mask( \
434 (__v4sf)(__m128)__A, (__v4di)__W, (__mmask8)__U, (int)__R))
435
436#define _mm256_maskz_cvtts_roundps_epi64(__U, __A, __R) \
437 ((__m256i)__builtin_ia32_vcvttps2qqs256_round_mask( \
438 (__v4sf)(__m128)__A, (__v4di)_mm256_setzero_si256(), (__mmask8)__U, \
439 (int)__R))
440
441// 128 bit : float -> ulong
442static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_cvttsps_epu64(__m128 __A) {
443 return ((__m128i)__builtin_ia32_vcvttps2uqqs128_mask(
444 (__v4sf)__A, (__v2di)_mm_undefined_si128(), (__mmask8)-1));
445}
446
447static __inline__ __m128i __DEFAULT_FN_ATTRS128
448_mm_mask_cvttsps_epu64(__m128i __W, __mmask8 __U, __m128 __A) {
449 return ((__m128i)__builtin_ia32_vcvttps2uqqs128_mask(
450 (__v4sf)__A, (__v2di)(__m128i)__W, (__mmask8)__U));
451}
452
453static __inline__ __m128i __DEFAULT_FN_ATTRS128
454_mm_maskz_cvttsps_epu64(__mmask8 __U, __m128 __A) {
455 return ((__m128i)__builtin_ia32_vcvttps2uqqs128_mask(
456 (__v4sf)__A, (__v2di)_mm_setzero_si128(), (__mmask8)__U));
457}
458// 256 bit : float -> ulong
459
460static __inline__ __m256i __DEFAULT_FN_ATTRS256
461_mm256_cvttsps_epu64(__m128 __A) {
462 return ((__m256i)__builtin_ia32_vcvttps2uqqs256_round_mask(
463 (__v4sf)__A, (__v4di)_mm256_undefined_si256(), (__mmask8)-1,
464 _MM_FROUND_CUR_DIRECTION));
465}
466
467static __inline__ __m256i __DEFAULT_FN_ATTRS256
468_mm256_mask_cvttsps_epu64(__m256i __W, __mmask8 __U, __m128 __A) {
469 return ((__m256i)__builtin_ia32_vcvttps2uqqs256_round_mask(
470 (__v4sf)__A, (__v4di)__W, __U, _MM_FROUND_CUR_DIRECTION));
471}
472
473static __inline__ __m256i __DEFAULT_FN_ATTRS256
474_mm256_maskz_cvttsps_epu64(__mmask8 __U, __m128 __A) {
475 return ((__m256i)__builtin_ia32_vcvttps2uqqs256_round_mask(
476 (__v4sf)__A, (__v4di)_mm256_setzero_si256(), __U,
477 _MM_FROUND_CUR_DIRECTION));
478}
479
480#define _mm256_cvtts_roundps_epu64(__A, __R) \
481 ((__m256i)__builtin_ia32_vcvttps2uqqs256_round_mask( \
482 (__v4sf)(__m128)__A, (__v4di)_mm256_undefined_si256(), (__mmask8) - 1, \
483 (int)__R))
484
485#define _mm256_mask_cvtts_roundps_epu64(__W, __U, __A, __R) \
486 ((__m256i)__builtin_ia32_vcvttps2uqqs256_round_mask( \
487 (__v4sf)(__m128)__A, (__v4di)__W, (__mmask8)__U, (int)__R))
488
489#define _mm256_maskz_cvtts_roundps_epu64(__U, __A, __R) \
490 ((__m256i)__builtin_ia32_vcvttps2uqqs256_round_mask( \
491 (__v4sf)(__m128)__A, (__v4di)_mm256_setzero_si256(), (__mmask8)__U, \
492 (int)__R))
493
494#undef __DEFAULT_FN_ATTRS128
495#undef __DEFAULT_FN_ATTRS256
496#endif // __AVX10_2SATCVTDSINTRIN_H
lib/include/avx10_2satcvtintrin.h created+444
......@@ -0,0 +1,444 @@
1/*===----------- avx10_2satcvtintrin.h - AVX10_2SATCVT intrinsics ----------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <avx10_2satcvtintrin.h> directly; include <immintrin.h> instead."
12#endif // __IMMINTRIN_H
13
14#ifndef __AVX10_2SATCVTINTRIN_H
15#define __AVX10_2SATCVTINTRIN_H
16
17#define _mm_ipcvtbf16_epi8(A) \
18 ((__m128i)__builtin_ia32_vcvtbf162ibs128((__v8bf)(__m128bh)(A)))
19
20#define _mm_mask_ipcvtbf16_epi8(W, U, A) \
21 ((__m128i)__builtin_ia32_selectw_128( \
22 (__mmask8)(U), (__v8hi)_mm_ipcvtbf16_epi8(A), (__v8hi)(__m128i)(W)))
23
24#define _mm_maskz_ipcvtbf16_epi8(U, A) \
25 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \
26 (__v8hi)_mm_ipcvtbf16_epi8(A), \
27 (__v8hi)_mm_setzero_si128()))
28
29#define _mm256_ipcvtbf16_epi8(A) \
30 ((__m256i)__builtin_ia32_vcvtbf162ibs256((__v16bf)(__m256bh)(A)))
31
32#define _mm256_mask_ipcvtbf16_epi8(W, U, A) \
33 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
34 (__v16hi)_mm256_ipcvtbf16_epi8(A), \
35 (__v16hi)(__m256i)(W)))
36
37#define _mm256_maskz_ipcvtbf16_epi8(U, A) \
38 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
39 (__v16hi)_mm256_ipcvtbf16_epi8(A), \
40 (__v16hi)_mm256_setzero_si256()))
41
42#define _mm_ipcvtbf16_epu8(A) \
43 ((__m128i)__builtin_ia32_vcvtbf162iubs128((__v8bf)(__m128bh)(A)))
44
45#define _mm_mask_ipcvtbf16_epu8(W, U, A) \
46 ((__m128i)__builtin_ia32_selectw_128( \
47 (__mmask8)(U), (__v8hi)_mm_ipcvtbf16_epu8(A), (__v8hi)(__m128i)(W)))
48
49#define _mm_maskz_ipcvtbf16_epu8(U, A) \
50 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \
51 (__v8hi)_mm_ipcvtbf16_epu8(A), \
52 (__v8hi)_mm_setzero_si128()))
53
54#define _mm256_ipcvtbf16_epu8(A) \
55 ((__m256i)__builtin_ia32_vcvtbf162iubs256((__v16bf)(__m256bh)(A)))
56
57#define _mm256_mask_ipcvtbf16_epu8(W, U, A) \
58 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
59 (__v16hi)_mm256_ipcvtbf16_epu8(A), \
60 (__v16hi)(__m256i)(W)))
61
62#define _mm256_maskz_ipcvtbf16_epu8(U, A) \
63 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
64 (__v16hi)_mm256_ipcvtbf16_epu8(A), \
65 (__v16hi)_mm256_setzero_si256()))
66
67#define _mm_ipcvtph_epi8(A) \
68 ((__m128i)__builtin_ia32_vcvtph2ibs128_mask( \
69 (__v8hf)(__m128h)(A), (__v8hu)_mm_setzero_si128(), (__mmask8)-1))
70
71#define _mm_mask_ipcvtph_epi8(W, U, A) \
72 ((__m128i)__builtin_ia32_vcvtph2ibs128_mask((__v8hf)(__m128h)(A), \
73 (__v8hu)(W), (__mmask8)(U)))
74
75#define _mm_maskz_ipcvtph_epi8(U, A) \
76 ((__m128i)__builtin_ia32_vcvtph2ibs128_mask( \
77 (__v8hf)(__m128h)(A), (__v8hu)(_mm_setzero_si128()), (__mmask8)(U)))
78
79#define _mm256_ipcvtph_epi8(A) \
80 ((__m256i)__builtin_ia32_vcvtph2ibs256_mask( \
81 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)-1, \
82 _MM_FROUND_CUR_DIRECTION))
83
84#define _mm256_mask_ipcvtph_epi8(W, U, A) \
85 ((__m256i)__builtin_ia32_vcvtph2ibs256_mask((__v16hf)(__m256h)(A), \
86 (__v16hu)(W), (__mmask16)(U), \
87 _MM_FROUND_CUR_DIRECTION))
88
89#define _mm256_maskz_ipcvtph_epi8(U, A) \
90 ((__m256i)__builtin_ia32_vcvtph2ibs256_mask( \
91 (__v16hf)(__m256h)(A), (__v16hu)(_mm256_setzero_si256()), \
92 (__mmask16)(U), _MM_FROUND_CUR_DIRECTION))
93
94#define _mm256_ipcvt_roundph_epi8(A, R) \
95 ((__m256i)__builtin_ia32_vcvtph2ibs256_mask((__v16hf)(__m256h)(A), \
96 (__v16hu)_mm256_setzero_si256(), \
97 (__mmask16)-1, (const int)R))
98
99#define _mm256_mask_ipcvt_roundph_epi8(W, U, A, R) \
100 ((__m256i)__builtin_ia32_vcvtph2ibs256_mask( \
101 (__v16hf)(__m256h)(A), (__v16hu)(W), (__mmask16)(U), (const int)R))
102
103#define _mm256_maskz_ipcvt_roundph_epi8(U, A, R) \
104 ((__m256i)__builtin_ia32_vcvtph2ibs256_mask((__v16hf)(__m256h)(A), \
105 (__v16hu)_mm256_setzero_si256(), \
106 (__mmask16)(U), (const int)R))
107
108#define _mm_ipcvtph_epu8(A) \
109 ((__m128i)__builtin_ia32_vcvtph2iubs128_mask( \
110 (__v8hf)(__m128h)(A), (__v8hu)_mm_setzero_si128(), (__mmask8)-1))
111
112#define _mm_mask_ipcvtph_epu8(W, U, A) \
113 ((__m128i)__builtin_ia32_vcvtph2iubs128_mask((__v8hf)(__m128h)(A), \
114 (__v8hu)(W), (__mmask8)(U)))
115
116#define _mm_maskz_ipcvtph_epu8(U, A) \
117 ((__m128i)__builtin_ia32_vcvtph2iubs128_mask( \
118 (__v8hf)(__m128h)(A), (__v8hu)(_mm_setzero_si128()), (__mmask8)(U)))
119
120#define _mm256_ipcvtph_epu8(A) \
121 ((__m256i)__builtin_ia32_vcvtph2iubs256_mask( \
122 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)-1, \
123 _MM_FROUND_CUR_DIRECTION))
124
125#define _mm256_mask_ipcvtph_epu8(W, U, A) \
126 ((__m256i)__builtin_ia32_vcvtph2iubs256_mask((__v16hf)(__m256h)(A), \
127 (__v16hu)(W), (__mmask16)(U), \
128 _MM_FROUND_CUR_DIRECTION))
129
130#define _mm256_maskz_ipcvtph_epu8(U, A) \
131 ((__m256i)__builtin_ia32_vcvtph2iubs256_mask( \
132 (__v16hf)(__m256h)(A), (__v16hu)(_mm256_setzero_si256()), \
133 (__mmask16)(U), _MM_FROUND_CUR_DIRECTION))
134
135#define _mm256_ipcvt_roundph_epu8(A, R) \
136 ((__m256i)__builtin_ia32_vcvtph2iubs256_mask( \
137 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)-1, \
138 (const int)R))
139
140#define _mm256_mask_ipcvt_roundph_epu8(W, U, A, R) \
141 ((__m256i)__builtin_ia32_vcvtph2iubs256_mask( \
142 (__v16hf)(__m256h)(A), (__v16hu)(W), (__mmask16)(U), (const int)R))
143
144#define _mm256_maskz_ipcvt_roundph_epu8(U, A, R) \
145 ((__m256i)__builtin_ia32_vcvtph2iubs256_mask( \
146 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)(U), \
147 (const int)R))
148
149#define _mm_ipcvtps_epi8(A) \
150 ((__m128i)__builtin_ia32_vcvtps2ibs128_mask( \
151 (__v4sf)(__m128)(A), (__v4su)_mm_setzero_si128(), (__mmask8)-1))
152
153#define _mm_mask_ipcvtps_epi8(W, U, A) \
154 ((__m128i)__builtin_ia32_vcvtps2ibs128_mask((__v4sf)(__m128)(A), \
155 (__v4su)(W), (__mmask8)(U)))
156
157#define _mm_maskz_ipcvtps_epi8(U, A) \
158 ((__m128i)__builtin_ia32_vcvtps2ibs128_mask( \
159 (__v4sf)(__m128)(A), (__v4su)(_mm_setzero_si128()), (__mmask8)(U)))
160
161#define _mm256_ipcvtps_epi8(A) \
162 ((__m256i)__builtin_ia32_vcvtps2ibs256_mask( \
163 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)-1, \
164 _MM_FROUND_CUR_DIRECTION))
165
166#define _mm256_mask_ipcvtps_epi8(W, U, A) \
167 ((__m256i)__builtin_ia32_vcvtps2ibs256_mask((__v8sf)(__m256)(A), \
168 (__v8su)(W), (__mmask8)(U), \
169 _MM_FROUND_CUR_DIRECTION))
170
171#define _mm256_maskz_ipcvtps_epi8(U, A) \
172 ((__m256i)__builtin_ia32_vcvtps2ibs256_mask( \
173 (__v8sf)(__m256)(A), (__v8su)(_mm256_setzero_si256()), (__mmask8)(U), \
174 _MM_FROUND_CUR_DIRECTION))
175
176#define _mm256_ipcvt_roundps_epi8(A, R) \
177 ((__m256i)__builtin_ia32_vcvtps2ibs256_mask((__v8sf)(__m256)(A), \
178 (__v8su)_mm256_setzero_si256(), \
179 (__mmask8)-1, (const int)R))
180
181#define _mm256_mask_ipcvt_roundps_epi8(W, U, A, R) \
182 ((__m256i)__builtin_ia32_vcvtps2ibs256_mask( \
183 (__v8sf)(__m256)(A), (__v8su)(W), (__mmask8)(U), (const int)R))
184
185#define _mm256_maskz_ipcvt_roundps_epi8(U, A, R) \
186 ((__m256i)__builtin_ia32_vcvtps2ibs256_mask((__v8sf)(__m256)(A), \
187 (__v8su)_mm256_setzero_si256(), \
188 (__mmask8)(U), (const int)R))
189
190#define _mm_ipcvtps_epu8(A) \
191 ((__m128i)__builtin_ia32_vcvtps2iubs128_mask( \
192 (__v4sf)(__m128)(A), (__v4su)_mm_setzero_si128(), (__mmask8)-1))
193
194#define _mm_mask_ipcvtps_epu8(W, U, A) \
195 ((__m128i)__builtin_ia32_vcvtps2iubs128_mask((__v4sf)(__m128)(A), \
196 (__v4su)(W), (__mmask8)(U)))
197
198#define _mm_maskz_ipcvtps_epu8(U, A) \
199 ((__m128i)__builtin_ia32_vcvtps2iubs128_mask( \
200 (__v4sf)(__m128)(A), (__v4su)(_mm_setzero_si128()), (__mmask8)(U)))
201
202#define _mm256_ipcvtps_epu8(A) \
203 ((__m256i)__builtin_ia32_vcvtps2iubs256_mask( \
204 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)-1, \
205 _MM_FROUND_CUR_DIRECTION))
206
207#define _mm256_mask_ipcvtps_epu8(W, U, A) \
208 ((__m256i)__builtin_ia32_vcvtps2iubs256_mask((__v8sf)(__m256)(A), \
209 (__v8su)(W), (__mmask8)(U), \
210 _MM_FROUND_CUR_DIRECTION))
211
212#define _mm256_maskz_ipcvtps_epu8(U, A) \
213 ((__m256i)__builtin_ia32_vcvtps2iubs256_mask( \
214 (__v8sf)(__m256)(A), (__v8su)(_mm256_setzero_si256()), (__mmask8)(U), \
215 _MM_FROUND_CUR_DIRECTION))
216
217#define _mm256_ipcvt_roundps_epu8(A, R) \
218 ((__m256i)__builtin_ia32_vcvtps2iubs256_mask((__v8sf)(__m256)(A), \
219 (__v8su)_mm256_setzero_si256(), \
220 (__mmask8)-1, (const int)R))
221
222#define _mm256_mask_ipcvt_roundps_epu8(W, U, A, R) \
223 ((__m256i)__builtin_ia32_vcvtps2iubs256_mask( \
224 (__v8sf)(__m256)(A), (__v8su)(W), (__mmask8)(U), (const int)R))
225
226#define _mm256_maskz_ipcvt_roundps_epu8(U, A, R) \
227 ((__m256i)__builtin_ia32_vcvtps2iubs256_mask((__v8sf)(__m256)(A), \
228 (__v8su)_mm256_setzero_si256(), \
229 (__mmask8)(U), (const int)R))
230
231#define _mm_ipcvttbf16_epi8(A) \
232 ((__m128i)__builtin_ia32_vcvttbf162ibs128((__v8bf)(__m128bh)(A)))
233
234#define _mm_mask_ipcvttbf16_epi8(W, U, A) \
235 ((__m128i)__builtin_ia32_selectw_128( \
236 (__mmask8)(U), (__v8hi)_mm_ipcvttbf16_epi8(A), (__v8hi)(__m128i)(W)))
237
238#define _mm_maskz_ipcvttbf16_epi8(U, A) \
239 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \
240 (__v8hi)_mm_ipcvttbf16_epi8(A), \
241 (__v8hi)_mm_setzero_si128()))
242
243#define _mm256_ipcvttbf16_epi8(A) \
244 ((__m256i)__builtin_ia32_vcvttbf162ibs256((__v16bf)(__m256bh)(A)))
245
246#define _mm256_mask_ipcvttbf16_epi8(W, U, A) \
247 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
248 (__v16hi)_mm256_ipcvttbf16_epi8(A), \
249 (__v16hi)(__m256i)(W)))
250
251#define _mm256_maskz_ipcvttbf16_epi8(U, A) \
252 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
253 (__v16hi)_mm256_ipcvttbf16_epi8(A), \
254 (__v16hi)_mm256_setzero_si256()))
255
256#define _mm_ipcvttbf16_epu8(A) \
257 ((__m128i)__builtin_ia32_vcvttbf162iubs128((__v8bf)(__m128bh)(A)))
258
259#define _mm_mask_ipcvttbf16_epu8(W, U, A) \
260 ((__m128i)__builtin_ia32_selectw_128( \
261 (__mmask8)(U), (__v8hi)_mm_ipcvttbf16_epu8(A), (__v8hi)(__m128i)(W)))
262
263#define _mm_maskz_ipcvttbf16_epu8(U, A) \
264 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \
265 (__v8hi)_mm_ipcvttbf16_epu8(A), \
266 (__v8hi)_mm_setzero_si128()))
267
268#define _mm256_ipcvttbf16_epu8(A) \
269 ((__m256i)__builtin_ia32_vcvttbf162iubs256((__v16bf)(__m256bh)(A)))
270
271#define _mm256_mask_ipcvttbf16_epu8(W, U, A) \
272 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
273 (__v16hi)_mm256_ipcvttbf16_epu8(A), \
274 (__v16hi)(__m256i)(W)))
275
276#define _mm256_maskz_ipcvttbf16_epu8(U, A) \
277 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
278 (__v16hi)_mm256_ipcvttbf16_epu8(A), \
279 (__v16hi)_mm256_setzero_si256()))
280
281#define _mm_ipcvttph_epi8(A) \
282 ((__m128i)__builtin_ia32_vcvttph2ibs128_mask( \
283 (__v8hf)(__m128h)(A), (__v8hu)_mm_setzero_si128(), (__mmask8)-1))
284
285#define _mm_mask_ipcvttph_epi8(W, U, A) \
286 ((__m128i)__builtin_ia32_vcvttph2ibs128_mask((__v8hf)(__m128h)(A), \
287 (__v8hu)(W), (__mmask8)(U)))
288
289#define _mm_maskz_ipcvttph_epi8(U, A) \
290 ((__m128i)__builtin_ia32_vcvttph2ibs128_mask( \
291 (__v8hf)(__m128h)(A), (__v8hu)(_mm_setzero_si128()), (__mmask8)(U)))
292
293#define _mm256_ipcvttph_epi8(A) \
294 ((__m256i)__builtin_ia32_vcvttph2ibs256_mask( \
295 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)-1, \
296 _MM_FROUND_CUR_DIRECTION))
297
298#define _mm256_mask_ipcvttph_epi8(W, U, A) \
299 ((__m256i)__builtin_ia32_vcvttph2ibs256_mask((__v16hf)(__m256h)(A), \
300 (__v16hu)(W), (__mmask16)(U), \
301 _MM_FROUND_CUR_DIRECTION))
302
303#define _mm256_maskz_ipcvttph_epi8(U, A) \
304 ((__m256i)__builtin_ia32_vcvttph2ibs256_mask( \
305 (__v16hf)(__m256h)(A), (__v16hu)(_mm256_setzero_si256()), \
306 (__mmask16)(U), _MM_FROUND_CUR_DIRECTION))
307
308#define _mm256_ipcvtt_roundph_epi8(A, R) \
309 ((__m256i)__builtin_ia32_vcvttph2ibs256_mask( \
310 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)-1, \
311 (const int)R))
312
313#define _mm256_mask_ipcvtt_roundph_epi8(W, U, A, R) \
314 ((__m256i)__builtin_ia32_vcvttph2ibs256_mask( \
315 (__v16hf)(__m256h)(A), (__v16hu)(W), (__mmask16)(U), (const int)R))
316
317#define _mm256_maskz_ipcvtt_roundph_epi8(U, A, R) \
318 ((__m256i)__builtin_ia32_vcvttph2ibs256_mask( \
319 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)(U), \
320 (const int)R))
321
322#define _mm_ipcvttph_epu8(A) \
323 ((__m128i)__builtin_ia32_vcvttph2iubs128_mask( \
324 (__v8hf)(__m128h)(A), (__v8hu)_mm_setzero_si128(), (__mmask8)-1))
325
326#define _mm_mask_ipcvttph_epu8(W, U, A) \
327 ((__m128i)__builtin_ia32_vcvttph2iubs128_mask((__v8hf)(__m128h)(A), \
328 (__v8hu)(W), (__mmask8)(U)))
329
330#define _mm_maskz_ipcvttph_epu8(U, A) \
331 ((__m128i)__builtin_ia32_vcvttph2iubs128_mask( \
332 (__v8hf)(__m128h)(A), (__v8hu)(_mm_setzero_si128()), (__mmask8)(U)))
333
334#define _mm256_ipcvttph_epu8(A) \
335 ((__m256i)__builtin_ia32_vcvttph2iubs256_mask( \
336 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)-1, \
337 _MM_FROUND_CUR_DIRECTION))
338
339#define _mm256_mask_ipcvttph_epu8(W, U, A) \
340 ((__m256i)__builtin_ia32_vcvttph2iubs256_mask((__v16hf)(__m256h)(A), \
341 (__v16hu)(W), (__mmask16)(U), \
342 _MM_FROUND_CUR_DIRECTION))
343
344#define _mm256_maskz_ipcvttph_epu8(U, A) \
345 ((__m256i)__builtin_ia32_vcvttph2iubs256_mask( \
346 (__v16hf)(__m256h)(A), (__v16hu)(_mm256_setzero_si256()), \
347 (__mmask16)(U), _MM_FROUND_CUR_DIRECTION))
348
349#define _mm256_ipcvtt_roundph_epu8(A, R) \
350 ((__m256i)__builtin_ia32_vcvttph2iubs256_mask( \
351 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)-1, \
352 (const int)R))
353
354#define _mm256_mask_ipcvtt_roundph_epu8(W, U, A, R) \
355 ((__m256i)__builtin_ia32_vcvttph2iubs256_mask( \
356 (__v16hf)(__m256h)(A), (__v16hu)(W), (__mmask16)(U), (const int)R))
357
358#define _mm256_maskz_ipcvtt_roundph_epu8(U, A, R) \
359 ((__m256i)__builtin_ia32_vcvttph2iubs256_mask( \
360 (__v16hf)(__m256h)(A), (__v16hu)_mm256_setzero_si256(), (__mmask16)(U), \
361 (const int)R))
362
363#define _mm_ipcvttps_epi8(A) \
364 ((__m128i)__builtin_ia32_vcvttps2ibs128_mask( \
365 (__v4sf)(__m128)(A), (__v4su)_mm_setzero_si128(), (__mmask8)-1))
366
367#define _mm_mask_ipcvttps_epi8(W, U, A) \
368 ((__m128i)__builtin_ia32_vcvttps2ibs128_mask((__v4sf)(__m128)(A), \
369 (__v4su)(W), (__mmask8)(U)))
370
371#define _mm_maskz_ipcvttps_epi8(U, A) \
372 ((__m128i)__builtin_ia32_vcvttps2ibs128_mask( \
373 (__v4sf)(__m128)(A), (__v4su)(_mm_setzero_si128()), (__mmask8)(U)))
374
375#define _mm256_ipcvttps_epi8(A) \
376 ((__m256i)__builtin_ia32_vcvttps2ibs256_mask( \
377 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)-1, \
378 _MM_FROUND_CUR_DIRECTION))
379
380#define _mm256_mask_ipcvttps_epi8(W, U, A) \
381 ((__m256i)__builtin_ia32_vcvttps2ibs256_mask((__v8sf)(__m256)(A), \
382 (__v8su)(W), (__mmask8)(U), \
383 _MM_FROUND_CUR_DIRECTION))
384
385#define _mm256_maskz_ipcvttps_epi8(U, A) \
386 ((__m256i)__builtin_ia32_vcvttps2ibs256_mask( \
387 (__v8sf)(__m256)(A), (__v8su)(_mm256_setzero_si256()), (__mmask8)(U), \
388 _MM_FROUND_CUR_DIRECTION))
389
390#define _mm256_ipcvtt_roundps_epi8(A, R) \
391 ((__m256i)__builtin_ia32_vcvttps2ibs256_mask((__v8sf)(__m256)(A), \
392 (__v8su)_mm256_setzero_si256(), \
393 (__mmask8)-1, (const int)R))
394
395#define _mm256_mask_ipcvtt_roundps_epi8(W, U, A, R) \
396 ((__m256i)__builtin_ia32_vcvttps2ibs256_mask( \
397 (__v8sf)(__m256)(A), (__v8su)(W), (__mmask8)(U), (const int)R))
398
399#define _mm256_maskz_ipcvtt_roundps_epi8(U, A, R) \
400 ((__m256i)__builtin_ia32_vcvttps2ibs256_mask((__v8sf)(__m256)(A), \
401 (__v8su)_mm256_setzero_si256(), \
402 (__mmask8)(U), (const int)R))
403
404#define _mm_ipcvttps_epu8(A) \
405 ((__m128i)__builtin_ia32_vcvttps2iubs128_mask( \
406 (__v4sf)(__m128)(A), (__v4su)_mm_setzero_si128(), (__mmask8)-1))
407
408#define _mm_mask_ipcvttps_epu8(W, U, A) \
409 ((__m128i)__builtin_ia32_vcvttps2iubs128_mask((__v4sf)(__m128)(A), \
410 (__v4su)(W), (__mmask8)(U)))
411
412#define _mm_maskz_ipcvttps_epu8(U, A) \
413 ((__m128i)__builtin_ia32_vcvttps2iubs128_mask( \
414 (__v4sf)(__m128)(A), (__v4su)(_mm_setzero_si128()), (__mmask8)(U)))
415
416#define _mm256_ipcvttps_epu8(A) \
417 ((__m256i)__builtin_ia32_vcvttps2iubs256_mask( \
418 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)-1, \
419 _MM_FROUND_CUR_DIRECTION))
420
421#define _mm256_mask_ipcvttps_epu8(W, U, A) \
422 ((__m256i)__builtin_ia32_vcvttps2iubs256_mask((__v8sf)(__m256)(A), \
423 (__v8su)(W), (__mmask8)(U), \
424 _MM_FROUND_CUR_DIRECTION))
425
426#define _mm256_maskz_ipcvttps_epu8(U, A) \
427 ((__m256i)__builtin_ia32_vcvttps2iubs256_mask( \
428 (__v8sf)(__m256)(A), (__v8su)(_mm256_setzero_si256()), (__mmask8)(U), \
429 _MM_FROUND_CUR_DIRECTION))
430
431#define _mm256_ipcvtt_roundps_epu8(A, R) \
432 ((__m256i)__builtin_ia32_vcvttps2iubs256_mask( \
433 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)-1, \
434 (const int)R))
435
436#define _mm256_mask_ipcvtt_roundps_epu8(W, U, A, R) \
437 ((__m256i)__builtin_ia32_vcvttps2iubs256_mask( \
438 (__v8sf)(__m256)(A), (__v8su)(W), (__mmask8)(U), (const int)R))
439
440#define _mm256_maskz_ipcvtt_roundps_epu8(U, A, R) \
441 ((__m256i)__builtin_ia32_vcvttps2iubs256_mask( \
442 (__v8sf)(__m256)(A), (__v8su)_mm256_setzero_si256(), (__mmask8)(U), \
443 (const int)R))
444#endif // __AVX10_2SATCVTINTRIN_H
lib/include/avx2intrin.h+9
......@@ -15,12 +15,21 @@
1515#define __AVX2INTRIN_H
1616
1717/* Define the default attributes for the functions in this file. */
18#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
1819#define __DEFAULT_FN_ATTRS256 \
1920 __attribute__((__always_inline__, __nodebug__, \
2021 __target__("avx2,no-evex512"), __min_vector_width__(256)))
2122#define __DEFAULT_FN_ATTRS128 \
2223 __attribute__((__always_inline__, __nodebug__, \
2324 __target__("avx2,no-evex512"), __min_vector_width__(128)))
25#else
26#define __DEFAULT_FN_ATTRS256 \
27 __attribute__((__always_inline__, __nodebug__, __target__("avx2"), \
28 __min_vector_width__(256)))
29#define __DEFAULT_FN_ATTRS128 \
30 __attribute__((__always_inline__, __nodebug__, __target__("avx2"), \
31 __min_vector_width__(128)))
32#endif
2433
2534/* SSE4 Multiple Packed Sums of Absolute Difference. */
2635/// Computes sixteen sum of absolute difference (SAD) operations on sets of
lib/include/avx512bitalgintrin.h+2-2
......@@ -23,7 +23,7 @@
2323static __inline__ __m512i __DEFAULT_FN_ATTRS
2424_mm512_popcnt_epi16(__m512i __A)
2525{
26 return (__m512i) __builtin_ia32_vpopcntw_512((__v32hi) __A);
26 return (__m512i)__builtin_elementwise_popcount((__v32hu)__A);
2727}
2828
2929static __inline__ __m512i __DEFAULT_FN_ATTRS
......@@ -45,7 +45,7 @@ _mm512_maskz_popcnt_epi16(__mmask32 __U, __m512i __B)
4545static __inline__ __m512i __DEFAULT_FN_ATTRS
4646_mm512_popcnt_epi8(__m512i __A)
4747{
48 return (__m512i) __builtin_ia32_vpopcntb_512((__v64qi) __A);
48 return (__m512i)__builtin_elementwise_popcount((__v64qu)__A);
4949}
5050
5151static __inline__ __m512i __DEFAULT_FN_ATTRS
lib/include/avx512fintrin.h+22-14
......@@ -175,12 +175,21 @@ typedef enum
175175 __attribute__((__always_inline__, __nodebug__, \
176176 __target__("avx512f,no-evex512")))
177177
178#if defined(__cplusplus) && (__cplusplus >= 201103L)
179#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
180#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr
181#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
182#else
183#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS128
184#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512
185#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS
186#endif
187
178188/* Create vectors with repeated elements */
179189
180static __inline __m512i __DEFAULT_FN_ATTRS512
181_mm512_setzero_si512(void)
182{
183 return __extension__ (__m512i)(__v8di){ 0, 0, 0, 0, 0, 0, 0, 0 };
190static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
191_mm512_setzero_si512(void) {
192 return __extension__(__m512i)(__v8di){0, 0, 0, 0, 0, 0, 0, 0};
184193}
185194
186195#define _mm512_setzero_epi32 _mm512_setzero_si512
......@@ -256,20 +265,16 @@ _mm512_maskz_broadcastq_epi64 (__mmask8 __M, __m128i __A)
256265 (__v8di) _mm512_setzero_si512());
257266}
258267
259
260static __inline __m512 __DEFAULT_FN_ATTRS512
261_mm512_setzero_ps(void)
262{
263 return __extension__ (__m512){ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f,
264 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
268static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_setzero_ps(void) {
269 return __extension__(__m512){0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f,
270 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
265271}
266272
267273#define _mm512_setzero _mm512_setzero_ps
268274
269static __inline __m512d __DEFAULT_FN_ATTRS512
270_mm512_setzero_pd(void)
271{
272 return __extension__ (__m512d){ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 };
275static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
276_mm512_setzero_pd(void) {
277 return __extension__(__m512d){0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
273278}
274279
275280static __inline __m512 __DEFAULT_FN_ATTRS512
......@@ -9775,5 +9780,8 @@ _mm512_cvtsi512_si32(__m512i __A) {
97759780#undef __DEFAULT_FN_ATTRS512
97769781#undef __DEFAULT_FN_ATTRS128
97779782#undef __DEFAULT_FN_ATTRS
9783#undef __DEFAULT_FN_ATTRS512_CONSTEXPR
9784#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
9785#undef __DEFAULT_FN_ATTRS_CONSTEXPR
97789786
97799787#endif /* __AVX512FINTRIN_H */
lib/include/avx512vlbitalgintrin.h+4-4
......@@ -27,7 +27,7 @@
2727static __inline__ __m256i __DEFAULT_FN_ATTRS256
2828_mm256_popcnt_epi16(__m256i __A)
2929{
30 return (__m256i) __builtin_ia32_vpopcntw_256((__v16hi) __A);
30 return (__m256i)__builtin_elementwise_popcount((__v16hu)__A);
3131}
3232
3333static __inline__ __m256i __DEFAULT_FN_ATTRS256
......@@ -49,7 +49,7 @@ _mm256_maskz_popcnt_epi16(__mmask16 __U, __m256i __B)
4949static __inline__ __m128i __DEFAULT_FN_ATTRS128
5050_mm_popcnt_epi16(__m128i __A)
5151{
52 return (__m128i) __builtin_ia32_vpopcntw_128((__v8hi) __A);
52 return (__m128i)__builtin_elementwise_popcount((__v8hu)__A);
5353}
5454
5555static __inline__ __m128i __DEFAULT_FN_ATTRS128
......@@ -71,7 +71,7 @@ _mm_maskz_popcnt_epi16(__mmask8 __U, __m128i __B)
7171static __inline__ __m256i __DEFAULT_FN_ATTRS256
7272_mm256_popcnt_epi8(__m256i __A)
7373{
74 return (__m256i) __builtin_ia32_vpopcntb_256((__v32qi) __A);
74 return (__m256i)__builtin_elementwise_popcount((__v32qu)__A);
7575}
7676
7777static __inline__ __m256i __DEFAULT_FN_ATTRS256
......@@ -93,7 +93,7 @@ _mm256_maskz_popcnt_epi8(__mmask32 __U, __m256i __B)
9393static __inline__ __m128i __DEFAULT_FN_ATTRS128
9494_mm_popcnt_epi8(__m128i __A)
9595{
96 return (__m128i) __builtin_ia32_vpopcntb_128((__v16qi) __A);
96 return (__m128i)__builtin_elementwise_popcount((__v16qu)__A);
9797}
9898
9999static __inline__ __m128i __DEFAULT_FN_ATTRS128
lib/include/avx512vpopcntdqintrin.h+12-4
......@@ -21,8 +21,15 @@
2121 __target__("avx512vpopcntdq,evex512"), \
2222 __min_vector_width__(512)))
2323
24static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_popcnt_epi64(__m512i __A) {
25 return (__m512i)__builtin_ia32_vpopcntq_512((__v8di)__A);
24#if defined(__cplusplus) && (__cplusplus >= 201103L)
25#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
26#else
27#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
28#endif
29
30static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
31_mm512_popcnt_epi64(__m512i __A) {
32 return (__m512i)__builtin_elementwise_popcount((__v8du)__A);
2633}
2734
2835static __inline__ __m512i __DEFAULT_FN_ATTRS
......@@ -36,8 +43,9 @@ _mm512_maskz_popcnt_epi64(__mmask8 __U, __m512i __A) {
3643 return _mm512_mask_popcnt_epi64((__m512i)_mm512_setzero_si512(), __U, __A);
3744}
3845
39static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_popcnt_epi32(__m512i __A) {
40 return (__m512i)__builtin_ia32_vpopcntd_512((__v16si)__A);
46static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
47_mm512_popcnt_epi32(__m512i __A) {
48 return (__m512i)__builtin_elementwise_popcount((__v16su)__A);
4149}
4250
4351static __inline__ __m512i __DEFAULT_FN_ATTRS
lib/include/avx512vpopcntdqvlintrin.h+16-8
......@@ -25,9 +25,17 @@
2525 __target__("avx512vpopcntdq,avx512vl,no-evex512"), \
2626 __min_vector_width__(256)))
2727
28static __inline__ __m128i __DEFAULT_FN_ATTRS128
28#if defined(__cplusplus) && (__cplusplus >= 201103L)
29#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
30#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
31#else
32#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
33#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
34#endif
35
36static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2937_mm_popcnt_epi64(__m128i __A) {
30 return (__m128i)__builtin_ia32_vpopcntq_128((__v2di)__A);
38 return (__m128i)__builtin_elementwise_popcount((__v2du)__A);
3139}
3240
3341static __inline__ __m128i __DEFAULT_FN_ATTRS128
......@@ -41,9 +49,9 @@ _mm_maskz_popcnt_epi64(__mmask8 __U, __m128i __A) {
4149 return _mm_mask_popcnt_epi64((__m128i)_mm_setzero_si128(), __U, __A);
4250}
4351
44static __inline__ __m128i __DEFAULT_FN_ATTRS128
52static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4553_mm_popcnt_epi32(__m128i __A) {
46 return (__m128i)__builtin_ia32_vpopcntd_128((__v4si)__A);
54 return (__m128i)__builtin_elementwise_popcount((__v4su)__A);
4755}
4856
4957static __inline__ __m128i __DEFAULT_FN_ATTRS128
......@@ -57,9 +65,9 @@ _mm_maskz_popcnt_epi32(__mmask8 __U, __m128i __A) {
5765 return _mm_mask_popcnt_epi32((__m128i)_mm_setzero_si128(), __U, __A);
5866}
5967
60static __inline__ __m256i __DEFAULT_FN_ATTRS256
68static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
6169_mm256_popcnt_epi64(__m256i __A) {
62 return (__m256i)__builtin_ia32_vpopcntq_256((__v4di)__A);
70 return (__m256i)__builtin_elementwise_popcount((__v4du)__A);
6371}
6472
6573static __inline__ __m256i __DEFAULT_FN_ATTRS256
......@@ -73,9 +81,9 @@ _mm256_maskz_popcnt_epi64(__mmask8 __U, __m256i __A) {
7381 return _mm256_mask_popcnt_epi64((__m256i)_mm256_setzero_si256(), __U, __A);
7482}
7583
76static __inline__ __m256i __DEFAULT_FN_ATTRS256
84static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
7785_mm256_popcnt_epi32(__m256i __A) {
78 return (__m256i)__builtin_ia32_vpopcntd_256((__v8si)__A);
86 return (__m256i)__builtin_elementwise_popcount((__v8su)__A);
7987}
8088
8189static __inline__ __m256i __DEFAULT_FN_ATTRS256
lib/include/avxintrin.h+30-16
......@@ -50,12 +50,29 @@ typedef __bf16 __m256bh __attribute__((__vector_size__(32), __aligned__(32)));
5050#endif
5151
5252/* Define the default attributes for the functions in this file. */
53#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
5354#define __DEFAULT_FN_ATTRS \
5455 __attribute__((__always_inline__, __nodebug__, __target__("avx,no-evex512"), \
5556 __min_vector_width__(256)))
5657#define __DEFAULT_FN_ATTRS128 \
5758 __attribute__((__always_inline__, __nodebug__, __target__("avx,no-evex512"), \
5859 __min_vector_width__(128)))
60#else
61#define __DEFAULT_FN_ATTRS \
62 __attribute__((__always_inline__, __nodebug__, __target__("avx"), \
63 __min_vector_width__(256)))
64#define __DEFAULT_FN_ATTRS128 \
65 __attribute__((__always_inline__, __nodebug__, __target__("avx"), \
66 __min_vector_width__(128)))
67#endif
68
69#if defined(__cplusplus) && (__cplusplus >= 201103L)
70#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
71#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
72#else
73#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS128
74#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS
75#endif
5976
6077/* Arithmetic */
6178/// Adds two 256-bit vectors of [4 x double].
......@@ -3689,7 +3706,7 @@ _mm256_undefined_si256(void)
36893706/// A double-precision floating-point value used to initialize bits [63:0]
36903707/// of the result.
36913708/// \returns An initialized 256-bit floating-point vector of [4 x double].
3692static __inline __m256d __DEFAULT_FN_ATTRS
3709static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
36933710_mm256_set_pd(double __a, double __b, double __c, double __d)
36943711{
36953712 return __extension__ (__m256d){ __d, __c, __b, __a };
......@@ -3728,7 +3745,7 @@ _mm256_set_pd(double __a, double __b, double __c, double __d)
37283745/// A single-precision floating-point value used to initialize bits [31:0]
37293746/// of the result.
37303747/// \returns An initialized 256-bit floating-point vector of [8 x float].
3731static __inline __m256 __DEFAULT_FN_ATTRS
3748static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
37323749_mm256_set_ps(float __a, float __b, float __c, float __d,
37333750 float __e, float __f, float __g, float __h)
37343751{
......@@ -3955,7 +3972,7 @@ _mm256_set_epi64x(long long __a, long long __b, long long __c, long long __d)
39553972/// A double-precision floating-point value used to initialize bits [255:192]
39563973/// of the result.
39573974/// \returns An initialized 256-bit floating-point vector of [4 x double].
3958static __inline __m256d __DEFAULT_FN_ATTRS
3975static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
39593976_mm256_setr_pd(double __a, double __b, double __c, double __d)
39603977{
39613978 return _mm256_set_pd(__d, __c, __b, __a);
......@@ -3995,7 +4012,7 @@ _mm256_setr_pd(double __a, double __b, double __c, double __d)
39954012/// A single-precision floating-point value used to initialize bits [255:224]
39964013/// of the result.
39974014/// \returns An initialized 256-bit floating-point vector of [8 x float].
3998static __inline __m256 __DEFAULT_FN_ATTRS
4015static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
39994016_mm256_setr_ps(float __a, float __b, float __c, float __d,
40004017 float __e, float __f, float __g, float __h)
40014018{
......@@ -4212,7 +4229,7 @@ _mm256_setr_epi64x(long long __a, long long __b, long long __c, long long __d)
42124229/// A double-precision floating-point value used to initialize each vector
42134230/// element of the result.
42144231/// \returns An initialized 256-bit floating-point vector of [4 x double].
4215static __inline __m256d __DEFAULT_FN_ATTRS
4232static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
42164233_mm256_set1_pd(double __w)
42174234{
42184235 return _mm256_set_pd(__w, __w, __w, __w);
......@@ -4231,7 +4248,7 @@ _mm256_set1_pd(double __w)
42314248/// A single-precision floating-point value used to initialize each vector
42324249/// element of the result.
42334250/// \returns An initialized 256-bit floating-point vector of [8 x float].
4234static __inline __m256 __DEFAULT_FN_ATTRS
4251static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
42354252_mm256_set1_ps(float __w)
42364253{
42374254 return _mm256_set_ps(__w, __w, __w, __w, __w, __w, __w, __w);
......@@ -4322,10 +4339,8 @@ _mm256_set1_epi64x(long long __q)
43224339/// This intrinsic corresponds to the <c> VXORPS </c> instruction.
43234340///
43244341/// \returns A 256-bit vector of [4 x double] with all elements set to zero.
4325static __inline __m256d __DEFAULT_FN_ATTRS
4326_mm256_setzero_pd(void)
4327{
4328 return __extension__ (__m256d){ 0.0, 0.0, 0.0, 0.0 };
4342static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_setzero_pd(void) {
4343 return __extension__(__m256d){0.0, 0.0, 0.0, 0.0};
43294344}
43304345
43314346/// Constructs a 256-bit floating-point vector of [8 x float] with all
......@@ -4336,9 +4351,7 @@ _mm256_setzero_pd(void)
43364351/// This intrinsic corresponds to the <c> VXORPS </c> instruction.
43374352///
43384353/// \returns A 256-bit vector of [8 x float] with all elements set to zero.
4339static __inline __m256 __DEFAULT_FN_ATTRS
4340_mm256_setzero_ps(void)
4341{
4354static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_setzero_ps(void) {
43424355 return __extension__ (__m256){ 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
43434356}
43444357
......@@ -4349,9 +4362,8 @@ _mm256_setzero_ps(void)
43494362/// This intrinsic corresponds to the <c> VXORPS </c> instruction.
43504363///
43514364/// \returns A 256-bit integer vector initialized to zero.
4352static __inline __m256i __DEFAULT_FN_ATTRS
4353_mm256_setzero_si256(void)
4354{
4365static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
4366_mm256_setzero_si256(void) {
43554367 return __extension__ (__m256i)(__v4di){ 0, 0, 0, 0 };
43564368}
43574369
......@@ -5121,6 +5133,8 @@ _mm256_storeu2_m128i(__m128i_u *__addr_hi, __m128i_u *__addr_lo, __m256i __a)
51215133}
51225134
51235135#undef __DEFAULT_FN_ATTRS
5136#undef __DEFAULT_FN_ATTRS_CONSTEXPR
51245137#undef __DEFAULT_FN_ATTRS128
5138#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
51255139
51265140#endif /* __AVXINTRIN_H */
lib/include/avxvnniint16intrin.h+36-77
......@@ -15,14 +15,6 @@
1515#ifndef __AVXVNNIINT16INTRIN_H
1616#define __AVXVNNIINT16INTRIN_H
1717
18/* Define the default attributes for the functions in this file. */
19#define __DEFAULT_FN_ATTRS128 \
20 __attribute__((__always_inline__, __nodebug__, __target__("avxvnniint16"), \
21 __min_vector_width__(128)))
22#define __DEFAULT_FN_ATTRS256 \
23 __attribute__((__always_inline__, __nodebug__, __target__("avxvnniint16"), \
24 __min_vector_width__(256)))
25
2618/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
2719/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
2820/// signed 16-bit results. Sum these 2 results with the corresponding
......@@ -53,12 +45,9 @@
5345/// ENDFOR
5446/// dst[MAX:128] := 0
5547/// \endcode
56static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsud_epi32(__m128i __W,
57 __m128i __A,
58 __m128i __B) {
59 return (__m128i)__builtin_ia32_vpdpwsud128((__v4si)__W, (__v4si)__A,
60 (__v4si)__B);
61}
48#define _mm_dpwsud_epi32(__W, __A, __B) \
49 ((__m128i)__builtin_ia32_vpdpwsud128((__v4si)(__W), (__v4si)(__A), \
50 (__v4si)(__B)))
6251
6352/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
6453/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
......@@ -90,11 +79,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsud_epi32(__m128i __W,
9079/// ENDFOR
9180/// dst[MAX:256] := 0
9281/// \endcode
93static __inline__ __m256i __DEFAULT_FN_ATTRS256
94_mm256_dpwsud_epi32(__m256i __W, __m256i __A, __m256i __B) {
95 return (__m256i)__builtin_ia32_vpdpwsud256((__v8si)__W, (__v8si)__A,
96 (__v8si)__B);
97}
82#define _mm256_dpwsud_epi32(__W, __A, __B) \
83 ((__m256i)__builtin_ia32_vpdpwsud256((__v8si)(__W), (__v8si)(__A), \
84 (__v8si)(__B)))
9885
9986/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
10087/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
......@@ -127,12 +114,9 @@ _mm256_dpwsud_epi32(__m256i __W, __m256i __A, __m256i __B) {
127114/// ENDFOR
128115/// dst[MAX:128] := 0
129116/// \endcode
130static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsuds_epi32(__m128i __W,
131 __m128i __A,
132 __m128i __B) {
133 return (__m128i)__builtin_ia32_vpdpwsuds128((__v4si)__W, (__v4si)__A,
134 (__v4si)__B);
135}
117#define _mm_dpwsuds_epi32(__W, __A, __B) \
118 ((__m128i)__builtin_ia32_vpdpwsuds128((__v4si)(__W), (__v4si)(__A), \
119 (__v4si)(__B)))
136120
137121/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
138122/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
......@@ -165,11 +149,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsuds_epi32(__m128i __W,
165149/// ENDFOR
166150/// dst[MAX:256] := 0
167151/// \endcode
168static __inline__ __m256i __DEFAULT_FN_ATTRS256
169_mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B) {
170 return (__m256i)__builtin_ia32_vpdpwsuds256((__v8si)__W, (__v8si)__A,
171 (__v8si)__B);
172}
152#define _mm256_dpwsuds_epi32(__W, __A, __B) \
153 ((__m256i)__builtin_ia32_vpdpwsuds256((__v8si)(__W), (__v8si)(__A), \
154 (__v8si)(__B)))
173155
174156/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
175157/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
......@@ -201,12 +183,9 @@ _mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B) {
201183/// ENDFOR
202184/// dst[MAX:128] := 0
203185/// \endcode
204static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusd_epi32(__m128i __W,
205 __m128i __A,
206 __m128i __B) {
207 return (__m128i)__builtin_ia32_vpdpwusd128((__v4si)__W, (__v4si)__A,
208 (__v4si)__B);
209}
186#define _mm_dpwusd_epi32(__W, __A, __B) \
187 ((__m128i)__builtin_ia32_vpdpwusd128((__v4si)(__W), (__v4si)(__A), \
188 (__v4si)(__B)))
210189
211190/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
212191/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
......@@ -238,11 +217,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusd_epi32(__m128i __W,
238217/// ENDFOR
239218/// dst[MAX:256] := 0
240219/// \endcode
241static __inline__ __m256i __DEFAULT_FN_ATTRS256
242_mm256_dpwusd_epi32(__m256i __W, __m256i __A, __m256i __B) {
243 return (__m256i)__builtin_ia32_vpdpwusd256((__v8si)__W, (__v8si)__A,
244 (__v8si)__B);
245}
220#define _mm256_dpwusd_epi32(__W, __A, __B) \
221 ((__m256i)__builtin_ia32_vpdpwusd256((__v8si)(__W), (__v8si)(__A), \
222 (__v8si)(__B)))
246223
247224/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
248225/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
......@@ -275,12 +252,9 @@ _mm256_dpwusd_epi32(__m256i __W, __m256i __A, __m256i __B) {
275252/// ENDFOR
276253/// dst[MAX:128] := 0
277254/// \endcode
278static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusds_epi32(__m128i __W,
279 __m128i __A,
280 __m128i __B) {
281 return (__m128i)__builtin_ia32_vpdpwusds128((__v4si)__W, (__v4si)__A,
282 (__v4si)__B);
283}
255#define _mm_dpwusds_epi32(__W, __A, __B) \
256 ((__m128i)__builtin_ia32_vpdpwusds128((__v4si)(__W), (__v4si)(__A), \
257 (__v4si)(__B)))
284258
285259/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
286260/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
......@@ -313,11 +287,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusds_epi32(__m128i __W,
313287/// ENDFOR
314288/// dst[MAX:256] := 0
315289/// \endcode
316static __inline__ __m256i __DEFAULT_FN_ATTRS256
317_mm256_dpwusds_epi32(__m256i __W, __m256i __A, __m256i __B) {
318 return (__m256i)__builtin_ia32_vpdpwusds256((__v8si)__W, (__v8si)__A,
319 (__v8si)__B);
320}
290#define _mm256_dpwusds_epi32(__W, __A, __B) \
291 ((__m256i)__builtin_ia32_vpdpwusds256((__v8si)(__W), (__v8si)(__A), \
292 (__v8si)(__B)))
321293
322294/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
323295/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
......@@ -349,12 +321,9 @@ _mm256_dpwusds_epi32(__m256i __W, __m256i __A, __m256i __B) {
349321/// ENDFOR
350322/// dst[MAX:128] := 0
351323/// \endcode
352static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuud_epi32(__m128i __W,
353 __m128i __A,
354 __m128i __B) {
355 return (__m128i)__builtin_ia32_vpdpwuud128((__v4si)__W, (__v4si)__A,
356 (__v4si)__B);
357}
324#define _mm_dpwuud_epi32(__W, __A, __B) \
325 ((__m128i)__builtin_ia32_vpdpwuud128((__v4si)(__W), (__v4si)(__A), \
326 (__v4si)(__B)))
358327
359328/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
360329/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
......@@ -386,11 +355,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuud_epi32(__m128i __W,
386355/// ENDFOR
387356/// dst[MAX:256] := 0
388357/// \endcode
389static __inline__ __m256i __DEFAULT_FN_ATTRS256
390_mm256_dpwuud_epi32(__m256i __W, __m256i __A, __m256i __B) {
391 return (__m256i)__builtin_ia32_vpdpwuud256((__v8si)__W, (__v8si)__A,
392 (__v8si)__B);
393}
358#define _mm256_dpwuud_epi32(__W, __A, __B) \
359 ((__m256i)__builtin_ia32_vpdpwuud256((__v8si)(__W), (__v8si)(__A), \
360 (__v8si)(__B)))
394361
395362/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
396363/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
......@@ -423,12 +390,9 @@ _mm256_dpwuud_epi32(__m256i __W, __m256i __A, __m256i __B) {
423390/// ENDFOR
424391/// dst[MAX:128] := 0
425392/// \endcode
426static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuuds_epi32(__m128i __W,
427 __m128i __A,
428 __m128i __B) {
429 return (__m128i)__builtin_ia32_vpdpwuuds128((__v4si)__W, (__v4si)__A,
430 (__v4si)__B);
431}
393#define _mm_dpwuuds_epi32(__W, __A, __B) \
394 ((__m128i)__builtin_ia32_vpdpwuuds128((__v4si)(__W), (__v4si)(__A), \
395 (__v4si)(__B)))
432396
433397/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
434398/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
......@@ -461,13 +425,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuuds_epi32(__m128i __W,
461425/// ENDFOR
462426/// dst[MAX:256] := 0
463427/// \endcode
464static __inline__ __m256i __DEFAULT_FN_ATTRS256
465_mm256_dpwuuds_epi32(__m256i __W, __m256i __A, __m256i __B) {
466 return (__m256i)__builtin_ia32_vpdpwuuds256((__v8si)__W, (__v8si)__A,
467 (__v8si)__B);
468}
469
470#undef __DEFAULT_FN_ATTRS128
471#undef __DEFAULT_FN_ATTRS256
428#define _mm256_dpwuuds_epi32(__W, __A, __B) \
429 ((__m256i)__builtin_ia32_vpdpwuuds256((__v8si)(__W), (__v8si)(__A), \
430 (__v8si)(__B)))
472431
473432#endif // __AVXVNNIINT16INTRIN_H
lib/include/avxvnniint8intrin.h+36-77
......@@ -14,14 +14,6 @@
1414#ifndef __AVXVNNIINT8INTRIN_H
1515#define __AVXVNNIINT8INTRIN_H
1616
17/* Define the default attributes for the functions in this file. */
18#define __DEFAULT_FN_ATTRS256 \
19 __attribute__((__always_inline__, __nodebug__, __target__("avxvnniint8"), \
20 __min_vector_width__(256)))
21#define __DEFAULT_FN_ATTRS128 \
22 __attribute__((__always_inline__, __nodebug__, __target__("avxvnniint8"), \
23 __min_vector_width__(128)))
24
2517/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
2618/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate
2719/// signed 16-bit results. Sum these 4 results with the corresponding
......@@ -52,12 +44,9 @@
5244/// ENDFOR
5345/// dst[MAX:128] := 0
5446/// \endcode
55static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbssd_epi32(__m128i __W,
56 __m128i __A,
57 __m128i __B) {
58 return (__m128i)__builtin_ia32_vpdpbssd128((__v4si)__W, (__v4si)__A,
59 (__v4si)__B);
60}
47#define _mm_dpbssd_epi32(__W, __A, __B) \
48 ((__m128i)__builtin_ia32_vpdpbssd128((__v4si)(__W), (__v4si)(__A), \
49 (__v4si)(__B)))
6150
6251/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
6352/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate
......@@ -89,11 +78,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbssd_epi32(__m128i __W,
8978/// ENDFOR
9079/// dst[MAX:256] := 0
9180/// \endcode
92static __inline__ __m256i __DEFAULT_FN_ATTRS256
93_mm256_dpbssd_epi32(__m256i __W, __m256i __A, __m256i __B) {
94 return (__m256i)__builtin_ia32_vpdpbssd256((__v8si)__W, (__v8si)__A,
95 (__v8si)__B);
96}
81#define _mm256_dpbssd_epi32(__W, __A, __B) \
82 ((__m256i)__builtin_ia32_vpdpbssd256((__v8si)(__W), (__v8si)(__A), \
83 (__v8si)(__B)))
9784
9885/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
9986/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate
......@@ -126,12 +113,9 @@ _mm256_dpbssd_epi32(__m256i __W, __m256i __A, __m256i __B) {
126113/// ENDFOR
127114/// dst[MAX:128] := 0
128115/// \endcode
129static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbssds_epi32(__m128i __W,
130 __m128i __A,
131 __m128i __B) {
132 return (__m128i)__builtin_ia32_vpdpbssds128((__v4si)__W, (__v4si)__A,
133 (__v4si)__B);
134}
116#define _mm_dpbssds_epi32(__W, __A, __B) \
117 ((__m128i)__builtin_ia32_vpdpbssds128((__v4si)(__W), (__v4si)(__A), \
118 (__v4si)(__B)))
135119
136120/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
137121/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate
......@@ -164,11 +148,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbssds_epi32(__m128i __W,
164148/// ENDFOR
165149/// dst[MAX:256] := 0
166150/// \endcode
167static __inline__ __m256i __DEFAULT_FN_ATTRS256
168_mm256_dpbssds_epi32(__m256i __W, __m256i __A, __m256i __B) {
169 return (__m256i)__builtin_ia32_vpdpbssds256((__v8si)__W, (__v8si)__A,
170 (__v8si)__B);
171}
151#define _mm256_dpbssds_epi32(__W, __A, __B) \
152 ((__m256i)__builtin_ia32_vpdpbssds256((__v8si)(__W), (__v8si)(__A), \
153 (__v8si)(__B)))
172154
173155/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
174156/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
......@@ -200,12 +182,9 @@ _mm256_dpbssds_epi32(__m256i __W, __m256i __A, __m256i __B) {
200182/// ENDFOR
201183/// dst[MAX:128] := 0
202184/// \endcode
203static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbsud_epi32(__m128i __W,
204 __m128i __A,
205 __m128i __B) {
206 return (__m128i)__builtin_ia32_vpdpbsud128((__v4si)__W, (__v4si)__A,
207 (__v4si)__B);
208}
185#define _mm_dpbsud_epi32(__W, __A, __B) \
186 ((__m128i)__builtin_ia32_vpdpbsud128((__v4si)(__W), (__v4si)(__A), \
187 (__v4si)(__B)))
209188
210189/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
211190/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
......@@ -237,11 +216,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbsud_epi32(__m128i __W,
237216/// ENDFOR
238217/// dst[MAX:256] := 0
239218/// \endcode
240static __inline__ __m256i __DEFAULT_FN_ATTRS256
241_mm256_dpbsud_epi32(__m256i __W, __m256i __A, __m256i __B) {
242 return (__m256i)__builtin_ia32_vpdpbsud256((__v8si)__W, (__v8si)__A,
243 (__v8si)__B);
244}
219#define _mm256_dpbsud_epi32(__W, __A, __B) \
220 ((__m256i)__builtin_ia32_vpdpbsud256((__v8si)(__W), (__v8si)(__A), \
221 (__v8si)(__B)))
245222
246223/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
247224/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
......@@ -274,12 +251,9 @@ _mm256_dpbsud_epi32(__m256i __W, __m256i __A, __m256i __B) {
274251/// ENDFOR
275252/// dst[MAX:128] := 0
276253/// \endcode
277static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbsuds_epi32(__m128i __W,
278 __m128i __A,
279 __m128i __B) {
280 return (__m128i)__builtin_ia32_vpdpbsuds128((__v4si)__W, (__v4si)__A,
281 (__v4si)__B);
282}
254#define _mm_dpbsuds_epi32(__W, __A, __B) \
255 ((__m128i)__builtin_ia32_vpdpbsuds128((__v4si)(__W), (__v4si)(__A), \
256 (__v4si)(__B)))
283257
284258/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
285259/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
......@@ -312,11 +286,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbsuds_epi32(__m128i __W,
312286/// ENDFOR
313287/// dst[MAX:256] := 0
314288/// \endcode
315static __inline__ __m256i __DEFAULT_FN_ATTRS256
316_mm256_dpbsuds_epi32(__m256i __W, __m256i __A, __m256i __B) {
317 return (__m256i)__builtin_ia32_vpdpbsuds256((__v8si)__W, (__v8si)__A,
318 (__v8si)__B);
319}
289#define _mm256_dpbsuds_epi32(__W, __A, __B) \
290 ((__m256i)__builtin_ia32_vpdpbsuds256((__v8si)(__W), (__v8si)(__A), \
291 (__v8si)(__B)))
320292
321293/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
322294/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
......@@ -348,12 +320,9 @@ _mm256_dpbsuds_epi32(__m256i __W, __m256i __A, __m256i __B) {
348320/// ENDFOR
349321/// dst[MAX:128] := 0
350322/// \endcode
351static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbuud_epi32(__m128i __W,
352 __m128i __A,
353 __m128i __B) {
354 return (__m128i)__builtin_ia32_vpdpbuud128((__v4si)__W, (__v4si)__A,
355 (__v4si)__B);
356}
323#define _mm_dpbuud_epi32(__W, __A, __B) \
324 ((__m128i)__builtin_ia32_vpdpbuud128((__v4si)(__W), (__v4si)(__A), \
325 (__v4si)(__B)))
357326
358327/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
359328/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
......@@ -385,11 +354,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbuud_epi32(__m128i __W,
385354/// ENDFOR
386355/// dst[MAX:256] := 0
387356/// \endcode
388static __inline__ __m256i __DEFAULT_FN_ATTRS256
389_mm256_dpbuud_epi32(__m256i __W, __m256i __A, __m256i __B) {
390 return (__m256i)__builtin_ia32_vpdpbuud256((__v8si)__W, (__v8si)__A,
391 (__v8si)__B);
392}
357#define _mm256_dpbuud_epi32(__W, __A, __B) \
358 ((__m256i)__builtin_ia32_vpdpbuud256((__v8si)(__W), (__v8si)(__A), \
359 (__v8si)(__B)))
393360
394361/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
395362/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
......@@ -422,14 +389,10 @@ _mm256_dpbuud_epi32(__m256i __W, __m256i __A, __m256i __B) {
422389/// ENDFOR
423390/// dst[MAX:128] := 0
424391/// \endcode
425static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbuuds_epi32(__m128i __W,
426 __m128i __A,
427 __m128i __B) {
428 return (__m128i)__builtin_ia32_vpdpbuuds128((__v4si)__W, (__v4si)__A,
429 (__v4si)__B);
430}
392#define _mm_dpbuuds_epi32(__W, __A, __B) \
393 ((__m128i)__builtin_ia32_vpdpbuuds128((__v4si)(__W), (__v4si)(__A), \
394 (__v4si)(__B)))
431395
432/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
433396/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
434397/// signed 16-bit results. Sum these 4 results with the corresponding
435398/// 32-bit integer in \a __W with signed saturation, and store the packed
......@@ -460,12 +423,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpbuuds_epi32(__m128i __W,
460423/// ENDFOR
461424/// dst[MAX:256] := 0
462425/// \endcode
463static __inline__ __m256i __DEFAULT_FN_ATTRS256
464_mm256_dpbuuds_epi32(__m256i __W, __m256i __A, __m256i __B) {
465 return (__m256i)__builtin_ia32_vpdpbuuds256((__v8si)__W, (__v8si)__A,
466 (__v8si)__B);
467}
468#undef __DEFAULT_FN_ATTRS128
469#undef __DEFAULT_FN_ATTRS256
426#define _mm256_dpbuuds_epi32(__W, __A, __B) \
427 ((__m256i)__builtin_ia32_vpdpbuuds256((__v8si)(__W), (__v8si)(__A), \
428 (__v8si)(__B)))
470429
471430#endif // __AVXVNNIINT8INTRIN_H
lib/include/bmi2intrin.h+15-17
......@@ -15,7 +15,13 @@
1515#define __BMI2INTRIN_H
1616
1717/* Define the default attributes for the functions in this file. */
18#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("bmi2")))
18#if defined(__cplusplus) && (__cplusplus >= 201103L)
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, __target__("bmi2"))) constexpr
21#else
22#define __DEFAULT_FN_ATTRS \
23 __attribute__((__always_inline__, __nodebug__, __target__("bmi2")))
24#endif
1925
2026/// Copies the unsigned 32-bit integer \a __X and zeroes the upper bits
2127/// starting at bit number \a __Y.
......@@ -38,8 +44,7 @@
3844/// The lower 8 bits specify the bit number of the lowest bit to zero.
3945/// \returns The partially zeroed 32-bit value.
4046static __inline__ unsigned int __DEFAULT_FN_ATTRS
41_bzhi_u32(unsigned int __X, unsigned int __Y)
42{
47_bzhi_u32(unsigned int __X, unsigned int __Y) {
4348 return __builtin_ia32_bzhi_si(__X, __Y);
4449}
4550
......@@ -68,8 +73,7 @@ _bzhi_u32(unsigned int __X, unsigned int __Y)
6873/// The 32-bit mask specifying where to deposit source bits.
6974/// \returns The 32-bit result.
7075static __inline__ unsigned int __DEFAULT_FN_ATTRS
71_pdep_u32(unsigned int __X, unsigned int __Y)
72{
76_pdep_u32(unsigned int __X, unsigned int __Y) {
7377 return __builtin_ia32_pdep_si(__X, __Y);
7478}
7579
......@@ -98,8 +102,7 @@ _pdep_u32(unsigned int __X, unsigned int __Y)
98102/// The 32-bit mask specifying which source bits to extract.
99103/// \returns The 32-bit result.
100104static __inline__ unsigned int __DEFAULT_FN_ATTRS
101_pext_u32(unsigned int __X, unsigned int __Y)
102{
105_pext_u32(unsigned int __X, unsigned int __Y) {
103106 return __builtin_ia32_pext_si(__X, __Y);
104107}
105108
......@@ -124,8 +127,7 @@ _pext_u32(unsigned int __X, unsigned int __Y)
124127/// A pointer to memory for storing the upper half of the product.
125128/// \returns The lower half of the product.
126129static __inline__ unsigned int __DEFAULT_FN_ATTRS
127_mulx_u32(unsigned int __X, unsigned int __Y, unsigned int *__P)
128{
130_mulx_u32(unsigned int __X, unsigned int __Y, unsigned int *__P) {
129131 unsigned long long __res = (unsigned long long) __X * __Y;
130132 *__P = (unsigned int)(__res >> 32);
131133 return (unsigned int)__res;
......@@ -154,8 +156,7 @@ _mulx_u32(unsigned int __X, unsigned int __Y, unsigned int *__P)
154156/// The lower 8 bits specify the bit number of the lowest bit to zero.
155157/// \returns The partially zeroed 64-bit value.
156158static __inline__ unsigned long long __DEFAULT_FN_ATTRS
157_bzhi_u64(unsigned long long __X, unsigned long long __Y)
158{
159_bzhi_u64(unsigned long long __X, unsigned long long __Y) {
159160 return __builtin_ia32_bzhi_di(__X, __Y);
160161}
161162
......@@ -184,8 +185,7 @@ _bzhi_u64(unsigned long long __X, unsigned long long __Y)
184185/// The 64-bit mask specifying where to deposit source bits.
185186/// \returns The 64-bit result.
186187static __inline__ unsigned long long __DEFAULT_FN_ATTRS
187_pdep_u64(unsigned long long __X, unsigned long long __Y)
188{
188_pdep_u64(unsigned long long __X, unsigned long long __Y) {
189189 return __builtin_ia32_pdep_di(__X, __Y);
190190}
191191
......@@ -214,8 +214,7 @@ _pdep_u64(unsigned long long __X, unsigned long long __Y)
214214/// The 64-bit mask specifying which source bits to extract.
215215/// \returns The 64-bit result.
216216static __inline__ unsigned long long __DEFAULT_FN_ATTRS
217_pext_u64(unsigned long long __X, unsigned long long __Y)
218{
217_pext_u64(unsigned long long __X, unsigned long long __Y) {
219218 return __builtin_ia32_pext_di(__X, __Y);
220219}
221220
......@@ -241,8 +240,7 @@ _pext_u64(unsigned long long __X, unsigned long long __Y)
241240/// \returns The lower half of the product.
242241static __inline__ unsigned long long __DEFAULT_FN_ATTRS
243242_mulx_u64 (unsigned long long __X, unsigned long long __Y,
244 unsigned long long *__P)
245{
243 unsigned long long *__P) {
246244 unsigned __int128 __res = (unsigned __int128) __X * __Y;
247245 *__P = (unsigned long long) (__res >> 64);
248246 return (unsigned long long) __res;
lib/include/bmiintrin.h+31-37
......@@ -17,7 +17,12 @@
1717/* Allow using the tzcnt intrinsics even for non-BMI targets. Since the TZCNT
1818 instruction behaves as BSF on non-BMI targets, there is code that expects
1919 to use it as a potentially faster version of BSF. */
20#if defined(__cplusplus) && (__cplusplus >= 201103L)
21#define __RELAXED_FN_ATTRS \
22 __attribute__((__always_inline__, __nodebug__)) constexpr
23#else
2024#define __RELAXED_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
25#endif
2126
2227/// Counts the number of trailing zero bits in the operand.
2328///
......@@ -31,8 +36,7 @@
3136/// bits in the operand.
3237/// \see _tzcnt_u16
3338static __inline__ unsigned short __RELAXED_FN_ATTRS
34__tzcnt_u16(unsigned short __X)
35{
39__tzcnt_u16(unsigned short __X) {
3640 return __builtin_ia32_tzcnt_u16(__X);
3741}
3842
......@@ -65,8 +69,7 @@ __tzcnt_u16(unsigned short __X)
6569/// bits in the operand.
6670/// \see { _mm_tzcnt_32 _tzcnt_u32 }
6771static __inline__ unsigned int __RELAXED_FN_ATTRS
68__tzcnt_u32(unsigned int __X)
69{
72__tzcnt_u32(unsigned int __X) {
7073 return __builtin_ia32_tzcnt_u32(__X);
7174}
7275
......@@ -82,8 +85,7 @@ __tzcnt_u32(unsigned int __X)
8285/// the operand.
8386/// \see { __tzcnt_u32 _tzcnt_u32 }
8487static __inline__ int __RELAXED_FN_ATTRS
85_mm_tzcnt_32(unsigned int __X)
86{
88_mm_tzcnt_32(unsigned int __X) {
8789 return (int)__builtin_ia32_tzcnt_u32(__X);
8890}
8991
......@@ -118,8 +120,7 @@ _mm_tzcnt_32(unsigned int __X)
118120/// bits in the operand.
119121/// \see { _mm_tzcnt_64 _tzcnt_u64 }
120122static __inline__ unsigned long long __RELAXED_FN_ATTRS
121__tzcnt_u64(unsigned long long __X)
122{
123__tzcnt_u64(unsigned long long __X) {
123124 return __builtin_ia32_tzcnt_u64(__X);
124125}
125126
......@@ -135,8 +136,7 @@ __tzcnt_u64(unsigned long long __X)
135136/// the operand.
136137/// \see { __tzcnt_u64 _tzcnt_u64 }
137138static __inline__ long long __RELAXED_FN_ATTRS
138_mm_tzcnt_64(unsigned long long __X)
139{
139_mm_tzcnt_64(unsigned long long __X) {
140140 return (long long)__builtin_ia32_tzcnt_u64(__X);
141141}
142142
......@@ -164,7 +164,13 @@ _mm_tzcnt_64(unsigned long long __X)
164164#if !defined(__SCE__) || __has_feature(modules) || defined(__BMI__)
165165
166166/* Define the default attributes for the functions in this file. */
167#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("bmi")))
167#if defined(__cplusplus) && (__cplusplus >= 201103L)
168#define __DEFAULT_FN_ATTRS \
169 __attribute__((__always_inline__, __nodebug__, __target__("bmi"))) constexpr
170#else
171#define __DEFAULT_FN_ATTRS \
172 __attribute__((__always_inline__, __nodebug__, __target__("bmi")))
173#endif
168174
169175/// Performs a bitwise AND of the second operand with the one's
170176/// complement of the first operand.
......@@ -181,8 +187,7 @@ _mm_tzcnt_64(unsigned long long __X)
181187/// operand with the one's complement of the first operand.
182188/// \see _andn_u32
183189static __inline__ unsigned int __DEFAULT_FN_ATTRS
184__andn_u32(unsigned int __X, unsigned int __Y)
185{
190__andn_u32(unsigned int __X, unsigned int __Y) {
186191 return ~__X & __Y;
187192}
188193
......@@ -224,8 +229,7 @@ __andn_u32(unsigned int __X, unsigned int __Y)
224229/// extracted bits.
225230/// \see _bextr_u32
226231static __inline__ unsigned int __DEFAULT_FN_ATTRS
227__bextr_u32(unsigned int __X, unsigned int __Y)
228{
232__bextr_u32(unsigned int __X, unsigned int __Y) {
229233 return __builtin_ia32_bextr_u32(__X, __Y);
230234}
231235
......@@ -249,9 +253,8 @@ __bextr_u32(unsigned int __X, unsigned int __Y)
249253/// extracted bits.
250254/// \see __bextr_u32
251255static __inline__ unsigned int __DEFAULT_FN_ATTRS
252_bextr_u32(unsigned int __X, unsigned int __Y, unsigned int __Z)
253{
254 return __builtin_ia32_bextr_u32 (__X, ((__Y & 0xff) | ((__Z & 0xff) << 8)));
256_bextr_u32(unsigned int __X, unsigned int __Y, unsigned int __Z) {
257 return __builtin_ia32_bextr_u32(__X, ((__Y & 0xff) | ((__Z & 0xff) << 8)));
255258}
256259
257260/* Intel-specified, single-leading-underscore version of BEXTR2 */
......@@ -289,8 +292,7 @@ _bextr2_u32(unsigned int __X, unsigned int __Y) {
289292/// the source operand.
290293/// \see _blsi_u32
291294static __inline__ unsigned int __DEFAULT_FN_ATTRS
292__blsi_u32(unsigned int __X)
293{
295__blsi_u32(unsigned int __X) {
294296 return __X & -__X;
295297}
296298
......@@ -325,8 +327,7 @@ __blsi_u32(unsigned int __X)
325327/// \returns An unsigned integer containing the newly created mask.
326328/// \see _blsmsk_u32
327329static __inline__ unsigned int __DEFAULT_FN_ATTRS
328__blsmsk_u32(unsigned int __X)
329{
330__blsmsk_u32(unsigned int __X) {
330331 return __X ^ (__X - 1);
331332}
332333
......@@ -361,8 +362,7 @@ __blsmsk_u32(unsigned int __X)
361362/// operand.
362363/// \see _blsr_u32
363364static __inline__ unsigned int __DEFAULT_FN_ATTRS
364__blsr_u32(unsigned int __X)
365{
365__blsr_u32(unsigned int __X) {
366366 return __X & (__X - 1);
367367}
368368
......@@ -401,8 +401,7 @@ __blsr_u32(unsigned int __X)
401401/// operand with the one's complement of the first operand.
402402/// \see _andn_u64
403403static __inline__ unsigned long long __DEFAULT_FN_ATTRS
404__andn_u64 (unsigned long long __X, unsigned long long __Y)
405{
404__andn_u64 (unsigned long long __X, unsigned long long __Y) {
406405 return ~__X & __Y;
407406}
408407
......@@ -445,8 +444,7 @@ __andn_u64 (unsigned long long __X, unsigned long long __Y)
445444/// extracted bits.
446445/// \see _bextr_u64
447446static __inline__ unsigned long long __DEFAULT_FN_ATTRS
448__bextr_u64(unsigned long long __X, unsigned long long __Y)
449{
447__bextr_u64(unsigned long long __X, unsigned long long __Y) {
450448 return __builtin_ia32_bextr_u64(__X, __Y);
451449}
452450
......@@ -470,9 +468,8 @@ __bextr_u64(unsigned long long __X, unsigned long long __Y)
470468/// extracted bits.
471469/// \see __bextr_u64
472470static __inline__ unsigned long long __DEFAULT_FN_ATTRS
473_bextr_u64(unsigned long long __X, unsigned int __Y, unsigned int __Z)
474{
475 return __builtin_ia32_bextr_u64 (__X, ((__Y & 0xff) | ((__Z & 0xff) << 8)));
471_bextr_u64(unsigned long long __X, unsigned int __Y, unsigned int __Z) {
472 return __builtin_ia32_bextr_u64(__X, ((__Y & 0xff) | ((__Z & 0xff) << 8)));
476473}
477474
478475/* Intel-specified, single-leading-underscore version of BEXTR2 */
......@@ -510,8 +507,7 @@ _bextr2_u64(unsigned long long __X, unsigned long long __Y) {
510507/// bits from the source operand.
511508/// \see _blsi_u64
512509static __inline__ unsigned long long __DEFAULT_FN_ATTRS
513__blsi_u64(unsigned long long __X)
514{
510__blsi_u64(unsigned long long __X) {
515511 return __X & -__X;
516512}
517513
......@@ -546,8 +542,7 @@ __blsi_u64(unsigned long long __X)
546542/// \returns An unsigned 64-bit integer containing the newly created mask.
547543/// \see _blsmsk_u64
548544static __inline__ unsigned long long __DEFAULT_FN_ATTRS
549__blsmsk_u64(unsigned long long __X)
550{
545__blsmsk_u64(unsigned long long __X) {
551546 return __X ^ (__X - 1);
552547}
553548
......@@ -582,8 +577,7 @@ __blsmsk_u64(unsigned long long __X)
582577/// source operand.
583578/// \see _blsr_u64
584579static __inline__ unsigned long long __DEFAULT_FN_ATTRS
585__blsr_u64(unsigned long long __X)
586{
580__blsr_u64(unsigned long long __X) {
587581 return __X & (__X - 1);
588582}
589583
lib/include/cmpccxaddintrin.h+1-1
......@@ -63,7 +63,7 @@ typedef enum {
6363 (int)(__D))))
6464
6565#define _cmpccxadd_epi64(__A, __B, __C, __D) \
66 ((long long)(__builtin_ia32_cmpccxadd64((void *)(__A), (long long)(__B), \
66 ((long long)(__builtin_ia32_cmpccxadd64((__A), (long long)(__B), \
6767 (long long)(__C), (int)(__D))))
6868
6969#endif // __x86_64__
lib/include/cpuid.h+12-11
......@@ -187,17 +187,18 @@
187187#define bit_ENQCMD 0x20000000
188188
189189/* Features in %edx for leaf 7 sub-leaf 0 */
190#define bit_AVX5124VNNIW 0x00000004
191#define bit_AVX5124FMAPS 0x00000008
192#define bit_UINTR 0x00000020
193#define bit_SERIALIZE 0x00004000
194#define bit_TSXLDTRK 0x00010000
195#define bit_PCONFIG 0x00040000
196#define bit_IBT 0x00100000
197#define bit_AMXBF16 0x00400000
198#define bit_AVX512FP16 0x00800000
199#define bit_AMXTILE 0x01000000
200#define bit_AMXINT8 0x02000000
190#define bit_AVX5124VNNIW 0x00000004
191#define bit_AVX5124FMAPS 0x00000008
192#define bit_UINTR 0x00000020
193#define bit_AVX512VP2INTERSECT 0x00000100
194#define bit_SERIALIZE 0x00004000
195#define bit_TSXLDTRK 0x00010000
196#define bit_PCONFIG 0x00040000
197#define bit_IBT 0x00100000
198#define bit_AMXBF16 0x00400000
199#define bit_AVX512FP16 0x00800000
200#define bit_AMXTILE 0x01000000
201#define bit_AMXINT8 0x02000000
201202
202203/* Features in %eax for leaf 7 sub-leaf 1 */
203204#define bit_SHA512 0x00000001
lib/include/emmintrin.h+141-107
......@@ -49,12 +49,27 @@ typedef __bf16 __m128bh __attribute__((__vector_size__(16), __aligned__(16)));
4949#endif
5050
5151/* Define the default attributes for the functions in this file. */
52#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
5253#define __DEFAULT_FN_ATTRS \
5354 __attribute__((__always_inline__, __nodebug__, \
5455 __target__("sse2,no-evex512"), __min_vector_width__(128)))
55#define __DEFAULT_FN_ATTRS_MMX \
56 __attribute__((__always_inline__, __nodebug__, \
57 __target__("mmx,sse2,no-evex512"), __min_vector_width__(64)))
56#else
57#define __DEFAULT_FN_ATTRS \
58 __attribute__((__always_inline__, __nodebug__, __target__("sse2"), \
59 __min_vector_width__(128)))
60#endif
61
62#if defined(__cplusplus) && (__cplusplus >= 201103L)
63#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
64#else
65#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
66#endif
67
68#define __trunc64(x) \
69 (__m64) __builtin_shufflevector((__v2di)(x), __extension__(__v2di){}, 0)
70#define __anyext128(x) \
71 (__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0, \
72 1, -1, -1)
5873
5974/// Adds lower double-precision values in both operands and returns the
6075/// sum in the lower 64 bits of the result. The upper 64 bits of the result
......@@ -71,8 +86,8 @@ typedef __bf16 __m128bh __attribute__((__vector_size__(16), __aligned__(16)));
7186/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the
7287/// sum of the lower 64 bits of both operands. The upper 64 bits are copied
7388/// from the upper 64 bits of the first source operand.
74static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_add_sd(__m128d __a,
75 __m128d __b) {
89static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_add_sd(__m128d __a,
90 __m128d __b) {
7691 __a[0] += __b[0];
7792 return __a;
7893}
......@@ -89,8 +104,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_add_sd(__m128d __a,
89104/// A 128-bit vector of [2 x double] containing one of the source operands.
90105/// \returns A 128-bit vector of [2 x double] containing the sums of both
91106/// operands.
92static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_add_pd(__m128d __a,
93 __m128d __b) {
107static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_add_pd(__m128d __a,
108 __m128d __b) {
94109 return (__m128d)((__v2df)__a + (__v2df)__b);
95110}
96111
......@@ -111,8 +126,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_add_pd(__m128d __a,
111126/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the
112127/// difference of the lower 64 bits of both operands. The upper 64 bits are
113128/// copied from the upper 64 bits of the first source operand.
114static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sub_sd(__m128d __a,
115 __m128d __b) {
129static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_sub_sd(__m128d __a,
130 __m128d __b) {
116131 __a[0] -= __b[0];
117132 return __a;
118133}
......@@ -129,8 +144,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sub_sd(__m128d __a,
129144/// A 128-bit vector of [2 x double] containing the subtrahend.
130145/// \returns A 128-bit vector of [2 x double] containing the differences between
131146/// both operands.
132static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sub_pd(__m128d __a,
133 __m128d __b) {
147static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_sub_pd(__m128d __a,
148 __m128d __b) {
134149 return (__m128d)((__v2df)__a - (__v2df)__b);
135150}
136151
......@@ -150,8 +165,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sub_pd(__m128d __a,
150165/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the
151166/// product of the lower 64 bits of both operands. The upper 64 bits are
152167/// copied from the upper 64 bits of the first source operand.
153static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_mul_sd(__m128d __a,
154 __m128d __b) {
168static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_mul_sd(__m128d __a,
169 __m128d __b) {
155170 __a[0] *= __b[0];
156171 return __a;
157172}
......@@ -168,8 +183,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_mul_sd(__m128d __a,
168183/// A 128-bit vector of [2 x double] containing one of the operands.
169184/// \returns A 128-bit vector of [2 x double] containing the products of both
170185/// operands.
171static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_mul_pd(__m128d __a,
172 __m128d __b) {
186static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_mul_pd(__m128d __a,
187 __m128d __b) {
173188 return (__m128d)((__v2df)__a * (__v2df)__b);
174189}
175190
......@@ -190,8 +205,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_mul_pd(__m128d __a,
190205/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the
191206/// quotient of the lower 64 bits of both operands. The upper 64 bits are
192207/// copied from the upper 64 bits of the first source operand.
193static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_div_sd(__m128d __a,
194 __m128d __b) {
208static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_div_sd(__m128d __a,
209 __m128d __b) {
195210 __a[0] /= __b[0];
196211 return __a;
197212}
......@@ -209,8 +224,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_div_sd(__m128d __a,
209224/// A 128-bit vector of [2 x double] containing the divisor.
210225/// \returns A 128-bit vector of [2 x double] containing the quotients of both
211226/// operands.
212static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_div_pd(__m128d __a,
213 __m128d __b) {
227static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_div_pd(__m128d __a,
228 __m128d __b) {
214229 return (__m128d)((__v2df)__a / (__v2df)__b);
215230}
216231
......@@ -358,8 +373,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_max_pd(__m128d __a,
358373/// A 128-bit vector of [2 x double] containing one of the source operands.
359374/// \returns A 128-bit vector of [2 x double] containing the bitwise AND of the
360375/// values between both operands.
361static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_and_pd(__m128d __a,
362 __m128d __b) {
376static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_and_pd(__m128d __a,
377 __m128d __b) {
363378 return (__m128d)((__v2du)__a & (__v2du)__b);
364379}
365380
......@@ -378,8 +393,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_and_pd(__m128d __a,
378393/// \returns A 128-bit vector of [2 x double] containing the bitwise AND of the
379394/// values in the second operand and the one's complement of the first
380395/// operand.
381static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_andnot_pd(__m128d __a,
382 __m128d __b) {
396static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
397_mm_andnot_pd(__m128d __a, __m128d __b) {
383398 return (__m128d)(~(__v2du)__a & (__v2du)__b);
384399}
385400
......@@ -395,8 +410,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_andnot_pd(__m128d __a,
395410/// A 128-bit vector of [2 x double] containing one of the source operands.
396411/// \returns A 128-bit vector of [2 x double] containing the bitwise OR of the
397412/// values between both operands.
398static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_or_pd(__m128d __a,
399 __m128d __b) {
413static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_or_pd(__m128d __a,
414 __m128d __b) {
400415 return (__m128d)((__v2du)__a | (__v2du)__b);
401416}
402417
......@@ -412,8 +427,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_or_pd(__m128d __a,
412427/// A 128-bit vector of [2 x double] containing one of the source operands.
413428/// \returns A 128-bit vector of [2 x double] containing the bitwise XOR of the
414429/// values between both operands.
415static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_xor_pd(__m128d __a,
416 __m128d __b) {
430static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_xor_pd(__m128d __a,
431 __m128d __b) {
417432 return (__m128d)((__v2du)__a ^ (__v2du)__b);
418433}
419434
......@@ -1291,7 +1306,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_cvtpd_ps(__m128d __a) {
12911306/// floating-point elements are converted to double-precision values. The
12921307/// upper two elements are unused.
12931308/// \returns A 128-bit vector of [2 x double] containing the converted values.
1294static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtps_pd(__m128 __a) {
1309static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
1310_mm_cvtps_pd(__m128 __a) {
12951311 return (__m128d) __builtin_convertvector(
12961312 __builtin_shufflevector((__v4sf)__a, (__v4sf)__a, 0, 1), __v2df);
12971313}
......@@ -1312,7 +1328,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtps_pd(__m128 __a) {
13121328///
13131329/// The upper two elements are unused.
13141330/// \returns A 128-bit vector of [2 x double] containing the converted values.
1315static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtepi32_pd(__m128i __a) {
1331static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
1332_mm_cvtepi32_pd(__m128i __a) {
13161333 return (__m128d) __builtin_convertvector(
13171334 __builtin_shufflevector((__v4si)__a, (__v4si)__a, 0, 1), __v2df);
13181335}
......@@ -1398,8 +1415,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_cvtsd_ss(__m128 __a,
13981415/// \returns A 128-bit vector of [2 x double]. The lower 64 bits contain the
13991416/// converted value from the second parameter. The upper 64 bits are copied
14001417/// from the upper 64 bits of the first parameter.
1401static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtsi32_sd(__m128d __a,
1402 int __b) {
1418static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
1419_mm_cvtsi32_sd(__m128d __a, int __b) {
14031420 __a[0] = __b;
14041421 return __a;
14051422}
......@@ -1423,8 +1440,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtsi32_sd(__m128d __a,
14231440/// \returns A 128-bit vector of [2 x double]. The lower 64 bits contain the
14241441/// converted value from the second parameter. The upper 64 bits are copied
14251442/// from the upper 64 bits of the first parameter.
1426static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtss_sd(__m128d __a,
1427 __m128 __b) {
1443static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
1444_mm_cvtss_sd(__m128d __a, __m128 __b) {
14281445 __a[0] = __b[0];
14291446 return __a;
14301447}
......@@ -1486,8 +1503,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_cvttsd_si32(__m128d __a) {
14861503/// \param __a
14871504/// A 128-bit vector of [2 x double].
14881505/// \returns A 64-bit vector of [2 x i32] containing the converted values.
1489static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_cvtpd_pi32(__m128d __a) {
1490 return (__m64)__builtin_ia32_cvtpd2pi((__v2df)__a);
1506static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_cvtpd_pi32(__m128d __a) {
1507 return __trunc64(__builtin_ia32_cvtpd2dq((__v2df)__a));
14911508}
14921509
14931510/// Converts the two double-precision floating-point elements of a
......@@ -1505,8 +1522,8 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_cvtpd_pi32(__m128d __a) {
15051522/// \param __a
15061523/// A 128-bit vector of [2 x double].
15071524/// \returns A 64-bit vector of [2 x i32] containing the converted values.
1508static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_cvttpd_pi32(__m128d __a) {
1509 return (__m64)__builtin_ia32_cvttpd2pi((__v2df)__a);
1525static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_cvttpd_pi32(__m128d __a) {
1526 return __trunc64(__builtin_ia32_cvttpd2dq((__v2df)__a));
15101527}
15111528
15121529/// Converts the two signed 32-bit integer elements of a 64-bit vector of
......@@ -1520,8 +1537,9 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_cvttpd_pi32(__m128d __a) {
15201537/// \param __a
15211538/// A 64-bit vector of [2 x i32].
15221539/// \returns A 128-bit vector of [2 x double] containing the converted values.
1523static __inline__ __m128d __DEFAULT_FN_ATTRS_MMX _mm_cvtpi32_pd(__m64 __a) {
1524 return __builtin_ia32_cvtpi2pd((__v2si)__a);
1540static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
1541_mm_cvtpi32_pd(__m64 __a) {
1542 return (__m128d) __builtin_convertvector((__v2si)__a, __v2df);
15251543}
15261544
15271545/// Returns the low-order element of a 128-bit vector of [2 x double] as
......@@ -1535,7 +1553,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS_MMX _mm_cvtpi32_pd(__m64 __a) {
15351553/// A 128-bit vector of [2 x double]. The lower 64 bits are returned.
15361554/// \returns A double-precision floating-point value copied from the lower 64
15371555/// bits of \a __a.
1538static __inline__ double __DEFAULT_FN_ATTRS _mm_cvtsd_f64(__m128d __a) {
1556static __inline__ double __DEFAULT_FN_ATTRS_CONSTEXPR
1557_mm_cvtsd_f64(__m128d __a) {
15391558 return __a[0];
15401559}
15411560
......@@ -1770,7 +1789,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_undefined_pd(void) {
17701789/// \returns An initialized 128-bit floating-point vector of [2 x double]. The
17711790/// lower 64 bits contain the value of the parameter. The upper 64 bits are
17721791/// set to zero.
1773static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_sd(double __w) {
1792static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set_sd(double __w) {
17741793 return __extension__(__m128d){__w, 0.0};
17751794}
17761795
......@@ -1786,7 +1805,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_sd(double __w) {
17861805/// A double-precision floating-point value used to initialize each vector
17871806/// element of the result.
17881807/// \returns An initialized 128-bit floating-point vector of [2 x double].
1789static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set1_pd(double __w) {
1808static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set1_pd(double __w) {
17901809 return __extension__(__m128d){__w, __w};
17911810}
17921811
......@@ -1802,7 +1821,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set1_pd(double __w) {
18021821/// A double-precision floating-point value used to initialize each vector
18031822/// element of the result.
18041823/// \returns An initialized 128-bit floating-point vector of [2 x double].
1805static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_pd1(double __w) {
1824static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set_pd1(double __w) {
18061825 return _mm_set1_pd(__w);
18071826}
18081827
......@@ -1820,8 +1839,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_pd1(double __w) {
18201839/// A double-precision floating-point value used to initialize the lower 64
18211840/// bits of the result.
18221841/// \returns An initialized 128-bit floating-point vector of [2 x double].
1823static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_pd(double __w,
1824 double __x) {
1842static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set_pd(double __w,
1843 double __x) {
18251844 return __extension__(__m128d){__x, __w};
18261845}
18271846
......@@ -1840,8 +1859,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_set_pd(double __w,
18401859/// A double-precision floating-point value used to initialize the upper 64
18411860/// bits of the result.
18421861/// \returns An initialized 128-bit floating-point vector of [2 x double].
1843static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_setr_pd(double __w,
1844 double __x) {
1862static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_setr_pd(double __w,
1863 double __x) {
18451864 return __extension__(__m128d){__w, __x};
18461865}
18471866
......@@ -1854,7 +1873,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_setr_pd(double __w,
18541873///
18551874/// \returns An initialized 128-bit floating-point vector of [2 x double] with
18561875/// all elements set to zero.
1857static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_setzero_pd(void) {
1876static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_setzero_pd(void) {
18581877 return __extension__(__m128d){0.0, 0.0};
18591878}
18601879
......@@ -1873,8 +1892,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_setzero_pd(void) {
18731892/// A 128-bit vector of [2 x double]. The lower 64 bits are written to the
18741893/// lower 64 bits of the result.
18751894/// \returns A 128-bit vector of [2 x double] containing the moved values.
1876static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_move_sd(__m128d __a,
1877 __m128d __b) {
1895static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
1896_mm_move_sd(__m128d __a, __m128d __b) {
18781897 __a[0] = __b[0];
18791898 return __a;
18801899}
......@@ -2091,8 +2110,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi16(__m128i __a,
20912110/// A 128-bit vector of [4 x i32].
20922111/// \returns A 128-bit vector of [4 x i32] containing the sums of both
20932112/// parameters.
2094static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi32(__m128i __a,
2095 __m128i __b) {
2113static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
2114_mm_add_epi32(__m128i __a, __m128i __b) {
20962115 return (__m128i)((__v4su)__a + (__v4su)__b);
20972116}
20982117
......@@ -2108,9 +2127,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi32(__m128i __a,
21082127/// \param __b
21092128/// A 64-bit integer.
21102129/// \returns A 64-bit integer containing the sum of both parameters.
2111static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_add_si64(__m64 __a,
2112 __m64 __b) {
2113 return (__m64)__builtin_ia32_paddq((__v1di)__a, (__v1di)__b);
2130static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_add_si64(__m64 __a, __m64 __b) {
2131 return (__m64)(((unsigned long long)__a) + ((unsigned long long)__b));
21142132}
21152133
21162134/// Adds the corresponding elements of two 128-bit vectors of [2 x i64],
......@@ -2129,8 +2147,8 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_add_si64(__m64 __a,
21292147/// A 128-bit vector of [2 x i64].
21302148/// \returns A 128-bit vector of [2 x i64] containing the sums of both
21312149/// parameters.
2132static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi64(__m128i __a,
2133 __m128i __b) {
2150static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
2151_mm_add_epi64(__m128i __a, __m128i __b) {
21342152 return (__m128i)((__v2du)__a + (__v2du)__b);
21352153}
21362154
......@@ -2431,9 +2449,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mullo_epi16(__m128i __a,
24312449/// \param __b
24322450/// A 64-bit integer containing one of the source operands.
24332451/// \returns A 64-bit integer vector containing the product of both operands.
2434static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_mul_su32(__m64 __a,
2435 __m64 __b) {
2436 return __builtin_ia32_pmuludq((__v2si)__a, (__v2si)__b);
2452static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_mul_su32(__m64 __a, __m64 __b) {
2453 return __trunc64(__builtin_ia32_pmuludq128((__v4si)__anyext128(__a),
2454 (__v4si)__anyext128(__b)));
24372455}
24382456
24392457/// Multiplies 32-bit unsigned integer values contained in the lower
......@@ -2521,8 +2539,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi16(__m128i __a,
25212539/// A 128-bit integer vector containing the subtrahends.
25222540/// \returns A 128-bit integer vector containing the differences of the values
25232541/// in the operands.
2524static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi32(__m128i __a,
2525 __m128i __b) {
2542static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
2543_mm_sub_epi32(__m128i __a, __m128i __b) {
25262544 return (__m128i)((__v4su)__a - (__v4su)__b);
25272545}
25282546
......@@ -2539,9 +2557,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi32(__m128i __a,
25392557/// A 64-bit integer vector containing the subtrahend.
25402558/// \returns A 64-bit integer vector containing the difference of the values in
25412559/// the operands.
2542static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_sub_si64(__m64 __a,
2543 __m64 __b) {
2544 return (__m64)__builtin_ia32_psubq((__v1di)__a, (__v1di)__b);
2560static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_sub_si64(__m64 __a, __m64 __b) {
2561 return (__m64)((unsigned long long)__a - (unsigned long long)__b);
25452562}
25462563
25472564/// Subtracts the corresponding elements of two [2 x i64] vectors.
......@@ -2556,8 +2573,8 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX _mm_sub_si64(__m64 __a,
25562573/// A 128-bit integer vector containing the subtrahends.
25572574/// \returns A 128-bit integer vector containing the differences of the values
25582575/// in the operands.
2559static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi64(__m128i __a,
2560 __m128i __b) {
2576static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
2577_mm_sub_epi64(__m128i __a, __m128i __b) {
25612578 return (__m128i)((__v2du)__a - (__v2du)__b);
25622579}
25632580
......@@ -3255,8 +3272,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmplt_epi32(__m128i __a,
32553272/// \returns A 128-bit vector of [2 x double] whose lower 64 bits contain the
32563273/// converted value of the second operand. The upper 64 bits are copied from
32573274/// the upper 64 bits of the first operand.
3258static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cvtsi64_sd(__m128d __a,
3259 long long __b) {
3275static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
3276_mm_cvtsi64_sd(__m128d __a, long long __b) {
32603277 __a[0] = __b;
32613278 return __a;
32623279}
......@@ -3310,7 +3327,8 @@ static __inline__ long long __DEFAULT_FN_ATTRS _mm_cvttsd_si64(__m128d __a) {
33103327/// \param __a
33113328/// A 128-bit integer vector.
33123329/// \returns A 128-bit vector of [4 x float] containing the converted values.
3313static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_cvtepi32_ps(__m128i __a) {
3330static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
3331_mm_cvtepi32_ps(__m128i __a) {
33143332 return (__m128) __builtin_convertvector((__v4si)__a, __v4sf);
33153333}
33163334
......@@ -3494,8 +3512,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_undefined_si128(void) {
34943512/// destination vector of [2 x i64].
34953513/// \returns An initialized 128-bit vector of [2 x i64] containing the values
34963514/// provided in the operands.
3497static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi64x(long long __q1,
3498 long long __q0) {
3515static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3516_mm_set_epi64x(long long __q1, long long __q0) {
34993517 return __extension__(__m128i)(__v2di){__q0, __q1};
35003518}
35013519
......@@ -3515,9 +3533,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi64x(long long __q1,
35153533/// destination vector of [2 x i64].
35163534/// \returns An initialized 128-bit vector of [2 x i64] containing the values
35173535/// provided in the operands.
3518static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi64(__m64 __q1,
3519 __m64 __q0) {
3520 return _mm_set_epi64x((long long)__q1, (long long)__q0);
3536static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3537_mm_set_epi64(__m64 __q1, __m64 __q0) {
3538 return _mm_set_epi64x((long long)__q1[0], (long long)__q0[0]);
35213539}
35223540
35233541/// Initializes the 32-bit values in a 128-bit vector of [4 x i32] with
......@@ -3542,8 +3560,10 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi64(__m64 __q1,
35423560/// vector.
35433561/// \returns An initialized 128-bit vector of [4 x i32] containing the values
35443562/// provided in the operands.
3545static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi32(int __i3, int __i2,
3546 int __i1, int __i0) {
3563static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set_epi32(int __i3,
3564 int __i2,
3565 int __i1,
3566 int __i0) {
35473567 return __extension__(__m128i)(__v4si){__i0, __i1, __i2, __i3};
35483568}
35493569
......@@ -3581,7 +3601,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set_epi32(int __i3, int __i2,
35813601/// vector.
35823602/// \returns An initialized 128-bit vector of [8 x i16] containing the values
35833603/// provided in the operands.
3584static __inline__ __m128i __DEFAULT_FN_ATTRS
3604static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
35853605_mm_set_epi16(short __w7, short __w6, short __w5, short __w4, short __w3,
35863606 short __w2, short __w1, short __w0) {
35873607 return __extension__(__m128i)(__v8hi){__w0, __w1, __w2, __w3,
......@@ -3630,7 +3650,7 @@ _mm_set_epi16(short __w7, short __w6, short __w5, short __w4, short __w3,
36303650/// Initializes bits [7:0] of the destination vector.
36313651/// \returns An initialized 128-bit vector of [16 x i8] containing the values
36323652/// provided in the operands.
3633static __inline__ __m128i __DEFAULT_FN_ATTRS
3653static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
36343654_mm_set_epi8(char __b15, char __b14, char __b13, char __b12, char __b11,
36353655 char __b10, char __b9, char __b8, char __b7, char __b6, char __b5,
36363656 char __b4, char __b3, char __b2, char __b1, char __b0) {
......@@ -3652,7 +3672,8 @@ _mm_set_epi8(char __b15, char __b14, char __b13, char __b12, char __b11,
36523672/// vector.
36533673/// \returns An initialized 128-bit integer vector of [2 x i64] with both
36543674/// elements containing the value provided in the operand.
3655static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi64x(long long __q) {
3675static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3676_mm_set1_epi64x(long long __q) {
36563677 return _mm_set_epi64x(__q, __q);
36573678}
36583679
......@@ -3669,7 +3690,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi64x(long long __q) {
36693690/// vector.
36703691/// \returns An initialized 128-bit vector of [2 x i64] with all elements
36713692/// containing the value provided in the operand.
3672static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi64(__m64 __q) {
3693static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3694_mm_set1_epi64(__m64 __q) {
36733695 return _mm_set_epi64(__q, __q);
36743696}
36753697
......@@ -3686,7 +3708,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi64(__m64 __q) {
36863708/// vector.
36873709/// \returns An initialized 128-bit vector of [4 x i32] with all elements
36883710/// containing the value provided in the operand.
3689static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi32(int __i) {
3711static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set1_epi32(int __i) {
36903712 return _mm_set_epi32(__i, __i, __i, __i);
36913713}
36923714
......@@ -3703,7 +3725,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi32(int __i) {
37033725/// vector.
37043726/// \returns An initialized 128-bit vector of [8 x i16] with all elements
37053727/// containing the value provided in the operand.
3706static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi16(short __w) {
3728static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3729_mm_set1_epi16(short __w) {
37073730 return _mm_set_epi16(__w, __w, __w, __w, __w, __w, __w, __w);
37083731}
37093732
......@@ -3720,7 +3743,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi16(short __w) {
37203743/// vector.
37213744/// \returns An initialized 128-bit vector of [16 x i8] with all elements
37223745/// containing the value provided in the operand.
3723static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi8(char __b) {
3746static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR _mm_set1_epi8(char __b) {
37243747 return _mm_set_epi8(__b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b,
37253748 __b, __b, __b, __b, __b);
37263749}
......@@ -3739,8 +3762,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_set1_epi8(char __b) {
37393762/// A 64-bit integral value used to initialize the upper 64 bits of the
37403763/// result.
37413764/// \returns An initialized 128-bit integer vector.
3742static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setr_epi64(__m64 __q0,
3743 __m64 __q1) {
3765static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3766_mm_setr_epi64(__m64 __q0, __m64 __q1) {
37443767 return _mm_set_epi64(__q1, __q0);
37453768}
37463769
......@@ -3761,9 +3784,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setr_epi64(__m64 __q0,
37613784/// \param __i3
37623785/// A 32-bit integral value used to initialize bits [127:96] of the result.
37633786/// \returns An initialized 128-bit integer vector.
3764static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setr_epi32(int __i0, int __i1,
3765 int __i2,
3766 int __i3) {
3787static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
3788_mm_setr_epi32(int __i0, int __i1, int __i2, int __i3) {
37673789 return _mm_set_epi32(__i3, __i2, __i1, __i0);
37683790}
37693791
......@@ -3792,7 +3814,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setr_epi32(int __i0, int __i1,
37923814/// \param __w7
37933815/// A 16-bit integral value used to initialize bits [127:112] of the result.
37943816/// \returns An initialized 128-bit integer vector.
3795static __inline__ __m128i __DEFAULT_FN_ATTRS
3817static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
37963818_mm_setr_epi16(short __w0, short __w1, short __w2, short __w3, short __w4,
37973819 short __w5, short __w6, short __w7) {
37983820 return _mm_set_epi16(__w7, __w6, __w5, __w4, __w3, __w2, __w1, __w0);
......@@ -3839,7 +3861,7 @@ _mm_setr_epi16(short __w0, short __w1, short __w2, short __w3, short __w4,
38393861/// \param __b15
38403862/// An 8-bit integral value used to initialize bits [127:120] of the result.
38413863/// \returns An initialized 128-bit integer vector.
3842static __inline__ __m128i __DEFAULT_FN_ATTRS
3864static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
38433865_mm_setr_epi8(char __b0, char __b1, char __b2, char __b3, char __b4, char __b5,
38443866 char __b6, char __b7, char __b8, char __b9, char __b10,
38453867 char __b11, char __b12, char __b13, char __b14, char __b15) {
......@@ -3855,7 +3877,7 @@ _mm_setr_epi8(char __b0, char __b1, char __b2, char __b3, char __b4, char __b5,
38553877///
38563878/// \returns An initialized 128-bit integer vector with all elements set to
38573879/// zero.
3858static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_setzero_si128(void) {
3880static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR _mm_setzero_si128(void) {
38593881 return __extension__(__m128i)(__v2di){0LL, 0LL};
38603882}
38613883
......@@ -4588,7 +4610,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi64(__m128i __a,
45884610/// A 128-bit integer vector operand. The lower 64 bits are moved to the
45894611/// destination.
45904612/// \returns A 64-bit integer containing the lower 64 bits of the parameter.
4591static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_movepi64_pi64(__m128i __a) {
4613static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR
4614_mm_movepi64_pi64(__m128i __a) {
45924615 return (__m64)__a[0];
45934616}
45944617
......@@ -4603,8 +4626,9 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_movepi64_pi64(__m128i __a) {
46034626/// A 64-bit value.
46044627/// \returns A 128-bit integer vector. The lower 64 bits contain the value from
46054628/// the operand. The upper 64 bits are assigned zeros.
4606static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_movpi64_epi64(__m64 __a) {
4607 return __extension__(__m128i)(__v2di){(long long)__a, 0};
4629static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
4630_mm_movpi64_epi64(__m64 __a) {
4631 return __builtin_shufflevector((__v1di)__a, _mm_setzero_si64(), 0, 1);
46084632}
46094633
46104634/// Moves the lower 64 bits of a 128-bit integer vector to a 128-bit
......@@ -4619,7 +4643,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_movpi64_epi64(__m64 __a) {
46194643/// destination.
46204644/// \returns A 128-bit integer vector. The lower 64 bits contain the value from
46214645/// the operand. The upper 64 bits are assigned zeros.
4622static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_move_epi64(__m128i __a) {
4646static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
4647_mm_move_epi64(__m128i __a) {
46234648 return __builtin_shufflevector((__v2di)__a, _mm_setzero_si128(), 0, 2);
46244649}
46254650
......@@ -4638,8 +4663,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_move_epi64(__m128i __a) {
46384663/// A 128-bit vector of [2 x double]. \n
46394664/// Bits [127:64] are written to bits [127:64] of the destination.
46404665/// \returns A 128-bit vector of [2 x double] containing the interleaved values.
4641static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_unpackhi_pd(__m128d __a,
4642 __m128d __b) {
4666static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
4667_mm_unpackhi_pd(__m128d __a, __m128d __b) {
46434668 return __builtin_shufflevector((__v2df)__a, (__v2df)__b, 1, 2 + 1);
46444669}
46454670
......@@ -4658,8 +4683,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_unpackhi_pd(__m128d __a,
46584683/// A 128-bit vector of [2 x double]. \n
46594684/// Bits [63:0] are written to bits [127:64] of the destination.
46604685/// \returns A 128-bit vector of [2 x double] containing the interleaved values.
4661static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_unpacklo_pd(__m128d __a,
4662 __m128d __b) {
4686static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
4687_mm_unpacklo_pd(__m128d __a, __m128d __b) {
46634688 return __builtin_shufflevector((__v2df)__a, (__v2df)__b, 0, 2 + 0);
46644689}
46654690
......@@ -4722,7 +4747,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_movemask_pd(__m128d __a) {
47224747/// A 128-bit floating-point vector of [2 x double].
47234748/// \returns A 128-bit floating-point vector of [4 x float] containing the same
47244749/// bitwise pattern as the parameter.
4725static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_castpd_ps(__m128d __a) {
4750static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
4751_mm_castpd_ps(__m128d __a) {
47264752 return (__m128)__a;
47274753}
47284754
......@@ -4737,7 +4763,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_castpd_ps(__m128d __a) {
47374763/// A 128-bit floating-point vector of [2 x double].
47384764/// \returns A 128-bit integer vector containing the same bitwise pattern as the
47394765/// parameter.
4740static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_castpd_si128(__m128d __a) {
4766static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
4767_mm_castpd_si128(__m128d __a) {
47414768 return (__m128i)__a;
47424769}
47434770
......@@ -4752,7 +4779,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_castpd_si128(__m128d __a) {
47524779/// A 128-bit floating-point vector of [4 x float].
47534780/// \returns A 128-bit floating-point vector of [2 x double] containing the same
47544781/// bitwise pattern as the parameter.
4755static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_castps_pd(__m128 __a) {
4782static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
4783_mm_castps_pd(__m128 __a) {
47564784 return (__m128d)__a;
47574785}
47584786
......@@ -4767,7 +4795,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_castps_pd(__m128 __a) {
47674795/// A 128-bit floating-point vector of [4 x float].
47684796/// \returns A 128-bit integer vector containing the same bitwise pattern as the
47694797/// parameter.
4770static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_castps_si128(__m128 __a) {
4798static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
4799_mm_castps_si128(__m128 __a) {
47714800 return (__m128i)__a;
47724801}
47734802
......@@ -4782,7 +4811,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_castps_si128(__m128 __a) {
47824811/// A 128-bit integer vector.
47834812/// \returns A 128-bit floating-point vector of [4 x float] containing the same
47844813/// bitwise pattern as the parameter.
4785static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_castsi128_ps(__m128i __a) {
4814static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
4815_mm_castsi128_ps(__m128i __a) {
47864816 return (__m128)__a;
47874817}
47884818
......@@ -4797,7 +4827,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_castsi128_ps(__m128i __a) {
47974827/// A 128-bit integer vector.
47984828/// \returns A 128-bit floating-point vector of [2 x double] containing the same
47994829/// bitwise pattern as the parameter.
4800static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_castsi128_pd(__m128i __a) {
4830static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
4831_mm_castsi128_pd(__m128i __a) {
48014832 return (__m128d)__a;
48024833}
48034834
......@@ -4889,8 +4920,11 @@ void _mm_pause(void);
48894920#if defined(__cplusplus)
48904921} // extern "C"
48914922#endif
4923
4924#undef __anyext128
4925#undef __trunc64
48924926#undef __DEFAULT_FN_ATTRS
4893#undef __DEFAULT_FN_ATTRS_MMX
4927#undef __DEFAULT_FN_ATTRS_CONSTEXPR
48944928
48954929#define _MM_SHUFFLE2(x, y) (((x) << 1) | (y))
48964930
lib/include/gfniintrin.h+32-10
......@@ -14,6 +14,7 @@
1414#ifndef __GFNIINTRIN_H
1515#define __GFNIINTRIN_H
1616
17#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
1718/* Default attributes for simple form (no masking). */
1819#define __DEFAULT_FN_ATTRS \
1920 __attribute__((__always_inline__, __nodebug__, \
......@@ -25,26 +26,47 @@
2526 __target__("avx,gfni,no-evex512"), \
2627 __min_vector_width__(256)))
2728
28/* Default attributes for ZMM unmasked forms. */
29#define __DEFAULT_FN_ATTRS_Z \
29/* Default attributes for VLX masked forms. */
30#define __DEFAULT_FN_ATTRS_VL128 \
3031 __attribute__((__always_inline__, __nodebug__, \
31 __target__("avx512f,evex512,gfni"), \
32 __min_vector_width__(512)))
33/* Default attributes for ZMM masked forms. */
34#define __DEFAULT_FN_ATTRS_Z_MASK \
32 __target__("avx512bw,avx512vl,gfni,no-evex512"), \
33 __min_vector_width__(128)))
34#define __DEFAULT_FN_ATTRS_VL256 \
3535 __attribute__((__always_inline__, __nodebug__, \
36 __target__("avx512bw,evex512,gfni"), \
37 __min_vector_width__(512)))
36 __target__("avx512bw,avx512vl,gfni,no-evex512"), \
37 __min_vector_width__(256)))
38#else
39/* Default attributes for simple form (no masking). */
40#define __DEFAULT_FN_ATTRS \
41 __attribute__((__always_inline__, __nodebug__, __target__("gfni"), \
42 __min_vector_width__(128)))
43
44/* Default attributes for YMM unmasked form. */
45#define __DEFAULT_FN_ATTRS_Y \
46 __attribute__((__always_inline__, __nodebug__, __target__("avx,gfni"), \
47 __min_vector_width__(256)))
3848
3949/* Default attributes for VLX masked forms. */
4050#define __DEFAULT_FN_ATTRS_VL128 \
4151 __attribute__((__always_inline__, __nodebug__, \
42 __target__("avx512bw,avx512vl,gfni,no-evex512"), \
52 __target__("avx512bw,avx512vl,gfni"), \
4353 __min_vector_width__(128)))
4454#define __DEFAULT_FN_ATTRS_VL256 \
4555 __attribute__((__always_inline__, __nodebug__, \
46 __target__("avx512bw,avx512vl,gfni,no-evex512"), \
56 __target__("avx512bw,avx512vl,gfni"), \
4757 __min_vector_width__(256)))
58#endif
59
60/* Default attributes for ZMM unmasked forms. */
61#define __DEFAULT_FN_ATTRS_Z \
62 __attribute__((__always_inline__, __nodebug__, \
63 __target__("avx512f,evex512,gfni"), \
64 __min_vector_width__(512)))
65/* Default attributes for ZMM masked forms. */
66#define __DEFAULT_FN_ATTRS_Z_MASK \
67 __attribute__((__always_inline__, __nodebug__, \
68 __target__("avx512bw,evex512,gfni"), \
69 __min_vector_width__(512)))
4870
4971#define _mm_gf2p8affineinv_epi64_epi8(A, B, I) \
5072 ((__m128i)__builtin_ia32_vgf2p8affineinvqb_v16qi((__v16qi)(__m128i)(A), \
lib/include/hexagon_types.h+8-4
......@@ -1,7 +1,11 @@
1/******************************************************************************/
2/* (c) 2020 Qualcomm Innovation Center, Inc. All rights reserved. */
3/* */
4/******************************************************************************/
1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
59#ifndef HEXAGON_TYPES_H
610#define HEXAGON_TYPES_H
711
lib/include/hvx_hexagon_protos.h+427
......@@ -5178,6 +5178,433 @@
51785178#define Q6_Vuh_vmpy_VuhVuh_rs16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhvs)(Vu,Vv)
51795179#endif /* __HEXAGON_ARCH___ >= 69 */
51805180
5181#if __HVX_ARCH__ >= 73
5182/* ==========================================================================
5183 Assembly Syntax: Vdd32.sf=vadd(Vu32.bf,Vv32.bf)
5184 C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vadd_VbfVbf(HVX_Vector Vu,
5185 HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5186 ========================================================================== */
5187
5188#define Q6_Wsf_vadd_VbfVbf(Vu, Vv) \
5189 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_sf_bf)(Vu, Vv)
5190#endif /* __HEXAGON_ARCH___ >= 73 */
5191
5192#if __HVX_ARCH__ >= 73
5193/* ==========================================================================
5194 Assembly Syntax: Vd32.h=Vu32.hf
5195 C Intrinsic Prototype: HVX_Vector Q6_Vh_equals_Vhf(HVX_Vector Vu)
5196 Instruction Type: CVI_VS
5197 Execution Slots: SLOT0123
5198 ========================================================================== */
5199
5200#define Q6_Vh_equals_Vhf(Vu) \
5201 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_h_hf)(Vu)
5202#endif /* __HEXAGON_ARCH___ >= 73 */
5203
5204#if __HVX_ARCH__ >= 73
5205/* ==========================================================================
5206 Assembly Syntax: Vd32.hf=Vu32.h
5207 C Intrinsic Prototype: HVX_Vector Q6_Vhf_equals_Vh(HVX_Vector Vu)
5208 Instruction Type: CVI_VS
5209 Execution Slots: SLOT0123
5210 ========================================================================== */
5211
5212#define Q6_Vhf_equals_Vh(Vu) \
5213 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_hf_h)(Vu)
5214#endif /* __HEXAGON_ARCH___ >= 73 */
5215
5216#if __HVX_ARCH__ >= 73
5217/* ==========================================================================
5218 Assembly Syntax: Vd32.sf=Vu32.w
5219 C Intrinsic Prototype: HVX_Vector Q6_Vsf_equals_Vw(HVX_Vector Vu)
5220 Instruction Type: CVI_VS
5221 Execution Slots: SLOT0123
5222 ========================================================================== */
5223
5224#define Q6_Vsf_equals_Vw(Vu) \
5225 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_sf_w)(Vu)
5226#endif /* __HEXAGON_ARCH___ >= 73 */
5227
5228#if __HVX_ARCH__ >= 73
5229/* ==========================================================================
5230 Assembly Syntax: Vd32.w=Vu32.sf
5231 C Intrinsic Prototype: HVX_Vector Q6_Vw_equals_Vsf(HVX_Vector Vu)
5232 Instruction Type: CVI_VS
5233 Execution Slots: SLOT0123
5234 ========================================================================== */
5235
5236#define Q6_Vw_equals_Vsf(Vu) \
5237 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_w_sf)(Vu)
5238#endif /* __HEXAGON_ARCH___ >= 73 */
5239
5240#if __HVX_ARCH__ >= 73
5241/* ==========================================================================
5242 Assembly Syntax: Vd32.bf=vcvt(Vu32.sf,Vv32.sf)
5243 C Intrinsic Prototype: HVX_Vector Q6_Vbf_vcvt_VsfVsf(HVX_Vector Vu,
5244 HVX_Vector Vv) Instruction Type: CVI_VX Execution Slots: SLOT23
5245 ========================================================================== */
5246
5247#define Q6_Vbf_vcvt_VsfVsf(Vu, Vv) \
5248 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_bf_sf)(Vu, Vv)
5249#endif /* __HEXAGON_ARCH___ >= 73 */
5250
5251#if __HVX_ARCH__ >= 73
5252/* ==========================================================================
5253 Assembly Syntax: Qd4=vcmp.gt(Vu32.bf,Vv32.bf)
5254 C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VbfVbf(HVX_Vector Vu,
5255 HVX_Vector Vv) Instruction Type: CVI_VA Execution Slots: SLOT0123
5256 ========================================================================== */
5257
5258#define Q6_Q_vcmp_gt_VbfVbf(Vu, Vv) \
5259 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt) \
5260 ((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtbf)(Vu, Vv)), -1)
5261#endif /* __HEXAGON_ARCH___ >= 73 */
5262
5263#if __HVX_ARCH__ >= 73
5264/* ==========================================================================
5265 Assembly Syntax: Qx4&=vcmp.gt(Vu32.bf,Vv32.bf)
5266 C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVbfVbf(HVX_VectorPred
5267 Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
5268 Slots: SLOT0123
5269 ========================================================================== */
5270
5271#define Q6_Q_vcmp_gtand_QVbfVbf(Qx, Vu, Vv) \
5272 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt) \
5273 ((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtbf_and)( \
5274 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
5275 Vv)), \
5276 -1)
5277#endif /* __HEXAGON_ARCH___ >= 73 */
5278
5279#if __HVX_ARCH__ >= 73
5280/* ==========================================================================
5281 Assembly Syntax: Qx4|=vcmp.gt(Vu32.bf,Vv32.bf)
5282 C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVbfVbf(HVX_VectorPred
5283 Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
5284 Slots: SLOT0123
5285 ========================================================================== */
5286
5287#define Q6_Q_vcmp_gtor_QVbfVbf(Qx, Vu, Vv) \
5288 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt) \
5289 ((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtbf_or)( \
5290 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
5291 Vv)), \
5292 -1)
5293#endif /* __HEXAGON_ARCH___ >= 73 */
5294
5295#if __HVX_ARCH__ >= 73
5296/* ==========================================================================
5297 Assembly Syntax: Qx4^=vcmp.gt(Vu32.bf,Vv32.bf)
5298 C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVbfVbf(HVX_VectorPred
5299 Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
5300 Slots: SLOT0123
5301 ========================================================================== */
5302
5303#define Q6_Q_vcmp_gtxacc_QVbfVbf(Qx, Vu, Vv) \
5304 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt) \
5305 ((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtbf_xor)( \
5306 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
5307 Vv)), \
5308 -1)
5309#endif /* __HEXAGON_ARCH___ >= 73 */
5310
5311#if __HVX_ARCH__ >= 73
5312/* ==========================================================================
5313 Assembly Syntax: Vd32.bf=vmax(Vu32.bf,Vv32.bf)
5314 C Intrinsic Prototype: HVX_Vector Q6_Vbf_vmax_VbfVbf(HVX_Vector Vu,
5315 HVX_Vector Vv) Instruction Type: CVI_VX_LATE Execution Slots: SLOT23
5316 ========================================================================== */
5317
5318#define Q6_Vbf_vmax_VbfVbf(Vu, Vv) \
5319 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmax_bf)(Vu, Vv)
5320#endif /* __HEXAGON_ARCH___ >= 73 */
5321
5322#if __HVX_ARCH__ >= 73
5323/* ==========================================================================
5324 Assembly Syntax: Vd32.bf=vmin(Vu32.bf,Vv32.bf)
5325 C Intrinsic Prototype: HVX_Vector Q6_Vbf_vmin_VbfVbf(HVX_Vector Vu,
5326 HVX_Vector Vv) Instruction Type: CVI_VX_LATE Execution Slots: SLOT23
5327 ========================================================================== */
5328
5329#define Q6_Vbf_vmin_VbfVbf(Vu, Vv) \
5330 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmin_bf)(Vu, Vv)
5331#endif /* __HEXAGON_ARCH___ >= 73 */
5332
5333#if __HVX_ARCH__ >= 73
5334/* ==========================================================================
5335 Assembly Syntax: Vdd32.sf=vmpy(Vu32.bf,Vv32.bf)
5336 C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpy_VbfVbf(HVX_Vector Vu,
5337 HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5338 ========================================================================== */
5339
5340#define Q6_Wsf_vmpy_VbfVbf(Vu, Vv) \
5341 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_bf)(Vu, Vv)
5342#endif /* __HEXAGON_ARCH___ >= 73 */
5343
5344#if __HVX_ARCH__ >= 73
5345/* ==========================================================================
5346 Assembly Syntax: Vxx32.sf+=vmpy(Vu32.bf,Vv32.bf)
5347 C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpyacc_WsfVbfVbf(HVX_VectorPair
5348 Vxx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution
5349 Slots: SLOT23
5350 ========================================================================== */
5351
5352#define Q6_Wsf_vmpyacc_WsfVbfVbf(Vxx, Vu, Vv) \
5353 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_bf_acc)(Vxx, Vu, Vv)
5354#endif /* __HEXAGON_ARCH___ >= 73 */
5355
5356#if __HVX_ARCH__ >= 73
5357/* ==========================================================================
5358 Assembly Syntax: Vdd32.sf=vsub(Vu32.bf,Vv32.bf)
5359 C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vsub_VbfVbf(HVX_Vector Vu,
5360 HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5361 ========================================================================== */
5362
5363#define Q6_Wsf_vsub_VbfVbf(Vu, Vv) \
5364 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_sf_bf)(Vu, Vv)
5365#endif /* __HEXAGON_ARCH___ >= 73 */
5366
5367#if __HVX_ARCH__ >= 79
5368/* ==========================================================================
5369 Assembly Syntax: Vd32=vgetqfext(Vu32.x,Rt32)
5370 C Intrinsic Prototype: HVX_Vector Q6_V_vgetqfext_VR(HVX_Vector Vu, Word32 Rt)
5371 Instruction Type: CVI_VX
5372 Execution Slots: SLOT23
5373 ========================================================================== */
5374
5375#define Q6_V_vgetqfext_VR(Vu, Rt) \
5376 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_get_qfext)(Vu, Rt)
5377#endif /* __HEXAGON_ARCH___ >= 79 */
5378
5379#if __HVX_ARCH__ >= 79
5380/* ==========================================================================
5381 Assembly Syntax: Vx32|=vgetqfext(Vu32.x,Rt32)
5382 C Intrinsic Prototype: HVX_Vector Q6_V_vgetqfextor_VVR(HVX_Vector Vx,
5383 HVX_Vector Vu, Word32 Rt) Instruction Type: CVI_VX Execution Slots:
5384 SLOT23
5385 ========================================================================== */
5386
5387#define Q6_V_vgetqfextor_VVR(Vx, Vu, Rt) \
5388 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_get_qfext_oracc)(Vx, Vu, Rt)
5389#endif /* __HEXAGON_ARCH___ >= 79 */
5390
5391#if __HVX_ARCH__ >= 79
5392/* ==========================================================================
5393 Assembly Syntax: Vd32.x=vsetqfext(Vu32,Rt32)
5394 C Intrinsic Prototype: HVX_Vector Q6_V_vsetqfext_VR(HVX_Vector Vu, Word32 Rt)
5395 Instruction Type: CVI_VX
5396 Execution Slots: SLOT23
5397 ========================================================================== */
5398
5399#define Q6_V_vsetqfext_VR(Vu, Rt) \
5400 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_set_qfext)(Vu, Rt)
5401#endif /* __HEXAGON_ARCH___ >= 79 */
5402
5403#if __HVX_ARCH__ >= 79
5404/* ==========================================================================
5405 Assembly Syntax: Vd32.f8=vabs(Vu32.f8)
5406 C Intrinsic Prototype: HVX_Vector Q6_V_vabs_V(HVX_Vector Vu)
5407 Instruction Type: CVI_VX_LATE
5408 Execution Slots: SLOT23
5409 ========================================================================== */
5410
5411#define Q6_V_vabs_V(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabs_f8)(Vu)
5412#endif /* __HEXAGON_ARCH___ >= 79 */
5413
5414#if __HVX_ARCH__ >= 79
5415/* ==========================================================================
5416 Assembly Syntax: Vdd32.hf=vadd(Vu32.f8,Vv32.f8)
5417 C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vadd_VV(HVX_Vector Vu,
5418 HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5419 ========================================================================== */
5420
5421#define Q6_Whf_vadd_VV(Vu, Vv) \
5422 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_hf_f8)(Vu, Vv)
5423#endif /* __HEXAGON_ARCH___ >= 79 */
5424
5425#if __HVX_ARCH__ >= 79
5426/* ==========================================================================
5427 Assembly Syntax: Vd32.b=vcvt2(Vu32.hf,Vv32.hf)
5428 C Intrinsic Prototype: HVX_Vector Q6_Vb_vcvt2_VhfVhf(HVX_Vector Vu,
5429 HVX_Vector Vv) Instruction Type: CVI_VX Execution Slots: SLOT23
5430 ========================================================================== */
5431
5432#define Q6_Vb_vcvt2_VhfVhf(Vu, Vv) \
5433 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt2_b_hf)(Vu, Vv)
5434#endif /* __HEXAGON_ARCH___ >= 79 */
5435
5436#if __HVX_ARCH__ >= 79
5437/* ==========================================================================
5438 Assembly Syntax: Vdd32.hf=vcvt2(Vu32.b)
5439 C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt2_Vb(HVX_Vector Vu)
5440 Instruction Type: CVI_VX_DV
5441 Execution Slots: SLOT23
5442 ========================================================================== */
5443
5444#define Q6_Whf_vcvt2_Vb(Vu) \
5445 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt2_hf_b)(Vu)
5446#endif /* __HEXAGON_ARCH___ >= 79 */
5447
5448#if __HVX_ARCH__ >= 79
5449/* ==========================================================================
5450 Assembly Syntax: Vdd32.hf=vcvt2(Vu32.ub)
5451 C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt2_Vub(HVX_Vector Vu)
5452 Instruction Type: CVI_VX_DV
5453 Execution Slots: SLOT23
5454 ========================================================================== */
5455
5456#define Q6_Whf_vcvt2_Vub(Vu) \
5457 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt2_hf_ub)(Vu)
5458#endif /* __HEXAGON_ARCH___ >= 79 */
5459
5460#if __HVX_ARCH__ >= 79
5461/* ==========================================================================
5462 Assembly Syntax: Vd32.ub=vcvt2(Vu32.hf,Vv32.hf)
5463 C Intrinsic Prototype: HVX_Vector Q6_Vub_vcvt2_VhfVhf(HVX_Vector Vu,
5464 HVX_Vector Vv) Instruction Type: CVI_VX Execution Slots: SLOT23
5465 ========================================================================== */
5466
5467#define Q6_Vub_vcvt2_VhfVhf(Vu, Vv) \
5468 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt2_ub_hf)(Vu, Vv)
5469#endif /* __HEXAGON_ARCH___ >= 79 */
5470
5471#if __HVX_ARCH__ >= 79
5472/* ==========================================================================
5473 Assembly Syntax: Vd32.f8=vcvt(Vu32.hf,Vv32.hf)
5474 C Intrinsic Prototype: HVX_Vector Q6_V_vcvt_VhfVhf(HVX_Vector Vu, HVX_Vector
5475 Vv) Instruction Type: CVI_VX Execution Slots: SLOT23
5476 ========================================================================== */
5477
5478#define Q6_V_vcvt_VhfVhf(Vu, Vv) \
5479 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_f8_hf)(Vu, Vv)
5480#endif /* __HEXAGON_ARCH___ >= 79 */
5481
5482#if __HVX_ARCH__ >= 79
5483/* ==========================================================================
5484 Assembly Syntax: Vdd32.hf=vcvt(Vu32.f8)
5485 C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt_V(HVX_Vector Vu)
5486 Instruction Type: CVI_VX_DV
5487 Execution Slots: SLOT23
5488 ========================================================================== */
5489
5490#define Q6_Whf_vcvt_V(Vu) \
5491 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_f8)(Vu)
5492#endif /* __HEXAGON_ARCH___ >= 79 */
5493
5494#if __HVX_ARCH__ >= 79
5495/* ==========================================================================
5496 Assembly Syntax: Vd32.f8=vfmax(Vu32.f8,Vv32.f8)
5497 C Intrinsic Prototype: HVX_Vector Q6_V_vfmax_VV(HVX_Vector Vu, HVX_Vector Vv)
5498 Instruction Type: CVI_VX_LATE
5499 Execution Slots: SLOT23
5500 ========================================================================== */
5501
5502#define Q6_V_vfmax_VV(Vu, Vv) \
5503 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmax_f8)(Vu, Vv)
5504#endif /* __HEXAGON_ARCH___ >= 79 */
5505
5506#if __HVX_ARCH__ >= 79
5507/* ==========================================================================
5508 Assembly Syntax: Vd32.f8=vfmin(Vu32.f8,Vv32.f8)
5509 C Intrinsic Prototype: HVX_Vector Q6_V_vfmin_VV(HVX_Vector Vu, HVX_Vector Vv)
5510 Instruction Type: CVI_VX_LATE
5511 Execution Slots: SLOT23
5512 ========================================================================== */
5513
5514#define Q6_V_vfmin_VV(Vu, Vv) \
5515 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmin_f8)(Vu, Vv)
5516#endif /* __HEXAGON_ARCH___ >= 79 */
5517
5518#if __HVX_ARCH__ >= 79
5519/* ==========================================================================
5520 Assembly Syntax: Vd32.f8=vfneg(Vu32.f8)
5521 C Intrinsic Prototype: HVX_Vector Q6_V_vfneg_V(HVX_Vector Vu)
5522 Instruction Type: CVI_VX_LATE
5523 Execution Slots: SLOT23
5524 ========================================================================== */
5525
5526#define Q6_V_vfneg_V(Vu) \
5527 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfneg_f8)(Vu)
5528#endif /* __HEXAGON_ARCH___ >= 79 */
5529
5530#if __HVX_ARCH__ >= 79
5531/* ==========================================================================
5532 Assembly Syntax: Vd32=vmerge(Vu32.x,Vv32.w)
5533 C Intrinsic Prototype: HVX_Vector Q6_V_vmerge_VVw(HVX_Vector Vu, HVX_Vector
5534 Vv) Instruction Type: CVI_VS Execution Slots: SLOT0123
5535 ========================================================================== */
5536
5537#define Q6_V_vmerge_VVw(Vu, Vv) \
5538 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmerge_qf)(Vu, Vv)
5539#endif /* __HEXAGON_ARCH___ >= 79 */
5540
5541#if __HVX_ARCH__ >= 79
5542/* ==========================================================================
5543 Assembly Syntax: Vdd32.hf=vmpy(Vu32.f8,Vv32.f8)
5544 C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vmpy_VV(HVX_Vector Vu,
5545 HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5546 ========================================================================== */
5547
5548#define Q6_Whf_vmpy_VV(Vu, Vv) \
5549 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_hf_f8)(Vu, Vv)
5550#endif /* __HEXAGON_ARCH___ >= 79 */
5551
5552#if __HVX_ARCH__ >= 79
5553/* ==========================================================================
5554 Assembly Syntax: Vxx32.hf+=vmpy(Vu32.f8,Vv32.f8)
5555 C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vmpyacc_WhfVV(HVX_VectorPair
5556 Vxx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution
5557 Slots: SLOT23
5558 ========================================================================== */
5559
5560#define Q6_Whf_vmpyacc_WhfVV(Vxx, Vu, Vv) \
5561 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_hf_f8_acc)(Vxx, Vu, Vv)
5562#endif /* __HEXAGON_ARCH___ >= 79 */
5563
5564#if __HVX_ARCH__ >= 79
5565/* ==========================================================================
5566 Assembly Syntax: Vd32.qf16=vmpy(Vu32.hf,Rt32.hf)
5567 C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_VhfRhf(HVX_Vector Vu, Word32
5568 Rt) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5569 ========================================================================== */
5570
5571#define Q6_Vqf16_vmpy_VhfRhf(Vu, Rt) \
5572 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_rt_hf)(Vu, Rt)
5573#endif /* __HEXAGON_ARCH___ >= 79 */
5574
5575#if __HVX_ARCH__ >= 79
5576/* ==========================================================================
5577 Assembly Syntax: Vd32.qf16=vmpy(Vu32.qf16,Rt32.hf)
5578 C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_Vqf16Rhf(HVX_Vector Vu,
5579 Word32 Rt) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5580 ========================================================================== */
5581
5582#define Q6_Vqf16_vmpy_Vqf16Rhf(Vu, Rt) \
5583 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_rt_qf16)(Vu, Rt)
5584#endif /* __HEXAGON_ARCH___ >= 79 */
5585
5586#if __HVX_ARCH__ >= 79
5587/* ==========================================================================
5588 Assembly Syntax: Vd32.qf32=vmpy(Vu32.sf,Rt32.sf)
5589 C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vmpy_VsfRsf(HVX_Vector Vu, Word32
5590 Rt) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5591 ========================================================================== */
5592
5593#define Q6_Vqf32_vmpy_VsfRsf(Vu, Rt) \
5594 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_rt_sf)(Vu, Rt)
5595#endif /* __HEXAGON_ARCH___ >= 79 */
5596
5597#if __HVX_ARCH__ >= 79
5598/* ==========================================================================
5599 Assembly Syntax: Vdd32.hf=vsub(Vu32.f8,Vv32.f8)
5600 C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vsub_VV(HVX_Vector Vu,
5601 HVX_Vector Vv) Instruction Type: CVI_VX_DV Execution Slots: SLOT23
5602 ========================================================================== */
5603
5604#define Q6_Whf_vsub_VV(Vu, Vv) \
5605 __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_hf_f8)(Vu, Vv)
5606#endif /* __HEXAGON_ARCH___ >= 79 */
5607
51815608#endif /* __HVX__ */
51825609
51835610#endif
lib/include/immintrin.h+87-3
......@@ -605,6 +605,20 @@ _storebe_i64(void * __P, long long __D) {
605605#include <movdirintrin.h>
606606#endif
607607
608#if !defined(__SCE__) || __has_feature(modules) || defined(__MOVRS__)
609#include <movrsintrin.h>
610#endif
611
612#if !defined(__SCE__) || __has_feature(modules) || \
613 (defined(__AVX10_2__) && defined(__MOVRS__))
614#include <movrs_avx10_2intrin.h>
615#endif
616
617#if !defined(__SCE__) || __has_feature(modules) || \
618 (defined(__AVX10_2_512__) && defined(__MOVRS__))
619#include <movrs_avx10_2_512intrin.h>
620#endif
621
608622#if !defined(__SCE__) || __has_feature(modules) || defined(__PCONFIG__)
609623#include <pconfigintrin.h>
610624#endif
......@@ -620,9 +634,6 @@ _storebe_i64(void * __P, long long __D) {
620634#if !defined(__SCE__) || __has_feature(modules) || defined(__INVPCID__)
621635#include <invpcidintrin.h>
622636#endif
623#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_FP16__)
624#include <amxfp16intrin.h>
625#endif
626637
627638#if !defined(__SCE__) || __has_feature(modules) || defined(__KL__) || \
628639 defined(__WIDEKL__)
......@@ -634,10 +645,59 @@ _storebe_i64(void * __P, long long __D) {
634645#include <amxintrin.h>
635646#endif
636647
648#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_FP16__)
649#include <amxfp16intrin.h>
650#endif
651
637652#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_COMPLEX__)
638653#include <amxcomplexintrin.h>
639654#endif
640655
656#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_FP8__)
657#include <amxfp8intrin.h>
658#endif
659
660#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_TRANSPOSE__)
661#include <amxtransposeintrin.h>
662#endif
663
664#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_MOVRS__)
665#include <amxmovrsintrin.h>
666#endif
667
668#if !defined(__SCE__) || __has_feature(modules) || \
669 (defined(__AMX_MOVRS__) && defined(__AMX_TRANSPOSE__))
670#include <amxmovrstransposeintrin.h>
671#endif
672
673#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_AVX512__)
674#include <amxavx512intrin.h>
675#endif
676
677#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_TF32__)
678#include <amxtf32intrin.h>
679#endif
680
681#if !defined(__SCE__) || __has_feature(modules) || \
682 (defined(__AMX_TF32__) && defined(__AMX_TRANSPOSE__))
683#include <amxtf32transposeintrin.h>
684#endif
685
686#if !defined(__SCE__) || __has_feature(modules) || \
687 (defined(__AMX_BF16__) && defined(__AMX_TRANSPOSE__))
688#include <amxbf16transposeintrin.h>
689#endif
690
691#if !defined(__SCE__) || __has_feature(modules) || \
692 (defined(__AMX_FP16__) && defined(__AMX_TRANSPOSE__))
693#include <amxfp16transposeintrin.h>
694#endif
695
696#if !defined(__SCE__) || __has_feature(modules) || \
697 (defined(__AMX_COMPLEX__) && defined(__AMX_TRANSPOSE__))
698#include <amxcomplextransposeintrin.h>
699#endif
700
641701#if !defined(__SCE__) || __has_feature(modules) || \
642702 defined(__AVX512VP2INTERSECT__)
643703#include <avx512vp2intersectintrin.h>
......@@ -648,6 +708,30 @@ _storebe_i64(void * __P, long long __D) {
648708#include <avx512vlvp2intersectintrin.h>
649709#endif
650710
711#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX10_2__)
712#include <avx10_2bf16intrin.h>
713#include <avx10_2convertintrin.h>
714#include <avx10_2copyintrin.h>
715#include <avx10_2minmaxintrin.h>
716#include <avx10_2niintrin.h>
717#include <avx10_2satcvtdsintrin.h>
718#include <avx10_2satcvtintrin.h>
719#endif
720
721#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX10_2_512__)
722#include <avx10_2_512bf16intrin.h>
723#include <avx10_2_512convertintrin.h>
724#include <avx10_2_512minmaxintrin.h>
725#include <avx10_2_512niintrin.h>
726#include <avx10_2_512satcvtdsintrin.h>
727#include <avx10_2_512satcvtintrin.h>
728#endif
729
730#if !defined(__SCE__) || __has_feature(modules) || \
731 (defined(__AVX10_2_512__) && defined(__SM4__))
732#include <sm4evexintrin.h>
733#endif
734
651735#if !defined(__SCE__) || __has_feature(modules) || defined(__ENQCMD__)
652736#include <enqcmdintrin.h>
653737#endif
lib/include/intrin.h+20-10
......@@ -94,8 +94,8 @@ void __outwordstring(unsigned short, unsigned short *, unsigned long);
9494unsigned long __readcr0(void);
9595unsigned long __readcr2(void);
9696unsigned __LPTRINT_TYPE__ __readcr3(void);
97unsigned long __readcr4(void);
98unsigned long __readcr8(void);
97unsigned __LPTRINT_TYPE__ __readcr4(void);
98unsigned __int64 __readcr8(void);
9999unsigned int __readdr(unsigned int);
100100#ifdef __i386__
101101unsigned char __readfsbyte(unsigned long);
......@@ -124,8 +124,8 @@ void __vmx_vmptrst(unsigned __int64 *);
124124void __wbinvd(void);
125125void __writecr0(unsigned int);
126126void __writecr3(unsigned __INTPTR_TYPE__);
127void __writecr4(unsigned int);
128void __writecr8(unsigned int);
127void __writecr4(unsigned __INTPTR_TYPE__);
128void __writecr8(unsigned __int64);
129129void __writedr(unsigned int, unsigned int);
130130void __writefsbyte(unsigned long, unsigned char);
131131void __writefsdword(unsigned long, unsigned long);
......@@ -330,33 +330,33 @@ static __inline__ void __DEFAULT_FN_ATTRS __halt(void) {
330330 __asm__ volatile("hlt");
331331}
332332
333static inline unsigned char __inbyte(unsigned short port) {
333static __inline__ unsigned char __inbyte(unsigned short port) {
334334 unsigned char ret;
335335 __asm__ __volatile__("inb %w1, %b0" : "=a"(ret) : "Nd"(port));
336336 return ret;
337337}
338338
339static inline unsigned short __inword(unsigned short port) {
339static __inline__ unsigned short __inword(unsigned short port) {
340340 unsigned short ret;
341341 __asm__ __volatile__("inw %w1, %w0" : "=a"(ret) : "Nd"(port));
342342 return ret;
343343}
344344
345static inline unsigned long __indword(unsigned short port) {
345static __inline__ unsigned long __indword(unsigned short port) {
346346 unsigned long ret;
347347 __asm__ __volatile__("inl %w1, %k0" : "=a"(ret) : "Nd"(port));
348348 return ret;
349349}
350350
351static inline void __outbyte(unsigned short port, unsigned char data) {
351static __inline__ void __outbyte(unsigned short port, unsigned char data) {
352352 __asm__ __volatile__("outb %b0, %w1" : : "a"(data), "Nd"(port));
353353}
354354
355static inline void __outword(unsigned short port, unsigned short data) {
355static __inline__ void __outword(unsigned short port, unsigned short data) {
356356 __asm__ __volatile__("outw %w0, %w1" : : "a"(data), "Nd"(port));
357357}
358358
359static inline void __outdword(unsigned short port, unsigned long data) {
359static __inline__ void __outdword(unsigned short port, unsigned long data) {
360360 __asm__ __volatile__("outl %k0, %w1" : : "a"(data), "Nd"(port));
361361}
362362#endif
......@@ -396,6 +396,16 @@ unsigned short __readx18word(unsigned long offset);
396396unsigned long __readx18dword(unsigned long offset);
397397unsigned __int64 __readx18qword(unsigned long offset);
398398
399void __addx18byte(unsigned long offset, unsigned char data);
400void __addx18word(unsigned long offset, unsigned short data);
401void __addx18dword(unsigned long offset, unsigned long data);
402void __addx18qword(unsigned long offset, unsigned __int64 data);
403
404void __incx18byte(unsigned long offset);
405void __incx18word(unsigned long offset);
406void __incx18dword(unsigned long offset);
407void __incx18qword(unsigned long offset);
408
399409double _CopyDoubleFromInt64(__int64);
400410float _CopyFloatFromInt32(__int32);
401411__int32 _CopyInt32FromFloat(float);
lib/include/intrin0.h+10-1
......@@ -44,7 +44,7 @@ unsigned char _InterlockedCompareExchange128_rel(__int64 volatile *_Destination,
4444 __int64 *_ComparandResult);
4545#endif
4646
47#ifdef __x86_64__ && !defined(__arm64ec__)
47#if defined(__x86_64__) && !defined(__arm64ec__)
4848unsigned __int64 _umul128(unsigned __int64, unsigned __int64,
4949 unsigned __int64 *);
5050unsigned __int64 __shiftleft128(unsigned __int64 _LowPart,
......@@ -207,6 +207,9 @@ long _InterlockedExchange_rel(long volatile *_Target, long _Value);
207207__int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value);
208208__int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value);
209209__int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value);
210void *_InterlockedExchangePointer_acq(void *volatile *_Target, void *_Value);
211void *_InterlockedExchangePointer_nf(void *volatile *_Target, void *_Value);
212void *_InterlockedExchangePointer_rel(void *volatile *_Target, void *_Value);
210213
211214/*----------------------------------------------------------------------------*\
212215|* Interlocked Compare Exchange
......@@ -237,6 +240,12 @@ __int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination,
237240__int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination,
238241 __int64 _Exchange,
239242 __int64 _Comparand);
243void *_InterlockedCompareExchangePointer_acq(void *volatile *_Destination,
244 void *_Exchange, void *_Comparand);
245void *_InterlockedCompareExchangePointer_nf(void *volatile *_Destination,
246 void *_Exchange, void *_Comparand);
247void *_InterlockedCompareExchangePointer_rel(void *volatile *_Destination,
248 void *_Exchange, void *_Comparand);
240249#endif
241250
242251#ifdef __cplusplus
lib/include/larchintrin.h+22-8
......@@ -228,17 +228,31 @@ extern __inline void
228228 ((void)__builtin_loongarch_ldpte_d((long int)(_1), (_2)))
229229#endif
230230
231#define __frecipe_s(/*float*/ _1) \
232 (float)__builtin_loongarch_frecipe_s((float)_1)
231#ifdef __loongarch_frecipe
232extern __inline float
233 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
234 __frecipe_s(float _1) {
235 return __builtin_loongarch_frecipe_s(_1);
236}
233237
234#define __frecipe_d(/*double*/ _1) \
235 (double)__builtin_loongarch_frecipe_d((double)_1)
238extern __inline double
239 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
240 __frecipe_d(double _1) {
241 return __builtin_loongarch_frecipe_d(_1);
242}
236243
237#define __frsqrte_s(/*float*/ _1) \
238 (float)__builtin_loongarch_frsqrte_s((float)_1)
244extern __inline float
245 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
246 __frsqrte_s(float _1) {
247 return __builtin_loongarch_frsqrte_s(_1);
248}
239249
240#define __frsqrte_d(/*double*/ _1) \
241 (double)__builtin_loongarch_frsqrte_d((double)_1)
250extern __inline double
251 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
252 __frsqrte_d(double _1) {
253 return __builtin_loongarch_frsqrte_d(_1);
254}
255#endif
242256
243257#ifdef __cplusplus
244258}
lib/include/lasxintrin.h+27-25
......@@ -1726,18 +1726,6 @@ extern __inline
17261726 return (__m256d)__builtin_lasx_xvfrecip_d((v4f64)_1);
17271727}
17281728
1729extern __inline
1730 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
1731 __lasx_xvfrecipe_s(__m256 _1) {
1732 return (__m256)__builtin_lasx_xvfrecipe_s((v8f32)_1);
1733}
1734
1735extern __inline
1736 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256d
1737 __lasx_xvfrecipe_d(__m256d _1) {
1738 return (__m256d)__builtin_lasx_xvfrecipe_d((v4f64)_1);
1739}
1740
17411729extern __inline
17421730 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
17431731 __lasx_xvfrint_s(__m256 _1) {
......@@ -1762,18 +1750,6 @@ extern __inline
17621750 return (__m256d)__builtin_lasx_xvfrsqrt_d((v4f64)_1);
17631751}
17641752
1765extern __inline
1766 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
1767 __lasx_xvfrsqrte_s(__m256 _1) {
1768 return (__m256)__builtin_lasx_xvfrsqrte_s((v8f32)_1);
1769}
1770
1771extern __inline
1772 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256d
1773 __lasx_xvfrsqrte_d(__m256d _1) {
1774 return (__m256d)__builtin_lasx_xvfrsqrte_d((v4f64)_1);
1775}
1776
17771753extern __inline
17781754 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
17791755 __lasx_xvflogb_s(__m256 _1) {
......@@ -2585,7 +2561,7 @@ extern __inline
25852561extern __inline
25862562 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256i
25872563 __lasx_xvorn_v(__m256i _1, __m256i _2) {
2588 return (__m256i)__builtin_lasx_xvorn_v((v32i8)_1, (v32i8)_2);
2564 return (__m256i)__builtin_lasx_xvorn_v((v32u8)_1, (v32u8)_2);
25892565}
25902566
25912567#define __lasx_xvldi(/*i13*/ _1) ((__m256i)__builtin_lasx_xvldi((_1)))
......@@ -3866,6 +3842,32 @@ extern __inline
38663842 return (__m256i)__builtin_lasx_xvfcmp_sun_s((v8f32)_1, (v8f32)_2);
38673843}
38683844
3845#if defined(__loongarch_frecipe)
3846extern __inline
3847 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
3848 __lasx_xvfrecipe_s(__m256 _1) {
3849 return (__m256)__builtin_lasx_xvfrecipe_s((v8f32)_1);
3850}
3851
3852extern __inline
3853 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256d
3854 __lasx_xvfrecipe_d(__m256d _1) {
3855 return (__m256d)__builtin_lasx_xvfrecipe_d((v4f64)_1);
3856}
3857
3858extern __inline
3859 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
3860 __lasx_xvfrsqrte_s(__m256 _1) {
3861 return (__m256)__builtin_lasx_xvfrsqrte_s((v8f32)_1);
3862}
3863
3864extern __inline
3865 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256d
3866 __lasx_xvfrsqrte_d(__m256d _1) {
3867 return (__m256d)__builtin_lasx_xvfrsqrte_d((v4f64)_1);
3868}
3869#endif
3870
38693871#define __lasx_xvpickve_d_f(/*__m256d*/ _1, /*ui2*/ _2) \
38703872 ((__m256d)__builtin_lasx_xvpickve_d_f((v4f64)(_1), (_2)))
38713873
lib/include/limits.h+7-4
......@@ -111,11 +111,14 @@
111111#define ULLONG_MAX (__LONG_LONG_MAX__*2ULL+1ULL)
112112#endif
113113
114/* LONG_LONG_MIN/LONG_LONG_MAX/ULONG_LONG_MAX are a GNU extension. It's too bad
115 that we don't have something like #pragma poison that could be used to
116 deprecate a macro - the code should just use LLONG_MAX and friends.
114/* LONG_LONG_MIN/LONG_LONG_MAX/ULONG_LONG_MAX are a GNU extension. Android's
115 bionic also defines them. It's too bad that we don't have something like
116 #pragma poison that could be used to deprecate a macro - the code should just
117 use LLONG_MAX and friends.
117118 */
118#if defined(__GNU_LIBRARY__) ? defined(__USE_GNU) : !defined(__STRICT_ANSI__)
119#if (defined(__GNU_LIBRARY__) ? defined(__USE_GNU) \
120 : !defined(__STRICT_ANSI__)) || \
121 defined(__BIONIC__)
119122
120123#undef LONG_LONG_MIN
121124#undef LONG_LONG_MAX
lib/include/llvm_libc_wrappers/ctype.h+38
......@@ -51,6 +51,19 @@
5151#pragma push_macro("toascii")
5252#pragma push_macro("tolower")
5353#pragma push_macro("toupper")
54#pragma push_macro("isalnum_l")
55#pragma push_macro("isalpha_l")
56#pragma push_macro("isascii_l")
57#pragma push_macro("isblank_l")
58#pragma push_macro("iscntrl_l")
59#pragma push_macro("isdigit_l")
60#pragma push_macro("isgraph_l")
61#pragma push_macro("islower_l")
62#pragma push_macro("isprint_l")
63#pragma push_macro("ispunct_l")
64#pragma push_macro("isspace_l")
65#pragma push_macro("isupper_l")
66#pragma push_macro("isxdigit_l")
5467
5568#undef isalnum
5669#undef isalpha
......@@ -68,6 +81,18 @@
6881#undef toascii
6982#undef tolower
7083#undef toupper
84#undef isalnum_l
85#undef isalpha_l
86#undef iscntrl_l
87#undef isdigit_l
88#undef islower_l
89#undef isgraph_l
90#undef isprint_l
91#undef ispunct_l
92#undef isspace_l
93#undef isupper_l
94#undef isblank_l
95#undef isxdigit_l
7196
7297#pragma omp begin declare target
7398
......@@ -93,6 +118,19 @@
93118#pragma pop_macro("toascii")
94119#pragma pop_macro("tolower")
95120#pragma pop_macro("toupper")
121#pragma pop_macro("isalnum_l")
122#pragma pop_macro("isalpha_l")
123#pragma pop_macro("isascii_l")
124#pragma pop_macro("isblank_l")
125#pragma pop_macro("iscntrl_l")
126#pragma pop_macro("isdigit_l")
127#pragma pop_macro("isgraph_l")
128#pragma pop_macro("islower_l")
129#pragma pop_macro("isprint_l")
130#pragma pop_macro("ispunct_l")
131#pragma pop_macro("isspace_l")
132#pragma pop_macro("isupper_l")
133#pragma pop_macro("isxdigit_l")
96134#endif
97135
98136#undef __LIBC_ATTRS
lib/include/llvm_libc_wrappers/stdlib.h+8
......@@ -34,8 +34,16 @@ _Static_assert(__builtin_offsetof(div_t, quot) == 0, "ABI mismatch!");
3434_Static_assert(__builtin_offsetof(ldiv_t, quot) == 0, "ABI mismatch!");
3535_Static_assert(__builtin_offsetof(lldiv_t, quot) == 0, "ABI mismatch!");
3636
37#if defined(__GLIBC__) && __cplusplus >= 201703L
38#define at_quick_exit atexit
39#endif
40
3741#include <llvm-libc-decls/stdlib.h>
3842
43#if defined(__GLIBC__) && __cplusplus >= 201703L
44#undef at_quick_exit
45#endif
46
3947#pragma omp end declare target
4048
4149#undef __LIBC_ATTRS
lib/include/lsxintrin.h+27-25
......@@ -1776,18 +1776,6 @@ extern __inline
17761776 return (__m128d)__builtin_lsx_vfrecip_d((v2f64)_1);
17771777}
17781778
1779extern __inline
1780 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
1781 __lsx_vfrecipe_s(__m128 _1) {
1782 return (__m128)__builtin_lsx_vfrecipe_s((v4f32)_1);
1783}
1784
1785extern __inline
1786 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128d
1787 __lsx_vfrecipe_d(__m128d _1) {
1788 return (__m128d)__builtin_lsx_vfrecipe_d((v2f64)_1);
1789}
1790
17911779extern __inline
17921780 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
17931781 __lsx_vfrint_s(__m128 _1) {
......@@ -1812,18 +1800,6 @@ extern __inline
18121800 return (__m128d)__builtin_lsx_vfrsqrt_d((v2f64)_1);
18131801}
18141802
1815extern __inline
1816 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
1817 __lsx_vfrsqrte_s(__m128 _1) {
1818 return (__m128)__builtin_lsx_vfrsqrte_s((v4f32)_1);
1819}
1820
1821extern __inline
1822 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128d
1823 __lsx_vfrsqrte_d(__m128d _1) {
1824 return (__m128d)__builtin_lsx_vfrsqrte_d((v2f64)_1);
1825}
1826
18271803extern __inline
18281804 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
18291805 __lsx_vflogb_s(__m128 _1) {
......@@ -3425,7 +3401,7 @@ extern __inline
34253401extern __inline
34263402 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128i
34273403 __lsx_vorn_v(__m128i _1, __m128i _2) {
3428 return (__m128i)__builtin_lsx_vorn_v((v16i8)_1, (v16i8)_2);
3404 return (__m128i)__builtin_lsx_vorn_v((v16u8)_1, (v16u8)_2);
34293405}
34303406
34313407#define __lsx_vldi(/*i13*/ _1) ((__m128i)__builtin_lsx_vldi((_1)))
......@@ -3738,6 +3714,32 @@ extern __inline
37383714 return (__m128i)__builtin_lsx_vfcmp_sun_s((v4f32)_1, (v4f32)_2);
37393715}
37403716
3717#if defined(__loongarch_frecipe)
3718extern __inline
3719 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
3720 __lsx_vfrecipe_s(__m128 _1) {
3721 return (__m128)__builtin_lsx_vfrecipe_s((v4f32)_1);
3722}
3723
3724extern __inline
3725 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128d
3726 __lsx_vfrecipe_d(__m128d _1) {
3727 return (__m128d)__builtin_lsx_vfrecipe_d((v2f64)_1);
3728}
3729
3730extern __inline
3731 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
3732 __lsx_vfrsqrte_s(__m128 _1) {
3733 return (__m128)__builtin_lsx_vfrsqrte_s((v4f32)_1);
3734}
3735
3736extern __inline
3737 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128d
3738 __lsx_vfrsqrte_d(__m128d _1) {
3739 return (__m128d)__builtin_lsx_vfrsqrte_d((v2f64)_1);
3740}
3741#endif
3742
37413743#define __lsx_vrepli_b(/*si10*/ _1) ((__m128i)__builtin_lsx_vrepli_b((_1)))
37423744
37433745#define __lsx_vrepli_d(/*si10*/ _1) ((__m128i)__builtin_lsx_vrepli_d((_1)))
lib/include/lzcntintrin.h+10-7
......@@ -15,7 +15,13 @@
1515#define __LZCNTINTRIN_H
1616
1717/* Define the default attributes for the functions in this file. */
18#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("lzcnt")))
18#if defined(__cplusplus) && (__cplusplus >= 201103L)
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, __target__("lzcnt"))) constexpr
21#else
22#define __DEFAULT_FN_ATTRS \
23 __attribute__((__always_inline__, __nodebug__, __target__("lzcnt")))
24#endif
1925
2026#ifndef _MSC_VER
2127/// Counts the number of leading zero bits in the operand.
......@@ -43,8 +49,7 @@
4349/// bits in the operand.
4450/// \see _lzcnt_u32
4551static __inline__ unsigned int __DEFAULT_FN_ATTRS
46__lzcnt32(unsigned int __X)
47{
52__lzcnt32(unsigned int __X) {
4853 return __builtin_ia32_lzcnt_u32(__X);
4954}
5055
......@@ -60,8 +65,7 @@ __lzcnt32(unsigned int __X)
6065/// bits in the operand.
6166/// \see __lzcnt32
6267static __inline__ unsigned int __DEFAULT_FN_ATTRS
63_lzcnt_u32(unsigned int __X)
64{
68_lzcnt_u32(unsigned int __X) {
6569 return __builtin_ia32_lzcnt_u32(__X);
6670}
6771
......@@ -93,8 +97,7 @@ _lzcnt_u32(unsigned int __X)
9397/// bits in the operand.
9498/// \see __lzcnt64
9599static __inline__ unsigned long long __DEFAULT_FN_ATTRS
96_lzcnt_u64(unsigned long long __X)
97{
100_lzcnt_u64(unsigned long long __X) {
98101 return __builtin_ia32_lzcnt_u64(__X);
99102}
100103#endif
lib/include/mmintrin.h+209-155
......@@ -21,10 +21,45 @@ typedef int __v2si __attribute__((__vector_size__(8)));
2121typedef short __v4hi __attribute__((__vector_size__(8)));
2222typedef char __v8qi __attribute__((__vector_size__(8)));
2323
24/* Unsigned types */
25typedef unsigned long long __v1du __attribute__ ((__vector_size__ (8)));
26typedef unsigned int __v2su __attribute__ ((__vector_size__ (8)));
27typedef unsigned short __v4hu __attribute__((__vector_size__(8)));
28typedef unsigned char __v8qu __attribute__((__vector_size__(8)));
29
30/* We need an explicitly signed variant for char. Note that this shouldn't
31 * appear in the interface though. */
32typedef signed char __v8qs __attribute__((__vector_size__(8)));
33
34/* SSE/SSE2 types */
35typedef long long __m128i __attribute__((__vector_size__(16), __aligned__(16)));
36typedef long long __v2di __attribute__ ((__vector_size__ (16)));
37typedef int __v4si __attribute__((__vector_size__(16)));
38typedef short __v8hi __attribute__((__vector_size__(16)));
39typedef char __v16qi __attribute__((__vector_size__(16)));
40
2441/* Define the default attributes for the functions in this file. */
25#define __DEFAULT_FN_ATTRS \
26 __attribute__((__always_inline__, __nodebug__, __target__("mmx,no-evex512"), \
27 __min_vector_width__(64)))
42#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
43#define __DEFAULT_FN_ATTRS_SSE2 \
44 __attribute__((__always_inline__, __nodebug__, \
45 __target__("sse2,no-evex512"), __min_vector_width__(128)))
46#else
47#define __DEFAULT_FN_ATTRS_SSE2 \
48 __attribute__((__always_inline__, __nodebug__, __target__("sse2"), \
49 __min_vector_width__(128)))
50#endif
51
52#if defined(__cplusplus) && (__cplusplus >= 201103L)
53#define __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR __DEFAULT_FN_ATTRS_SSE2 constexpr
54#else
55#define __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR __DEFAULT_FN_ATTRS_SSE2
56#endif
57
58#define __trunc64(x) \
59 (__m64) __builtin_shufflevector((__v2di)(x), __extension__(__v2di){}, 0)
60#define __anyext128(x) \
61 (__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0, \
62 1, -1, -1)
2863
2964/// Clears the MMX state by setting the state of the x87 stack registers
3065/// to empty.
......@@ -50,10 +85,10 @@ _mm_empty(void) {
5085/// A 32-bit integer value.
5186/// \returns A 64-bit integer vector. The lower 32 bits contain the value of the
5287/// parameter. The upper 32 bits are set to 0.
53static __inline__ __m64 __DEFAULT_FN_ATTRS
88static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
5489_mm_cvtsi32_si64(int __i)
5590{
56 return (__m64)__builtin_ia32_vec_init_v2si(__i, 0);
91 return __extension__ (__m64)(__v2si){__i, 0};
5792}
5893
5994/// Returns the lower 32 bits of a 64-bit integer vector as a 32-bit
......@@ -67,10 +102,10 @@ _mm_cvtsi32_si64(int __i)
67102/// A 64-bit integer vector.
68103/// \returns A 32-bit signed integer value containing the lower 32 bits of the
69104/// parameter.
70static __inline__ int __DEFAULT_FN_ATTRS
105static __inline__ int __DEFAULT_FN_ATTRS_SSE2
71106_mm_cvtsi64_si32(__m64 __m)
72107{
73 return __builtin_ia32_vec_ext_v2si((__v2si)__m, 0);
108 return ((__v2si)__m)[0];
74109}
75110
76111/// Casts a 64-bit signed integer value into a 64-bit integer vector.
......@@ -83,7 +118,7 @@ _mm_cvtsi64_si32(__m64 __m)
83118/// A 64-bit signed integer.
84119/// \returns A 64-bit integer vector containing the same bitwise pattern as the
85120/// parameter.
86static __inline__ __m64 __DEFAULT_FN_ATTRS
121static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
87122_mm_cvtsi64_m64(long long __i)
88123{
89124 return (__m64)__i;
......@@ -99,7 +134,7 @@ _mm_cvtsi64_m64(long long __i)
99134/// A 64-bit integer vector.
100135/// \returns A 64-bit signed integer containing the same bitwise pattern as the
101136/// parameter.
102static __inline__ long long __DEFAULT_FN_ATTRS
137static __inline__ long long __DEFAULT_FN_ATTRS_SSE2
103138_mm_cvtm64_si64(__m64 __m)
104139{
105140 return (long long)__m;
......@@ -124,10 +159,11 @@ _mm_cvtm64_si64(__m64 __m)
124159/// written to the upper 32 bits of the result.
125160/// \returns A 64-bit integer vector of [8 x i8] containing the converted
126161/// values.
127static __inline__ __m64 __DEFAULT_FN_ATTRS
162static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
128163_mm_packs_pi16(__m64 __m1, __m64 __m2)
129164{
130 return (__m64)__builtin_ia32_packsswb((__v4hi)__m1, (__v4hi)__m2);
165 return __trunc64(__builtin_ia32_packsswb128(
166 (__v8hi)__builtin_shufflevector(__m1, __m2, 0, 1), (__v8hi){}));
131167}
132168
133169/// Converts, with saturation, 32-bit signed integers from both 64-bit integer
......@@ -149,10 +185,11 @@ _mm_packs_pi16(__m64 __m1, __m64 __m2)
149185/// written to the upper 32 bits of the result.
150186/// \returns A 64-bit integer vector of [4 x i16] containing the converted
151187/// values.
152static __inline__ __m64 __DEFAULT_FN_ATTRS
188static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
153189_mm_packs_pi32(__m64 __m1, __m64 __m2)
154190{
155 return (__m64)__builtin_ia32_packssdw((__v2si)__m1, (__v2si)__m2);
191 return __trunc64(__builtin_ia32_packssdw128(
192 (__v4si)__builtin_shufflevector(__m1, __m2, 0, 1), (__v4si){}));
156193}
157194
158195/// Converts, with saturation, 16-bit signed integers from both 64-bit integer
......@@ -174,10 +211,11 @@ _mm_packs_pi32(__m64 __m1, __m64 __m2)
174211/// written to the upper 32 bits of the result.
175212/// \returns A 64-bit integer vector of [8 x i8] containing the converted
176213/// values.
177static __inline__ __m64 __DEFAULT_FN_ATTRS
214static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
178215_mm_packs_pu16(__m64 __m1, __m64 __m2)
179216{
180 return (__m64)__builtin_ia32_packuswb((__v4hi)__m1, (__v4hi)__m2);
217 return __trunc64(__builtin_ia32_packuswb128(
218 (__v8hi)__builtin_shufflevector(__m1, __m2, 0, 1), (__v8hi){}));
181219}
182220
183221/// Unpacks the upper 32 bits from two 64-bit integer vectors of [8 x i8]
......@@ -201,10 +239,11 @@ _mm_packs_pu16(__m64 __m1, __m64 __m2)
201239/// Bits [63:56] are written to bits [63:56] of the result.
202240/// \returns A 64-bit integer vector of [8 x i8] containing the interleaved
203241/// values.
204static __inline__ __m64 __DEFAULT_FN_ATTRS
242static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
205243_mm_unpackhi_pi8(__m64 __m1, __m64 __m2)
206244{
207 return (__m64)__builtin_ia32_punpckhbw((__v8qi)__m1, (__v8qi)__m2);
245 return (__m64)__builtin_shufflevector((__v8qi)__m1, (__v8qi)__m2,
246 4, 12, 5, 13, 6, 14, 7, 15);
208247}
209248
210249/// Unpacks the upper 32 bits from two 64-bit integer vectors of
......@@ -224,10 +263,11 @@ _mm_unpackhi_pi8(__m64 __m1, __m64 __m2)
224263/// Bits [63:48] are written to bits [63:48] of the result.
225264/// \returns A 64-bit integer vector of [4 x i16] containing the interleaved
226265/// values.
227static __inline__ __m64 __DEFAULT_FN_ATTRS
266static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
228267_mm_unpackhi_pi16(__m64 __m1, __m64 __m2)
229268{
230 return (__m64)__builtin_ia32_punpckhwd((__v4hi)__m1, (__v4hi)__m2);
269 return (__m64)__builtin_shufflevector((__v4hi)__m1, (__v4hi)__m2,
270 2, 6, 3, 7);
231271}
232272
233273/// Unpacks the upper 32 bits from two 64-bit integer vectors of
......@@ -245,10 +285,10 @@ _mm_unpackhi_pi16(__m64 __m1, __m64 __m2)
245285/// the upper 32 bits of the result.
246286/// \returns A 64-bit integer vector of [2 x i32] containing the interleaved
247287/// values.
248static __inline__ __m64 __DEFAULT_FN_ATTRS
288static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
249289_mm_unpackhi_pi32(__m64 __m1, __m64 __m2)
250290{
251 return (__m64)__builtin_ia32_punpckhdq((__v2si)__m1, (__v2si)__m2);
291 return (__m64)__builtin_shufflevector((__v2si)__m1, (__v2si)__m2, 1, 3);
252292}
253293
254294/// Unpacks the lower 32 bits from two 64-bit integer vectors of [8 x i8]
......@@ -272,10 +312,11 @@ _mm_unpackhi_pi32(__m64 __m1, __m64 __m2)
272312/// Bits [31:24] are written to bits [63:56] of the result.
273313/// \returns A 64-bit integer vector of [8 x i8] containing the interleaved
274314/// values.
275static __inline__ __m64 __DEFAULT_FN_ATTRS
315static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
276316_mm_unpacklo_pi8(__m64 __m1, __m64 __m2)
277317{
278 return (__m64)__builtin_ia32_punpcklbw((__v8qi)__m1, (__v8qi)__m2);
318 return (__m64)__builtin_shufflevector((__v8qi)__m1, (__v8qi)__m2,
319 0, 8, 1, 9, 2, 10, 3, 11);
279320}
280321
281322/// Unpacks the lower 32 bits from two 64-bit integer vectors of
......@@ -295,10 +336,11 @@ _mm_unpacklo_pi8(__m64 __m1, __m64 __m2)
295336/// Bits [31:16] are written to bits [63:48] of the result.
296337/// \returns A 64-bit integer vector of [4 x i16] containing the interleaved
297338/// values.
298static __inline__ __m64 __DEFAULT_FN_ATTRS
339static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
299340_mm_unpacklo_pi16(__m64 __m1, __m64 __m2)
300341{
301 return (__m64)__builtin_ia32_punpcklwd((__v4hi)__m1, (__v4hi)__m2);
342 return (__m64)__builtin_shufflevector((__v4hi)__m1, (__v4hi)__m2,
343 0, 4, 1, 5);
302344}
303345
304346/// Unpacks the lower 32 bits from two 64-bit integer vectors of
......@@ -316,10 +358,10 @@ _mm_unpacklo_pi16(__m64 __m1, __m64 __m2)
316358/// the upper 32 bits of the result.
317359/// \returns A 64-bit integer vector of [2 x i32] containing the interleaved
318360/// values.
319static __inline__ __m64 __DEFAULT_FN_ATTRS
361static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
320362_mm_unpacklo_pi32(__m64 __m1, __m64 __m2)
321363{
322 return (__m64)__builtin_ia32_punpckldq((__v2si)__m1, (__v2si)__m2);
364 return (__m64)__builtin_shufflevector((__v2si)__m1, (__v2si)__m2, 0, 2);
323365}
324366
325367/// Adds each 8-bit integer element of the first 64-bit integer vector
......@@ -337,10 +379,10 @@ _mm_unpacklo_pi32(__m64 __m1, __m64 __m2)
337379/// A 64-bit integer vector of [8 x i8].
338380/// \returns A 64-bit integer vector of [8 x i8] containing the sums of both
339381/// parameters.
340static __inline__ __m64 __DEFAULT_FN_ATTRS
382static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
341383_mm_add_pi8(__m64 __m1, __m64 __m2)
342384{
343 return (__m64)__builtin_ia32_paddb((__v8qi)__m1, (__v8qi)__m2);
385 return (__m64)(((__v8qu)__m1) + ((__v8qu)__m2));
344386}
345387
346388/// Adds each 16-bit integer element of the first 64-bit integer vector
......@@ -358,10 +400,10 @@ _mm_add_pi8(__m64 __m1, __m64 __m2)
358400/// A 64-bit integer vector of [4 x i16].
359401/// \returns A 64-bit integer vector of [4 x i16] containing the sums of both
360402/// parameters.
361static __inline__ __m64 __DEFAULT_FN_ATTRS
403static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
362404_mm_add_pi16(__m64 __m1, __m64 __m2)
363405{
364 return (__m64)__builtin_ia32_paddw((__v4hi)__m1, (__v4hi)__m2);
406 return (__m64)(((__v4hu)__m1) + ((__v4hu)__m2));
365407}
366408
367409/// Adds each 32-bit integer element of the first 64-bit integer vector
......@@ -379,10 +421,10 @@ _mm_add_pi16(__m64 __m1, __m64 __m2)
379421/// A 64-bit integer vector of [2 x i32].
380422/// \returns A 64-bit integer vector of [2 x i32] containing the sums of both
381423/// parameters.
382static __inline__ __m64 __DEFAULT_FN_ATTRS
424static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
383425_mm_add_pi32(__m64 __m1, __m64 __m2)
384426{
385 return (__m64)__builtin_ia32_paddd((__v2si)__m1, (__v2si)__m2);
427 return (__m64)(((__v2su)__m1) + ((__v2su)__m2));
386428}
387429
388430/// Adds, with saturation, each 8-bit signed integer element of the first
......@@ -403,10 +445,10 @@ _mm_add_pi32(__m64 __m1, __m64 __m2)
403445/// A 64-bit integer vector of [8 x i8].
404446/// \returns A 64-bit integer vector of [8 x i8] containing the saturated sums
405447/// of both parameters.
406static __inline__ __m64 __DEFAULT_FN_ATTRS
448static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
407449_mm_adds_pi8(__m64 __m1, __m64 __m2)
408450{
409 return (__m64)__builtin_ia32_paddsb((__v8qi)__m1, (__v8qi)__m2);
451 return (__m64)__builtin_elementwise_add_sat((__v8qs)__m1, (__v8qs)__m2);
410452}
411453
412454/// Adds, with saturation, each 16-bit signed integer element of the first
......@@ -427,10 +469,10 @@ _mm_adds_pi8(__m64 __m1, __m64 __m2)
427469/// A 64-bit integer vector of [4 x i16].
428470/// \returns A 64-bit integer vector of [4 x i16] containing the saturated sums
429471/// of both parameters.
430static __inline__ __m64 __DEFAULT_FN_ATTRS
472static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
431473_mm_adds_pi16(__m64 __m1, __m64 __m2)
432474{
433 return (__m64)__builtin_ia32_paddsw((__v4hi)__m1, (__v4hi)__m2);
475 return (__m64)__builtin_elementwise_add_sat((__v4hi)__m1, (__v4hi)__m2);
434476}
435477
436478/// Adds, with saturation, each 8-bit unsigned integer element of the first
......@@ -450,10 +492,10 @@ _mm_adds_pi16(__m64 __m1, __m64 __m2)
450492/// A 64-bit integer vector of [8 x i8].
451493/// \returns A 64-bit integer vector of [8 x i8] containing the saturated
452494/// unsigned sums of both parameters.
453static __inline__ __m64 __DEFAULT_FN_ATTRS
495static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
454496_mm_adds_pu8(__m64 __m1, __m64 __m2)
455497{
456 return (__m64)__builtin_ia32_paddusb((__v8qi)__m1, (__v8qi)__m2);
498 return (__m64)__builtin_elementwise_add_sat((__v8qu)__m1, (__v8qu)__m2);
457499}
458500
459501/// Adds, with saturation, each 16-bit unsigned integer element of the first
......@@ -473,10 +515,10 @@ _mm_adds_pu8(__m64 __m1, __m64 __m2)
473515/// A 64-bit integer vector of [4 x i16].
474516/// \returns A 64-bit integer vector of [4 x i16] containing the saturated
475517/// unsigned sums of both parameters.
476static __inline__ __m64 __DEFAULT_FN_ATTRS
518static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
477519_mm_adds_pu16(__m64 __m1, __m64 __m2)
478520{
479 return (__m64)__builtin_ia32_paddusw((__v4hi)__m1, (__v4hi)__m2);
521 return (__m64)__builtin_elementwise_add_sat((__v4hu)__m1, (__v4hu)__m2);
480522}
481523
482524/// Subtracts each 8-bit integer element of the second 64-bit integer
......@@ -494,10 +536,10 @@ _mm_adds_pu16(__m64 __m1, __m64 __m2)
494536/// A 64-bit integer vector of [8 x i8] containing the subtrahends.
495537/// \returns A 64-bit integer vector of [8 x i8] containing the differences of
496538/// both parameters.
497static __inline__ __m64 __DEFAULT_FN_ATTRS
539static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
498540_mm_sub_pi8(__m64 __m1, __m64 __m2)
499541{
500 return (__m64)__builtin_ia32_psubb((__v8qi)__m1, (__v8qi)__m2);
542 return (__m64)(((__v8qu)__m1) - ((__v8qu)__m2));
501543}
502544
503545/// Subtracts each 16-bit integer element of the second 64-bit integer
......@@ -515,10 +557,10 @@ _mm_sub_pi8(__m64 __m1, __m64 __m2)
515557/// A 64-bit integer vector of [4 x i16] containing the subtrahends.
516558/// \returns A 64-bit integer vector of [4 x i16] containing the differences of
517559/// both parameters.
518static __inline__ __m64 __DEFAULT_FN_ATTRS
560static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
519561_mm_sub_pi16(__m64 __m1, __m64 __m2)
520562{
521 return (__m64)__builtin_ia32_psubw((__v4hi)__m1, (__v4hi)__m2);
563 return (__m64)(((__v4hu)__m1) - ((__v4hu)__m2));
522564}
523565
524566/// Subtracts each 32-bit integer element of the second 64-bit integer
......@@ -536,10 +578,10 @@ _mm_sub_pi16(__m64 __m1, __m64 __m2)
536578/// A 64-bit integer vector of [2 x i32] containing the subtrahends.
537579/// \returns A 64-bit integer vector of [2 x i32] containing the differences of
538580/// both parameters.
539static __inline__ __m64 __DEFAULT_FN_ATTRS
581static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
540582_mm_sub_pi32(__m64 __m1, __m64 __m2)
541583{
542 return (__m64)__builtin_ia32_psubd((__v2si)__m1, (__v2si)__m2);
584 return (__m64)(((__v2su)__m1) - ((__v2su)__m2));
543585}
544586
545587/// Subtracts, with saturation, each 8-bit signed integer element of the second
......@@ -560,10 +602,10 @@ _mm_sub_pi32(__m64 __m1, __m64 __m2)
560602/// A 64-bit integer vector of [8 x i8] containing the subtrahends.
561603/// \returns A 64-bit integer vector of [8 x i8] containing the saturated
562604/// differences of both parameters.
563static __inline__ __m64 __DEFAULT_FN_ATTRS
605static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
564606_mm_subs_pi8(__m64 __m1, __m64 __m2)
565607{
566 return (__m64)__builtin_ia32_psubsb((__v8qi)__m1, (__v8qi)__m2);
608 return (__m64)__builtin_elementwise_sub_sat((__v8qs)__m1, (__v8qs)__m2);
567609}
568610
569611/// Subtracts, with saturation, each 16-bit signed integer element of the
......@@ -584,10 +626,10 @@ _mm_subs_pi8(__m64 __m1, __m64 __m2)
584626/// A 64-bit integer vector of [4 x i16] containing the subtrahends.
585627/// \returns A 64-bit integer vector of [4 x i16] containing the saturated
586628/// differences of both parameters.
587static __inline__ __m64 __DEFAULT_FN_ATTRS
629static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
588630_mm_subs_pi16(__m64 __m1, __m64 __m2)
589631{
590 return (__m64)__builtin_ia32_psubsw((__v4hi)__m1, (__v4hi)__m2);
632 return (__m64)__builtin_elementwise_sub_sat((__v4hi)__m1, (__v4hi)__m2);
591633}
592634
593635/// Subtracts each 8-bit unsigned integer element of the second 64-bit
......@@ -608,10 +650,10 @@ _mm_subs_pi16(__m64 __m1, __m64 __m2)
608650/// A 64-bit integer vector of [8 x i8] containing the subtrahends.
609651/// \returns A 64-bit integer vector of [8 x i8] containing the saturated
610652/// differences of both parameters.
611static __inline__ __m64 __DEFAULT_FN_ATTRS
653static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
612654_mm_subs_pu8(__m64 __m1, __m64 __m2)
613655{
614 return (__m64)__builtin_ia32_psubusb((__v8qi)__m1, (__v8qi)__m2);
656 return (__m64)__builtin_elementwise_sub_sat((__v8qu)__m1, (__v8qu)__m2);
615657}
616658
617659/// Subtracts each 16-bit unsigned integer element of the second 64-bit
......@@ -632,10 +674,10 @@ _mm_subs_pu8(__m64 __m1, __m64 __m2)
632674/// A 64-bit integer vector of [4 x i16] containing the subtrahends.
633675/// \returns A 64-bit integer vector of [4 x i16] containing the saturated
634676/// differences of both parameters.
635static __inline__ __m64 __DEFAULT_FN_ATTRS
677static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
636678_mm_subs_pu16(__m64 __m1, __m64 __m2)
637679{
638 return (__m64)__builtin_ia32_psubusw((__v4hi)__m1, (__v4hi)__m2);
680 return (__m64)__builtin_elementwise_sub_sat((__v4hu)__m1, (__v4hu)__m2);
639681}
640682
641683/// Multiplies each 16-bit signed integer element of the first 64-bit
......@@ -659,10 +701,11 @@ _mm_subs_pu16(__m64 __m1, __m64 __m2)
659701/// A 64-bit integer vector of [4 x i16].
660702/// \returns A 64-bit integer vector of [2 x i32] containing the sums of
661703/// products of both parameters.
662static __inline__ __m64 __DEFAULT_FN_ATTRS
704static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
663705_mm_madd_pi16(__m64 __m1, __m64 __m2)
664706{
665 return (__m64)__builtin_ia32_pmaddwd((__v4hi)__m1, (__v4hi)__m2);
707 return __trunc64(__builtin_ia32_pmaddwd128((__v8hi)__anyext128(__m1),
708 (__v8hi)__anyext128(__m2)));
666709}
667710
668711/// Multiplies each 16-bit signed integer element of the first 64-bit
......@@ -680,10 +723,11 @@ _mm_madd_pi16(__m64 __m1, __m64 __m2)
680723/// A 64-bit integer vector of [4 x i16].
681724/// \returns A 64-bit integer vector of [4 x i16] containing the upper 16 bits
682725/// of the products of both parameters.
683static __inline__ __m64 __DEFAULT_FN_ATTRS
726static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
684727_mm_mulhi_pi16(__m64 __m1, __m64 __m2)
685728{
686 return (__m64)__builtin_ia32_pmulhw((__v4hi)__m1, (__v4hi)__m2);
729 return __trunc64(__builtin_ia32_pmulhw128((__v8hi)__anyext128(__m1),
730 (__v8hi)__anyext128(__m2)));
687731}
688732
689733/// Multiplies each 16-bit signed integer element of the first 64-bit
......@@ -701,10 +745,10 @@ _mm_mulhi_pi16(__m64 __m1, __m64 __m2)
701745/// A 64-bit integer vector of [4 x i16].
702746/// \returns A 64-bit integer vector of [4 x i16] containing the lower 16 bits
703747/// of the products of both parameters.
704static __inline__ __m64 __DEFAULT_FN_ATTRS
748static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
705749_mm_mullo_pi16(__m64 __m1, __m64 __m2)
706750{
707 return (__m64)__builtin_ia32_pmullw((__v4hi)__m1, (__v4hi)__m2);
751 return (__m64)(((__v4hu)__m1) * ((__v4hu)__m2));
708752}
709753
710754/// Left-shifts each 16-bit signed integer element of the first
......@@ -724,10 +768,11 @@ _mm_mullo_pi16(__m64 __m1, __m64 __m2)
724768/// \returns A 64-bit integer vector of [4 x i16] containing the left-shifted
725769/// values. If \a __count is greater or equal to 16, the result is set to all
726770/// 0.
727static __inline__ __m64 __DEFAULT_FN_ATTRS
771static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
728772_mm_sll_pi16(__m64 __m, __m64 __count)
729773{
730 return (__m64)__builtin_ia32_psllw((__v4hi)__m, __count);
774 return __trunc64(__builtin_ia32_psllw128((__v8hi)__anyext128(__m),
775 (__v8hi)__anyext128(__count)));
731776}
732777
733778/// Left-shifts each 16-bit signed integer element of a 64-bit integer
......@@ -746,10 +791,11 @@ _mm_sll_pi16(__m64 __m, __m64 __count)
746791/// \returns A 64-bit integer vector of [4 x i16] containing the left-shifted
747792/// values. If \a __count is greater or equal to 16, the result is set to all
748793/// 0.
749static __inline__ __m64 __DEFAULT_FN_ATTRS
794static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
750795_mm_slli_pi16(__m64 __m, int __count)
751796{
752 return (__m64)__builtin_ia32_psllwi((__v4hi)__m, __count);
797 return __trunc64(__builtin_ia32_psllwi128((__v8hi)__anyext128(__m),
798 __count));
753799}
754800
755801/// Left-shifts each 32-bit signed integer element of the first
......@@ -769,10 +815,11 @@ _mm_slli_pi16(__m64 __m, int __count)
769815/// \returns A 64-bit integer vector of [2 x i32] containing the left-shifted
770816/// values. If \a __count is greater or equal to 32, the result is set to all
771817/// 0.
772static __inline__ __m64 __DEFAULT_FN_ATTRS
818static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
773819_mm_sll_pi32(__m64 __m, __m64 __count)
774820{
775 return (__m64)__builtin_ia32_pslld((__v2si)__m, __count);
821 return __trunc64(__builtin_ia32_pslld128((__v4si)__anyext128(__m),
822 (__v4si)__anyext128(__count)));
776823}
777824
778825/// Left-shifts each 32-bit signed integer element of a 64-bit integer
......@@ -791,10 +838,11 @@ _mm_sll_pi32(__m64 __m, __m64 __count)
791838/// \returns A 64-bit integer vector of [2 x i32] containing the left-shifted
792839/// values. If \a __count is greater or equal to 32, the result is set to all
793840/// 0.
794static __inline__ __m64 __DEFAULT_FN_ATTRS
841static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
795842_mm_slli_pi32(__m64 __m, int __count)
796843{
797 return (__m64)__builtin_ia32_pslldi((__v2si)__m, __count);
844 return __trunc64(__builtin_ia32_pslldi128((__v4si)__anyext128(__m),
845 __count));
798846}
799847
800848/// Left-shifts the first 64-bit integer parameter by the number of bits
......@@ -811,10 +859,11 @@ _mm_slli_pi32(__m64 __m, int __count)
811859/// A 64-bit integer vector interpreted as a single 64-bit integer.
812860/// \returns A 64-bit integer vector containing the left-shifted value. If
813861/// \a __count is greater or equal to 64, the result is set to 0.
814static __inline__ __m64 __DEFAULT_FN_ATTRS
862static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
815863_mm_sll_si64(__m64 __m, __m64 __count)
816864{
817 return (__m64)__builtin_ia32_psllq((__v1di)__m, __count);
865 return __trunc64(__builtin_ia32_psllq128((__v2di)__anyext128(__m),
866 (__v2di)__anyext128(__count)));
818867}
819868
820869/// Left-shifts the first parameter, which is a 64-bit integer, by the
......@@ -831,10 +880,11 @@ _mm_sll_si64(__m64 __m, __m64 __count)
831880/// A 32-bit integer value.
832881/// \returns A 64-bit integer vector containing the left-shifted value. If
833882/// \a __count is greater or equal to 64, the result is set to 0.
834static __inline__ __m64 __DEFAULT_FN_ATTRS
883static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
835884_mm_slli_si64(__m64 __m, int __count)
836885{
837 return (__m64)__builtin_ia32_psllqi((__v1di)__m, __count);
886 return __trunc64(__builtin_ia32_psllqi128((__v2di)__anyext128(__m),
887 __count));
838888}
839889
840890/// Right-shifts each 16-bit integer element of the first parameter,
......@@ -855,10 +905,11 @@ _mm_slli_si64(__m64 __m, int __count)
855905/// A 64-bit integer vector interpreted as a single 64-bit integer.
856906/// \returns A 64-bit integer vector of [4 x i16] containing the right-shifted
857907/// values.
858static __inline__ __m64 __DEFAULT_FN_ATTRS
908static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
859909_mm_sra_pi16(__m64 __m, __m64 __count)
860910{
861 return (__m64)__builtin_ia32_psraw((__v4hi)__m, __count);
911 return __trunc64(__builtin_ia32_psraw128((__v8hi)__anyext128(__m),
912 (__v8hi)__anyext128(__count)));
862913}
863914
864915/// Right-shifts each 16-bit integer element of a 64-bit integer vector
......@@ -878,10 +929,11 @@ _mm_sra_pi16(__m64 __m, __m64 __count)
878929/// A 32-bit integer value.
879930/// \returns A 64-bit integer vector of [4 x i16] containing the right-shifted
880931/// values.
881static __inline__ __m64 __DEFAULT_FN_ATTRS
932static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
882933_mm_srai_pi16(__m64 __m, int __count)
883934{
884 return (__m64)__builtin_ia32_psrawi((__v4hi)__m, __count);
935 return __trunc64(__builtin_ia32_psrawi128((__v8hi)__anyext128(__m),
936 __count));
885937}
886938
887939/// Right-shifts each 32-bit integer element of the first parameter,
......@@ -902,10 +954,11 @@ _mm_srai_pi16(__m64 __m, int __count)
902954/// A 64-bit integer vector interpreted as a single 64-bit integer.
903955/// \returns A 64-bit integer vector of [2 x i32] containing the right-shifted
904956/// values.
905static __inline__ __m64 __DEFAULT_FN_ATTRS
957static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
906958_mm_sra_pi32(__m64 __m, __m64 __count)
907959{
908 return (__m64)__builtin_ia32_psrad((__v2si)__m, __count);
960 return __trunc64(__builtin_ia32_psrad128((__v4si)__anyext128(__m),
961 (__v4si)__anyext128(__count)));
909962}
910963
911964/// Right-shifts each 32-bit integer element of a 64-bit integer vector
......@@ -925,10 +978,11 @@ _mm_sra_pi32(__m64 __m, __m64 __count)
925978/// A 32-bit integer value.
926979/// \returns A 64-bit integer vector of [2 x i32] containing the right-shifted
927980/// values.
928static __inline__ __m64 __DEFAULT_FN_ATTRS
981static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
929982_mm_srai_pi32(__m64 __m, int __count)
930983{
931 return (__m64)__builtin_ia32_psradi((__v2si)__m, __count);
984 return __trunc64(__builtin_ia32_psradi128((__v4si)__anyext128(__m),
985 __count));
932986}
933987
934988/// Right-shifts each 16-bit integer element of the first parameter,
......@@ -948,10 +1002,11 @@ _mm_srai_pi32(__m64 __m, int __count)
9481002/// A 64-bit integer vector interpreted as a single 64-bit integer.
9491003/// \returns A 64-bit integer vector of [4 x i16] containing the right-shifted
9501004/// values.
951static __inline__ __m64 __DEFAULT_FN_ATTRS
1005static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
9521006_mm_srl_pi16(__m64 __m, __m64 __count)
9531007{
954 return (__m64)__builtin_ia32_psrlw((__v4hi)__m, __count);
1008 return __trunc64(__builtin_ia32_psrlw128((__v8hi)__anyext128(__m),
1009 (__v8hi)__anyext128(__count)));
9551010}
9561011
9571012/// Right-shifts each 16-bit integer element of a 64-bit integer vector
......@@ -970,10 +1025,11 @@ _mm_srl_pi16(__m64 __m, __m64 __count)
9701025/// A 32-bit integer value.
9711026/// \returns A 64-bit integer vector of [4 x i16] containing the right-shifted
9721027/// values.
973static __inline__ __m64 __DEFAULT_FN_ATTRS
1028static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
9741029_mm_srli_pi16(__m64 __m, int __count)
9751030{
976 return (__m64)__builtin_ia32_psrlwi((__v4hi)__m, __count);
1031 return __trunc64(__builtin_ia32_psrlwi128((__v8hi)__anyext128(__m),
1032 __count));
9771033}
9781034
9791035/// Right-shifts each 32-bit integer element of the first parameter,
......@@ -993,10 +1049,11 @@ _mm_srli_pi16(__m64 __m, int __count)
9931049/// A 64-bit integer vector interpreted as a single 64-bit integer.
9941050/// \returns A 64-bit integer vector of [2 x i32] containing the right-shifted
9951051/// values.
996static __inline__ __m64 __DEFAULT_FN_ATTRS
1052static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
9971053_mm_srl_pi32(__m64 __m, __m64 __count)
9981054{
999 return (__m64)__builtin_ia32_psrld((__v2si)__m, __count);
1055 return __trunc64(__builtin_ia32_psrld128((__v4si)__anyext128(__m),
1056 (__v4si)__anyext128(__count)));
10001057}
10011058
10021059/// Right-shifts each 32-bit integer element of a 64-bit integer vector
......@@ -1015,10 +1072,11 @@ _mm_srl_pi32(__m64 __m, __m64 __count)
10151072/// A 32-bit integer value.
10161073/// \returns A 64-bit integer vector of [2 x i32] containing the right-shifted
10171074/// values.
1018static __inline__ __m64 __DEFAULT_FN_ATTRS
1075static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
10191076_mm_srli_pi32(__m64 __m, int __count)
10201077{
1021 return (__m64)__builtin_ia32_psrldi((__v2si)__m, __count);
1078 return __trunc64(__builtin_ia32_psrldi128((__v4si)__anyext128(__m),
1079 __count));
10221080}
10231081
10241082/// Right-shifts the first 64-bit integer parameter by the number of bits
......@@ -1035,10 +1093,11 @@ _mm_srli_pi32(__m64 __m, int __count)
10351093/// \param __count
10361094/// A 64-bit integer vector interpreted as a single 64-bit integer.
10371095/// \returns A 64-bit integer vector containing the right-shifted value.
1038static __inline__ __m64 __DEFAULT_FN_ATTRS
1096static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
10391097_mm_srl_si64(__m64 __m, __m64 __count)
10401098{
1041 return (__m64)__builtin_ia32_psrlq((__v1di)__m, __count);
1099 return __trunc64(__builtin_ia32_psrlq128((__v2di)__anyext128(__m),
1100 (__v2di)__anyext128(__count)));
10421101}
10431102
10441103/// Right-shifts the first parameter, which is a 64-bit integer, by the
......@@ -1056,10 +1115,11 @@ _mm_srl_si64(__m64 __m, __m64 __count)
10561115/// \param __count
10571116/// A 32-bit integer value.
10581117/// \returns A 64-bit integer vector containing the right-shifted value.
1059static __inline__ __m64 __DEFAULT_FN_ATTRS
1118static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
10601119_mm_srli_si64(__m64 __m, int __count)
10611120{
1062 return (__m64)__builtin_ia32_psrlqi((__v1di)__m, __count);
1121 return __trunc64(__builtin_ia32_psrlqi128((__v2di)__anyext128(__m),
1122 __count));
10631123}
10641124
10651125/// Performs a bitwise AND of two 64-bit integer vectors.
......@@ -1074,10 +1134,10 @@ _mm_srli_si64(__m64 __m, int __count)
10741134/// A 64-bit integer vector.
10751135/// \returns A 64-bit integer vector containing the bitwise AND of both
10761136/// parameters.
1077static __inline__ __m64 __DEFAULT_FN_ATTRS
1137static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
10781138_mm_and_si64(__m64 __m1, __m64 __m2)
10791139{
1080 return __builtin_ia32_pand((__v1di)__m1, (__v1di)__m2);
1140 return (__m64)(((__v1du)__m1) & ((__v1du)__m2));
10811141}
10821142
10831143/// Performs a bitwise NOT of the first 64-bit integer vector, and then
......@@ -1095,10 +1155,10 @@ _mm_and_si64(__m64 __m1, __m64 __m2)
10951155/// A 64-bit integer vector.
10961156/// \returns A 64-bit integer vector containing the bitwise AND of the second
10971157/// parameter and the one's complement of the first parameter.
1098static __inline__ __m64 __DEFAULT_FN_ATTRS
1158static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
10991159_mm_andnot_si64(__m64 __m1, __m64 __m2)
11001160{
1101 return __builtin_ia32_pandn((__v1di)__m1, (__v1di)__m2);
1161 return (__m64)(~((__v1du)__m1) & ((__v1du)__m2));
11021162}
11031163
11041164/// Performs a bitwise OR of two 64-bit integer vectors.
......@@ -1113,10 +1173,10 @@ _mm_andnot_si64(__m64 __m1, __m64 __m2)
11131173/// A 64-bit integer vector.
11141174/// \returns A 64-bit integer vector containing the bitwise OR of both
11151175/// parameters.
1116static __inline__ __m64 __DEFAULT_FN_ATTRS
1176static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
11171177_mm_or_si64(__m64 __m1, __m64 __m2)
11181178{
1119 return __builtin_ia32_por((__v1di)__m1, (__v1di)__m2);
1179 return (__m64)(((__v1du)__m1) | ((__v1du)__m2));
11201180}
11211181
11221182/// Performs a bitwise exclusive OR of two 64-bit integer vectors.
......@@ -1131,10 +1191,10 @@ _mm_or_si64(__m64 __m1, __m64 __m2)
11311191/// A 64-bit integer vector.
11321192/// \returns A 64-bit integer vector containing the bitwise exclusive OR of both
11331193/// parameters.
1134static __inline__ __m64 __DEFAULT_FN_ATTRS
1194static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
11351195_mm_xor_si64(__m64 __m1, __m64 __m2)
11361196{
1137 return __builtin_ia32_pxor((__v1di)__m1, (__v1di)__m2);
1197 return (__m64)(((__v1du)__m1) ^ ((__v1du)__m2));
11381198}
11391199
11401200/// Compares the 8-bit integer elements of two 64-bit integer vectors of
......@@ -1153,10 +1213,10 @@ _mm_xor_si64(__m64 __m1, __m64 __m2)
11531213/// A 64-bit integer vector of [8 x i8].
11541214/// \returns A 64-bit integer vector of [8 x i8] containing the comparison
11551215/// results.
1156static __inline__ __m64 __DEFAULT_FN_ATTRS
1216static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
11571217_mm_cmpeq_pi8(__m64 __m1, __m64 __m2)
11581218{
1159 return (__m64)__builtin_ia32_pcmpeqb((__v8qi)__m1, (__v8qi)__m2);
1219 return (__m64)(((__v8qi)__m1) == ((__v8qi)__m2));
11601220}
11611221
11621222/// Compares the 16-bit integer elements of two 64-bit integer vectors of
......@@ -1175,10 +1235,10 @@ _mm_cmpeq_pi8(__m64 __m1, __m64 __m2)
11751235/// A 64-bit integer vector of [4 x i16].
11761236/// \returns A 64-bit integer vector of [4 x i16] containing the comparison
11771237/// results.
1178static __inline__ __m64 __DEFAULT_FN_ATTRS
1238static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
11791239_mm_cmpeq_pi16(__m64 __m1, __m64 __m2)
11801240{
1181 return (__m64)__builtin_ia32_pcmpeqw((__v4hi)__m1, (__v4hi)__m2);
1241 return (__m64)(((__v4hi)__m1) == ((__v4hi)__m2));
11821242}
11831243
11841244/// Compares the 32-bit integer elements of two 64-bit integer vectors of
......@@ -1197,10 +1257,10 @@ _mm_cmpeq_pi16(__m64 __m1, __m64 __m2)
11971257/// A 64-bit integer vector of [2 x i32].
11981258/// \returns A 64-bit integer vector of [2 x i32] containing the comparison
11991259/// results.
1200static __inline__ __m64 __DEFAULT_FN_ATTRS
1260static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
12011261_mm_cmpeq_pi32(__m64 __m1, __m64 __m2)
12021262{
1203 return (__m64)__builtin_ia32_pcmpeqd((__v2si)__m1, (__v2si)__m2);
1263 return (__m64)(((__v2si)__m1) == ((__v2si)__m2));
12041264}
12051265
12061266/// Compares the 8-bit integer elements of two 64-bit integer vectors of
......@@ -1219,10 +1279,12 @@ _mm_cmpeq_pi32(__m64 __m1, __m64 __m2)
12191279/// A 64-bit integer vector of [8 x i8].
12201280/// \returns A 64-bit integer vector of [8 x i8] containing the comparison
12211281/// results.
1222static __inline__ __m64 __DEFAULT_FN_ATTRS
1282static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
12231283_mm_cmpgt_pi8(__m64 __m1, __m64 __m2)
12241284{
1225 return (__m64)__builtin_ia32_pcmpgtb((__v8qi)__m1, (__v8qi)__m2);
1285 /* This function always performs a signed comparison, but __v8qi is a char
1286 which may be signed or unsigned, so use __v8qs. */
1287 return (__m64)((__v8qs)__m1 > (__v8qs)__m2);
12261288}
12271289
12281290/// Compares the 16-bit integer elements of two 64-bit integer vectors of
......@@ -1241,10 +1303,10 @@ _mm_cmpgt_pi8(__m64 __m1, __m64 __m2)
12411303/// A 64-bit integer vector of [4 x i16].
12421304/// \returns A 64-bit integer vector of [4 x i16] containing the comparison
12431305/// results.
1244static __inline__ __m64 __DEFAULT_FN_ATTRS
1306static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
12451307_mm_cmpgt_pi16(__m64 __m1, __m64 __m2)
12461308{
1247 return (__m64)__builtin_ia32_pcmpgtw((__v4hi)__m1, (__v4hi)__m2);
1309 return (__m64)((__v4hi)__m1 > (__v4hi)__m2);
12481310}
12491311
12501312/// Compares the 32-bit integer elements of two 64-bit integer vectors of
......@@ -1263,10 +1325,10 @@ _mm_cmpgt_pi16(__m64 __m1, __m64 __m2)
12631325/// A 64-bit integer vector of [2 x i32].
12641326/// \returns A 64-bit integer vector of [2 x i32] containing the comparison
12651327/// results.
1266static __inline__ __m64 __DEFAULT_FN_ATTRS
1328static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
12671329_mm_cmpgt_pi32(__m64 __m1, __m64 __m2)
12681330{
1269 return (__m64)__builtin_ia32_pcmpgtd((__v2si)__m1, (__v2si)__m2);
1331 return (__m64)((__v2si)__m1 > (__v2si)__m2);
12701332}
12711333
12721334/// Constructs a 64-bit integer vector initialized to zero.
......@@ -1276,10 +1338,9 @@ _mm_cmpgt_pi32(__m64 __m1, __m64 __m2)
12761338/// This intrinsic corresponds to the <c> PXOR </c> instruction.
12771339///
12781340/// \returns An initialized 64-bit integer vector with all elements set to zero.
1279static __inline__ __m64 __DEFAULT_FN_ATTRS
1280_mm_setzero_si64(void)
1281{
1282 return __extension__ (__m64){ 0LL };
1341static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1342_mm_setzero_si64(void) {
1343 return __extension__(__m64){0LL};
12831344}
12841345
12851346/// Constructs a 64-bit integer vector initialized with the specified
......@@ -1297,10 +1358,9 @@ _mm_setzero_si64(void)
12971358/// A 32-bit integer value used to initialize the lower 32 bits of the
12981359/// result.
12991360/// \returns An initialized 64-bit integer vector.
1300static __inline__ __m64 __DEFAULT_FN_ATTRS
1301_mm_set_pi32(int __i1, int __i0)
1302{
1303 return (__m64)__builtin_ia32_vec_init_v2si(__i0, __i1);
1361static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1362_mm_set_pi32(int __i1, int __i0) {
1363 return __extension__(__m64)(__v2si){__i0, __i1};
13041364}
13051365
13061366/// Constructs a 64-bit integer vector initialized with the specified
......@@ -1320,10 +1380,9 @@ _mm_set_pi32(int __i1, int __i0)
13201380/// \param __s0
13211381/// A 16-bit integer value used to initialize bits [15:0] of the result.
13221382/// \returns An initialized 64-bit integer vector.
1323static __inline__ __m64 __DEFAULT_FN_ATTRS
1324_mm_set_pi16(short __s3, short __s2, short __s1, short __s0)
1325{
1326 return (__m64)__builtin_ia32_vec_init_v4hi(__s0, __s1, __s2, __s3);
1383static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1384_mm_set_pi16(short __s3, short __s2, short __s1, short __s0) {
1385 return __extension__(__m64)(__v4hi){__s0, __s1, __s2, __s3};
13271386}
13281387
13291388/// Constructs a 64-bit integer vector initialized with the specified
......@@ -1351,12 +1410,11 @@ _mm_set_pi16(short __s3, short __s2, short __s1, short __s0)
13511410/// \param __b0
13521411/// An 8-bit integer value used to initialize bits [7:0] of the result.
13531412/// \returns An initialized 64-bit integer vector.
1354static __inline__ __m64 __DEFAULT_FN_ATTRS
1413static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
13551414_mm_set_pi8(char __b7, char __b6, char __b5, char __b4, char __b3, char __b2,
1356 char __b1, char __b0)
1357{
1358 return (__m64)__builtin_ia32_vec_init_v8qi(__b0, __b1, __b2, __b3,
1359 __b4, __b5, __b6, __b7);
1415 char __b1, char __b0) {
1416 return __extension__(__m64)(__v8qi){__b0, __b1, __b2, __b3,
1417 __b4, __b5, __b6, __b7};
13601418}
13611419
13621420/// Constructs a 64-bit integer vector of [2 x i32], with each of the
......@@ -1372,10 +1430,9 @@ _mm_set_pi8(char __b7, char __b6, char __b5, char __b4, char __b3, char __b2,
13721430/// A 32-bit integer value used to initialize each vector element of the
13731431/// result.
13741432/// \returns An initialized 64-bit integer vector of [2 x i32].
1375static __inline__ __m64 __DEFAULT_FN_ATTRS
1376_mm_set1_pi32(int __i)
1377{
1378 return _mm_set_pi32(__i, __i);
1433static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1434_mm_set1_pi32(int __i) {
1435 return _mm_set_pi32(__i, __i);
13791436}
13801437
13811438/// Constructs a 64-bit integer vector of [4 x i16], with each of the
......@@ -1391,10 +1448,9 @@ _mm_set1_pi32(int __i)
13911448/// A 16-bit integer value used to initialize each vector element of the
13921449/// result.
13931450/// \returns An initialized 64-bit integer vector of [4 x i16].
1394static __inline__ __m64 __DEFAULT_FN_ATTRS
1395_mm_set1_pi16(short __w)
1396{
1397 return _mm_set_pi16(__w, __w, __w, __w);
1451static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1452_mm_set1_pi16(short __w) {
1453 return _mm_set_pi16(__w, __w, __w, __w);
13981454}
13991455
14001456/// Constructs a 64-bit integer vector of [8 x i8], with each of the
......@@ -1409,10 +1465,9 @@ _mm_set1_pi16(short __w)
14091465/// An 8-bit integer value used to initialize each vector element of the
14101466/// result.
14111467/// \returns An initialized 64-bit integer vector of [8 x i8].
1412static __inline__ __m64 __DEFAULT_FN_ATTRS
1413_mm_set1_pi8(char __b)
1414{
1415 return _mm_set_pi8(__b, __b, __b, __b, __b, __b, __b, __b);
1468static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1469_mm_set1_pi8(char __b) {
1470 return _mm_set_pi8(__b, __b, __b, __b, __b, __b, __b, __b);
14161471}
14171472
14181473/// Constructs a 64-bit integer vector, initialized in reverse order with
......@@ -1430,10 +1485,9 @@ _mm_set1_pi8(char __b)
14301485/// A 32-bit integer value used to initialize the upper 32 bits of the
14311486/// result.
14321487/// \returns An initialized 64-bit integer vector.
1433static __inline__ __m64 __DEFAULT_FN_ATTRS
1434_mm_setr_pi32(int __i0, int __i1)
1435{
1436 return _mm_set_pi32(__i1, __i0);
1488static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1489_mm_setr_pi32(int __i0, int __i1) {
1490 return _mm_set_pi32(__i1, __i0);
14371491}
14381492
14391493/// Constructs a 64-bit integer vector, initialized in reverse order with
......@@ -1453,10 +1507,9 @@ _mm_setr_pi32(int __i0, int __i1)
14531507/// \param __w3
14541508/// A 16-bit integer value used to initialize bits [63:48] of the result.
14551509/// \returns An initialized 64-bit integer vector.
1456static __inline__ __m64 __DEFAULT_FN_ATTRS
1457_mm_setr_pi16(short __w0, short __w1, short __w2, short __w3)
1458{
1459 return _mm_set_pi16(__w3, __w2, __w1, __w0);
1510static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1511_mm_setr_pi16(short __w0, short __w1, short __w2, short __w3) {
1512 return _mm_set_pi16(__w3, __w2, __w1, __w0);
14601513}
14611514
14621515/// Constructs a 64-bit integer vector, initialized in reverse order with
......@@ -1484,14 +1537,15 @@ _mm_setr_pi16(short __w0, short __w1, short __w2, short __w3)
14841537/// \param __b7
14851538/// An 8-bit integer value used to initialize bits [63:56] of the result.
14861539/// \returns An initialized 64-bit integer vector.
1487static __inline__ __m64 __DEFAULT_FN_ATTRS
1540static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
14881541_mm_setr_pi8(char __b0, char __b1, char __b2, char __b3, char __b4, char __b5,
1489 char __b6, char __b7)
1490{
1491 return _mm_set_pi8(__b7, __b6, __b5, __b4, __b3, __b2, __b1, __b0);
1542 char __b6, char __b7) {
1543 return _mm_set_pi8(__b7, __b6, __b5, __b4, __b3, __b2, __b1, __b0);
14921544}
14931545
1494#undef __DEFAULT_FN_ATTRS
1546#undef __anyext128
1547#undef __trunc64
1548#undef __DEFAULT_FN_ATTRS_SSE2
14951549
14961550/* Aliases for compatibility. */
14971551#define _m_empty _mm_empty
lib/include/module.modulemap+2-4
......@@ -66,6 +66,8 @@ module _Builtin_intrinsics [system] [extern_c] {
6666 textual header "__wmmintrin_aes.h"
6767 textual header "__wmmintrin_pclmul.h"
6868
69 textual header "mm3dnow.h"
70
6971 explicit module mm_malloc {
7072 requires !freestanding
7173 header "mm_malloc.h"
......@@ -122,10 +124,6 @@ module _Builtin_intrinsics [system] [extern_c] {
122124 header "popcntintrin.h"
123125 }
124126
125 explicit module mm3dnow {
126 header "mm3dnow.h"
127 }
128
129127 explicit module aes_pclmul {
130128 header "wmmintrin.h"
131129 export aes
lib/include/movrs_avx10_2_512intrin.h created+98
......@@ -0,0 +1,98 @@
1/*===----- movrs_avx10_2_512intrin.h - AVX10.2-512-MOVRS intrinsics --------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <movrs_avx10_2_512intrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifndef __MOVRS_AVX10_2_512INTRIN_H
15#define __MOVRS_AVX10_2_512INTRIN_H
16#ifdef __x86_64__
17
18/* Define the default attributes for the functions in this file. */
19#define __DEFAULT_FN_ATTRS512 \
20 __attribute__((__always_inline__, __nodebug__, \
21 __target__("movrs, avx10.2-512"), __min_vector_width__(512)))
22
23static __inline__ __m512i __DEFAULT_FN_ATTRS512
24_mm512_loadrs_epi8(void const *__A) {
25 return (__m512i)__builtin_ia32_vmovrsb512((const __v64qi *)(__A));
26}
27
28static __inline__ __m512i __DEFAULT_FN_ATTRS512
29_mm512_mask_loadrs_epi8(__m512i __W, __mmask64 __U, void const *__A) {
30 return (__m512i)__builtin_ia32_selectb_512(
31 (__mmask64)__U, (__v64qi)_mm512_loadrs_epi8(__A), (__v64qi)__W);
32}
33
34static __inline__ __m512i __DEFAULT_FN_ATTRS512
35_mm512_maskz_loadrs_epi8(__mmask64 __U, void const *__A) {
36 return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
37 (__v64qi)_mm512_loadrs_epi8(__A),
38 (__v64qi)_mm512_setzero_si512());
39}
40
41static __inline__ __m512i __DEFAULT_FN_ATTRS512
42_mm512_loadrs_epi32(void const *__A) {
43 return (__m512i)__builtin_ia32_vmovrsd512((const __v16si *)(__A));
44}
45
46static __inline__ __m512i __DEFAULT_FN_ATTRS512
47_mm512_mask_loadrs_epi32(__m512i __W, __mmask16 __U, void const *__A) {
48 return (__m512i)__builtin_ia32_selectd_512(
49 (__mmask16)__U, (__v16si)_mm512_loadrs_epi32(__A), (__v16si)__W);
50}
51
52static __inline__ __m512i __DEFAULT_FN_ATTRS512
53_mm512_maskz_loadrs_epi32(__mmask16 __U, void const *__A) {
54 return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
55 (__v16si)_mm512_loadrs_epi32(__A),
56 (__v16si)_mm512_setzero_si512());
57}
58
59static __inline__ __m512i __DEFAULT_FN_ATTRS512
60_mm512_loadrs_epi64(void const *__A) {
61 return (__m512i)__builtin_ia32_vmovrsq512((const __v8di *)(__A));
62}
63
64static __inline__ __m512i __DEFAULT_FN_ATTRS512
65_mm512_mask_loadrs_epi64(__m512i __W, __mmask8 __U, void const *__A) {
66 return (__m512i)__builtin_ia32_selectq_512(
67 (__mmask8)__U, (__v8di)_mm512_loadrs_epi64(__A), (__v8di)__W);
68}
69
70static __inline__ __m512i __DEFAULT_FN_ATTRS512
71_mm512_maskz_loadrs_epi64(__mmask8 __U, void const *__A) {
72 return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
73 (__v8di)_mm512_loadrs_epi64(__A),
74 (__v8di)_mm512_setzero_si512());
75}
76
77static __inline__ __m512i __DEFAULT_FN_ATTRS512
78_mm512_loadrs_epi16(void const *__A) {
79 return (__m512i)__builtin_ia32_vmovrsw512((const __v32hi *)(__A));
80}
81
82static __inline__ __m512i __DEFAULT_FN_ATTRS512
83_mm512_mask_loadrs_epi16(__m512i __W, __mmask32 __U, void const *__A) {
84 return (__m512i)__builtin_ia32_selectw_512(
85 (__mmask32)__U, (__v32hi)_mm512_loadrs_epi16(__A), (__v32hi)__W);
86}
87
88static __inline__ __m512i __DEFAULT_FN_ATTRS512
89_mm512_maskz_loadrs_epi16(__mmask32 __U, void const *__A) {
90 return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
91 (__v32hi)_mm512_loadrs_epi16(__A),
92 (__v32hi)_mm512_setzero_si512());
93}
94
95#undef __DEFAULT_FN_ATTRS512
96
97#endif /* __x86_64__ */
98#endif /* __MOVRS_AVX10_2_512INTRIN_H */
lib/include/movrs_avx10_2intrin.h created+174
......@@ -0,0 +1,174 @@
1/*===--------- movrs_avx10_2intrin.h - AVX10.2-MOVRS intrinsics ------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error \
11 "Never use <movrs_avx10_2intrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifndef __MOVRS_AVX10_2INTRIN_H
15#define __MOVRS_AVX10_2INTRIN_H
16#ifdef __x86_64__
17
18/* Define the default attributes for the functions in this file. */
19#define __DEFAULT_FN_ATTRS128 \
20 __attribute__((__always_inline__, __nodebug__, \
21 __target__("movrs,avx10.2-256"), __min_vector_width__(128)))
22#define __DEFAULT_FN_ATTRS256 \
23 __attribute__((__always_inline__, __nodebug__, \
24 __target__("movrs,avx10.2-256"), __min_vector_width__(256)))
25
26static __inline__ __m128i __DEFAULT_FN_ATTRS128
27_mm_loadrs_epi8(void const *__A) {
28 return (__m128i)__builtin_ia32_vmovrsb128((const __v16qi *)(__A));
29}
30
31static __inline__ __m128i __DEFAULT_FN_ATTRS128
32_mm_mask_loadrs_epi8(__m128i __W, __mmask16 __U, void const *__A) {
33 return (__m128i)__builtin_ia32_selectb_128(
34 (__mmask16)__U, (__v16qi)_mm_loadrs_epi8(__A), (__v16qi)__W);
35}
36
37static __inline__ __m128i __DEFAULT_FN_ATTRS128
38_mm_maskz_loadrs_epi8(__mmask16 __U, void const *__A) {
39 return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
40 (__v16qi)_mm_loadrs_epi8(__A),
41 (__v16qi)_mm_setzero_si128());
42}
43
44static __inline__ __m256i __DEFAULT_FN_ATTRS256
45_mm256_loadrs_epi8(void const *__A) {
46 return (__m256i)__builtin_ia32_vmovrsb256((const __v32qi *)(__A));
47}
48
49static __inline__ __m256i __DEFAULT_FN_ATTRS256
50_mm256_mask_loadrs_epi8(__m256i __W, __mmask32 __U, void const *__A) {
51 return (__m256i)__builtin_ia32_selectb_256(
52 (__mmask32)__U, (__v32qi)_mm256_loadrs_epi8(__A), (__v32qi)__W);
53}
54
55static __inline__ __m256i __DEFAULT_FN_ATTRS256
56_mm256_maskz_loadrs_epi8(__mmask32 __U, void const *__A) {
57 return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
58 (__v32qi)_mm256_loadrs_epi8(__A),
59 (__v32qi)_mm256_setzero_si256());
60}
61
62static __inline__ __m128i __DEFAULT_FN_ATTRS128
63_mm_loadrs_epi32(void const *__A) {
64 return (__m128i)__builtin_ia32_vmovrsd128((const __v4si *)(__A));
65}
66
67static __inline__ __m128i __DEFAULT_FN_ATTRS128
68_mm_mask_loadrs_epi32(__m128i __W, __mmask8 __U, void const *__A) {
69 return (__m128i)__builtin_ia32_selectd_128(
70 (__mmask8)__U, (__v4si)_mm_loadrs_epi32(__A), (__v4si)__W);
71}
72
73static __inline__ __m128i __DEFAULT_FN_ATTRS128
74_mm_maskz_loadrs_epi32(__mmask8 __U, void const *__A) {
75 return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
76 (__v4si)_mm_loadrs_epi32(__A),
77 (__v4si)_mm_setzero_si128());
78}
79
80static __inline__ __m256i __DEFAULT_FN_ATTRS256
81_mm256_loadrs_epi32(void const *__A) {
82 return (__m256i)__builtin_ia32_vmovrsd256((const __v8si *)(__A));
83}
84
85static __inline__ __m256i __DEFAULT_FN_ATTRS256
86_mm256_mask_loadrs_epi32(__m256i __W, __mmask8 __U, void const *__A) {
87 return (__m256i)__builtin_ia32_selectd_256(
88 (__mmask8)__U, (__v8si)_mm256_loadrs_epi32(__A), (__v8si)__W);
89}
90
91static __inline__ __m256i __DEFAULT_FN_ATTRS256
92_mm256_maskz_loadrs_epi32(__mmask8 __U, void const *__A) {
93 return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
94 (__v8si)_mm256_loadrs_epi32(__A),
95 (__v8si)_mm256_setzero_si256());
96}
97
98static __inline__ __m128i __DEFAULT_FN_ATTRS128
99_mm_loadrs_epi64(void const *__A) {
100 return (__m128i)__builtin_ia32_vmovrsq128((const __v2di *)(__A));
101}
102
103static __inline__ __m128i __DEFAULT_FN_ATTRS128
104_mm_mask_loadrs_epi64(__m128i __W, __mmask8 __U, void const *__A) {
105 return (__m128i)__builtin_ia32_selectq_128(
106 (__mmask8)__U, (__v2di)_mm_loadrs_epi64(__A), (__v2di)__W);
107}
108
109static __inline__ __m128i __DEFAULT_FN_ATTRS128
110_mm_maskz_loadrs_epi64(__mmask8 __U, void const *__A) {
111 return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
112 (__v2di)_mm_loadrs_epi64(__A),
113 (__v2di)_mm_setzero_si128());
114}
115
116static __inline__ __m256i __DEFAULT_FN_ATTRS256
117_mm256_loadrs_epi64(void const *__A) {
118 return (__m256i)__builtin_ia32_vmovrsq256((const __v4di *)(__A));
119}
120
121static __inline__ __m256i __DEFAULT_FN_ATTRS256
122_mm256_mask_loadrs_epi64(__m256i __W, __mmask8 __U, void const *__A) {
123 return (__m256i)__builtin_ia32_selectq_256(
124 (__mmask8)__U, (__v4di)_mm256_loadrs_epi64(__A), (__v4di)__W);
125}
126
127static __inline__ __m256i __DEFAULT_FN_ATTRS256
128_mm256_maskz_loadrs_epi64(__mmask8 __U, void const *__A) {
129 return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
130 (__v4di)_mm256_loadrs_epi64(__A),
131 (__v4di)_mm256_setzero_si256());
132}
133
134static __inline__ __m128i __DEFAULT_FN_ATTRS128
135_mm_loadrs_epi16(void const *__A) {
136 return (__m128i)__builtin_ia32_vmovrsw128((const __v8hi *)(__A));
137}
138
139static __inline__ __m128i __DEFAULT_FN_ATTRS128
140_mm_mask_loadrs_epi16(__m128i __W, __mmask8 __U, void const *__A) {
141 return (__m128i)__builtin_ia32_selectw_128(
142 (__mmask8)__U, (__v8hi)_mm_loadrs_epi16(__A), (__v8hi)__W);
143}
144
145static __inline__ __m128i __DEFAULT_FN_ATTRS128
146_mm_maskz_loadrs_epi16(__mmask8 __U, void const *__A) {
147 return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
148 (__v8hi)_mm_loadrs_epi16(__A),
149 (__v8hi)_mm_setzero_si128());
150}
151
152static __inline__ __m256i __DEFAULT_FN_ATTRS256
153_mm256_loadrs_epi16(void const *__A) {
154 return (__m256i)__builtin_ia32_vmovrsw256((const __v16hi *)(__A));
155}
156
157static __inline__ __m256i __DEFAULT_FN_ATTRS256
158_mm256_mask_loadrs_epi16(__m256i __W, __mmask16 __U, void const *__A) {
159 return (__m256i)__builtin_ia32_selectw_256(
160 (__mmask16)__U, (__v16hi)_mm256_loadrs_epi16(__A), (__v16hi)__W);
161}
162
163static __inline__ __m256i __DEFAULT_FN_ATTRS256
164_mm256_maskz_loadrs_epi16(__mmask16 __U, void const *__A) {
165 return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
166 (__v16hi)_mm256_loadrs_epi16(__A),
167 (__v16hi)_mm256_setzero_si256());
168}
169
170#undef __DEFAULT_FN_ATTRS128
171#undef __DEFAULT_FN_ATTRS256
172
173#endif /* __x86_64__ */
174#endif /* __MOVRS_AVX10_2INTRIN_H */
lib/include/movrsintrin.h created+59
......@@ -0,0 +1,59 @@
1/*===---------------- movrsintrin.h - MOVRS intrinsics ----------------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===----------------------------------------------------------------------===*/
8
9#ifndef __IMMINTRIN_H
10#error "Never use <movrsintrin.h> directly; include <immintrin.h> instead."
11#endif // __IMMINTRIN_H
12
13#ifndef __MOVRSINTRIN_H
14#define __MOVRSINTRIN_H
15
16#define __DEFAULT_FN_ATTRS \
17 __attribute__((__always_inline__, __nodebug__, __target__("movrs")))
18
19#ifdef __x86_64__
20static __inline__ char __DEFAULT_FN_ATTRS _movrs_i8(const void *__A) {
21 return (char)__builtin_ia32_movrsqi((const void *)__A);
22}
23
24static __inline__ short __DEFAULT_FN_ATTRS _movrs_i16(const void *__A) {
25 return (short)__builtin_ia32_movrshi((const void *)__A);
26}
27
28static __inline__ int __DEFAULT_FN_ATTRS _movrs_i32(const void *__A) {
29 return (int)__builtin_ia32_movrssi((const void *)__A);
30}
31
32static __inline__ long long __DEFAULT_FN_ATTRS _movrs_i64(const void *__A) {
33 return (long long)__builtin_ia32_movrsdi((const void *)__A);
34}
35#endif // __x86_64__
36
37// Loads a memory sequence containing the specified memory address into
38/// the L3 data cache. Data will be shared (read/written) to by requesting
39/// core and other cores.
40///
41/// Note that the effect of this intrinsic is dependent on the processor
42/// implementation.
43///
44/// \headerfile <x86intrin.h>
45///
46/// This intrinsic corresponds to the \c PREFETCHRS instruction.
47///
48/// \param __P
49/// A pointer specifying the memory address to be prefetched.
50static __inline__ void __DEFAULT_FN_ATTRS
51_m_prefetchrs(volatile const void *__P) {
52#pragma clang diagnostic push
53#pragma clang diagnostic ignored "-Wcast-qual"
54 __builtin_ia32_prefetchrs((const void *)__P);
55#pragma clang diagnostic pop
56}
57
58#undef __DEFAULT_FN_ATTRS
59#endif // __MOVRSINTRIN_H
\ No newline at end of file
lib/include/openmp_wrappers/__clang_openmp_device_functions.h+4-5
......@@ -10,17 +10,15 @@
1010#ifndef __CLANG_OPENMP_DEVICE_FUNCTIONS_H__
1111#define __CLANG_OPENMP_DEVICE_FUNCTIONS_H__
1212
13#ifndef _OPENMP
14#error "This file is for OpenMP compilation only."
15#endif
16
1713#ifdef __cplusplus
1814extern "C" {
1915#endif
2016
17#ifdef __NVPTX__
2118#pragma omp begin declare variant match( \
2219 device = {arch(nvptx, nvptx64)}, implementation = {extension(match_any)})
2320
21#pragma push_macro("__CUDA__")
2422#define __CUDA__
2523#define __OPENMP_NVPTX__
2624
......@@ -31,9 +29,10 @@ extern "C" {
3129#include <__clang_cuda_device_functions.h>
3230
3331#undef __OPENMP_NVPTX__
34#undef __CUDA__
32#pragma pop_macro("__CUDA__")
3533
3634#pragma omp end declare variant
35#endif
3736
3837#ifdef __AMDGCN__
3938#pragma omp begin declare variant match(device = {arch(amdgcn)})
lib/include/openmp_wrappers/complex_cmath.h+7-2
......@@ -64,8 +64,13 @@ template <class _Tp> __DEVICE__ _Tp norm(const std::complex<_Tp> &__c) {
6464}
6565
6666// conj
67
68template <class _Tp> std::complex<_Tp> conj(const std::complex<_Tp> &__c) {
67#ifdef _GLIBCXX20_CONSTEXPR
68#define CXX20_CONSTEXPR_DEVICE __DEVICE__
69#else
70#define CXX20_CONSTEXPR_DEVICE
71#endif
72template <class _Tp>
73CXX20_CONSTEXPR_DEVICE std::complex<_Tp> conj(const std::complex<_Tp> &__c) {
6974 return std::complex<_Tp>(__c.real(), -__c.imag());
7075}
7176
lib/include/pmmintrin.h+16-3
......@@ -17,9 +17,21 @@
1717#include <emmintrin.h>
1818
1919/* Define the default attributes for the functions in this file. */
20#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
2021#define __DEFAULT_FN_ATTRS \
2122 __attribute__((__always_inline__, __nodebug__, \
2223 __target__("sse3,no-evex512"), __min_vector_width__(128)))
24#else
25#define __DEFAULT_FN_ATTRS \
26 __attribute__((__always_inline__, __nodebug__, __target__("sse3"), \
27 __min_vector_width__(128)))
28#endif
29
30#if defined(__cplusplus) && (__cplusplus >= 201103L)
31#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
32#else
33#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
34#endif
2335
2436/// Loads data from an unaligned memory location to elements in a 128-bit
2537/// vector.
......@@ -122,7 +134,7 @@ _mm_hsub_ps(__m128 __a, __m128 __b)
122134/// destination.
123135/// \returns A 128-bit vector of [4 x float] containing the moved and duplicated
124136/// values.
125static __inline__ __m128 __DEFAULT_FN_ATTRS
137static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
126138_mm_movehdup_ps(__m128 __a)
127139{
128140 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__a, 1, 1, 3, 3);
......@@ -143,7 +155,7 @@ _mm_movehdup_ps(__m128 __a)
143155/// destination.
144156/// \returns A 128-bit vector of [4 x float] containing the moved and duplicated
145157/// values.
146static __inline__ __m128 __DEFAULT_FN_ATTRS
158static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
147159_mm_moveldup_ps(__m128 __a)
148160{
149161 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__a, 0, 0, 2, 2);
......@@ -244,7 +256,7 @@ _mm_hsub_pd(__m128d __a, __m128d __b)
244256/// [127:64] and [63:0] of the destination.
245257/// \returns A 128-bit vector of [2 x double] containing the moved and
246258/// duplicated values.
247static __inline__ __m128d __DEFAULT_FN_ATTRS
259static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
248260_mm_movedup_pd(__m128d __a)
249261{
250262 return __builtin_shufflevector((__v2df)__a, (__v2df)__a, 0, 0);
......@@ -297,5 +309,6 @@ _mm_mwait(unsigned __extensions, unsigned __hints)
297309}
298310
299311#undef __DEFAULT_FN_ATTRS
312#undef __DEFAULT_FN_ATTRS_CONSTEXPR
300313
301314#endif /* __PMMINTRIN_H */
lib/include/popcntintrin.h+7-7
......@@ -11,12 +11,13 @@
1111#define __POPCNTINTRIN_H
1212
1313/* Define the default attributes for the functions in this file. */
14#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("popcnt")))
15
1614#if defined(__cplusplus) && (__cplusplus >= 201103L)
17#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
15#define __DEFAULT_FN_ATTRS \
16 __attribute__((__always_inline__, __nodebug__, \
17 __target__("popcnt"))) constexpr
1818#else
19#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, __target__("popcnt")))
2021#endif
2122
2223/// Counts the number of bits in the source operand having a value of 1.
......@@ -29,7 +30,7 @@
2930/// An unsigned 32-bit integer operand.
3031/// \returns A 32-bit integer containing the number of bits with value 1 in the
3132/// source operand.
32static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
33static __inline__ int __DEFAULT_FN_ATTRS
3334_mm_popcnt_u32(unsigned int __A)
3435{
3536 return __builtin_popcount(__A);
......@@ -46,7 +47,7 @@ _mm_popcnt_u32(unsigned int __A)
4647/// An unsigned 64-bit integer operand.
4748/// \returns A 64-bit integer containing the number of bits with value 1 in the
4849/// source operand.
49static __inline__ long long __DEFAULT_FN_ATTRS_CONSTEXPR
50static __inline__ long long __DEFAULT_FN_ATTRS
5051_mm_popcnt_u64(unsigned long long __A)
5152{
5253 return __builtin_popcountll(__A);
......@@ -54,6 +55,5 @@ _mm_popcnt_u64(unsigned long long __A)
5455#endif /* __x86_64__ */
5556
5657#undef __DEFAULT_FN_ATTRS
57#undef __DEFAULT_FN_ATTRS_CONSTEXPR
5858
5959#endif /* __POPCNTINTRIN_H */
lib/include/ptrauth.h+6
......@@ -42,6 +42,9 @@ typedef enum {
4242 The extra data is always 0. */
4343 ptrauth_key_cxx_vtable_pointer = ptrauth_key_process_independent_data,
4444
45 /* The key used to sign pointers in ELF .init_array/.fini_array. */
46 ptrauth_key_init_fini_pointer = ptrauth_key_process_independent_code,
47
4548 /* Other pointers signed under the ABI use private ABI rules. */
4649
4750} ptrauth_key;
......@@ -253,6 +256,9 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
253256 [[clang::ptrauth_vtable_pointer(key, address_discrimination, \
254257 extra_discrimination)]]
255258
259/* The value is ptrauth_string_discriminator("init_fini") */
260#define __ptrauth_init_fini_discriminator 0xd9d4
261
256262#else
257263
258264#define ptrauth_strip(__value, __key) \
lib/include/riscv_corev_alu.h created+128
......@@ -0,0 +1,128 @@
1/*===---- riscv_corev_alu.h - CORE-V ALU intrinsics ------------------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10#ifndef __RISCV_COREV_ALU_H
11#define __RISCV_COREV_ALU_H
12
13#include <stdint.h>
14
15#if defined(__cplusplus)
16extern "C" {
17#endif
18
19#if defined(__riscv_xcvalu)
20
21#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
22
23static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_abs(long a) {
24 return __builtin_abs(a);
25}
26
27static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_slet(long a, long b) {
28 return __builtin_riscv_cv_alu_slet(a, b);
29}
30
31static __inline__ long __DEFAULT_FN_ATTRS
32__riscv_cv_alu_sletu(unsigned long a, unsigned long b) {
33 return __builtin_riscv_cv_alu_sletu(a, b);
34}
35
36static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_min(long a, long b) {
37 return __builtin_elementwise_min(a, b);
38}
39
40static __inline__ unsigned long __DEFAULT_FN_ATTRS
41__riscv_cv_alu_minu(unsigned long a, unsigned long b) {
42 return __builtin_elementwise_min(a, b);
43}
44
45static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_max(long a, long b) {
46 return __builtin_elementwise_max(a, b);
47}
48
49static __inline__ unsigned long __DEFAULT_FN_ATTRS
50__riscv_cv_alu_maxu(unsigned long a, unsigned long b) {
51 return __builtin_elementwise_max(a, b);
52}
53
54static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_exths(int16_t a) {
55 return __builtin_riscv_cv_alu_exths(a);
56}
57
58static __inline__ unsigned long __DEFAULT_FN_ATTRS
59__riscv_cv_alu_exthz(uint16_t a) {
60 return __builtin_riscv_cv_alu_exthz(a);
61}
62
63static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_extbs(int8_t a) {
64 return __builtin_riscv_cv_alu_extbs(a);
65}
66
67static __inline__ unsigned long __DEFAULT_FN_ATTRS
68__riscv_cv_alu_extbz(uint8_t a) {
69 return __builtin_riscv_cv_alu_extbz(a);
70}
71
72static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_clip(long a,
73 unsigned long b) {
74 return __builtin_riscv_cv_alu_clip(a, b);
75}
76
77static __inline__ unsigned long __DEFAULT_FN_ATTRS
78__riscv_cv_alu_clipu(unsigned long a, unsigned long b) {
79 return __builtin_riscv_cv_alu_clipu(a, b);
80}
81
82static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_addN(long a, long b,
83 uint8_t shft) {
84 return __builtin_riscv_cv_alu_addN(a, b, shft);
85}
86
87static __inline__ unsigned long __DEFAULT_FN_ATTRS
88__riscv_cv_alu_adduN(unsigned long a, unsigned long b, uint8_t shft) {
89 return __builtin_riscv_cv_alu_adduN(a, b, shft);
90}
91
92static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_addRN(long a, long b,
93 uint8_t shft) {
94 return __builtin_riscv_cv_alu_addRN(a, b, shft);
95}
96
97static __inline__ unsigned long __DEFAULT_FN_ATTRS
98__riscv_cv_alu_adduRN(unsigned long a, unsigned long b, uint8_t shft) {
99 return __builtin_riscv_cv_alu_adduRN(a, b, shft);
100}
101
102static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_subN(long a, long b,
103 uint8_t shft) {
104 return __builtin_riscv_cv_alu_subN(a, b, shft);
105}
106
107static __inline__ unsigned long __DEFAULT_FN_ATTRS
108__riscv_cv_alu_subuN(unsigned long a, unsigned long b, uint8_t shft) {
109 return __builtin_riscv_cv_alu_subuN(a, b, shft);
110}
111
112static __inline__ long __DEFAULT_FN_ATTRS __riscv_cv_alu_subRN(long a, long b,
113 uint8_t shft) {
114 return __builtin_riscv_cv_alu_subRN(a, b, shft);
115}
116
117static __inline__ unsigned long __DEFAULT_FN_ATTRS
118__riscv_cv_alu_subuRN(unsigned long a, unsigned long b, uint8_t shft) {
119 return __builtin_riscv_cv_alu_subuRN(a, b, shft);
120}
121
122#endif // defined(__riscv_xcvalu)
123
124#if defined(__cplusplus)
125}
126#endif
127
128#endif // define __RISCV_COREV_ALU_H
lib/include/riscv_vector.h-1
......@@ -419,7 +419,6 @@ typedef __rvv_bfloat16m2x4_t vbfloat16m2x4_t;
419419typedef __rvv_bfloat16m4_t vbfloat16m4_t;
420420typedef __rvv_bfloat16m4x2_t vbfloat16m4x2_t;
421421typedef __rvv_bfloat16m8_t vbfloat16m8_t;
422#define __riscv_v_intrinsic_overloading 1
423422
424423#ifdef __cplusplus
425424}
lib/include/sm4evexintrin.h created+32
......@@ -0,0 +1,32 @@
1/*===--------------- sm4evexintrin.h - SM4 EVEX intrinsics -----------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error "Never use <sm4evexintrin.h> directly; include <immintrin.h> instead."
11#endif // __IMMINTRIN_H
12
13#ifndef __SM4EVEXINTRIN_H
14#define __SM4EVEXINTRIN_H
15
16#define __DEFAULT_FN_ATTRS512 \
17 __attribute__((__always_inline__, __nodebug__, \
18 __target__("sm4,avx10.2-512"), __min_vector_width__(512)))
19
20static __inline__ __m512i __DEFAULT_FN_ATTRS512
21_mm512_sm4key4_epi32(__m512i __A, __m512i __B) {
22 return (__m512i)__builtin_ia32_vsm4key4512((__v16su)__A, (__v16su)__B);
23}
24
25static __inline__ __m512i __DEFAULT_FN_ATTRS512
26_mm512_sm4rnds4_epi32(__m512i __A, __m512i __B) {
27 return (__m512i)__builtin_ia32_vsm4rnds4512((__v16su)__A, (__v16su)__B);
28}
29
30#undef __DEFAULT_FN_ATTRS512
31
32#endif // __SM4EVEXINTRIN_H
lib/include/smmintrin.h+6
......@@ -17,9 +17,15 @@
1717#include <tmmintrin.h>
1818
1919/* Define the default attributes for the functions in this file. */
20#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
2021#define __DEFAULT_FN_ATTRS \
2122 __attribute__((__always_inline__, __nodebug__, \
2223 __target__("sse4.1,no-evex512"), __min_vector_width__(128)))
24#else
25#define __DEFAULT_FN_ATTRS \
26 __attribute__((__always_inline__, __nodebug__, __target__("sse4.1"), \
27 __min_vector_width__(128)))
28#endif
2329
2430/* SSE4 Rounding macros. */
2531#define _MM_FROUND_TO_NEAREST_INT 0x00
lib/include/stdalign.h-5
......@@ -10,10 +10,6 @@
1010#ifndef __STDALIGN_H
1111#define __STDALIGN_H
1212
13#if defined(__MVS__) && __has_include_next(<stdalign.h>)
14#include_next <stdalign.h>
15#else
16
1713#if defined(__cplusplus) || \
1814 (defined(__STDC_VERSION__) && __STDC_VERSION__ < 202311L)
1915#ifndef __cplusplus
......@@ -25,5 +21,4 @@
2521#define __alignof_is_defined 1
2622#endif /* __STDC_VERSION__ */
2723
28#endif /* __MVS__ */
2924#endif /* __STDALIGN_H */
lib/include/tbmintrin.h+25-37
......@@ -15,63 +15,60 @@
1515#define __TBMINTRIN_H
1616
1717/* Define the default attributes for the functions in this file. */
18#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("tbm")))
18#if defined(__cplusplus) && (__cplusplus >= 201103L)
19#define __DEFAULT_FN_ATTRS \
20 __attribute__((__always_inline__, __nodebug__, __target__("tbm"))) constexpr
21#else
22#define __DEFAULT_FN_ATTRS \
23 __attribute__((__always_inline__, __nodebug__, __target__("tbm")))
24#endif
1925
2026#define __bextri_u32(a, b) \
2127 ((unsigned int)__builtin_ia32_bextri_u32((unsigned int)(a), \
2228 (unsigned int)(b)))
2329
2430static __inline__ unsigned int __DEFAULT_FN_ATTRS
25__blcfill_u32(unsigned int __a)
26{
31__blcfill_u32(unsigned int __a) {
2732 return __a & (__a + 1);
2833}
2934
3035static __inline__ unsigned int __DEFAULT_FN_ATTRS
31__blci_u32(unsigned int __a)
32{
36__blci_u32(unsigned int __a) {
3337 return __a | ~(__a + 1);
3438}
3539
3640static __inline__ unsigned int __DEFAULT_FN_ATTRS
37__blcic_u32(unsigned int __a)
38{
41__blcic_u32(unsigned int __a) {
3942 return ~__a & (__a + 1);
4043}
4144
4245static __inline__ unsigned int __DEFAULT_FN_ATTRS
43__blcmsk_u32(unsigned int __a)
44{
46__blcmsk_u32(unsigned int __a) {
4547 return __a ^ (__a + 1);
4648}
4749
4850static __inline__ unsigned int __DEFAULT_FN_ATTRS
49__blcs_u32(unsigned int __a)
50{
51__blcs_u32(unsigned int __a) {
5152 return __a | (__a + 1);
5253}
5354
5455static __inline__ unsigned int __DEFAULT_FN_ATTRS
55__blsfill_u32(unsigned int __a)
56{
56__blsfill_u32(unsigned int __a) {
5757 return __a | (__a - 1);
5858}
5959
6060static __inline__ unsigned int __DEFAULT_FN_ATTRS
61__blsic_u32(unsigned int __a)
62{
61__blsic_u32(unsigned int __a) {
6362 return ~__a | (__a - 1);
6463}
6564
6665static __inline__ unsigned int __DEFAULT_FN_ATTRS
67__t1mskc_u32(unsigned int __a)
68{
66__t1mskc_u32(unsigned int __a) {
6967 return ~__a | (__a + 1);
7068}
7169
7270static __inline__ unsigned int __DEFAULT_FN_ATTRS
73__tzmsk_u32(unsigned int __a)
74{
71__tzmsk_u32(unsigned int __a) {
7572 return ~__a & (__a - 1);
7673}
7774
......@@ -81,56 +78,47 @@ __tzmsk_u32(unsigned int __a)
8178 (unsigned long long)(b)))
8279
8380static __inline__ unsigned long long __DEFAULT_FN_ATTRS
84__blcfill_u64(unsigned long long __a)
85{
81__blcfill_u64(unsigned long long __a) {
8682 return __a & (__a + 1);
8783}
8884
8985static __inline__ unsigned long long __DEFAULT_FN_ATTRS
90__blci_u64(unsigned long long __a)
91{
86__blci_u64(unsigned long long __a) {
9287 return __a | ~(__a + 1);
9388}
9489
9590static __inline__ unsigned long long __DEFAULT_FN_ATTRS
96__blcic_u64(unsigned long long __a)
97{
91__blcic_u64(unsigned long long __a) {
9892 return ~__a & (__a + 1);
9993}
10094
10195static __inline__ unsigned long long __DEFAULT_FN_ATTRS
102__blcmsk_u64(unsigned long long __a)
103{
96__blcmsk_u64(unsigned long long __a) {
10497 return __a ^ (__a + 1);
10598}
10699
107100static __inline__ unsigned long long __DEFAULT_FN_ATTRS
108__blcs_u64(unsigned long long __a)
109{
101__blcs_u64(unsigned long long __a) {
110102 return __a | (__a + 1);
111103}
112104
113105static __inline__ unsigned long long __DEFAULT_FN_ATTRS
114__blsfill_u64(unsigned long long __a)
115{
106__blsfill_u64(unsigned long long __a) {
116107 return __a | (__a - 1);
117108}
118109
119110static __inline__ unsigned long long __DEFAULT_FN_ATTRS
120__blsic_u64(unsigned long long __a)
121{
111__blsic_u64(unsigned long long __a) {
122112 return ~__a | (__a - 1);
123113}
124114
125115static __inline__ unsigned long long __DEFAULT_FN_ATTRS
126__t1mskc_u64(unsigned long long __a)
127{
116__t1mskc_u64(unsigned long long __a) {
128117 return ~__a | (__a + 1);
129118}
130119
131120static __inline__ unsigned long long __DEFAULT_FN_ATTRS
132__tzmsk_u64(unsigned long long __a)
133{
121__tzmsk_u64(unsigned long long __a) {
134122 return ~__a & (__a - 1);
135123}
136124#endif
lib/include/tmmintrin.h+64-38
......@@ -17,13 +17,21 @@
1717#include <pmmintrin.h>
1818
1919/* Define the default attributes for the functions in this file. */
20#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
2021#define __DEFAULT_FN_ATTRS \
2122 __attribute__((__always_inline__, __nodebug__, \
22 __target__("ssse3,no-evex512"), __min_vector_width__(64)))
23#define __DEFAULT_FN_ATTRS_MMX \
24 __attribute__((__always_inline__, __nodebug__, \
25 __target__("mmx,ssse3,no-evex512"), \
26 __min_vector_width__(64)))
23 __target__("ssse3,no-evex512"), __min_vector_width__(128)))
24#else
25#define __DEFAULT_FN_ATTRS \
26 __attribute__((__always_inline__, __nodebug__, __target__("ssse3"), \
27 __min_vector_width__(128)))
28#endif
29
30#define __trunc64(x) \
31 (__m64) __builtin_shufflevector((__v2di)(x), __extension__(__v2di){}, 0)
32#define __anyext128(x) \
33 (__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0, \
34 1, -1, -1)
2735
2836/// Computes the absolute value of each of the packed 8-bit signed
2937/// integers in the source operand and stores the 8-bit unsigned integer
......@@ -37,10 +45,10 @@
3745/// A 64-bit vector of [8 x i8].
3846/// \returns A 64-bit integer vector containing the absolute values of the
3947/// elements in the operand.
40static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
48static __inline__ __m64 __DEFAULT_FN_ATTRS
4149_mm_abs_pi8(__m64 __a)
4250{
43 return (__m64)__builtin_ia32_pabsb((__v8qi)__a);
51 return (__m64)__builtin_elementwise_abs((__v8qs)__a);
4452}
4553
4654/// Computes the absolute value of each of the packed 8-bit signed
......@@ -73,10 +81,10 @@ _mm_abs_epi8(__m128i __a)
7381/// A 64-bit vector of [4 x i16].
7482/// \returns A 64-bit integer vector containing the absolute values of the
7583/// elements in the operand.
76static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
84static __inline__ __m64 __DEFAULT_FN_ATTRS
7785_mm_abs_pi16(__m64 __a)
7886{
79 return (__m64)__builtin_ia32_pabsw((__v4hi)__a);
87 return (__m64)__builtin_elementwise_abs((__v4hi)__a);
8088}
8189
8290/// Computes the absolute value of each of the packed 16-bit signed
......@@ -109,10 +117,10 @@ _mm_abs_epi16(__m128i __a)
109117/// A 64-bit vector of [2 x i32].
110118/// \returns A 64-bit integer vector containing the absolute values of the
111119/// elements in the operand.
112static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
120static __inline__ __m64 __DEFAULT_FN_ATTRS
113121_mm_abs_pi32(__m64 __a)
114122{
115 return (__m64)__builtin_ia32_pabsd((__v2si)__a);
123 return (__m64)__builtin_elementwise_abs((__v2si)__a);
116124}
117125
118126/// Computes the absolute value of each of the packed 32-bit signed
......@@ -177,7 +185,10 @@ _mm_abs_epi32(__m128i __a)
177185/// \returns A 64-bit integer vector containing the concatenated right-shifted
178186/// value.
179187#define _mm_alignr_pi8(a, b, n) \
180 ((__m64)__builtin_ia32_palignr((__v8qi)(__m64)(a), (__v8qi)(__m64)(b), (n)))
188 ((__m64)__builtin_shufflevector( \
189 __builtin_ia32_psrldqi128_byteshift( \
190 __builtin_shufflevector((__v1di)(a), (__v1di)(b), 1, 0), \
191 (n)), __extension__ (__v2di){}, 0))
181192
182193/// Horizontally adds the adjacent pairs of values contained in 2 packed
183194/// 128-bit vectors of [8 x i16].
......@@ -242,10 +253,11 @@ _mm_hadd_epi32(__m128i __a, __m128i __b)
242253/// destination.
243254/// \returns A 64-bit vector of [4 x i16] containing the horizontal sums of both
244255/// operands.
245static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
256static __inline__ __m64 __DEFAULT_FN_ATTRS
246257_mm_hadd_pi16(__m64 __a, __m64 __b)
247258{
248 return (__m64)__builtin_ia32_phaddw((__v4hi)__a, (__v4hi)__b);
259 return __trunc64(__builtin_ia32_phaddw128(
260 (__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
249261}
250262
251263/// Horizontally adds the adjacent pairs of values contained in 2 packed
......@@ -265,10 +277,11 @@ _mm_hadd_pi16(__m64 __a, __m64 __b)
265277/// destination.
266278/// \returns A 64-bit vector of [2 x i32] containing the horizontal sums of both
267279/// operands.
268static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
280static __inline__ __m64 __DEFAULT_FN_ATTRS
269281_mm_hadd_pi32(__m64 __a, __m64 __b)
270282{
271 return (__m64)__builtin_ia32_phaddd((__v2si)__a, (__v2si)__b);
283 return __trunc64(__builtin_ia32_phaddd128(
284 (__v4si)__builtin_shufflevector(__a, __b, 0, 1), (__v4si){}));
272285}
273286
274287/// Horizontally adds, with saturation, the adjacent pairs of values contained
......@@ -317,10 +330,11 @@ _mm_hadds_epi16(__m128i __a, __m128i __b)
317330/// destination.
318331/// \returns A 64-bit vector of [4 x i16] containing the horizontal saturated
319332/// sums of both operands.
320static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
333static __inline__ __m64 __DEFAULT_FN_ATTRS
321334_mm_hadds_pi16(__m64 __a, __m64 __b)
322335{
323 return (__m64)__builtin_ia32_phaddsw((__v4hi)__a, (__v4hi)__b);
336 return __trunc64(__builtin_ia32_phaddsw128(
337 (__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
324338}
325339
326340/// Horizontally subtracts the adjacent pairs of values contained in 2
......@@ -386,10 +400,11 @@ _mm_hsub_epi32(__m128i __a, __m128i __b)
386400/// the destination.
387401/// \returns A 64-bit vector of [4 x i16] containing the horizontal differences
388402/// of both operands.
389static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
403static __inline__ __m64 __DEFAULT_FN_ATTRS
390404_mm_hsub_pi16(__m64 __a, __m64 __b)
391405{
392 return (__m64)__builtin_ia32_phsubw((__v4hi)__a, (__v4hi)__b);
406 return __trunc64(__builtin_ia32_phsubw128(
407 (__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
393408}
394409
395410/// Horizontally subtracts the adjacent pairs of values contained in 2
......@@ -409,10 +424,11 @@ _mm_hsub_pi16(__m64 __a, __m64 __b)
409424/// the destination.
410425/// \returns A 64-bit vector of [2 x i32] containing the horizontal differences
411426/// of both operands.
412static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
427static __inline__ __m64 __DEFAULT_FN_ATTRS
413428_mm_hsub_pi32(__m64 __a, __m64 __b)
414429{
415 return (__m64)__builtin_ia32_phsubd((__v2si)__a, (__v2si)__b);
430 return __trunc64(__builtin_ia32_phsubd128(
431 (__v4si)__builtin_shufflevector(__a, __b, 0, 1), (__v4si){}));
416432}
417433
418434/// Horizontally subtracts, with saturation, the adjacent pairs of values
......@@ -461,10 +477,11 @@ _mm_hsubs_epi16(__m128i __a, __m128i __b)
461477/// the destination.
462478/// \returns A 64-bit vector of [4 x i16] containing the horizontal saturated
463479/// differences of both operands.
464static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
480static __inline__ __m64 __DEFAULT_FN_ATTRS
465481_mm_hsubs_pi16(__m64 __a, __m64 __b)
466482{
467 return (__m64)__builtin_ia32_phsubsw((__v4hi)__a, (__v4hi)__b);
483 return __trunc64(__builtin_ia32_phsubsw128(
484 (__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
468485}
469486
470487/// Multiplies corresponding pairs of packed 8-bit unsigned integer
......@@ -525,10 +542,11 @@ _mm_maddubs_epi16(__m128i __a, __m128i __b)
525542/// \a R1 := (\a __a2 * \a __b2) + (\a __a3 * \a __b3) \n
526543/// \a R2 := (\a __a4 * \a __b4) + (\a __a5 * \a __b5) \n
527544/// \a R3 := (\a __a6 * \a __b6) + (\a __a7 * \a __b7)
528static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
545static __inline__ __m64 __DEFAULT_FN_ATTRS
529546_mm_maddubs_pi16(__m64 __a, __m64 __b)
530547{
531 return (__m64)__builtin_ia32_pmaddubsw((__v8qi)__a, (__v8qi)__b);
548 return __trunc64(__builtin_ia32_pmaddubsw128((__v16qi)__anyext128(__a),
549 (__v16qi)__anyext128(__b)));
532550}
533551
534552/// Multiplies packed 16-bit signed integer values, truncates the 32-bit
......@@ -565,10 +583,11 @@ _mm_mulhrs_epi16(__m128i __a, __m128i __b)
565583/// A 64-bit vector of [4 x i16] containing one of the source operands.
566584/// \returns A 64-bit vector of [4 x i16] containing the rounded and scaled
567585/// products of both operands.
568static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
586static __inline__ __m64 __DEFAULT_FN_ATTRS
569587_mm_mulhrs_pi16(__m64 __a, __m64 __b)
570588{
571 return (__m64)__builtin_ia32_pmulhrsw((__v4hi)__a, (__v4hi)__b);
589 return __trunc64(__builtin_ia32_pmulhrsw128((__v8hi)__anyext128(__a),
590 (__v8hi)__anyext128(__b)));
572591}
573592
574593/// Copies the 8-bit integers from a 128-bit integer vector to the
......@@ -614,12 +633,15 @@ _mm_shuffle_epi8(__m128i __a, __m128i __b)
614633/// 1: Clear the corresponding byte in the destination. \n
615634/// 0: Copy the selected source byte to the corresponding byte in the
616635/// destination. \n
617/// Bits [3:0] select the source byte to be copied.
636/// Bits [2:0] select the source byte to be copied.
618637/// \returns A 64-bit integer vector containing the copied or cleared values.
619static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
638static __inline__ __m64 __DEFAULT_FN_ATTRS
620639_mm_shuffle_pi8(__m64 __a, __m64 __b)
621640{
622 return (__m64)__builtin_ia32_pshufb((__v8qi)__a, (__v8qi)__b);
641 return __trunc64(__builtin_ia32_pshufb128(
642 (__v16qi)__builtin_shufflevector(
643 (__v2si)(__a), __extension__ (__v2si){}, 0, 1, 0, 1),
644 (__v16qi)__anyext128(__b)));
623645}
624646
625647/// For each 8-bit integer in the first source operand, perform one of
......@@ -720,10 +742,11 @@ _mm_sign_epi32(__m128i __a, __m128i __b)
720742/// A 64-bit integer vector containing control bytes corresponding to
721743/// positions in the destination.
722744/// \returns A 64-bit integer vector containing the resultant values.
723static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
745static __inline__ __m64 __DEFAULT_FN_ATTRS
724746_mm_sign_pi8(__m64 __a, __m64 __b)
725747{
726 return (__m64)__builtin_ia32_psignb((__v8qi)__a, (__v8qi)__b);
748 return __trunc64(__builtin_ia32_psignb128((__v16qi)__anyext128(__a),
749 (__v16qi)__anyext128(__b)));
727750}
728751
729752/// For each 16-bit integer in the first source operand, perform one of
......@@ -746,10 +769,11 @@ _mm_sign_pi8(__m64 __a, __m64 __b)
746769/// A 64-bit integer vector containing control words corresponding to
747770/// positions in the destination.
748771/// \returns A 64-bit integer vector containing the resultant values.
749static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
772static __inline__ __m64 __DEFAULT_FN_ATTRS
750773_mm_sign_pi16(__m64 __a, __m64 __b)
751774{
752 return (__m64)__builtin_ia32_psignw((__v4hi)__a, (__v4hi)__b);
775 return __trunc64(__builtin_ia32_psignw128((__v8hi)__anyext128(__a),
776 (__v8hi)__anyext128(__b)));
753777}
754778
755779/// For each 32-bit integer in the first source operand, perform one of
......@@ -772,13 +796,15 @@ _mm_sign_pi16(__m64 __a, __m64 __b)
772796/// A 64-bit integer vector containing two control doublewords corresponding
773797/// to positions in the destination.
774798/// \returns A 64-bit integer vector containing the resultant values.
775static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
799static __inline__ __m64 __DEFAULT_FN_ATTRS
776800_mm_sign_pi32(__m64 __a, __m64 __b)
777801{
778 return (__m64)__builtin_ia32_psignd((__v2si)__a, (__v2si)__b);
802 return __trunc64(__builtin_ia32_psignd128((__v4si)__anyext128(__a),
803 (__v4si)__anyext128(__b)));
779804}
780805
806#undef __anyext128
807#undef __trunc64
781808#undef __DEFAULT_FN_ATTRS
782#undef __DEFAULT_FN_ATTRS_MMX
783809
784810#endif /* __TMMINTRIN_H */
lib/include/vecintrin.h+1778-60
......@@ -468,6 +468,27 @@ vec_perm(__vector __bool long long __a, __vector __bool long long __b,
468468 (__vector unsigned char)__a, (__vector unsigned char)__b, __c);
469469}
470470
471static inline __ATTRS_o_ai __vector signed __int128
472vec_perm(__vector signed __int128 __a, __vector signed __int128 __b,
473 __vector unsigned char __c) {
474 return (__vector signed __int128)__builtin_s390_vperm(
475 (__vector unsigned char)__a, (__vector unsigned char)__b, __c);
476}
477
478static inline __ATTRS_o_ai __vector unsigned __int128
479vec_perm(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
480 __vector unsigned char __c) {
481 return (__vector unsigned __int128)__builtin_s390_vperm(
482 (__vector unsigned char)__a, (__vector unsigned char)__b, __c);
483}
484
485static inline __ATTRS_o_ai __vector __bool __int128
486vec_perm(__vector __bool __int128 __a, __vector __bool __int128 __b,
487 __vector unsigned char __c) {
488 return (__vector __bool __int128)__builtin_s390_vperm(
489 (__vector unsigned char)__a, (__vector unsigned char)__b, __c);
490}
491
471492#if __ARCH__ >= 12
472493static inline __ATTRS_o_ai __vector float
473494vec_perm(__vector float __a, __vector float __b,
......@@ -514,9 +535,19 @@ vec_permi(__vector double __a, __vector double __b, int __c)
514535 (__vector unsigned long long)(Y), \
515536 (((Z) & 2) << 1) | ((Z) & 1)))
516537
538/*-- vec_bperm --------------------------------------------------------------*/
539
540#if __ARCH__ >= 12
541static inline __ATTRS_ai __vector unsigned long long
542vec_bperm(__vector unsigned __int128 __a, __vector unsigned char __b) {
543 return __builtin_s390_vbperm((__vector unsigned char)__a, __b);
544}
545#endif
546
517547/*-- vec_bperm_u128 ---------------------------------------------------------*/
518548
519549#if __ARCH__ >= 12
550// This prototype is deprecated.
520551static inline __ATTRS_ai __vector unsigned long long
521552vec_bperm_u128(__vector unsigned char __a, __vector unsigned char __b) {
522553 return __builtin_s390_vbperm(__a, __b);
......@@ -558,6 +589,18 @@ vec_revb(__vector unsigned long long __vec) {
558589 return __builtin_s390_vlbrg(__vec);
559590}
560591
592static inline __ATTRS_o_ai __vector signed __int128
593vec_revb(__vector signed __int128 __vec) {
594 return (__vector signed __int128)
595 __builtin_s390_vlbrq((unsigned __int128)__vec);
596}
597
598static inline __ATTRS_o_ai __vector unsigned __int128
599vec_revb(__vector unsigned __int128 __vec) {
600 return (__vector unsigned __int128)
601 __builtin_s390_vlbrq((unsigned __int128)__vec);
602}
603
561604#if __ARCH__ >= 12
562605static inline __ATTRS_o_ai __vector float
563606vec_revb(__vector float __vec) {
......@@ -820,6 +863,46 @@ vec_sel(__vector unsigned long long __a, __vector unsigned long long __b,
820863 (~(__vector unsigned long long)__c & __a));
821864}
822865
866static inline __ATTRS_o_ai __vector signed __int128
867vec_sel(__vector signed __int128 __a, __vector signed __int128 __b,
868 __vector unsigned __int128 __c) {
869 return (((__vector signed __int128)__c & __b) |
870 (~(__vector signed __int128)__c & __a));
871}
872
873static inline __ATTRS_o_ai __vector signed __int128
874vec_sel(__vector signed __int128 __a, __vector signed __int128 __b,
875 __vector __bool __int128 __c) {
876 return (((__vector signed __int128)__c & __b) |
877 (~(__vector signed __int128)__c & __a));
878}
879
880static inline __ATTRS_o_ai __vector __bool __int128
881vec_sel(__vector __bool __int128 __a, __vector __bool __int128 __b,
882 __vector unsigned __int128 __c) {
883 return (((__vector __bool __int128)__c & __b) |
884 (~(__vector __bool __int128)__c & __a));
885}
886
887static inline __ATTRS_o_ai __vector __bool __int128
888vec_sel(__vector __bool __int128 __a, __vector __bool __int128 __b,
889 __vector __bool __int128 __c) {
890 return (__c & __b) | (~__c & __a);
891}
892
893static inline __ATTRS_o_ai __vector unsigned __int128
894vec_sel(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
895 __vector unsigned __int128 __c) {
896 return (__c & __b) | (~__c & __a);
897}
898
899static inline __ATTRS_o_ai __vector unsigned __int128
900vec_sel(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
901 __vector __bool __int128 __c) {
902 return (((__vector unsigned __int128)__c & __b) |
903 (~(__vector unsigned __int128)__c & __a));
904}
905
823906#if __ARCH__ >= 12
824907static inline __ATTRS_o_ai __vector float
825908vec_sel(__vector float __a, __vector float __b, __vector unsigned int __c) {
......@@ -1078,6 +1161,22 @@ vec_xl(long __offset, const unsigned long long *__ptr) {
10781161 return V;
10791162}
10801163
1164static inline __ATTRS_o_ai __vector signed __int128
1165vec_xl(long __offset, const signed __int128 *__ptr) {
1166 __vector signed __int128 V;
1167 __builtin_memcpy(&V, ((const char *)__ptr + __offset),
1168 sizeof(__vector signed __int128));
1169 return V;
1170}
1171
1172static inline __ATTRS_o_ai __vector unsigned __int128
1173vec_xl(long __offset, const unsigned __int128 *__ptr) {
1174 __vector unsigned __int128 V;
1175 __builtin_memcpy(&V, ((const char *)__ptr + __offset),
1176 sizeof(__vector unsigned __int128));
1177 return V;
1178}
1179
10811180#if __ARCH__ >= 12
10821181static inline __ATTRS_o_ai __vector float
10831182vec_xl(long __offset, const float *__ptr) {
......@@ -1294,6 +1393,22 @@ vec_xst(__vector unsigned long long __vec, long __offset,
12941393 sizeof(__vector unsigned long long));
12951394}
12961395
1396static inline __ATTRS_o_ai void
1397vec_xst(__vector signed __int128 __vec, long __offset,
1398 signed __int128 *__ptr) {
1399 __vector signed __int128 V = __vec;
1400 __builtin_memcpy(((char *)__ptr + __offset), &V,
1401 sizeof(__vector signed __int128));
1402}
1403
1404static inline __ATTRS_o_ai void
1405vec_xst(__vector unsigned __int128 __vec, long __offset,
1406 unsigned __int128 *__ptr) {
1407 __vector unsigned __int128 V = __vec;
1408 __builtin_memcpy(((char *)__ptr + __offset), &V,
1409 sizeof(__vector unsigned __int128));
1410}
1411
12971412#if __ARCH__ >= 12
12981413static inline __ATTRS_o_ai void
12991414vec_xst(__vector float __vec, long __offset, float *__ptr) {
......@@ -1465,6 +1580,14 @@ extern __ATTRS_o __vector unsigned long long
14651580vec_load_bndry(const unsigned long long *__ptr, unsigned short __len)
14661581 __constant_pow2_range(__len, 64, 4096);
14671582
1583extern __ATTRS_o __vector signed __int128
1584vec_load_bndry(const signed __int128 *__ptr, unsigned short __len)
1585 __constant_pow2_range(__len, 64, 4096);
1586
1587extern __ATTRS_o __vector unsigned __int128
1588vec_load_bndry(const unsigned __int128 *__ptr, unsigned short __len)
1589 __constant_pow2_range(__len, 64, 4096);
1590
14681591#if __ARCH__ >= 12
14691592extern __ATTRS_o __vector float
14701593vec_load_bndry(const float *__ptr, unsigned short __len)
......@@ -1496,43 +1619,51 @@ vec_load_len(const unsigned char *__ptr, unsigned int __len) {
14961619 return (__vector unsigned char)__builtin_s390_vll(__len, __ptr);
14971620}
14981621
1622// This prototype is deprecated.
14991623static inline __ATTRS_o_ai __vector signed short
15001624vec_load_len(const signed short *__ptr, unsigned int __len) {
15011625 return (__vector signed short)__builtin_s390_vll(__len, __ptr);
15021626}
15031627
1628// This prototype is deprecated.
15041629static inline __ATTRS_o_ai __vector unsigned short
15051630vec_load_len(const unsigned short *__ptr, unsigned int __len) {
15061631 return (__vector unsigned short)__builtin_s390_vll(__len, __ptr);
15071632}
15081633
1634// This prototype is deprecated.
15091635static inline __ATTRS_o_ai __vector signed int
15101636vec_load_len(const signed int *__ptr, unsigned int __len) {
15111637 return (__vector signed int)__builtin_s390_vll(__len, __ptr);
15121638}
15131639
1640// This prototype is deprecated.
15141641static inline __ATTRS_o_ai __vector unsigned int
15151642vec_load_len(const unsigned int *__ptr, unsigned int __len) {
15161643 return (__vector unsigned int)__builtin_s390_vll(__len, __ptr);
15171644}
15181645
1646// This prototype is deprecated.
15191647static inline __ATTRS_o_ai __vector signed long long
15201648vec_load_len(const signed long long *__ptr, unsigned int __len) {
15211649 return (__vector signed long long)__builtin_s390_vll(__len, __ptr);
15221650}
15231651
1652// This prototype is deprecated.
15241653static inline __ATTRS_o_ai __vector unsigned long long
15251654vec_load_len(const unsigned long long *__ptr, unsigned int __len) {
15261655 return (__vector unsigned long long)__builtin_s390_vll(__len, __ptr);
15271656}
15281657
15291658#if __ARCH__ >= 12
1659// This prototype is deprecated.
15301660static inline __ATTRS_o_ai __vector float
15311661vec_load_len(const float *__ptr, unsigned int __len) {
15321662 return (__vector float)__builtin_s390_vll(__len, __ptr);
15331663}
15341664#endif
15351665
1666// This prototype is deprecated.
15361667static inline __ATTRS_o_ai __vector double
15371668vec_load_len(const double *__ptr, unsigned int __len) {
15381669 return (__vector double)__builtin_s390_vll(__len, __ptr);
......@@ -1541,7 +1672,12 @@ vec_load_len(const double *__ptr, unsigned int __len) {
15411672/*-- vec_load_len_r ---------------------------------------------------------*/
15421673
15431674#if __ARCH__ >= 12
1544static inline __ATTRS_ai __vector unsigned char
1675static inline __ATTRS_o_ai __vector signed char
1676vec_load_len_r(const signed char *__ptr, unsigned int __len) {
1677 return (__vector signed char)__builtin_s390_vlrlr(__len, __ptr);
1678}
1679
1680static inline __ATTRS_o_ai __vector unsigned char
15451681vec_load_len_r(const unsigned char *__ptr, unsigned int __len) {
15461682 return (__vector unsigned char)__builtin_s390_vlrlr(__len, __ptr);
15471683}
......@@ -1561,36 +1697,42 @@ vec_store_len(__vector unsigned char __vec, unsigned char *__ptr,
15611697 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);
15621698}
15631699
1700// This prototype is deprecated.
15641701static inline __ATTRS_o_ai void
15651702vec_store_len(__vector signed short __vec, signed short *__ptr,
15661703 unsigned int __len) {
15671704 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);
15681705}
15691706
1707// This prototype is deprecated.
15701708static inline __ATTRS_o_ai void
15711709vec_store_len(__vector unsigned short __vec, unsigned short *__ptr,
15721710 unsigned int __len) {
15731711 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);
15741712}
15751713
1714// This prototype is deprecated.
15761715static inline __ATTRS_o_ai void
15771716vec_store_len(__vector signed int __vec, signed int *__ptr,
15781717 unsigned int __len) {
15791718 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);
15801719}
15811720
1721// This prototype is deprecated.
15821722static inline __ATTRS_o_ai void
15831723vec_store_len(__vector unsigned int __vec, unsigned int *__ptr,
15841724 unsigned int __len) {
15851725 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);
15861726}
15871727
1728// This prototype is deprecated.
15881729static inline __ATTRS_o_ai void
15891730vec_store_len(__vector signed long long __vec, signed long long *__ptr,
15901731 unsigned int __len) {
15911732 __builtin_s390_vstl((__vector signed char)__vec, __len, __ptr);
15921733}
15931734
1735// This prototype is deprecated.
15941736static inline __ATTRS_o_ai void
15951737vec_store_len(__vector unsigned long long __vec, unsigned long long *__ptr,
15961738 unsigned int __len) {
......@@ -1598,6 +1740,7 @@ vec_store_len(__vector unsigned long long __vec, unsigned long long *__ptr,
15981740}
15991741
16001742#if __ARCH__ >= 12
1743// This prototype is deprecated.
16011744static inline __ATTRS_o_ai void
16021745vec_store_len(__vector float __vec, float *__ptr,
16031746 unsigned int __len) {
......@@ -1605,6 +1748,7 @@ vec_store_len(__vector float __vec, float *__ptr,
16051748}
16061749#endif
16071750
1751// This prototype is deprecated.
16081752static inline __ATTRS_o_ai void
16091753vec_store_len(__vector double __vec, double *__ptr,
16101754 unsigned int __len) {
......@@ -1614,7 +1758,13 @@ vec_store_len(__vector double __vec, double *__ptr,
16141758/*-- vec_store_len_r --------------------------------------------------------*/
16151759
16161760#if __ARCH__ >= 12
1617static inline __ATTRS_ai void
1761static inline __ATTRS_o_ai void
1762vec_store_len_r(__vector signed char __vec, signed char *__ptr,
1763 unsigned int __len) {
1764 __builtin_s390_vstrlr(__vec, __len, __ptr);
1765}
1766
1767static inline __ATTRS_o_ai void
16181768vec_store_len_r(__vector unsigned char __vec, unsigned char *__ptr,
16191769 unsigned int __len) {
16201770 __builtin_s390_vstrlr((__vector signed char)__vec, __len, __ptr);
......@@ -1711,6 +1861,35 @@ vec_genmasks_64(unsigned char __first, unsigned char __last)
17111861 return (__vector unsigned long long)__value;
17121862}
17131863
1864/*-- vec_gen_element_masks_* ------------------------------------------------*/
1865
1866#if __ARCH__ >= 15
1867static inline __ATTRS_ai __vector unsigned char
1868vec_gen_element_masks_8(__vector unsigned short __mask) {
1869 return __builtin_s390_vgemb(__mask);
1870}
1871
1872static inline __ATTRS_ai __vector unsigned short
1873vec_gen_element_masks_16(__vector unsigned char __mask) {
1874 return __builtin_s390_vgemh(__mask);
1875}
1876
1877static inline __ATTRS_ai __vector unsigned int
1878vec_gen_element_masks_32(__vector unsigned char __mask) {
1879 return __builtin_s390_vgemf(__mask);
1880}
1881
1882static inline __ATTRS_ai __vector unsigned long long
1883vec_gen_element_masks_64(__vector unsigned char __mask) {
1884 return __builtin_s390_vgemg(__mask);
1885}
1886
1887static inline __ATTRS_ai __vector unsigned __int128
1888vec_gen_element_masks_128(__vector unsigned char __mask) {
1889 return (__vector unsigned __int128)__builtin_s390_vgemq(__mask);
1890}
1891#endif
1892
17141893/*-- vec_splat --------------------------------------------------------------*/
17151894
17161895static inline __ATTRS_o_ai __vector signed char
......@@ -1894,6 +2073,16 @@ vec_splats(unsigned long long __scalar) {
18942073 return (__vector unsigned long long)__scalar;
18952074}
18962075
2076static inline __ATTRS_o_ai __vector signed __int128
2077vec_splats(signed __int128 __scalar) {
2078 return (__vector signed __int128)__scalar;
2079}
2080
2081static inline __ATTRS_o_ai __vector unsigned __int128
2082vec_splats(unsigned __int128 __scalar) {
2083 return (__vector unsigned __int128)__scalar;
2084}
2085
18972086#if __ARCH__ >= 12
18982087static inline __ATTRS_o_ai __vector float
18992088vec_splats(float __scalar) {
......@@ -2166,6 +2355,27 @@ vec_pack(__vector unsigned long long __a, __vector unsigned long long __b) {
21662355 return (__vector unsigned int)(__ac[1], __ac[3], __bc[1], __bc[3]);
21672356}
21682357
2358static inline __ATTRS_o_ai __vector signed long long
2359vec_pack(__vector signed __int128 __a, __vector signed __int128 __b) {
2360 __vector signed long long __ac = (__vector signed long long)__a;
2361 __vector signed long long __bc = (__vector signed long long)__b;
2362 return (__vector signed long long)(__ac[1], __bc[1]);
2363}
2364
2365static inline __ATTRS_o_ai __vector __bool long long
2366vec_pack(__vector __bool __int128 __a, __vector __bool __int128 __b) {
2367 __vector __bool long long __ac = (__vector __bool long long)__a;
2368 __vector __bool long long __bc = (__vector __bool long long)__b;
2369 return (__vector __bool long long)(__ac[1], __bc[1]);
2370}
2371
2372static inline __ATTRS_o_ai __vector unsigned long long
2373vec_pack(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
2374 __vector unsigned long long __ac = (__vector unsigned long long)__a;
2375 __vector unsigned long long __bc = (__vector unsigned long long)__b;
2376 return (__vector unsigned long long)(__ac[1], __bc[1]);
2377}
2378
21692379/*-- vec_packs --------------------------------------------------------------*/
21702380
21712381static inline __ATTRS_o_ai __vector signed char
......@@ -2344,6 +2554,24 @@ vec_unpackh(__vector unsigned int __a) {
23442554 return __builtin_s390_vuplhf(__a);
23452555}
23462556
2557#if __ARCH__ >= 15
2558static inline __ATTRS_o_ai __vector signed __int128
2559vec_unpackh(__vector signed long long __a) {
2560 return (__vector signed __int128)__builtin_s390_vuphg(__a);
2561}
2562
2563static inline __ATTRS_o_ai __vector __bool __int128
2564vec_unpackh(__vector __bool long long __a) {
2565 return ((__vector __bool __int128)
2566 __builtin_s390_vuphg((__vector signed long long)__a));
2567}
2568
2569static inline __ATTRS_o_ai __vector unsigned __int128
2570vec_unpackh(__vector unsigned long long __a) {
2571 return (__vector unsigned __int128)__builtin_s390_vuplhg(__a);
2572}
2573#endif
2574
23472575/*-- vec_unpackl ------------------------------------------------------------*/
23482576
23492577static inline __ATTRS_o_ai __vector signed short
......@@ -2394,6 +2622,24 @@ vec_unpackl(__vector unsigned int __a) {
23942622 return __builtin_s390_vupllf(__a);
23952623}
23962624
2625#if __ARCH__ >= 15
2626static inline __ATTRS_o_ai __vector signed __int128
2627vec_unpackl(__vector signed long long __a) {
2628 return (__vector signed __int128)__builtin_s390_vuplg(__a);
2629}
2630
2631static inline __ATTRS_o_ai __vector __bool __int128
2632vec_unpackl(__vector __bool long long __a) {
2633 return ((__vector __bool __int128)
2634 __builtin_s390_vuplg((__vector signed long long)__a));
2635}
2636
2637static inline __ATTRS_o_ai __vector unsigned __int128
2638vec_unpackl(__vector unsigned long long __a) {
2639 return (__vector unsigned __int128)__builtin_s390_vupllg(__a);
2640}
2641#endif
2642
23972643/*-- vec_cmpeq --------------------------------------------------------------*/
23982644
23992645static inline __ATTRS_o_ai __vector __bool char
......@@ -2456,6 +2702,21 @@ vec_cmpeq(__vector unsigned long long __a, __vector unsigned long long __b) {
24562702 return (__vector __bool long long)(__a == __b);
24572703}
24582704
2705static inline __ATTRS_o_ai __vector __bool __int128
2706vec_cmpeq(__vector __bool __int128 __a, __vector __bool __int128 __b) {
2707 return (__vector __bool __int128)(__a == __b);
2708}
2709
2710static inline __ATTRS_o_ai __vector __bool __int128
2711vec_cmpeq(__vector signed __int128 __a, __vector signed __int128 __b) {
2712 return (__vector __bool __int128)(__a == __b);
2713}
2714
2715static inline __ATTRS_o_ai __vector __bool __int128
2716vec_cmpeq(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
2717 return (__vector __bool __int128)(__a == __b);
2718}
2719
24592720#if __ARCH__ >= 12
24602721static inline __ATTRS_o_ai __vector __bool int
24612722vec_cmpeq(__vector float __a, __vector float __b) {
......@@ -2510,6 +2771,16 @@ vec_cmpge(__vector unsigned long long __a, __vector unsigned long long __b) {
25102771 return (__vector __bool long long)(__a >= __b);
25112772}
25122773
2774static inline __ATTRS_o_ai __vector __bool __int128
2775vec_cmpge(__vector signed __int128 __a, __vector signed __int128 __b) {
2776 return (__vector __bool __int128)(__a >= __b);
2777}
2778
2779static inline __ATTRS_o_ai __vector __bool __int128
2780vec_cmpge(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
2781 return (__vector __bool __int128)(__a >= __b);
2782}
2783
25132784#if __ARCH__ >= 12
25142785static inline __ATTRS_o_ai __vector __bool int
25152786vec_cmpge(__vector float __a, __vector float __b) {
......@@ -2564,6 +2835,16 @@ vec_cmpgt(__vector unsigned long long __a, __vector unsigned long long __b) {
25642835 return (__vector __bool long long)(__a > __b);
25652836}
25662837
2838static inline __ATTRS_o_ai __vector __bool __int128
2839vec_cmpgt(__vector signed __int128 __a, __vector signed __int128 __b) {
2840 return (__vector __bool __int128)(__a > __b);
2841}
2842
2843static inline __ATTRS_o_ai __vector __bool __int128
2844vec_cmpgt(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
2845 return (__vector __bool __int128)(__a > __b);
2846}
2847
25672848#if __ARCH__ >= 12
25682849static inline __ATTRS_o_ai __vector __bool int
25692850vec_cmpgt(__vector float __a, __vector float __b) {
......@@ -2618,6 +2899,16 @@ vec_cmple(__vector unsigned long long __a, __vector unsigned long long __b) {
26182899 return (__vector __bool long long)(__a <= __b);
26192900}
26202901
2902static inline __ATTRS_o_ai __vector __bool __int128
2903vec_cmple(__vector signed __int128 __a, __vector signed __int128 __b) {
2904 return (__vector __bool __int128)(__a <= __b);
2905}
2906
2907static inline __ATTRS_o_ai __vector __bool __int128
2908vec_cmple(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
2909 return (__vector __bool __int128)(__a <= __b);
2910}
2911
26212912#if __ARCH__ >= 12
26222913static inline __ATTRS_o_ai __vector __bool int
26232914vec_cmple(__vector float __a, __vector float __b) {
......@@ -2672,6 +2963,16 @@ vec_cmplt(__vector unsigned long long __a, __vector unsigned long long __b) {
26722963 return (__vector __bool long long)(__a < __b);
26732964}
26742965
2966static inline __ATTRS_o_ai __vector __bool __int128
2967vec_cmplt(__vector signed __int128 __a, __vector signed __int128 __b) {
2968 return (__vector __bool __int128)(__a < __b);
2969}
2970
2971static inline __ATTRS_o_ai __vector __bool __int128
2972vec_cmplt(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
2973 return (__vector __bool __int128)(__a < __b);
2974}
2975
26752976#if __ARCH__ >= 12
26762977static inline __ATTRS_o_ai __vector __bool int
26772978vec_cmplt(__vector float __a, __vector float __b) {
......@@ -2914,6 +3215,29 @@ vec_all_eq(__vector __bool long long __a, __vector __bool long long __b) {
29143215 return __cc == 0;
29153216}
29163217
3218#if __ARCH__ >= 15
3219static inline __ATTRS_o_ai int
3220vec_all_eq(__vector signed __int128 __a, __vector signed __int128 __b) {
3221 int __cc;
3222 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
3223 return __cc == 0;
3224}
3225
3226static inline __ATTRS_o_ai int
3227vec_all_eq(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
3228 int __cc;
3229 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
3230 return __cc == 0;
3231}
3232
3233static inline __ATTRS_o_ai int
3234vec_all_eq(__vector __bool __int128 __a, __vector __bool __int128 __b) {
3235 int __cc;
3236 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
3237 return __cc == 0;
3238}
3239#endif
3240
29173241#if __ARCH__ >= 12
29183242static inline __ATTRS_o_ai int
29193243vec_all_eq(__vector float __a, __vector float __b) {
......@@ -3161,6 +3485,29 @@ vec_all_ne(__vector __bool long long __a, __vector __bool long long __b) {
31613485 return __cc == 3;
31623486}
31633487
3488#if __ARCH__ >= 15
3489static inline __ATTRS_o_ai int
3490vec_all_ne(__vector signed __int128 __a, __vector signed __int128 __b) {
3491 int __cc;
3492 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
3493 return __cc == 3;
3494}
3495
3496static inline __ATTRS_o_ai int
3497vec_all_ne(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
3498 int __cc;
3499 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
3500 return __cc == 3;
3501}
3502
3503static inline __ATTRS_o_ai int
3504vec_all_ne(__vector __bool __int128 __a, __vector __bool __int128 __b) {
3505 int __cc;
3506 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
3507 return __cc == 3;
3508}
3509#endif
3510
31643511#if __ARCH__ >= 12
31653512static inline __ATTRS_o_ai int
31663513vec_all_ne(__vector float __a, __vector float __b) {
......@@ -3399,6 +3746,22 @@ vec_all_ge(__vector __bool long long __a, __vector __bool long long __b) {
33993746 return __cc == 3;
34003747}
34013748
3749#if __ARCH__ >= 15
3750static inline __ATTRS_o_ai int
3751vec_all_ge(__vector signed __int128 __a, __vector signed __int128 __b) {
3752 int __cc;
3753 __builtin_s390_vchqs((signed __int128)__b, (signed __int128)__a, &__cc);
3754 return __cc == 3;
3755}
3756
3757static inline __ATTRS_o_ai int
3758vec_all_ge(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
3759 int __cc;
3760 __builtin_s390_vchlqs((unsigned __int128)__b, (unsigned __int128)__a, &__cc);
3761 return __cc == 3;
3762}
3763#endif
3764
34023765#if __ARCH__ >= 12
34033766static inline __ATTRS_o_ai int
34043767vec_all_ge(__vector float __a, __vector float __b) {
......@@ -3637,6 +4000,22 @@ vec_all_gt(__vector __bool long long __a, __vector __bool long long __b) {
36374000 return __cc == 0;
36384001}
36394002
4003#if __ARCH__ >= 15
4004static inline __ATTRS_o_ai int
4005vec_all_gt(__vector signed __int128 __a, __vector signed __int128 __b) {
4006 int __cc;
4007 __builtin_s390_vchqs((signed __int128)__a, (signed __int128)__b, &__cc);
4008 return __cc == 0;
4009}
4010
4011static inline __ATTRS_o_ai int
4012vec_all_gt(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
4013 int __cc;
4014 __builtin_s390_vchlqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
4015 return __cc == 0;
4016}
4017#endif
4018
36404019#if __ARCH__ >= 12
36414020static inline __ATTRS_o_ai int
36424021vec_all_gt(__vector float __a, __vector float __b) {
......@@ -3875,6 +4254,22 @@ vec_all_le(__vector __bool long long __a, __vector __bool long long __b) {
38754254 return __cc == 3;
38764255}
38774256
4257#if __ARCH__ >= 15
4258static inline __ATTRS_o_ai int
4259vec_all_le(__vector signed __int128 __a, __vector signed __int128 __b) {
4260 int __cc;
4261 __builtin_s390_vchqs((signed __int128)__a, (signed __int128)__b, &__cc);
4262 return __cc == 3;
4263}
4264
4265static inline __ATTRS_o_ai int
4266vec_all_le(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
4267 int __cc;
4268 __builtin_s390_vchlqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
4269 return __cc == 3;
4270}
4271#endif
4272
38784273#if __ARCH__ >= 12
38794274static inline __ATTRS_o_ai int
38804275vec_all_le(__vector float __a, __vector float __b) {
......@@ -4113,6 +4508,22 @@ vec_all_lt(__vector __bool long long __a, __vector __bool long long __b) {
41134508 return __cc == 0;
41144509}
41154510
4511#if __ARCH__ >= 15
4512static inline __ATTRS_o_ai int
4513vec_all_lt(__vector signed __int128 __a, __vector signed __int128 __b) {
4514 int __cc;
4515 __builtin_s390_vchqs((signed __int128)__b, (signed __int128)__a, &__cc);
4516 return __cc == 0;
4517}
4518
4519static inline __ATTRS_o_ai int
4520vec_all_lt(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
4521 int __cc;
4522 __builtin_s390_vchlqs((unsigned __int128)__b, (unsigned __int128)__a, &__cc);
4523 return __cc == 0;
4524}
4525#endif
4526
41164527#if __ARCH__ >= 12
41174528static inline __ATTRS_o_ai int
41184529vec_all_lt(__vector float __a, __vector float __b) {
......@@ -4467,6 +4878,29 @@ vec_any_eq(__vector __bool long long __a, __vector __bool long long __b) {
44674878 return __cc <= 1;
44684879}
44694880
4881#if __ARCH__ >= 15
4882static inline __ATTRS_o_ai int
4883vec_any_eq(__vector signed __int128 __a, __vector signed __int128 __b) {
4884 int __cc;
4885 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
4886 return __cc <= 1;
4887}
4888
4889static inline __ATTRS_o_ai int
4890vec_any_eq(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
4891 int __cc;
4892 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
4893 return __cc <= 1;
4894}
4895
4896static inline __ATTRS_o_ai int
4897vec_any_eq(__vector __bool __int128 __a, __vector __bool __int128 __b) {
4898 int __cc;
4899 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
4900 return __cc <= 1;
4901}
4902#endif
4903
44704904#if __ARCH__ >= 12
44714905static inline __ATTRS_o_ai int
44724906vec_any_eq(__vector float __a, __vector float __b) {
......@@ -4713,28 +5147,51 @@ vec_any_ne(__vector __bool long long __a, __vector __bool long long __b) {
47135147 return __cc != 0;
47145148}
47155149
4716#if __ARCH__ >= 12
5150#if __ARCH__ >= 15
47175151static inline __ATTRS_o_ai int
4718vec_any_ne(__vector float __a, __vector float __b) {
5152vec_any_ne(__vector signed __int128 __a, __vector signed __int128 __b) {
47195153 int __cc;
4720 __builtin_s390_vfcesbs(__a, __b, &__cc);
5154 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
47215155 return __cc != 0;
47225156}
4723#endif
47245157
47255158static inline __ATTRS_o_ai int
4726vec_any_ne(__vector double __a, __vector double __b) {
5159vec_any_ne(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
47275160 int __cc;
4728 __builtin_s390_vfcedbs(__a, __b, &__cc);
5161 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
47295162 return __cc != 0;
47305163}
47315164
4732/*-- vec_any_ge -------------------------------------------------------------*/
4733
47345165static inline __ATTRS_o_ai int
4735vec_any_ge(__vector signed char __a, __vector signed char __b) {
5166vec_any_ne(__vector __bool __int128 __a, __vector __bool __int128 __b) {
47365167 int __cc;
4737 __builtin_s390_vchbs(__b, __a, &__cc);
5168 __builtin_s390_vceqqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
5169 return __cc != 0;
5170}
5171#endif
5172
5173#if __ARCH__ >= 12
5174static inline __ATTRS_o_ai int
5175vec_any_ne(__vector float __a, __vector float __b) {
5176 int __cc;
5177 __builtin_s390_vfcesbs(__a, __b, &__cc);
5178 return __cc != 0;
5179}
5180#endif
5181
5182static inline __ATTRS_o_ai int
5183vec_any_ne(__vector double __a, __vector double __b) {
5184 int __cc;
5185 __builtin_s390_vfcedbs(__a, __b, &__cc);
5186 return __cc != 0;
5187}
5188
5189/*-- vec_any_ge -------------------------------------------------------------*/
5190
5191static inline __ATTRS_o_ai int
5192vec_any_ge(__vector signed char __a, __vector signed char __b) {
5193 int __cc;
5194 __builtin_s390_vchbs(__b, __a, &__cc);
47385195 return __cc != 0;
47395196}
47405197
......@@ -4951,6 +5408,22 @@ vec_any_ge(__vector __bool long long __a, __vector __bool long long __b) {
49515408 return __cc != 0;
49525409}
49535410
5411#if __ARCH__ >= 15
5412static inline __ATTRS_o_ai int
5413vec_any_ge(__vector signed __int128 __a, __vector signed __int128 __b) {
5414 int __cc;
5415 __builtin_s390_vchqs((signed __int128)__b, (signed __int128)__a, &__cc);
5416 return __cc != 0;
5417}
5418
5419static inline __ATTRS_o_ai int
5420vec_any_ge(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
5421 int __cc;
5422 __builtin_s390_vchlqs((unsigned __int128)__b, (unsigned __int128)__a, &__cc);
5423 return __cc != 0;
5424}
5425#endif
5426
49545427#if __ARCH__ >= 12
49555428static inline __ATTRS_o_ai int
49565429vec_any_ge(__vector float __a, __vector float __b) {
......@@ -5189,6 +5662,22 @@ vec_any_gt(__vector __bool long long __a, __vector __bool long long __b) {
51895662 return __cc <= 1;
51905663}
51915664
5665#if __ARCH__ >= 15
5666static inline __ATTRS_o_ai int
5667vec_any_gt(__vector signed __int128 __a, __vector signed __int128 __b) {
5668 int __cc;
5669 __builtin_s390_vchqs((signed __int128)__a, (signed __int128)__b, &__cc);
5670 return __cc <= 1;
5671}
5672
5673static inline __ATTRS_o_ai int
5674vec_any_gt(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
5675 int __cc;
5676 __builtin_s390_vchlqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
5677 return __cc <= 1;
5678}
5679#endif
5680
51925681#if __ARCH__ >= 12
51935682static inline __ATTRS_o_ai int
51945683vec_any_gt(__vector float __a, __vector float __b) {
......@@ -5427,6 +5916,22 @@ vec_any_le(__vector __bool long long __a, __vector __bool long long __b) {
54275916 return __cc != 0;
54285917}
54295918
5919#if __ARCH__ >= 15
5920static inline __ATTRS_o_ai int
5921vec_any_le(__vector signed __int128 __a, __vector signed __int128 __b) {
5922 int __cc;
5923 __builtin_s390_vchqs((signed __int128)__a, (signed __int128)__b, &__cc);
5924 return __cc != 0;
5925}
5926
5927static inline __ATTRS_o_ai int
5928vec_any_le(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
5929 int __cc;
5930 __builtin_s390_vchlqs((unsigned __int128)__a, (unsigned __int128)__b, &__cc);
5931 return __cc != 0;
5932}
5933#endif
5934
54305935#if __ARCH__ >= 12
54315936static inline __ATTRS_o_ai int
54325937vec_any_le(__vector float __a, __vector float __b) {
......@@ -5665,6 +6170,22 @@ vec_any_lt(__vector __bool long long __a, __vector __bool long long __b) {
56656170 return __cc <= 1;
56666171}
56676172
6173#if __ARCH__ >= 15
6174static inline __ATTRS_o_ai int
6175vec_any_lt(__vector signed __int128 __a, __vector signed __int128 __b) {
6176 int __cc;
6177 __builtin_s390_vchqs((signed __int128)__b, (signed __int128)__a, &__cc);
6178 return __cc <= 1;
6179}
6180
6181static inline __ATTRS_o_ai int
6182vec_any_lt(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
6183 int __cc;
6184 __builtin_s390_vchlqs((unsigned __int128)__b, (unsigned __int128)__a, &__cc);
6185 return __cc <= 1;
6186}
6187#endif
6188
56686189#if __ARCH__ >= 12
56696190static inline __ATTRS_o_ai int
56706191vec_any_lt(__vector float __a, __vector float __b) {
......@@ -5753,40 +6274,419 @@ vec_any_nlt(__vector double __a, __vector double __b) {
57536274 return __cc != 0;
57546275}
57556276
5756/*-- vec_any_nan ------------------------------------------------------------*/
6277/*-- vec_any_nan ------------------------------------------------------------*/
6278
6279#if __ARCH__ >= 12
6280static inline __ATTRS_o_ai int
6281vec_any_nan(__vector float __a) {
6282 int __cc;
6283 __builtin_s390_vftcisb(__a, 15, &__cc);
6284 return __cc != 3;
6285}
6286#endif
6287
6288static inline __ATTRS_o_ai int
6289vec_any_nan(__vector double __a) {
6290 int __cc;
6291 __builtin_s390_vftcidb(__a, 15, &__cc);
6292 return __cc != 3;
6293}
6294
6295/*-- vec_any_numeric --------------------------------------------------------*/
6296
6297#if __ARCH__ >= 12
6298static inline __ATTRS_o_ai int
6299vec_any_numeric(__vector float __a) {
6300 int __cc;
6301 __builtin_s390_vftcisb(__a, 15, &__cc);
6302 return __cc != 0;
6303}
6304#endif
6305
6306static inline __ATTRS_o_ai int
6307vec_any_numeric(__vector double __a) {
6308 int __cc;
6309 __builtin_s390_vftcidb(__a, 15, &__cc);
6310 return __cc != 0;
6311}
6312
6313/*-- vec_blend --------------------------------------------------------------*/
6314
6315#if __ARCH__ >= 15
6316static inline __ATTRS_o_ai __vector signed char
6317vec_blend(__vector signed char __a, __vector signed char __b,
6318 __vector signed char __c) {
6319 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed char)0));
6320}
6321
6322static inline __ATTRS_o_ai __vector __bool char
6323vec_blend(__vector __bool char __a, __vector __bool char __b,
6324 __vector signed char __c) {
6325 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed char)0));
6326}
6327
6328static inline __ATTRS_o_ai __vector unsigned char
6329vec_blend(__vector unsigned char __a, __vector unsigned char __b,
6330 __vector signed char __c) {
6331 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed char)0));
6332}
6333
6334static inline __ATTRS_o_ai __vector signed short
6335vec_blend(__vector signed short __a, __vector signed short __b,
6336 __vector signed short __c) {
6337 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed short)0));
6338}
6339
6340static inline __ATTRS_o_ai __vector __bool short
6341vec_blend(__vector __bool short __a, __vector __bool short __b,
6342 __vector signed short __c) {
6343 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed short)0));
6344}
6345
6346static inline __ATTRS_o_ai __vector unsigned short
6347vec_blend(__vector unsigned short __a, __vector unsigned short __b,
6348 __vector signed short __c) {
6349 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed short)0));
6350}
6351
6352static inline __ATTRS_o_ai __vector signed int
6353vec_blend(__vector signed int __a, __vector signed int __b,
6354 __vector signed int __c) {
6355 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed int)0));
6356}
6357
6358static inline __ATTRS_o_ai __vector __bool int
6359vec_blend(__vector __bool int __a, __vector __bool int __b,
6360 __vector signed int __c) {
6361 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed int)0));
6362}
6363
6364static inline __ATTRS_o_ai __vector unsigned int
6365vec_blend(__vector unsigned int __a, __vector unsigned int __b,
6366 __vector signed int __c) {
6367 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed int)0));
6368}
6369
6370static inline __ATTRS_o_ai __vector signed long long
6371vec_blend(__vector signed long long __a, __vector signed long long __b,
6372 __vector signed long long __c) {
6373 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed long long)0));
6374}
6375
6376static inline __ATTRS_o_ai __vector __bool long long
6377vec_blend(__vector __bool long long __a, __vector __bool long long __b,
6378 __vector signed long long __c) {
6379 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed long long)0));
6380}
6381
6382static inline __ATTRS_o_ai __vector unsigned long long
6383vec_blend(__vector unsigned long long __a, __vector unsigned long long __b,
6384 __vector signed long long __c) {
6385 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed long long)0));
6386}
6387
6388static inline __ATTRS_o_ai __vector signed __int128
6389vec_blend(__vector signed __int128 __a, __vector signed __int128 __b,
6390 __vector signed __int128 __c) {
6391 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed __int128)0));
6392}
6393
6394static inline __ATTRS_o_ai __vector __bool __int128
6395vec_blend(__vector __bool __int128 __a, __vector __bool __int128 __b,
6396 __vector signed __int128 __c) {
6397 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed __int128)0));
6398}
6399
6400static inline __ATTRS_o_ai __vector unsigned __int128
6401vec_blend(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
6402 __vector signed __int128 __c) {
6403 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed __int128)0));
6404}
6405
6406static inline __ATTRS_o_ai __vector float
6407vec_blend(__vector float __a, __vector float __b,
6408 __vector signed int __c) {
6409 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed int)0));
6410}
6411
6412static inline __ATTRS_o_ai __vector double
6413vec_blend(__vector double __a, __vector double __b,
6414 __vector signed long long __c) {
6415 return vec_sel(__a, __b, vec_cmplt(__c, (__vector signed long long)0));
6416}
6417#endif
6418
6419/*-- vec_and ---------------------------------------------------------------*/
6420
6421static inline __ATTRS_o_ai __vector __bool char
6422vec_and(__vector __bool char __a, __vector __bool char __b) {
6423 return __a & __b;
6424}
6425
6426static inline __ATTRS_o_ai __vector signed char
6427vec_and(__vector signed char __a, __vector signed char __b) {
6428 return __a & __b;
6429}
6430
6431static inline __ATTRS_o_ai __vector unsigned char
6432vec_and(__vector unsigned char __a, __vector unsigned char __b) {
6433 return __a & __b;
6434}
6435
6436static inline __ATTRS_o_ai __vector __bool short
6437vec_and(__vector __bool short __a, __vector __bool short __b) {
6438 return __a & __b;
6439}
6440
6441static inline __ATTRS_o_ai __vector signed short
6442vec_and(__vector signed short __a, __vector signed short __b) {
6443 return __a & __b;
6444}
6445
6446static inline __ATTRS_o_ai __vector unsigned short
6447vec_and(__vector unsigned short __a, __vector unsigned short __b) {
6448 return __a & __b;
6449}
6450
6451static inline __ATTRS_o_ai __vector __bool int
6452vec_and(__vector __bool int __a, __vector __bool int __b) {
6453 return __a & __b;
6454}
6455
6456static inline __ATTRS_o_ai __vector signed int
6457vec_and(__vector signed int __a, __vector signed int __b) {
6458 return __a & __b;
6459}
6460
6461static inline __ATTRS_o_ai __vector unsigned int
6462vec_and(__vector unsigned int __a, __vector unsigned int __b) {
6463 return __a & __b;
6464}
6465
6466static inline __ATTRS_o_ai __vector __bool long long
6467vec_and(__vector __bool long long __a, __vector __bool long long __b) {
6468 return __a & __b;
6469}
6470
6471static inline __ATTRS_o_ai __vector signed long long
6472vec_and(__vector signed long long __a, __vector signed long long __b) {
6473 return __a & __b;
6474}
6475
6476static inline __ATTRS_o_ai __vector unsigned long long
6477vec_and(__vector unsigned long long __a, __vector unsigned long long __b) {
6478 return __a & __b;
6479}
6480
6481static inline __ATTRS_o_ai __vector __bool __int128
6482vec_and(__vector __bool __int128 __a, __vector __bool __int128 __b) {
6483 return __a & __b;
6484}
6485
6486static inline __ATTRS_o_ai __vector signed __int128
6487vec_and(__vector signed __int128 __a, __vector signed __int128 __b) {
6488 return __a & __b;
6489}
6490
6491static inline __ATTRS_o_ai __vector unsigned __int128
6492vec_and(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
6493 return __a & __b;
6494}
6495
6496#if __ARCH__ >= 12
6497static inline __ATTRS_o_ai __vector float
6498vec_and(__vector float __a, __vector float __b) {
6499 return (__vector float)((__vector unsigned int)__a &
6500 (__vector unsigned int)__b);
6501}
6502#endif
6503
6504static inline __ATTRS_o_ai __vector double
6505vec_and(__vector double __a, __vector double __b) {
6506 return (__vector double)((__vector unsigned long long)__a &
6507 (__vector unsigned long long)__b);
6508}
6509
6510/*-- vec_or ----------------------------------------------------------------*/
6511
6512static inline __ATTRS_o_ai __vector __bool char
6513vec_or(__vector __bool char __a, __vector __bool char __b) {
6514 return __a | __b;
6515}
6516
6517static inline __ATTRS_o_ai __vector signed char
6518vec_or(__vector signed char __a, __vector signed char __b) {
6519 return __a | __b;
6520}
6521
6522static inline __ATTRS_o_ai __vector unsigned char
6523vec_or(__vector unsigned char __a, __vector unsigned char __b) {
6524 return __a | __b;
6525}
6526
6527static inline __ATTRS_o_ai __vector __bool short
6528vec_or(__vector __bool short __a, __vector __bool short __b) {
6529 return __a | __b;
6530}
6531
6532static inline __ATTRS_o_ai __vector signed short
6533vec_or(__vector signed short __a, __vector signed short __b) {
6534 return __a | __b;
6535}
6536
6537static inline __ATTRS_o_ai __vector unsigned short
6538vec_or(__vector unsigned short __a, __vector unsigned short __b) {
6539 return __a | __b;
6540}
6541
6542static inline __ATTRS_o_ai __vector __bool int
6543vec_or(__vector __bool int __a, __vector __bool int __b) {
6544 return __a | __b;
6545}
6546
6547static inline __ATTRS_o_ai __vector signed int
6548vec_or(__vector signed int __a, __vector signed int __b) {
6549 return __a | __b;
6550}
6551
6552static inline __ATTRS_o_ai __vector unsigned int
6553vec_or(__vector unsigned int __a, __vector unsigned int __b) {
6554 return __a | __b;
6555}
6556
6557static inline __ATTRS_o_ai __vector __bool long long
6558vec_or(__vector __bool long long __a, __vector __bool long long __b) {
6559 return __a | __b;
6560}
6561
6562static inline __ATTRS_o_ai __vector signed long long
6563vec_or(__vector signed long long __a, __vector signed long long __b) {
6564 return __a | __b;
6565}
6566
6567static inline __ATTRS_o_ai __vector unsigned long long
6568vec_or(__vector unsigned long long __a, __vector unsigned long long __b) {
6569 return __a | __b;
6570}
6571
6572static inline __ATTRS_o_ai __vector __bool __int128
6573vec_or(__vector __bool __int128 __a, __vector __bool __int128 __b) {
6574 return __a | __b;
6575}
6576
6577static inline __ATTRS_o_ai __vector signed __int128
6578vec_or(__vector signed __int128 __a, __vector signed __int128 __b) {
6579 return __a | __b;
6580}
6581
6582static inline __ATTRS_o_ai __vector unsigned __int128
6583vec_or(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
6584 return __a | __b;
6585}
6586
6587#if __ARCH__ >= 12
6588static inline __ATTRS_o_ai __vector float
6589vec_or(__vector float __a, __vector float __b) {
6590 return (__vector float)((__vector unsigned int)__a |
6591 (__vector unsigned int)__b);
6592}
6593#endif
6594
6595static inline __ATTRS_o_ai __vector double
6596vec_or(__vector double __a, __vector double __b) {
6597 return (__vector double)((__vector unsigned long long)__a |
6598 (__vector unsigned long long)__b);
6599}
6600
6601/*-- vec_xor ----------------------------------------------------------------*/
6602
6603static inline __ATTRS_o_ai __vector __bool char
6604vec_xor(__vector __bool char __a, __vector __bool char __b) {
6605 return __a ^ __b;
6606}
6607
6608static inline __ATTRS_o_ai __vector signed char
6609vec_xor(__vector signed char __a, __vector signed char __b) {
6610 return __a ^ __b;
6611}
6612
6613static inline __ATTRS_o_ai __vector unsigned char
6614vec_xor(__vector unsigned char __a, __vector unsigned char __b) {
6615 return __a ^ __b;
6616}
6617
6618static inline __ATTRS_o_ai __vector __bool short
6619vec_xor(__vector __bool short __a, __vector __bool short __b) {
6620 return __a ^ __b;
6621}
6622
6623static inline __ATTRS_o_ai __vector signed short
6624vec_xor(__vector signed short __a, __vector signed short __b) {
6625 return __a ^ __b;
6626}
6627
6628static inline __ATTRS_o_ai __vector unsigned short
6629vec_xor(__vector unsigned short __a, __vector unsigned short __b) {
6630 return __a ^ __b;
6631}
6632
6633static inline __ATTRS_o_ai __vector __bool int
6634vec_xor(__vector __bool int __a, __vector __bool int __b) {
6635 return __a ^ __b;
6636}
6637
6638static inline __ATTRS_o_ai __vector signed int
6639vec_xor(__vector signed int __a, __vector signed int __b) {
6640 return __a ^ __b;
6641}
6642
6643static inline __ATTRS_o_ai __vector unsigned int
6644vec_xor(__vector unsigned int __a, __vector unsigned int __b) {
6645 return __a ^ __b;
6646}
6647
6648static inline __ATTRS_o_ai __vector __bool long long
6649vec_xor(__vector __bool long long __a, __vector __bool long long __b) {
6650 return __a ^ __b;
6651}
6652
6653static inline __ATTRS_o_ai __vector signed long long
6654vec_xor(__vector signed long long __a, __vector signed long long __b) {
6655 return __a ^ __b;
6656}
6657
6658static inline __ATTRS_o_ai __vector unsigned long long
6659vec_xor(__vector unsigned long long __a, __vector unsigned long long __b) {
6660 return __a ^ __b;
6661}
57576662
5758#if __ARCH__ >= 12
5759static inline __ATTRS_o_ai int
5760vec_any_nan(__vector float __a) {
5761 int __cc;
5762 __builtin_s390_vftcisb(__a, 15, &__cc);
5763 return __cc != 3;
6663static inline __ATTRS_o_ai __vector __bool __int128
6664vec_xor(__vector __bool __int128 __a, __vector __bool __int128 __b) {
6665 return __a ^ __b;
57646666}
5765#endif
57666667
5767static inline __ATTRS_o_ai int
5768vec_any_nan(__vector double __a) {
5769 int __cc;
5770 __builtin_s390_vftcidb(__a, 15, &__cc);
5771 return __cc != 3;
6668static inline __ATTRS_o_ai __vector signed __int128
6669vec_xor(__vector signed __int128 __a, __vector signed __int128 __b) {
6670 return __a ^ __b;
57726671}
57736672
5774/*-- vec_any_numeric --------------------------------------------------------*/
6673static inline __ATTRS_o_ai __vector unsigned __int128
6674vec_xor(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
6675 return __a ^ __b;
6676}
57756677
57766678#if __ARCH__ >= 12
5777static inline __ATTRS_o_ai int
5778vec_any_numeric(__vector float __a) {
5779 int __cc;
5780 __builtin_s390_vftcisb(__a, 15, &__cc);
5781 return __cc != 0;
6679static inline __ATTRS_o_ai __vector float
6680vec_xor(__vector float __a, __vector float __b) {
6681 return (__vector float)((__vector unsigned int)__a ^
6682 (__vector unsigned int)__b);
57826683}
57836684#endif
57846685
5785static inline __ATTRS_o_ai int
5786vec_any_numeric(__vector double __a) {
5787 int __cc;
5788 __builtin_s390_vftcidb(__a, 15, &__cc);
5789 return __cc != 0;
6686static inline __ATTRS_o_ai __vector double
6687vec_xor(__vector double __a, __vector double __b) {
6688 return (__vector double)((__vector unsigned long long)__a ^
6689 (__vector unsigned long long)__b);
57906690}
57916691
57926692/*-- vec_andc ---------------------------------------------------------------*/
......@@ -5947,6 +6847,21 @@ vec_andc(__vector unsigned long long __a, __vector __bool long long __b) {
59476847 return __a & ~__b;
59486848}
59496849
6850static inline __ATTRS_o_ai __vector __bool __int128
6851vec_andc(__vector __bool __int128 __a, __vector __bool __int128 __b) {
6852 return __a & ~__b;
6853}
6854
6855static inline __ATTRS_o_ai __vector signed __int128
6856vec_andc(__vector signed __int128 __a, __vector signed __int128 __b) {
6857 return __a & ~__b;
6858}
6859
6860static inline __ATTRS_o_ai __vector unsigned __int128
6861vec_andc(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
6862 return __a & ~__b;
6863}
6864
59506865#if __ARCH__ >= 12
59516866static inline __ATTRS_o_ai __vector float
59526867vec_andc(__vector float __a, __vector float __b) {
......@@ -6133,6 +7048,21 @@ vec_nor(__vector unsigned long long __a, __vector __bool long long __b) {
61337048 return ~(__a | __b);
61347049}
61357050
7051static inline __ATTRS_o_ai __vector __bool __int128
7052vec_nor(__vector __bool __int128 __a, __vector __bool __int128 __b) {
7053 return ~(__a | __b);
7054}
7055
7056static inline __ATTRS_o_ai __vector signed __int128
7057vec_nor(__vector signed __int128 __a, __vector signed __int128 __b) {
7058 return ~(__a | __b);
7059}
7060
7061static inline __ATTRS_o_ai __vector unsigned __int128
7062vec_nor(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
7063 return ~(__a | __b);
7064}
7065
61367066#if __ARCH__ >= 12
61377067static inline __ATTRS_o_ai __vector float
61387068vec_nor(__vector float __a, __vector float __b) {
......@@ -6224,6 +7154,21 @@ vec_orc(__vector unsigned long long __a, __vector unsigned long long __b) {
62247154 return __a | ~__b;
62257155}
62267156
7157static inline __ATTRS_o_ai __vector __bool __int128
7158vec_orc(__vector __bool __int128 __a, __vector __bool __int128 __b) {
7159 return __a | ~__b;
7160}
7161
7162static inline __ATTRS_o_ai __vector signed __int128
7163vec_orc(__vector signed __int128 __a, __vector signed __int128 __b) {
7164 return __a | ~__b;
7165}
7166
7167static inline __ATTRS_o_ai __vector unsigned __int128
7168vec_orc(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
7169 return __a | ~__b;
7170}
7171
62277172static inline __ATTRS_o_ai __vector float
62287173vec_orc(__vector float __a, __vector float __b) {
62297174 return (__vector float)((__vector unsigned int)__a |
......@@ -6300,6 +7245,21 @@ vec_nand(__vector unsigned long long __a, __vector unsigned long long __b) {
63007245 return ~(__a & __b);
63017246}
63027247
7248static inline __ATTRS_o_ai __vector __bool __int128
7249vec_nand(__vector __bool __int128 __a, __vector __bool __int128 __b) {
7250 return ~(__a & __b);
7251}
7252
7253static inline __ATTRS_o_ai __vector signed __int128
7254vec_nand(__vector signed __int128 __a, __vector signed __int128 __b) {
7255 return ~(__a & __b);
7256}
7257
7258static inline __ATTRS_o_ai __vector unsigned __int128
7259vec_nand(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
7260 return ~(__a & __b);
7261}
7262
63037263static inline __ATTRS_o_ai __vector float
63047264vec_nand(__vector float __a, __vector float __b) {
63057265 return (__vector float)~((__vector unsigned int)__a &
......@@ -6376,6 +7336,21 @@ vec_eqv(__vector unsigned long long __a, __vector unsigned long long __b) {
63767336 return ~(__a ^ __b);
63777337}
63787338
7339static inline __ATTRS_o_ai __vector __bool __int128
7340vec_eqv(__vector __bool __int128 __a, __vector __bool __int128 __b) {
7341 return ~(__a ^ __b);
7342}
7343
7344static inline __ATTRS_o_ai __vector signed __int128
7345vec_eqv(__vector signed __int128 __a, __vector signed __int128 __b) {
7346 return ~(__a ^ __b);
7347}
7348
7349static inline __ATTRS_o_ai __vector unsigned __int128
7350vec_eqv(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
7351 return ~(__a ^ __b);
7352}
7353
63797354static inline __ATTRS_o_ai __vector float
63807355vec_eqv(__vector float __a, __vector float __b) {
63817356 return (__vector float)~((__vector unsigned int)__a ^
......@@ -6389,6 +7364,91 @@ vec_eqv(__vector double __a, __vector double __b) {
63897364}
63907365#endif
63917366
7367/*-- vec_evaluate -----------------------------------------------------------*/
7368
7369#if __ARCH__ >= 15
7370extern __ATTRS_o __vector signed char
7371vec_evaluate(__vector signed char __a, __vector signed char __b,
7372 __vector signed char __c, unsigned char __d)
7373 __constant(__d);
7374
7375extern __ATTRS_o __vector unsigned char
7376vec_evaluate(__vector unsigned char __a, __vector unsigned char __b,
7377 __vector unsigned char __c, unsigned char __d)
7378 __constant(__d);
7379
7380extern __ATTRS_o __vector __bool char
7381vec_evaluate(__vector __bool char __a, __vector __bool char __b,
7382 __vector __bool char __c, unsigned char __d)
7383 __constant(__d);
7384
7385extern __ATTRS_o __vector signed short
7386vec_evaluate(__vector signed short __a, __vector signed short __b,
7387 __vector signed short __c, unsigned char __d)
7388 __constant(__d);
7389
7390extern __ATTRS_o __vector unsigned short
7391vec_evaluate(__vector unsigned short __a, __vector unsigned short __b,
7392 __vector unsigned short __c, unsigned char __d)
7393 __constant(__d);
7394
7395extern __ATTRS_o __vector __bool short
7396vec_evaluate(__vector __bool short __a, __vector __bool short __b,
7397 __vector __bool short __c, unsigned char __d)
7398 __constant(__d);
7399
7400extern __ATTRS_o __vector signed int
7401vec_evaluate(__vector signed int __a, __vector signed int __b,
7402 __vector signed int __c, unsigned char __d)
7403 __constant(__d);
7404
7405extern __ATTRS_o __vector unsigned int
7406vec_evaluate(__vector unsigned int __a, __vector unsigned int __b,
7407 __vector unsigned int __c, unsigned char __d)
7408 __constant(__d);
7409
7410extern __ATTRS_o __vector __bool int
7411vec_evaluate(__vector __bool int __a, __vector __bool int __b,
7412 __vector __bool int __c, unsigned char __d)
7413 __constant(__d);
7414
7415extern __ATTRS_o __vector signed long long
7416vec_evaluate(__vector signed long long __a, __vector signed long long __b,
7417 __vector signed long long __c, unsigned char __d)
7418 __constant(__d);
7419
7420extern __ATTRS_o __vector unsigned long long
7421vec_evaluate(__vector unsigned long long __a, __vector unsigned long long __b,
7422 __vector unsigned long long __c, unsigned char __d)
7423 __constant(__d);
7424
7425extern __ATTRS_o __vector __bool long long
7426vec_evaluate(__vector __bool long long __a, __vector __bool long long __b,
7427 __vector __bool long long __c, unsigned char __d)
7428 __constant(__d);
7429
7430extern __ATTRS_o __vector signed __int128
7431vec_evaluate(__vector signed __int128 __a, __vector signed __int128 __b,
7432 __vector signed __int128 __c, unsigned char __d)
7433 __constant(__d);
7434
7435extern __ATTRS_o __vector unsigned __int128
7436vec_evaluate(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
7437 __vector unsigned __int128 __c, unsigned char __d)
7438 __constant(__d);
7439
7440extern __ATTRS_o __vector __bool __int128
7441vec_evaluate(__vector __bool __int128 __a, __vector __bool __int128 __b,
7442 __vector __bool __int128 __c, unsigned char __d)
7443 __constant(__d);
7444
7445#define vec_evaluate(A, B, C, D) \
7446 ((__typeof__((vec_evaluate)((A), (B), (C), (D)))) \
7447 __builtin_s390_veval((__vector unsigned char)(A), \
7448 (__vector unsigned char)(B), \
7449 (__vector unsigned char)(C), (D)))
7450#endif
7451
63927452/*-- vec_cntlz --------------------------------------------------------------*/
63937453
63947454static inline __ATTRS_o_ai __vector unsigned char
......@@ -6431,6 +7491,20 @@ vec_cntlz(__vector unsigned long long __a) {
64317491 return __builtin_s390_vclzg(__a);
64327492}
64337493
7494#if __ARCH__ >= 15
7495static inline __ATTRS_o_ai __vector unsigned __int128
7496vec_cntlz(__vector signed __int128 __a) {
7497 return (__vector unsigned __int128)
7498 __builtin_s390_vclzq((unsigned __int128)__a);
7499}
7500
7501static inline __ATTRS_o_ai __vector unsigned __int128
7502vec_cntlz(__vector unsigned __int128 __a) {
7503 return (__vector unsigned __int128)
7504 __builtin_s390_vclzq((unsigned __int128)__a);
7505}
7506#endif
7507
64347508/*-- vec_cnttz --------------------------------------------------------------*/
64357509
64367510static inline __ATTRS_o_ai __vector unsigned char
......@@ -6473,46 +7547,60 @@ vec_cnttz(__vector unsigned long long __a) {
64737547 return __builtin_s390_vctzg(__a);
64747548}
64757549
7550#if __ARCH__ >= 15
7551static inline __ATTRS_o_ai __vector unsigned __int128
7552vec_cnttz(__vector signed __int128 __a) {
7553 return (__vector unsigned __int128)
7554 __builtin_s390_vctzq((unsigned __int128)__a);
7555}
7556
7557static inline __ATTRS_o_ai __vector unsigned __int128
7558vec_cnttz(__vector unsigned __int128 __a) {
7559 return (__vector unsigned __int128)
7560 __builtin_s390_vctzq((unsigned __int128)__a);
7561}
7562#endif
7563
64767564/*-- vec_popcnt -------------------------------------------------------------*/
64777565
64787566static inline __ATTRS_o_ai __vector unsigned char
64797567vec_popcnt(__vector signed char __a) {
6480 return __builtin_s390_vpopctb((__vector unsigned char)__a);
7568 return __builtin_elementwise_popcount((__vector unsigned char)__a);
64817569}
64827570
64837571static inline __ATTRS_o_ai __vector unsigned char
64847572vec_popcnt(__vector unsigned char __a) {
6485 return __builtin_s390_vpopctb(__a);
7573 return __builtin_elementwise_popcount(__a);
64867574}
64877575
64887576static inline __ATTRS_o_ai __vector unsigned short
64897577vec_popcnt(__vector signed short __a) {
6490 return __builtin_s390_vpopcth((__vector unsigned short)__a);
7578 return __builtin_elementwise_popcount((__vector unsigned short)__a);
64917579}
64927580
64937581static inline __ATTRS_o_ai __vector unsigned short
64947582vec_popcnt(__vector unsigned short __a) {
6495 return __builtin_s390_vpopcth(__a);
7583 return __builtin_elementwise_popcount(__a);
64967584}
64977585
64987586static inline __ATTRS_o_ai __vector unsigned int
64997587vec_popcnt(__vector signed int __a) {
6500 return __builtin_s390_vpopctf((__vector unsigned int)__a);
7588 return __builtin_elementwise_popcount((__vector unsigned int)__a);
65017589}
65027590
65037591static inline __ATTRS_o_ai __vector unsigned int
65047592vec_popcnt(__vector unsigned int __a) {
6505 return __builtin_s390_vpopctf(__a);
7593 return __builtin_elementwise_popcount(__a);
65067594}
65077595
65087596static inline __ATTRS_o_ai __vector unsigned long long
65097597vec_popcnt(__vector signed long long __a) {
6510 return __builtin_s390_vpopctg((__vector unsigned long long)__a);
7598 return __builtin_elementwise_popcount((__vector unsigned long long)__a);
65117599}
65127600
65137601static inline __ATTRS_o_ai __vector unsigned long long
65147602vec_popcnt(__vector unsigned long long __a) {
6515 return __builtin_s390_vpopctg(__a);
7603 return __builtin_elementwise_popcount(__a);
65167604}
65177605
65187606/*-- vec_rl -----------------------------------------------------------------*/
......@@ -6904,8 +7992,21 @@ vec_sll(__vector unsigned long long __a, __vector unsigned int __b) {
69047992 (__vector unsigned char)__a, (__vector unsigned char)__b);
69057993}
69067994
7995static inline __ATTRS_o_ai __vector signed __int128
7996vec_sll(__vector signed __int128 __a, __vector unsigned char __b) {
7997 return (__vector signed __int128)__builtin_s390_vsl(
7998 (__vector unsigned char)__a, __b);
7999}
8000
8001static inline __ATTRS_o_ai __vector unsigned __int128
8002vec_sll(__vector unsigned __int128 __a, __vector unsigned char __b) {
8003 return (__vector unsigned __int128)__builtin_s390_vsl(
8004 (__vector unsigned char)__a, __b);
8005}
8006
69078007/*-- vec_slb ----------------------------------------------------------------*/
69088008
8009// This prototype is deprecated.
69098010static inline __ATTRS_o_ai __vector signed char
69108011vec_slb(__vector signed char __a, __vector signed char __b) {
69118012 return (__vector signed char)__builtin_s390_vslb(
......@@ -6918,6 +8019,7 @@ vec_slb(__vector signed char __a, __vector unsigned char __b) {
69188019 (__vector unsigned char)__a, __b);
69198020}
69208021
8022// This prototype is deprecated.
69218023static inline __ATTRS_o_ai __vector unsigned char
69228024vec_slb(__vector unsigned char __a, __vector signed char __b) {
69238025 return __builtin_s390_vslb(__a, (__vector unsigned char)__b);
......@@ -6928,110 +8030,187 @@ vec_slb(__vector unsigned char __a, __vector unsigned char __b) {
69288030 return __builtin_s390_vslb(__a, __b);
69298031}
69308032
8033// This prototype is deprecated.
69318034static inline __ATTRS_o_ai __vector signed short
69328035vec_slb(__vector signed short __a, __vector signed short __b) {
69338036 return (__vector signed short)__builtin_s390_vslb(
69348037 (__vector unsigned char)__a, (__vector unsigned char)__b);
69358038}
69368039
8040// This prototype is deprecated.
69378041static inline __ATTRS_o_ai __vector signed short
69388042vec_slb(__vector signed short __a, __vector unsigned short __b) {
69398043 return (__vector signed short)__builtin_s390_vslb(
69408044 (__vector unsigned char)__a, (__vector unsigned char)__b);
69418045}
69428046
8047static inline __ATTRS_o_ai __vector signed short
8048vec_slb(__vector signed short __a, __vector unsigned char __b) {
8049 return (__vector signed short)__builtin_s390_vslb(
8050 (__vector unsigned char)__a, __b);
8051}
8052
8053// This prototype is deprecated.
69438054static inline __ATTRS_o_ai __vector unsigned short
69448055vec_slb(__vector unsigned short __a, __vector signed short __b) {
69458056 return (__vector unsigned short)__builtin_s390_vslb(
69468057 (__vector unsigned char)__a, (__vector unsigned char)__b);
69478058}
69488059
8060// This prototype is deprecated.
69498061static inline __ATTRS_o_ai __vector unsigned short
69508062vec_slb(__vector unsigned short __a, __vector unsigned short __b) {
69518063 return (__vector unsigned short)__builtin_s390_vslb(
69528064 (__vector unsigned char)__a, (__vector unsigned char)__b);
69538065}
69548066
8067static inline __ATTRS_o_ai __vector unsigned short
8068vec_slb(__vector unsigned short __a, __vector unsigned char __b) {
8069 return (__vector unsigned short)__builtin_s390_vslb(
8070 (__vector unsigned char)__a, __b);
8071}
8072
8073// This prototype is deprecated.
69558074static inline __ATTRS_o_ai __vector signed int
69568075vec_slb(__vector signed int __a, __vector signed int __b) {
69578076 return (__vector signed int)__builtin_s390_vslb(
69588077 (__vector unsigned char)__a, (__vector unsigned char)__b);
69598078}
69608079
8080// This prototype is deprecated.
69618081static inline __ATTRS_o_ai __vector signed int
69628082vec_slb(__vector signed int __a, __vector unsigned int __b) {
69638083 return (__vector signed int)__builtin_s390_vslb(
69648084 (__vector unsigned char)__a, (__vector unsigned char)__b);
69658085}
69668086
8087static inline __ATTRS_o_ai __vector signed int
8088vec_slb(__vector signed int __a, __vector unsigned char __b) {
8089 return (__vector signed int)__builtin_s390_vslb(
8090 (__vector unsigned char)__a, __b);
8091}
8092
8093// This prototype is deprecated.
69678094static inline __ATTRS_o_ai __vector unsigned int
69688095vec_slb(__vector unsigned int __a, __vector signed int __b) {
69698096 return (__vector unsigned int)__builtin_s390_vslb(
69708097 (__vector unsigned char)__a, (__vector unsigned char)__b);
69718098}
69728099
8100// This prototype is deprecated.
69738101static inline __ATTRS_o_ai __vector unsigned int
69748102vec_slb(__vector unsigned int __a, __vector unsigned int __b) {
69758103 return (__vector unsigned int)__builtin_s390_vslb(
69768104 (__vector unsigned char)__a, (__vector unsigned char)__b);
69778105}
69788106
8107static inline __ATTRS_o_ai __vector unsigned int
8108vec_slb(__vector unsigned int __a, __vector unsigned char __b) {
8109 return (__vector unsigned int)__builtin_s390_vslb(
8110 (__vector unsigned char)__a, __b);
8111}
8112
8113// This prototype is deprecated.
69798114static inline __ATTRS_o_ai __vector signed long long
69808115vec_slb(__vector signed long long __a, __vector signed long long __b) {
69818116 return (__vector signed long long)__builtin_s390_vslb(
69828117 (__vector unsigned char)__a, (__vector unsigned char)__b);
69838118}
69848119
8120// This prototype is deprecated.
69858121static inline __ATTRS_o_ai __vector signed long long
69868122vec_slb(__vector signed long long __a, __vector unsigned long long __b) {
69878123 return (__vector signed long long)__builtin_s390_vslb(
69888124 (__vector unsigned char)__a, (__vector unsigned char)__b);
69898125}
69908126
8127static inline __ATTRS_o_ai __vector signed long long
8128vec_slb(__vector signed long long __a, __vector unsigned char __b) {
8129 return (__vector signed long long)__builtin_s390_vslb(
8130 (__vector unsigned char)__a, __b);
8131}
8132
8133// This prototype is deprecated.
69918134static inline __ATTRS_o_ai __vector unsigned long long
69928135vec_slb(__vector unsigned long long __a, __vector signed long long __b) {
69938136 return (__vector unsigned long long)__builtin_s390_vslb(
69948137 (__vector unsigned char)__a, (__vector unsigned char)__b);
69958138}
69968139
8140// This prototype is deprecated.
69978141static inline __ATTRS_o_ai __vector unsigned long long
69988142vec_slb(__vector unsigned long long __a, __vector unsigned long long __b) {
69998143 return (__vector unsigned long long)__builtin_s390_vslb(
70008144 (__vector unsigned char)__a, (__vector unsigned char)__b);
70018145}
70028146
8147static inline __ATTRS_o_ai __vector unsigned long long
8148vec_slb(__vector unsigned long long __a, __vector unsigned char __b) {
8149 return (__vector unsigned long long)__builtin_s390_vslb(
8150 (__vector unsigned char)__a, __b);
8151}
8152
8153static inline __ATTRS_o_ai __vector signed __int128
8154vec_slb(__vector signed __int128 __a, __vector unsigned char __b) {
8155 return (__vector signed __int128)__builtin_s390_vslb(
8156 (__vector unsigned char)__a, __b);
8157}
8158
8159static inline __ATTRS_o_ai __vector unsigned __int128
8160vec_slb(__vector unsigned __int128 __a, __vector unsigned char __b) {
8161 return (__vector unsigned __int128)__builtin_s390_vslb(
8162 (__vector unsigned char)__a, __b);
8163}
8164
70038165#if __ARCH__ >= 12
8166// This prototype is deprecated.
70048167static inline __ATTRS_o_ai __vector float
70058168vec_slb(__vector float __a, __vector signed int __b) {
70068169 return (__vector float)__builtin_s390_vslb(
70078170 (__vector unsigned char)__a, (__vector unsigned char)__b);
70088171}
70098172
8173// This prototype is deprecated.
70108174static inline __ATTRS_o_ai __vector float
70118175vec_slb(__vector float __a, __vector unsigned int __b) {
70128176 return (__vector float)__builtin_s390_vslb(
70138177 (__vector unsigned char)__a, (__vector unsigned char)__b);
70148178}
8179
8180static inline __ATTRS_o_ai __vector float
8181vec_slb(__vector float __a, __vector unsigned char __b) {
8182 return (__vector float)__builtin_s390_vslb(
8183 (__vector unsigned char)__a, __b);
8184}
70158185#endif
70168186
8187// This prototype is deprecated.
70178188static inline __ATTRS_o_ai __vector double
70188189vec_slb(__vector double __a, __vector signed long long __b) {
70198190 return (__vector double)__builtin_s390_vslb(
70208191 (__vector unsigned char)__a, (__vector unsigned char)__b);
70218192}
70228193
8194// This prototype is deprecated.
70238195static inline __ATTRS_o_ai __vector double
70248196vec_slb(__vector double __a, __vector unsigned long long __b) {
70258197 return (__vector double)__builtin_s390_vslb(
70268198 (__vector unsigned char)__a, (__vector unsigned char)__b);
70278199}
70288200
8201static inline __ATTRS_o_ai __vector double
8202vec_slb(__vector double __a, __vector unsigned char __b) {
8203 return (__vector double)__builtin_s390_vslb(
8204 (__vector unsigned char)__a, __b);
8205}
8206
70298207/*-- vec_sld ----------------------------------------------------------------*/
70308208
70318209extern __ATTRS_o __vector signed char
70328210vec_sld(__vector signed char __a, __vector signed char __b, int __c)
70338211 __constant_range(__c, 0, 15);
70348212
8213// This prototype is deprecated.
70358214extern __ATTRS_o __vector __bool char
70368215vec_sld(__vector __bool char __a, __vector __bool char __b, int __c)
70378216 __constant_range(__c, 0, 15);
......@@ -7044,6 +8223,7 @@ extern __ATTRS_o __vector signed short
70448223vec_sld(__vector signed short __a, __vector signed short __b, int __c)
70458224 __constant_range(__c, 0, 15);
70468225
8226// This prototype is deprecated.
70478227extern __ATTRS_o __vector __bool short
70488228vec_sld(__vector __bool short __a, __vector __bool short __b, int __c)
70498229 __constant_range(__c, 0, 15);
......@@ -7056,6 +8236,7 @@ extern __ATTRS_o __vector signed int
70568236vec_sld(__vector signed int __a, __vector signed int __b, int __c)
70578237 __constant_range(__c, 0, 15);
70588238
8239// This prototype is deprecated.
70598240extern __ATTRS_o __vector __bool int
70608241vec_sld(__vector __bool int __a, __vector __bool int __b, int __c)
70618242 __constant_range(__c, 0, 15);
......@@ -7068,6 +8249,7 @@ extern __ATTRS_o __vector signed long long
70688249vec_sld(__vector signed long long __a, __vector signed long long __b, int __c)
70698250 __constant_range(__c, 0, 15);
70708251
8252// This prototype is deprecated.
70718253extern __ATTRS_o __vector __bool long long
70728254vec_sld(__vector __bool long long __a, __vector __bool long long __b, int __c)
70738255 __constant_range(__c, 0, 15);
......@@ -7077,6 +8259,15 @@ vec_sld(__vector unsigned long long __a, __vector unsigned long long __b,
70778259 int __c)
70788260 __constant_range(__c, 0, 15);
70798261
8262extern __ATTRS_o __vector signed __int128
8263vec_sld(__vector signed __int128 __a, __vector signed __int128 __b, int __c)
8264 __constant_range(__c, 0, 15);
8265
8266extern __ATTRS_o __vector unsigned __int128
8267vec_sld(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
8268 int __c)
8269 __constant_range(__c, 0, 15);
8270
70808271#if __ARCH__ >= 12
70818272extern __ATTRS_o __vector float
70828273vec_sld(__vector float __a, __vector float __b, int __c)
......@@ -7126,6 +8317,15 @@ vec_sldw(__vector unsigned long long __a, __vector unsigned long long __b,
71268317 int __c)
71278318 __constant_range(__c, 0, 3);
71288319
8320extern __ATTRS_o __vector signed __int128
8321vec_sldw(__vector signed __int128 __a, __vector signed __int128 __b, int __c)
8322 __constant_range(__c, 0, 3);
8323
8324extern __ATTRS_o __vector unsigned __int128
8325vec_sldw(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
8326 int __c)
8327 __constant_range(__c, 0, 3);
8328
71298329// This prototype is deprecated.
71308330extern __ATTRS_o __vector double
71318331vec_sldw(__vector double __a, __vector double __b, int __c)
......@@ -7172,6 +8372,15 @@ vec_sldb(__vector unsigned long long __a, __vector unsigned long long __b,
71728372 int __c)
71738373 __constant_range(__c, 0, 7);
71748374
8375extern __ATTRS_o __vector signed __int128
8376vec_sldb(__vector signed __int128 __a, __vector signed __int128 __b, int __c)
8377 __constant_range(__c, 0, 7);
8378
8379extern __ATTRS_o __vector unsigned __int128
8380vec_sldb(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
8381 int __c)
8382 __constant_range(__c, 0, 7);
8383
71758384extern __ATTRS_o __vector float
71768385vec_sldb(__vector float __a, __vector float __b, int __c)
71778386 __constant_range(__c, 0, 7);
......@@ -7429,8 +8638,21 @@ vec_sral(__vector unsigned long long __a, __vector unsigned int __b) {
74298638 (__vector unsigned char)__a, (__vector unsigned char)__b);
74308639}
74318640
8641static inline __ATTRS_o_ai __vector signed __int128
8642vec_sral(__vector signed __int128 __a, __vector unsigned char __b) {
8643 return (__vector signed __int128)__builtin_s390_vsra(
8644 (__vector unsigned char)__a, __b);
8645}
8646
8647static inline __ATTRS_o_ai __vector unsigned __int128
8648vec_sral(__vector unsigned __int128 __a, __vector unsigned char __b) {
8649 return (__vector unsigned __int128)__builtin_s390_vsra(
8650 (__vector unsigned char)__a, __b);
8651}
8652
74328653/*-- vec_srab ---------------------------------------------------------------*/
74338654
8655// This prototype is deprecated.
74348656static inline __ATTRS_o_ai __vector signed char
74358657vec_srab(__vector signed char __a, __vector signed char __b) {
74368658 return (__vector signed char)__builtin_s390_vsrab(
......@@ -7443,6 +8665,7 @@ vec_srab(__vector signed char __a, __vector unsigned char __b) {
74438665 (__vector unsigned char)__a, __b);
74448666}
74458667
8668// This prototype is deprecated.
74468669static inline __ATTRS_o_ai __vector unsigned char
74478670vec_srab(__vector unsigned char __a, __vector signed char __b) {
74488671 return __builtin_s390_vsrab(__a, (__vector unsigned char)__b);
......@@ -7453,104 +8676,180 @@ vec_srab(__vector unsigned char __a, __vector unsigned char __b) {
74538676 return __builtin_s390_vsrab(__a, __b);
74548677}
74558678
8679// This prototype is deprecated.
74568680static inline __ATTRS_o_ai __vector signed short
74578681vec_srab(__vector signed short __a, __vector signed short __b) {
74588682 return (__vector signed short)__builtin_s390_vsrab(
74598683 (__vector unsigned char)__a, (__vector unsigned char)__b);
74608684}
74618685
8686// This prototype is deprecated.
74628687static inline __ATTRS_o_ai __vector signed short
74638688vec_srab(__vector signed short __a, __vector unsigned short __b) {
74648689 return (__vector signed short)__builtin_s390_vsrab(
74658690 (__vector unsigned char)__a, (__vector unsigned char)__b);
74668691}
74678692
8693static inline __ATTRS_o_ai __vector signed short
8694vec_srab(__vector signed short __a, __vector unsigned char __b) {
8695 return (__vector signed short)__builtin_s390_vsrab(
8696 (__vector unsigned char)__a, __b);
8697}
8698
8699// This prototype is deprecated.
74688700static inline __ATTRS_o_ai __vector unsigned short
74698701vec_srab(__vector unsigned short __a, __vector signed short __b) {
74708702 return (__vector unsigned short)__builtin_s390_vsrab(
74718703 (__vector unsigned char)__a, (__vector unsigned char)__b);
74728704}
74738705
8706// This prototype is deprecated.
74748707static inline __ATTRS_o_ai __vector unsigned short
74758708vec_srab(__vector unsigned short __a, __vector unsigned short __b) {
74768709 return (__vector unsigned short)__builtin_s390_vsrab(
74778710 (__vector unsigned char)__a, (__vector unsigned char)__b);
74788711}
74798712
8713static inline __ATTRS_o_ai __vector unsigned short
8714vec_srab(__vector unsigned short __a, __vector unsigned char __b) {
8715 return (__vector unsigned short)__builtin_s390_vsrab(
8716 (__vector unsigned char)__a, __b);
8717}
8718
8719// This prototype is deprecated.
74808720static inline __ATTRS_o_ai __vector signed int
74818721vec_srab(__vector signed int __a, __vector signed int __b) {
74828722 return (__vector signed int)__builtin_s390_vsrab(
74838723 (__vector unsigned char)__a, (__vector unsigned char)__b);
74848724}
74858725
8726// This prototype is deprecated.
74868727static inline __ATTRS_o_ai __vector signed int
74878728vec_srab(__vector signed int __a, __vector unsigned int __b) {
74888729 return (__vector signed int)__builtin_s390_vsrab(
74898730 (__vector unsigned char)__a, (__vector unsigned char)__b);
74908731}
74918732
8733static inline __ATTRS_o_ai __vector signed int
8734vec_srab(__vector signed int __a, __vector unsigned char __b) {
8735 return (__vector signed int)__builtin_s390_vsrab(
8736 (__vector unsigned char)__a, __b);
8737}
8738
8739// This prototype is deprecated.
74928740static inline __ATTRS_o_ai __vector unsigned int
74938741vec_srab(__vector unsigned int __a, __vector signed int __b) {
74948742 return (__vector unsigned int)__builtin_s390_vsrab(
74958743 (__vector unsigned char)__a, (__vector unsigned char)__b);
74968744}
74978745
8746// This prototype is deprecated.
74988747static inline __ATTRS_o_ai __vector unsigned int
74998748vec_srab(__vector unsigned int __a, __vector unsigned int __b) {
75008749 return (__vector unsigned int)__builtin_s390_vsrab(
75018750 (__vector unsigned char)__a, (__vector unsigned char)__b);
75028751}
75038752
8753static inline __ATTRS_o_ai __vector unsigned int
8754vec_srab(__vector unsigned int __a, __vector unsigned char __b) {
8755 return (__vector unsigned int)__builtin_s390_vsrab(
8756 (__vector unsigned char)__a, __b);
8757}
8758
8759// This prototype is deprecated.
75048760static inline __ATTRS_o_ai __vector signed long long
75058761vec_srab(__vector signed long long __a, __vector signed long long __b) {
75068762 return (__vector signed long long)__builtin_s390_vsrab(
75078763 (__vector unsigned char)__a, (__vector unsigned char)__b);
75088764}
75098765
8766// This prototype is deprecated.
75108767static inline __ATTRS_o_ai __vector signed long long
75118768vec_srab(__vector signed long long __a, __vector unsigned long long __b) {
75128769 return (__vector signed long long)__builtin_s390_vsrab(
75138770 (__vector unsigned char)__a, (__vector unsigned char)__b);
75148771}
75158772
8773static inline __ATTRS_o_ai __vector signed long long
8774vec_srab(__vector signed long long __a, __vector unsigned char __b) {
8775 return (__vector signed long long)__builtin_s390_vsrab(
8776 (__vector unsigned char)__a, __b);
8777}
8778
8779// This prototype is deprecated.
75168780static inline __ATTRS_o_ai __vector unsigned long long
75178781vec_srab(__vector unsigned long long __a, __vector signed long long __b) {
75188782 return (__vector unsigned long long)__builtin_s390_vsrab(
75198783 (__vector unsigned char)__a, (__vector unsigned char)__b);
75208784}
75218785
7522static inline __ATTRS_o_ai __vector unsigned long long
7523vec_srab(__vector unsigned long long __a, __vector unsigned long long __b) {
7524 return (__vector unsigned long long)__builtin_s390_vsrab(
7525 (__vector unsigned char)__a, (__vector unsigned char)__b);
8786// This prototype is deprecated.
8787static inline __ATTRS_o_ai __vector unsigned long long
8788vec_srab(__vector unsigned long long __a, __vector unsigned long long __b) {
8789 return (__vector unsigned long long)__builtin_s390_vsrab(
8790 (__vector unsigned char)__a, (__vector unsigned char)__b);
8791}
8792
8793static inline __ATTRS_o_ai __vector unsigned long long
8794vec_srab(__vector unsigned long long __a, __vector unsigned char __b) {
8795 return (__vector unsigned long long)__builtin_s390_vsrab(
8796 (__vector unsigned char)__a, __b);
8797}
8798
8799static inline __ATTRS_o_ai __vector signed __int128
8800vec_srab(__vector signed __int128 __a, __vector unsigned char __b) {
8801 return (__vector signed __int128)__builtin_s390_vsrab(
8802 (__vector unsigned char)__a, __b);
8803}
8804
8805static inline __ATTRS_o_ai __vector unsigned __int128
8806vec_srab(__vector unsigned __int128 __a, __vector unsigned char __b) {
8807 return (__vector unsigned __int128)__builtin_s390_vsrab(
8808 (__vector unsigned char)__a, __b);
75268809}
75278810
75288811#if __ARCH__ >= 12
8812// This prototype is deprecated.
75298813static inline __ATTRS_o_ai __vector float
75308814vec_srab(__vector float __a, __vector signed int __b) {
75318815 return (__vector float)__builtin_s390_vsrab(
75328816 (__vector unsigned char)__a, (__vector unsigned char)__b);
75338817}
75348818
8819// This prototype is deprecated.
75358820static inline __ATTRS_o_ai __vector float
75368821vec_srab(__vector float __a, __vector unsigned int __b) {
75378822 return (__vector float)__builtin_s390_vsrab(
75388823 (__vector unsigned char)__a, (__vector unsigned char)__b);
75398824}
8825
8826static inline __ATTRS_o_ai __vector float
8827vec_srab(__vector float __a, __vector unsigned char __b) {
8828 return (__vector float)__builtin_s390_vsrab(
8829 (__vector unsigned char)__a, __b);
8830}
75408831#endif
75418832
8833// This prototype is deprecated.
75428834static inline __ATTRS_o_ai __vector double
75438835vec_srab(__vector double __a, __vector signed long long __b) {
75448836 return (__vector double)__builtin_s390_vsrab(
75458837 (__vector unsigned char)__a, (__vector unsigned char)__b);
75468838}
75478839
8840// This prototype is deprecated.
75488841static inline __ATTRS_o_ai __vector double
75498842vec_srab(__vector double __a, __vector unsigned long long __b) {
75508843 return (__vector double)__builtin_s390_vsrab(
75518844 (__vector unsigned char)__a, (__vector unsigned char)__b);
75528845}
75538846
8847static inline __ATTRS_o_ai __vector double
8848vec_srab(__vector double __a, __vector unsigned char __b) {
8849 return (__vector double)__builtin_s390_vsrab(
8850 (__vector unsigned char)__a, __b);
8851}
8852
75548853/*-- vec_srl ----------------------------------------------------------------*/
75558854
75568855static inline __ATTRS_o_ai __vector signed char
......@@ -7794,8 +9093,21 @@ vec_srl(__vector unsigned long long __a, __vector unsigned int __b) {
77949093 (__vector unsigned char)__a, (__vector unsigned char)__b);
77959094}
77969095
9096static inline __ATTRS_o_ai __vector signed __int128
9097vec_srl(__vector signed __int128 __a, __vector unsigned char __b) {
9098 return (__vector signed __int128)__builtin_s390_vsrl(
9099 (__vector unsigned char)__a, __b);
9100}
9101
9102static inline __ATTRS_o_ai __vector unsigned __int128
9103vec_srl(__vector unsigned __int128 __a, __vector unsigned char __b) {
9104 return (__vector unsigned __int128)__builtin_s390_vsrl(
9105 (__vector unsigned char)__a, __b);
9106}
9107
77979108/*-- vec_srb ----------------------------------------------------------------*/
77989109
9110// This prototype is deprecated.
77999111static inline __ATTRS_o_ai __vector signed char
78009112vec_srb(__vector signed char __a, __vector signed char __b) {
78019113 return (__vector signed char)__builtin_s390_vsrlb(
......@@ -7808,6 +9120,7 @@ vec_srb(__vector signed char __a, __vector unsigned char __b) {
78089120 (__vector unsigned char)__a, __b);
78099121}
78109122
9123// This prototype is deprecated.
78119124static inline __ATTRS_o_ai __vector unsigned char
78129125vec_srb(__vector unsigned char __a, __vector signed char __b) {
78139126 return __builtin_s390_vsrlb(__a, (__vector unsigned char)__b);
......@@ -7818,104 +9131,180 @@ vec_srb(__vector unsigned char __a, __vector unsigned char __b) {
78189131 return __builtin_s390_vsrlb(__a, __b);
78199132}
78209133
9134// This prototype is deprecated.
78219135static inline __ATTRS_o_ai __vector signed short
78229136vec_srb(__vector signed short __a, __vector signed short __b) {
78239137 return (__vector signed short)__builtin_s390_vsrlb(
78249138 (__vector unsigned char)__a, (__vector unsigned char)__b);
78259139}
78269140
9141// This prototype is deprecated.
78279142static inline __ATTRS_o_ai __vector signed short
78289143vec_srb(__vector signed short __a, __vector unsigned short __b) {
78299144 return (__vector signed short)__builtin_s390_vsrlb(
78309145 (__vector unsigned char)__a, (__vector unsigned char)__b);
78319146}
78329147
9148static inline __ATTRS_o_ai __vector signed short
9149vec_srb(__vector signed short __a, __vector unsigned char __b) {
9150 return (__vector signed short)__builtin_s390_vsrlb(
9151 (__vector unsigned char)__a, __b);
9152}
9153
9154// This prototype is deprecated.
78339155static inline __ATTRS_o_ai __vector unsigned short
78349156vec_srb(__vector unsigned short __a, __vector signed short __b) {
78359157 return (__vector unsigned short)__builtin_s390_vsrlb(
78369158 (__vector unsigned char)__a, (__vector unsigned char)__b);
78379159}
78389160
9161// This prototype is deprecated.
78399162static inline __ATTRS_o_ai __vector unsigned short
78409163vec_srb(__vector unsigned short __a, __vector unsigned short __b) {
78419164 return (__vector unsigned short)__builtin_s390_vsrlb(
78429165 (__vector unsigned char)__a, (__vector unsigned char)__b);
78439166}
78449167
9168static inline __ATTRS_o_ai __vector unsigned short
9169vec_srb(__vector unsigned short __a, __vector unsigned char __b) {
9170 return (__vector unsigned short)__builtin_s390_vsrlb(
9171 (__vector unsigned char)__a, __b);
9172}
9173
9174// This prototype is deprecated.
78459175static inline __ATTRS_o_ai __vector signed int
78469176vec_srb(__vector signed int __a, __vector signed int __b) {
78479177 return (__vector signed int)__builtin_s390_vsrlb(
78489178 (__vector unsigned char)__a, (__vector unsigned char)__b);
78499179}
78509180
9181// This prototype is deprecated.
78519182static inline __ATTRS_o_ai __vector signed int
78529183vec_srb(__vector signed int __a, __vector unsigned int __b) {
78539184 return (__vector signed int)__builtin_s390_vsrlb(
78549185 (__vector unsigned char)__a, (__vector unsigned char)__b);
78559186}
78569187
9188static inline __ATTRS_o_ai __vector signed int
9189vec_srb(__vector signed int __a, __vector unsigned char __b) {
9190 return (__vector signed int)__builtin_s390_vsrlb(
9191 (__vector unsigned char)__a, __b);
9192}
9193
9194// This prototype is deprecated.
78579195static inline __ATTRS_o_ai __vector unsigned int
78589196vec_srb(__vector unsigned int __a, __vector signed int __b) {
78599197 return (__vector unsigned int)__builtin_s390_vsrlb(
78609198 (__vector unsigned char)__a, (__vector unsigned char)__b);
78619199}
78629200
9201// This prototype is deprecated.
78639202static inline __ATTRS_o_ai __vector unsigned int
78649203vec_srb(__vector unsigned int __a, __vector unsigned int __b) {
78659204 return (__vector unsigned int)__builtin_s390_vsrlb(
78669205 (__vector unsigned char)__a, (__vector unsigned char)__b);
78679206}
78689207
9208static inline __ATTRS_o_ai __vector unsigned int
9209vec_srb(__vector unsigned int __a, __vector unsigned char __b) {
9210 return (__vector unsigned int)__builtin_s390_vsrlb(
9211 (__vector unsigned char)__a, __b);
9212}
9213
9214// This prototype is deprecated.
78699215static inline __ATTRS_o_ai __vector signed long long
78709216vec_srb(__vector signed long long __a, __vector signed long long __b) {
78719217 return (__vector signed long long)__builtin_s390_vsrlb(
78729218 (__vector unsigned char)__a, (__vector unsigned char)__b);
78739219}
78749220
9221// This prototype is deprecated.
78759222static inline __ATTRS_o_ai __vector signed long long
78769223vec_srb(__vector signed long long __a, __vector unsigned long long __b) {
78779224 return (__vector signed long long)__builtin_s390_vsrlb(
78789225 (__vector unsigned char)__a, (__vector unsigned char)__b);
78799226}
78809227
9228static inline __ATTRS_o_ai __vector signed long long
9229vec_srb(__vector signed long long __a, __vector unsigned char __b) {
9230 return (__vector signed long long)__builtin_s390_vsrlb(
9231 (__vector unsigned char)__a, __b);
9232}
9233
9234// This prototype is deprecated.
78819235static inline __ATTRS_o_ai __vector unsigned long long
78829236vec_srb(__vector unsigned long long __a, __vector signed long long __b) {
78839237 return (__vector unsigned long long)__builtin_s390_vsrlb(
78849238 (__vector unsigned char)__a, (__vector unsigned char)__b);
78859239}
78869240
9241// This prototype is deprecated.
78879242static inline __ATTRS_o_ai __vector unsigned long long
78889243vec_srb(__vector unsigned long long __a, __vector unsigned long long __b) {
78899244 return (__vector unsigned long long)__builtin_s390_vsrlb(
78909245 (__vector unsigned char)__a, (__vector unsigned char)__b);
78919246}
78929247
9248static inline __ATTRS_o_ai __vector unsigned long long
9249vec_srb(__vector unsigned long long __a, __vector unsigned char __b) {
9250 return (__vector unsigned long long)__builtin_s390_vsrlb(
9251 (__vector unsigned char)__a, __b);
9252}
9253
9254static inline __ATTRS_o_ai __vector signed __int128
9255vec_srb(__vector signed __int128 __a, __vector unsigned char __b) {
9256 return (__vector signed __int128)__builtin_s390_vsrlb(
9257 (__vector unsigned char)__a, __b);
9258}
9259
9260static inline __ATTRS_o_ai __vector unsigned __int128
9261vec_srb(__vector unsigned __int128 __a, __vector unsigned char __b) {
9262 return (__vector unsigned __int128)__builtin_s390_vsrlb(
9263 (__vector unsigned char)__a, __b);
9264}
9265
78939266#if __ARCH__ >= 12
9267// This prototype is deprecated.
78949268static inline __ATTRS_o_ai __vector float
78959269vec_srb(__vector float __a, __vector signed int __b) {
78969270 return (__vector float)__builtin_s390_vsrlb(
78979271 (__vector unsigned char)__a, (__vector unsigned char)__b);
78989272}
78999273
9274// This prototype is deprecated.
79009275static inline __ATTRS_o_ai __vector float
79019276vec_srb(__vector float __a, __vector unsigned int __b) {
79029277 return (__vector float)__builtin_s390_vsrlb(
79039278 (__vector unsigned char)__a, (__vector unsigned char)__b);
79049279}
9280
9281static inline __ATTRS_o_ai __vector float
9282vec_srb(__vector float __a, __vector unsigned char __b) {
9283 return (__vector float)__builtin_s390_vsrlb(
9284 (__vector unsigned char)__a, __b);
9285}
79059286#endif
79069287
9288// This prototype is deprecated.
79079289static inline __ATTRS_o_ai __vector double
79089290vec_srb(__vector double __a, __vector signed long long __b) {
79099291 return (__vector double)__builtin_s390_vsrlb(
79109292 (__vector unsigned char)__a, (__vector unsigned char)__b);
79119293}
79129294
9295// This prototype is deprecated.
79139296static inline __ATTRS_o_ai __vector double
79149297vec_srb(__vector double __a, __vector unsigned long long __b) {
79159298 return (__vector double)__builtin_s390_vsrlb(
79169299 (__vector unsigned char)__a, (__vector unsigned char)__b);
79179300}
79189301
9302static inline __ATTRS_o_ai __vector double
9303vec_srb(__vector double __a, __vector unsigned char __b) {
9304 return (__vector double)__builtin_s390_vsrlb(
9305 (__vector unsigned char)__a, __b);
9306}
9307
79199308/*-- vec_srdb ---------------------------------------------------------------*/
79209309
79219310#if __ARCH__ >= 13
......@@ -7953,6 +9342,15 @@ vec_srdb(__vector unsigned long long __a, __vector unsigned long long __b,
79539342 int __c)
79549343 __constant_range(__c, 0, 7);
79559344
9345extern __ATTRS_o __vector signed __int128
9346vec_srdb(__vector signed __int128 __a, __vector signed __int128 __b, int __c)
9347 __constant_range(__c, 0, 7);
9348
9349extern __ATTRS_o __vector unsigned __int128
9350vec_srdb(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
9351 int __c)
9352 __constant_range(__c, 0, 7);
9353
79569354extern __ATTRS_o __vector float
79579355vec_srdb(__vector float __a, __vector float __b, int __c)
79589356 __constant_range(__c, 0, 7);
......@@ -7989,6 +9387,11 @@ vec_abs(__vector signed long long __a) {
79899387 return vec_sel(__a, -__a, vec_cmplt(__a, (__vector signed long long)0));
79909388}
79919389
9390static inline __ATTRS_o_ai __vector signed __int128
9391vec_abs(__vector signed __int128 __a) {
9392 return vec_sel(__a, -__a, vec_cmplt(__a, (__vector signed __int128)0));
9393}
9394
79929395#if __ARCH__ >= 12
79939396static inline __ATTRS_o_ai __vector float
79949397vec_abs(__vector float __a) {
......@@ -8169,6 +9572,16 @@ vec_max(__vector __bool long long __a, __vector unsigned long long __b) {
81699572 return vec_sel(__b, __ac, vec_cmpgt(__ac, __b));
81709573}
81719574
9575static inline __ATTRS_o_ai __vector signed __int128
9576vec_max(__vector signed __int128 __a, __vector signed __int128 __b) {
9577 return vec_sel(__b, __a, vec_cmpgt(__a, __b));
9578}
9579
9580static inline __ATTRS_o_ai __vector unsigned __int128
9581vec_max(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
9582 return vec_sel(__b, __a, vec_cmpgt(__a, __b));
9583}
9584
81729585#if __ARCH__ >= 12
81739586static inline __ATTRS_o_ai __vector float
81749587vec_max(__vector float __a, __vector float __b) {
......@@ -8339,6 +9752,16 @@ vec_min(__vector __bool long long __a, __vector unsigned long long __b) {
83399752 return vec_sel(__ac, __b, vec_cmpgt(__ac, __b));
83409753}
83419754
9755static inline __ATTRS_o_ai __vector signed __int128
9756vec_min(__vector signed __int128 __a, __vector signed __int128 __b) {
9757 return vec_sel(__a, __b, vec_cmpgt(__a, __b));
9758}
9759
9760static inline __ATTRS_o_ai __vector unsigned __int128
9761vec_min(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
9762 return vec_sel(__a, __b, vec_cmpgt(__a, __b));
9763}
9764
83429765#if __ARCH__ >= 12
83439766static inline __ATTRS_o_ai __vector float
83449767vec_min(__vector float __a, __vector float __b) {
......@@ -8357,9 +9780,11 @@ vec_min(__vector double __a, __vector double __b) {
83579780
83589781/*-- vec_add_u128 -----------------------------------------------------------*/
83599782
9783// This prototype is deprecated.
83609784static inline __ATTRS_ai __vector unsigned char
83619785vec_add_u128(__vector unsigned char __a, __vector unsigned char __b) {
8362 return (__vector unsigned char)((__int128)__a + (__int128)__b);
9786 return (__vector unsigned char)(__vector unsigned __int128)
9787 ((__int128)__a + (__int128)__b);
83639788}
83649789
83659790/*-- vec_addc ---------------------------------------------------------------*/
......@@ -8384,30 +9809,59 @@ vec_addc(__vector unsigned long long __a, __vector unsigned long long __b) {
83849809 return __builtin_s390_vaccg(__a, __b);
83859810}
83869811
9812static inline __ATTRS_o_ai __vector unsigned __int128
9813vec_addc(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
9814 return (__vector unsigned __int128)
9815 __builtin_s390_vaccq((unsigned __int128)__a, (unsigned __int128)__b);
9816}
9817
83879818/*-- vec_addc_u128 ----------------------------------------------------------*/
83889819
9820// This prototype is deprecated.
83899821static inline __ATTRS_ai __vector unsigned char
83909822vec_addc_u128(__vector unsigned char __a, __vector unsigned char __b) {
8391 return (__vector unsigned char)
9823 return (__vector unsigned char)(__vector unsigned __int128)
83929824 __builtin_s390_vaccq((unsigned __int128)__a, (unsigned __int128)__b);
83939825}
83949826
9827/*-- vec_adde ---------------------------------------------------------------*/
9828
9829static inline __ATTRS_ai __vector unsigned __int128
9830vec_adde(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
9831 __vector unsigned __int128 __c) {
9832 return (__vector unsigned __int128)
9833 __builtin_s390_vacq((unsigned __int128)__a, (unsigned __int128)__b,
9834 (unsigned __int128)__c);
9835}
9836
83959837/*-- vec_adde_u128 ----------------------------------------------------------*/
83969838
9839// This prototype is deprecated.
83979840static inline __ATTRS_ai __vector unsigned char
83989841vec_adde_u128(__vector unsigned char __a, __vector unsigned char __b,
83999842 __vector unsigned char __c) {
8400 return (__vector unsigned char)
9843 return (__vector unsigned char)(__vector unsigned __int128)
84019844 __builtin_s390_vacq((unsigned __int128)__a, (unsigned __int128)__b,
84029845 (unsigned __int128)__c);
84039846}
84049847
9848/*-- vec_addec --------------------------------------------------------------*/
9849
9850static inline __ATTRS_ai __vector unsigned __int128
9851vec_addec(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
9852 __vector unsigned __int128 __c) {
9853 return (__vector unsigned __int128)
9854 __builtin_s390_vacccq((unsigned __int128)__a, (unsigned __int128)__b,
9855 (unsigned __int128)__c);
9856}
9857
84059858/*-- vec_addec_u128 ---------------------------------------------------------*/
84069859
9860// This prototype is deprecated.
84079861static inline __ATTRS_ai __vector unsigned char
84089862vec_addec_u128(__vector unsigned char __a, __vector unsigned char __b,
84099863 __vector unsigned char __c) {
8410 return (__vector unsigned char)
9864 return (__vector unsigned char)(__vector unsigned __int128)
84119865 __builtin_s390_vacccq((unsigned __int128)__a, (unsigned __int128)__b,
84129866 (unsigned __int128)__c);
84139867}
......@@ -8434,6 +9888,14 @@ vec_avg(__vector signed long long __a, __vector signed long long __b) {
84349888 return __builtin_s390_vavgg(__a, __b);
84359889}
84369890
9891#if __ARCH__ >= 15
9892static inline __ATTRS_o_ai __vector signed __int128
9893vec_avg(__vector signed __int128 __a, __vector signed __int128 __b) {
9894 return (__vector signed __int128)
9895 __builtin_s390_vavgq((signed __int128)__a, (signed __int128)__b);
9896}
9897#endif
9898
84379899static inline __ATTRS_o_ai __vector unsigned char
84389900vec_avg(__vector unsigned char __a, __vector unsigned char __b) {
84399901 return __builtin_s390_vavglb(__a, __b);
......@@ -8454,6 +9916,14 @@ vec_avg(__vector unsigned long long __a, __vector unsigned long long __b) {
84549916 return __builtin_s390_vavglg(__a, __b);
84559917}
84569918
9919#if __ARCH__ >= 15
9920static inline __ATTRS_o_ai __vector unsigned __int128
9921vec_avg(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
9922 return (__vector unsigned __int128)
9923 __builtin_s390_vavglq((unsigned __int128)__a, (unsigned __int128)__b);
9924}
9925#endif
9926
84579927/*-- vec_checksum -----------------------------------------------------------*/
84589928
84599929static inline __ATTRS_ai __vector unsigned int
......@@ -8478,12 +9948,19 @@ vec_gfmsum(__vector unsigned int __a, __vector unsigned int __b) {
84789948 return __builtin_s390_vgfmf(__a, __b);
84799949}
84809950
9951static inline __ATTRS_o_ai __vector unsigned __int128
9952vec_gfmsum(__vector unsigned long long __a, __vector unsigned long long __b) {
9953 return (__vector unsigned __int128)__builtin_s390_vgfmg(__a, __b);
9954}
9955
84819956/*-- vec_gfmsum_128 ---------------------------------------------------------*/
84829957
9958// This prototype is deprecated.
84839959static inline __ATTRS_o_ai __vector unsigned char
84849960vec_gfmsum_128(__vector unsigned long long __a,
84859961 __vector unsigned long long __b) {
8486 return (__vector unsigned char)__builtin_s390_vgfmg(__a, __b);
9962 return (__vector unsigned char)(__vector unsigned __int128)
9963 __builtin_s390_vgfmg(__a, __b);
84879964}
84889965
84899966/*-- vec_gfmsum_accum -------------------------------------------------------*/
......@@ -8506,13 +9983,21 @@ vec_gfmsum_accum(__vector unsigned int __a, __vector unsigned int __b,
85069983 return __builtin_s390_vgfmaf(__a, __b, __c);
85079984}
85089985
9986static inline __ATTRS_o_ai __vector unsigned __int128
9987vec_gfmsum_accum(__vector unsigned long long __a, __vector unsigned long long __b,
9988 __vector unsigned __int128 __c) {
9989 return (__vector unsigned __int128)
9990 __builtin_s390_vgfmag(__a, __b, (unsigned __int128)__c);
9991}
9992
85099993/*-- vec_gfmsum_accum_128 ---------------------------------------------------*/
85109994
9995// This prototype is deprecated.
85119996static inline __ATTRS_o_ai __vector unsigned char
85129997vec_gfmsum_accum_128(__vector unsigned long long __a,
85139998 __vector unsigned long long __b,
85149999 __vector unsigned char __c) {
8515 return (__vector unsigned char)
10000 return (__vector unsigned char)(__vector unsigned __int128)
851610001 __builtin_s390_vgfmag(__a, __b, (unsigned __int128)__c);
851710002}
851810003
......@@ -8590,6 +10075,56 @@ vec_mladd(__vector unsigned int __a, __vector unsigned int __b,
859010075 return __a * __b + __c;
859110076}
859210077
10078#if __ARCH__ >= 15
10079static inline __ATTRS_o_ai __vector signed long long
10080vec_mladd(__vector signed long long __a, __vector signed long long __b,
10081 __vector signed long long __c) {
10082 return __a * __b + __c;
10083}
10084
10085static inline __ATTRS_o_ai __vector signed long long
10086vec_mladd(__vector unsigned long long __a, __vector signed long long __b,
10087 __vector signed long long __c) {
10088 return (__vector signed long long)__a * __b + __c;
10089}
10090
10091static inline __ATTRS_o_ai __vector signed long long
10092vec_mladd(__vector signed long long __a, __vector unsigned long long __b,
10093 __vector unsigned long long __c) {
10094 return __a * (__vector signed long long)__b + (__vector signed long long)__c;
10095}
10096
10097static inline __ATTRS_o_ai __vector unsigned long long
10098vec_mladd(__vector unsigned long long __a, __vector unsigned long long __b,
10099 __vector unsigned long long __c) {
10100 return __a * __b + __c;
10101}
10102
10103static inline __ATTRS_o_ai __vector signed __int128
10104vec_mladd(__vector signed __int128 __a, __vector signed __int128 __b,
10105 __vector signed __int128 __c) {
10106 return __a * __b + __c;
10107}
10108
10109static inline __ATTRS_o_ai __vector signed __int128
10110vec_mladd(__vector unsigned __int128 __a, __vector signed __int128 __b,
10111 __vector signed __int128 __c) {
10112 return (__vector signed __int128)__a * __b + __c;
10113}
10114
10115static inline __ATTRS_o_ai __vector signed __int128
10116vec_mladd(__vector signed __int128 __a, __vector unsigned __int128 __b,
10117 __vector unsigned __int128 __c) {
10118 return __a * (__vector signed __int128)__b + (__vector signed __int128)__c;
10119}
10120
10121static inline __ATTRS_o_ai __vector unsigned __int128
10122vec_mladd(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
10123 __vector unsigned __int128 __c) {
10124 return __a * __b + __c;
10125}
10126#endif
10127
859310128/*-- vec_mhadd --------------------------------------------------------------*/
859410129
859510130static inline __ATTRS_o_ai __vector signed char
......@@ -8628,6 +10163,34 @@ vec_mhadd(__vector unsigned int __a, __vector unsigned int __b,
862810163 return __builtin_s390_vmalhf(__a, __b, __c);
862910164}
863010165
10166#if __ARCH__ >= 15
10167static inline __ATTRS_o_ai __vector signed long long
10168vec_mhadd(__vector signed long long __a, __vector signed long long __b,
10169 __vector signed long long __c) {
10170 return __builtin_s390_vmahg(__a, __b, __c);
10171}
10172
10173static inline __ATTRS_o_ai __vector unsigned long long
10174vec_mhadd(__vector unsigned long long __a, __vector unsigned long long __b,
10175 __vector unsigned long long __c) {
10176 return __builtin_s390_vmalhg(__a, __b, __c);
10177}
10178
10179static inline __ATTRS_o_ai __vector signed __int128
10180vec_mhadd(__vector signed __int128 __a, __vector signed __int128 __b,
10181 __vector signed __int128 __c) {
10182 return (__vector signed __int128)
10183 __builtin_s390_vmahq((signed __int128)__a, (signed __int128)__b, (signed __int128)__c);
10184}
10185
10186static inline __ATTRS_o_ai __vector unsigned __int128
10187vec_mhadd(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
10188 __vector unsigned __int128 __c) {
10189 return (__vector unsigned __int128)
10190 __builtin_s390_vmalhq((unsigned __int128)__a, (unsigned __int128)__b, (unsigned __int128)__c);
10191}
10192#endif
10193
863110194/*-- vec_meadd --------------------------------------------------------------*/
863210195
863310196static inline __ATTRS_o_ai __vector signed short
......@@ -8666,6 +10229,22 @@ vec_meadd(__vector unsigned int __a, __vector unsigned int __b,
866610229 return __builtin_s390_vmalef(__a, __b, __c);
866710230}
866810231
10232#if __ARCH__ >= 15
10233static inline __ATTRS_o_ai __vector signed __int128
10234vec_meadd(__vector signed long long __a, __vector signed long long __b,
10235 __vector signed __int128 __c) {
10236 return (__vector signed __int128)
10237 __builtin_s390_vmaeg(__a, __b, (signed __int128)__c);
10238}
10239
10240static inline __ATTRS_o_ai __vector unsigned __int128
10241vec_meadd(__vector unsigned long long __a, __vector unsigned long long __b,
10242 __vector unsigned __int128 __c) {
10243 return (__vector unsigned __int128)
10244 __builtin_s390_vmaleg(__a, __b, (unsigned __int128)__c);
10245}
10246#endif
10247
866910248/*-- vec_moadd --------------------------------------------------------------*/
867010249
867110250static inline __ATTRS_o_ai __vector signed short
......@@ -8704,6 +10283,22 @@ vec_moadd(__vector unsigned int __a, __vector unsigned int __b,
870410283 return __builtin_s390_vmalof(__a, __b, __c);
870510284}
870610285
10286#if __ARCH__ >= 15
10287static inline __ATTRS_o_ai __vector signed __int128
10288vec_moadd(__vector signed long long __a, __vector signed long long __b,
10289 __vector signed __int128 __c) {
10290 return (__vector signed __int128)
10291 __builtin_s390_vmaog(__a, __b, (signed __int128)__c);
10292}
10293
10294static inline __ATTRS_o_ai __vector unsigned __int128
10295vec_moadd(__vector unsigned long long __a, __vector unsigned long long __b,
10296 __vector unsigned __int128 __c) {
10297 return (__vector unsigned __int128)
10298 __builtin_s390_vmalog(__a, __b, (unsigned __int128)__c);
10299}
10300#endif
10301
870710302/*-- vec_mulh ---------------------------------------------------------------*/
870810303
870910304static inline __ATTRS_o_ai __vector signed char
......@@ -8736,6 +10331,30 @@ vec_mulh(__vector unsigned int __a, __vector unsigned int __b) {
873610331 return __builtin_s390_vmlhf(__a, __b);
873710332}
873810333
10334#if __ARCH__ >= 15
10335static inline __ATTRS_o_ai __vector signed long long
10336vec_mulh(__vector signed long long __a, __vector signed long long __b) {
10337 return __builtin_s390_vmhg(__a, __b);
10338}
10339
10340static inline __ATTRS_o_ai __vector unsigned long long
10341vec_mulh(__vector unsigned long long __a, __vector unsigned long long __b) {
10342 return __builtin_s390_vmlhg(__a, __b);
10343}
10344
10345static inline __ATTRS_o_ai __vector signed __int128
10346vec_mulh(__vector signed __int128 __a, __vector signed __int128 __b) {
10347 return (__vector signed __int128)
10348 __builtin_s390_vmhq((signed __int128)__a, (signed __int128)__b);
10349}
10350
10351static inline __ATTRS_o_ai __vector unsigned __int128
10352vec_mulh(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
10353 return (__vector unsigned __int128)
10354 __builtin_s390_vmlhq((unsigned __int128)__a, (unsigned __int128)__b);
10355}
10356#endif
10357
873910358/*-- vec_mule ---------------------------------------------------------------*/
874010359
874110360static inline __ATTRS_o_ai __vector signed short
......@@ -8768,6 +10387,18 @@ vec_mule(__vector unsigned int __a, __vector unsigned int __b) {
876810387 return __builtin_s390_vmlef(__a, __b);
876910388}
877010389
10390#if __ARCH__ >= 15
10391static inline __ATTRS_o_ai __vector signed __int128
10392vec_mule(__vector signed long long __a, __vector signed long long __b) {
10393 return (__vector signed __int128)__builtin_s390_vmeg(__a, __b);
10394}
10395
10396static inline __ATTRS_o_ai __vector unsigned __int128
10397vec_mule(__vector unsigned long long __a, __vector unsigned long long __b) {
10398 return (__vector unsigned __int128)__builtin_s390_vmleg(__a, __b);
10399}
10400#endif
10401
877110402/*-- vec_mulo ---------------------------------------------------------------*/
877210403
877310404static inline __ATTRS_o_ai __vector signed short
......@@ -8800,9 +10431,35 @@ vec_mulo(__vector unsigned int __a, __vector unsigned int __b) {
880010431 return __builtin_s390_vmlof(__a, __b);
880110432}
880210433
10434#if __ARCH__ >= 15
10435static inline __ATTRS_o_ai __vector signed __int128
10436vec_mulo(__vector signed long long __a, __vector signed long long __b) {
10437 return (__vector signed __int128)__builtin_s390_vmog(__a, __b);
10438}
10439
10440static inline __ATTRS_o_ai __vector unsigned __int128
10441vec_mulo(__vector unsigned long long __a, __vector unsigned long long __b) {
10442 return (__vector unsigned __int128)__builtin_s390_vmlog(__a, __b);
10443}
10444#endif
10445
10446/*-- vec_msum ---------------------------------------------------------------*/
10447
10448#if __ARCH__ >= 12
10449extern __ATTRS_o __vector unsigned __int128
10450vec_msum(__vector unsigned long long __a, __vector unsigned long long __b,
10451 __vector unsigned __int128 __c, int __d)
10452 __constant_range(__d, 0, 15);
10453
10454#define vec_msum(X, Y, Z, W) \
10455 ((__typeof__((vec_msum)((X), (Y), (Z), (W)))) \
10456 __builtin_s390_vmslg((X), (Y), (unsigned __int128)(Z), (W)))
10457#endif
10458
880310459/*-- vec_msum_u128 ----------------------------------------------------------*/
880410460
880510461#if __ARCH__ >= 12
10462// This prototype is deprecated.
880610463extern __ATTRS_o __vector unsigned char
880710464vec_msum_u128(__vector unsigned long long __a, __vector unsigned long long __b,
880810465 __vector unsigned char __c, int __d)
......@@ -8810,14 +10467,17 @@ vec_msum_u128(__vector unsigned long long __a, __vector unsigned long long __b,
881010467
881110468#define vec_msum_u128(X, Y, Z, W) \
881210469 ((__typeof__((vec_msum_u128)((X), (Y), (Z), (W)))) \
10470 (__vector unsigned __int128) \
881310471 __builtin_s390_vmslg((X), (Y), (unsigned __int128)(Z), (W)))
881410472#endif
881510473
881610474/*-- vec_sub_u128 -----------------------------------------------------------*/
881710475
10476// This prototype is deprecated.
881810477static inline __ATTRS_ai __vector unsigned char
881910478vec_sub_u128(__vector unsigned char __a, __vector unsigned char __b) {
8820 return (__vector unsigned char)((__int128)__a - (__int128)__b);
10479 return (__vector unsigned char)(__vector unsigned __int128)
10480 ((__int128)__a - (__int128)__b);
882110481}
882210482
882310483/*-- vec_subc ---------------------------------------------------------------*/
......@@ -8842,30 +10502,59 @@ vec_subc(__vector unsigned long long __a, __vector unsigned long long __b) {
884210502 return __builtin_s390_vscbig(__a, __b);
884310503}
884410504
10505static inline __ATTRS_o_ai __vector unsigned __int128
10506vec_subc(__vector unsigned __int128 __a, __vector unsigned __int128 __b) {
10507 return (__vector unsigned __int128)
10508 __builtin_s390_vscbiq((unsigned __int128)__a, (unsigned __int128)__b);
10509}
10510
884510511/*-- vec_subc_u128 ----------------------------------------------------------*/
884610512
10513// This prototype is deprecated.
884710514static inline __ATTRS_ai __vector unsigned char
884810515vec_subc_u128(__vector unsigned char __a, __vector unsigned char __b) {
8849 return (__vector unsigned char)
10516 return (__vector unsigned char)(__vector unsigned __int128)
885010517 __builtin_s390_vscbiq((unsigned __int128)__a, (unsigned __int128)__b);
885110518}
885210519
10520/*-- vec_sube ---------------------------------------------------------------*/
10521
10522static inline __ATTRS_ai __vector unsigned __int128
10523vec_sube(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
10524 __vector unsigned __int128 __c) {
10525 return (__vector unsigned __int128)
10526 __builtin_s390_vsbiq((unsigned __int128)__a, (unsigned __int128)__b,
10527 (unsigned __int128)__c);
10528}
10529
885310530/*-- vec_sube_u128 ----------------------------------------------------------*/
885410531
10532// This prototype is deprecated.
885510533static inline __ATTRS_ai __vector unsigned char
885610534vec_sube_u128(__vector unsigned char __a, __vector unsigned char __b,
885710535 __vector unsigned char __c) {
8858 return (__vector unsigned char)
10536 return (__vector unsigned char)(__vector unsigned __int128)
885910537 __builtin_s390_vsbiq((unsigned __int128)__a, (unsigned __int128)__b,
886010538 (unsigned __int128)__c);
886110539}
886210540
10541/*-- vec_subec --------------------------------------------------------------*/
10542
10543static inline __ATTRS_ai __vector unsigned __int128
10544vec_subec(__vector unsigned __int128 __a, __vector unsigned __int128 __b,
10545 __vector unsigned __int128 __c) {
10546 return (__vector unsigned __int128)
10547 __builtin_s390_vsbcbiq((unsigned __int128)__a, (unsigned __int128)__b,
10548 (unsigned __int128)__c);
10549}
10550
886310551/*-- vec_subec_u128 ---------------------------------------------------------*/
886410552
10553// This prototype is deprecated.
886510554static inline __ATTRS_ai __vector unsigned char
886610555vec_subec_u128(__vector unsigned char __a, __vector unsigned char __b,
886710556 __vector unsigned char __c) {
8868 return (__vector unsigned char)
10557 return (__vector unsigned char)(__vector unsigned __int128)
886910558 __builtin_s390_vsbcbiq((unsigned __int128)__a, (unsigned __int128)__b,
887010559 (unsigned __int128)__c);
887110560}
......@@ -8882,16 +10571,32 @@ vec_sum2(__vector unsigned int __a, __vector unsigned int __b) {
888210571 return __builtin_s390_vsumgf(__a, __b);
888310572}
888410573
10574/*-- vec_sum ----------------------------------------------------------------*/
10575
10576static inline __ATTRS_o_ai __vector unsigned __int128
10577vec_sum(__vector unsigned int __a, __vector unsigned int __b) {
10578 return (__vector unsigned __int128)__builtin_s390_vsumqf(__a, __b);
10579}
10580
10581static inline __ATTRS_o_ai __vector unsigned __int128
10582vec_sum(__vector unsigned long long __a, __vector unsigned long long __b) {
10583 return (__vector unsigned __int128)__builtin_s390_vsumqg(__a, __b);
10584}
10585
888510586/*-- vec_sum_u128 -----------------------------------------------------------*/
888610587
10588// This prototype is deprecated.
888710589static inline __ATTRS_o_ai __vector unsigned char
888810590vec_sum_u128(__vector unsigned int __a, __vector unsigned int __b) {
8889 return (__vector unsigned char)__builtin_s390_vsumqf(__a, __b);
10591 return (__vector unsigned char)(__vector unsigned __int128)
10592 __builtin_s390_vsumqf(__a, __b);
889010593}
889110594
10595// This prototype is deprecated.
889210596static inline __ATTRS_o_ai __vector unsigned char
889310597vec_sum_u128(__vector unsigned long long __a, __vector unsigned long long __b) {
8894 return (__vector unsigned char)__builtin_s390_vsumqg(__a, __b);
10598 return (__vector unsigned char)(__vector unsigned __int128)
10599 __builtin_s390_vsumqg(__a, __b);
889510600}
889610601
889710602/*-- vec_sum4 ---------------------------------------------------------------*/
......@@ -8956,6 +10661,19 @@ vec_test_mask(__vector unsigned long long __a,
895610661 (__vector unsigned char)__b);
895710662}
895810663
10664static inline __ATTRS_o_ai int
10665vec_test_mask(__vector signed __int128 __a, __vector unsigned __int128 __b) {
10666 return __builtin_s390_vtm((__vector unsigned char)__a,
10667 (__vector unsigned char)__b);
10668}
10669
10670static inline __ATTRS_o_ai int
10671vec_test_mask(__vector unsigned __int128 __a,
10672 __vector unsigned __int128 __b) {
10673 return __builtin_s390_vtm((__vector unsigned char)__a,
10674 (__vector unsigned char)__b);
10675}
10676
895910677#if __ARCH__ >= 12
896010678static inline __ATTRS_o_ai int
896110679vec_test_mask(__vector float __a, __vector unsigned int __b) {
lib/include/wasm_simd128.h+167-21
......@@ -33,6 +33,7 @@ typedef unsigned long long __u64x2
3333 __attribute__((__vector_size__(16), __aligned__(16)));
3434typedef float __f32x4 __attribute__((__vector_size__(16), __aligned__(16)));
3535typedef double __f64x2 __attribute__((__vector_size__(16), __aligned__(16)));
36typedef __fp16 __f16x8 __attribute__((__vector_size__(16), __aligned__(16)));
3637
3738typedef signed char __i8x8 __attribute__((__vector_size__(8), __aligned__(8)));
3839typedef unsigned char __u8x8
......@@ -956,7 +957,7 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i8x16_bitmask(v128_t __a) {
956957}
957958
958959static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_popcnt(v128_t __a) {
959 return (v128_t)__builtin_wasm_popcnt_i8x16((__i8x16)__a);
960 return (v128_t)__builtin_elementwise_popcount((__i8x16)__a);
960961}
961962
962963static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shl(v128_t __a,
......@@ -981,12 +982,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_add(v128_t __a,
981982
982983static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_add_sat(v128_t __a,
983984 v128_t __b) {
984 return (v128_t)__builtin_wasm_add_sat_s_i8x16((__i8x16)__a, (__i8x16)__b);
985 return (v128_t)__builtin_elementwise_add_sat((__i8x16)__a, (__i8x16)__b);
985986}
986987
987988static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_add_sat(v128_t __a,
988989 v128_t __b) {
989 return (v128_t)__builtin_wasm_add_sat_u_i8x16((__u8x16)__a, (__u8x16)__b);
990 return (v128_t)__builtin_elementwise_add_sat((__u8x16)__a, (__u8x16)__b);
990991}
991992
992993static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_sub(v128_t __a,
......@@ -996,32 +997,32 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_sub(v128_t __a,
996997
997998static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_sub_sat(v128_t __a,
998999 v128_t __b) {
999 return (v128_t)__builtin_wasm_sub_sat_s_i8x16((__i8x16)__a, (__i8x16)__b);
1000 return (v128_t)__builtin_elementwise_sub_sat((__i8x16)__a, (__i8x16)__b);
10001001}
10011002
10021003static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_sub_sat(v128_t __a,
10031004 v128_t __b) {
1004 return (v128_t)__builtin_wasm_sub_sat_u_i8x16((__u8x16)__a, (__u8x16)__b);
1005 return (v128_t)__builtin_elementwise_sub_sat((__u8x16)__a, (__u8x16)__b);
10051006}
10061007
10071008static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_min(v128_t __a,
10081009 v128_t __b) {
1009 return (v128_t)__builtin_wasm_min_s_i8x16((__i8x16)__a, (__i8x16)__b);
1010 return (v128_t)__builtin_elementwise_min((__i8x16)__a, (__i8x16)__b);
10101011}
10111012
10121013static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_min(v128_t __a,
10131014 v128_t __b) {
1014 return (v128_t)__builtin_wasm_min_u_i8x16((__u8x16)__a, (__u8x16)__b);
1015 return (v128_t)__builtin_elementwise_min((__u8x16)__a, (__u8x16)__b);
10151016}
10161017
10171018static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_max(v128_t __a,
10181019 v128_t __b) {
1019 return (v128_t)__builtin_wasm_max_s_i8x16((__i8x16)__a, (__i8x16)__b);
1020 return (v128_t)__builtin_elementwise_max((__i8x16)__a, (__i8x16)__b);
10201021}
10211022
10221023static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_max(v128_t __a,
10231024 v128_t __b) {
1024 return (v128_t)__builtin_wasm_max_u_i8x16((__u8x16)__a, (__u8x16)__b);
1025 return (v128_t)__builtin_elementwise_max((__u8x16)__a, (__u8x16)__b);
10251026}
10261027
10271028static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_avgr(v128_t __a,
......@@ -1067,12 +1068,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_add(v128_t __a,
10671068
10681069static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_add_sat(v128_t __a,
10691070 v128_t __b) {
1070 return (v128_t)__builtin_wasm_add_sat_s_i16x8((__i16x8)__a, (__i16x8)__b);
1071 return (v128_t)__builtin_elementwise_add_sat((__i16x8)__a, (__i16x8)__b);
10711072}
10721073
10731074static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_add_sat(v128_t __a,
10741075 v128_t __b) {
1075 return (v128_t)__builtin_wasm_add_sat_u_i16x8((__u16x8)__a, (__u16x8)__b);
1076 return (v128_t)__builtin_elementwise_add_sat((__u16x8)__a, (__u16x8)__b);
10761077}
10771078
10781079static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_sub(v128_t __a,
......@@ -1082,12 +1083,12 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_sub(v128_t __a,
10821083
10831084static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_sub_sat(v128_t __a,
10841085 v128_t __b) {
1085 return (v128_t)__builtin_wasm_sub_sat_s_i16x8((__i16x8)__a, (__i16x8)__b);
1086 return (v128_t)__builtin_elementwise_sub_sat((__i16x8)__a, (__i16x8)__b);
10861087}
10871088
10881089static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_sub_sat(v128_t __a,
10891090 v128_t __b) {
1090 return (v128_t)__builtin_wasm_sub_sat_u_i16x8((__u16x8)__a, (__u16x8)__b);
1091 return (v128_t)__builtin_elementwise_sub_sat((__u16x8)__a, (__u16x8)__b);
10911092}
10921093
10931094static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_mul(v128_t __a,
......@@ -1097,22 +1098,22 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_mul(v128_t __a,
10971098
10981099static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_min(v128_t __a,
10991100 v128_t __b) {
1100 return (v128_t)__builtin_wasm_min_s_i16x8((__i16x8)__a, (__i16x8)__b);
1101 return (v128_t)__builtin_elementwise_min((__i16x8)__a, (__i16x8)__b);
11011102}
11021103
11031104static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_min(v128_t __a,
11041105 v128_t __b) {
1105 return (v128_t)__builtin_wasm_min_u_i16x8((__u16x8)__a, (__u16x8)__b);
1106 return (v128_t)__builtin_elementwise_min((__u16x8)__a, (__u16x8)__b);
11061107}
11071108
11081109static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_max(v128_t __a,
11091110 v128_t __b) {
1110 return (v128_t)__builtin_wasm_max_s_i16x8((__i16x8)__a, (__i16x8)__b);
1111 return (v128_t)__builtin_elementwise_max((__i16x8)__a, (__i16x8)__b);
11111112}
11121113
11131114static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_max(v128_t __a,
11141115 v128_t __b) {
1115 return (v128_t)__builtin_wasm_max_u_i16x8((__u16x8)__a, (__u16x8)__b);
1116 return (v128_t)__builtin_elementwise_max((__u16x8)__a, (__u16x8)__b);
11161117}
11171118
11181119static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_avgr(v128_t __a,
......@@ -1168,22 +1169,22 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_mul(v128_t __a,
11681169
11691170static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_min(v128_t __a,
11701171 v128_t __b) {
1171 return (v128_t)__builtin_wasm_min_s_i32x4((__i32x4)__a, (__i32x4)__b);
1172 return (v128_t)__builtin_elementwise_min((__i32x4)__a, (__i32x4)__b);
11721173}
11731174
11741175static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_min(v128_t __a,
11751176 v128_t __b) {
1176 return (v128_t)__builtin_wasm_min_u_i32x4((__u32x4)__a, (__u32x4)__b);
1177 return (v128_t)__builtin_elementwise_min((__u32x4)__a, (__u32x4)__b);
11771178}
11781179
11791180static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_max(v128_t __a,
11801181 v128_t __b) {
1181 return (v128_t)__builtin_wasm_max_s_i32x4((__i32x4)__a, (__i32x4)__b);
1182 return (v128_t)__builtin_elementwise_max((__i32x4)__a, (__i32x4)__b);
11821183}
11831184
11841185static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_max(v128_t __a,
11851186 v128_t __b) {
1186 return (v128_t)__builtin_wasm_max_u_i32x4((__u32x4)__a, (__u32x4)__b);
1187 return (v128_t)__builtin_elementwise_max((__u32x4)__a, (__u32x4)__b);
11871188}
11881189
11891190static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_dot_i16x8(v128_t __a,
......@@ -1878,6 +1879,151 @@ wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v128_t __a, v128_t __b, v128_t __c) {
18781879 (__i8x16)__a, (__i8x16)__b, (__i32x4)__c);
18791880}
18801881
1882// FP16 intrinsics
1883#define __FP16_FN_ATTRS \
1884 __attribute__((__always_inline__, __nodebug__, __target__("fp16"), \
1885 __min_vector_width__(128)))
1886
1887static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_splat(float __a) {
1888 return (v128_t)__builtin_wasm_splat_f16x8(__a);
1889}
1890
1891#ifdef __wasm_fp16__
1892// TODO Replace the following macros with regular C functions and use normal
1893// target-independent vector code like the other replace/extract instructions.
1894
1895#define wasm_f16x8_extract_lane(__a, __i) \
1896 (__builtin_wasm_extract_lane_f16x8((__f16x8)(__a), __i))
1897
1898#define wasm_f16x8_replace_lane(__a, __i, __b) \
1899 ((v128_t)__builtin_wasm_replace_lane_f16x8((__f16x8)(__a), __i, __b))
1900
1901#endif
1902
1903static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_abs(v128_t __a) {
1904 return (v128_t)__builtin_wasm_abs_f16x8((__f16x8)__a);
1905}
1906
1907static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_neg(v128_t __a) {
1908 return (v128_t)(-(__f16x8)__a);
1909}
1910
1911static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_sqrt(v128_t __a) {
1912 return (v128_t)__builtin_wasm_sqrt_f16x8((__f16x8)__a);
1913}
1914
1915static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_ceil(v128_t __a) {
1916 return (v128_t)__builtin_wasm_ceil_f16x8((__f16x8)__a);
1917}
1918
1919static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_floor(v128_t __a) {
1920 return (v128_t)__builtin_wasm_floor_f16x8((__f16x8)__a);
1921}
1922
1923static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_trunc(v128_t __a) {
1924 return (v128_t)__builtin_wasm_trunc_f16x8((__f16x8)__a);
1925}
1926
1927static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_nearest(v128_t __a) {
1928 return (v128_t)__builtin_wasm_nearest_f16x8((__f16x8)__a);
1929}
1930
1931static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_eq(v128_t __a, v128_t __b) {
1932 return (v128_t)((__f16x8)__a == (__f16x8)__b);
1933}
1934
1935static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_ne(v128_t __a, v128_t __b) {
1936 return (v128_t)((__f16x8)__a != (__f16x8)__b);
1937}
1938
1939static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_lt(v128_t __a, v128_t __b) {
1940 return (v128_t)((__f16x8)__a < (__f16x8)__b);
1941}
1942
1943static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_gt(v128_t __a, v128_t __b) {
1944 return (v128_t)((__f16x8)__a > (__f16x8)__b);
1945}
1946
1947static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_le(v128_t __a, v128_t __b) {
1948 return (v128_t)((__f16x8)__a <= (__f16x8)__b);
1949}
1950
1951static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_ge(v128_t __a, v128_t __b) {
1952 return (v128_t)((__f16x8)__a >= (__f16x8)__b);
1953}
1954
1955static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_add(v128_t __a,
1956 v128_t __b) {
1957 return (v128_t)((__f16x8)__a + (__f16x8)__b);
1958}
1959
1960static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_sub(v128_t __a,
1961 v128_t __b) {
1962 return (v128_t)((__f16x8)__a - (__f16x8)__b);
1963}
1964
1965static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_mul(v128_t __a,
1966 v128_t __b) {
1967 return (v128_t)((__f16x8)__a * (__f16x8)__b);
1968}
1969
1970static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_div(v128_t __a,
1971 v128_t __b) {
1972 return (v128_t)((__f16x8)__a / (__f16x8)__b);
1973}
1974
1975static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_min(v128_t __a,
1976 v128_t __b) {
1977 return (v128_t)__builtin_wasm_min_f16x8((__f16x8)__a, (__f16x8)__b);
1978}
1979
1980static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_max(v128_t __a,
1981 v128_t __b) {
1982 return (v128_t)__builtin_wasm_max_f16x8((__f16x8)__a, (__f16x8)__b);
1983}
1984
1985static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_pmin(v128_t __a,
1986 v128_t __b) {
1987 return (v128_t)__builtin_wasm_pmin_f16x8((__f16x8)__a, (__f16x8)__b);
1988}
1989
1990static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_pmax(v128_t __a,
1991 v128_t __b) {
1992 return (v128_t)__builtin_wasm_pmax_f16x8((__f16x8)__a, (__f16x8)__b);
1993}
1994
1995static __inline__ v128_t __FP16_FN_ATTRS
1996wasm_i16x8_trunc_sat_f16x8(v128_t __a) {
1997 return (v128_t)__builtin_wasm_trunc_saturate_s_i16x8_f16x8((__f16x8)__a);
1998}
1999
2000static __inline__ v128_t __FP16_FN_ATTRS
2001wasm_u16x8_trunc_sat_f16x8(v128_t __a) {
2002 return (v128_t)__builtin_wasm_trunc_saturate_u_i16x8_f16x8((__f16x8)__a);
2003}
2004
2005static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_convert_i16x8(v128_t __a) {
2006 return (v128_t) __builtin_convertvector((__i16x8)__a, __f16x8);
2007}
2008
2009static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_convert_u16x8(v128_t __a) {
2010 return (v128_t) __builtin_convertvector((__u16x8)__a, __f16x8);
2011}
2012
2013static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_relaxed_madd(v128_t __a,
2014 v128_t __b,
2015 v128_t __c) {
2016 return (v128_t)__builtin_wasm_relaxed_madd_f16x8((__f16x8)__a, (__f16x8)__b,
2017 (__f16x8)__c);
2018}
2019
2020static __inline__ v128_t __FP16_FN_ATTRS wasm_f16x8_relaxed_nmadd(v128_t __a,
2021 v128_t __b,
2022 v128_t __c) {
2023 return (v128_t)__builtin_wasm_relaxed_nmadd_f16x8((__f16x8)__a, (__f16x8)__b,
2024 (__f16x8)__c);
2025}
2026
18812027// Deprecated intrinsics
18822028
18832029static __inline__ v128_t __DEPRECATED_FN_ATTRS("wasm_i8x16_swizzle")
lib/include/xmmintrin.h+170-177
......@@ -32,12 +32,41 @@ typedef unsigned int __v4su __attribute__((__vector_size__(16)));
3232#endif
3333
3434/* Define the default attributes for the functions in this file. */
35#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
3536#define __DEFAULT_FN_ATTRS \
3637 __attribute__((__always_inline__, __nodebug__, __target__("sse,no-evex512"), \
3738 __min_vector_width__(128)))
38#define __DEFAULT_FN_ATTRS_MMX \
39#define __DEFAULT_FN_ATTRS_SSE2 \
3940 __attribute__((__always_inline__, __nodebug__, \
40 __target__("mmx,sse,no-evex512"), __min_vector_width__(64)))
41 __target__("sse2,no-evex512"), __min_vector_width__(128)))
42#else
43#define __DEFAULT_FN_ATTRS \
44 __attribute__((__always_inline__, __nodebug__, __target__("sse"), \
45 __min_vector_width__(128)))
46#define __DEFAULT_FN_ATTRS_SSE2 \
47 __attribute__((__always_inline__, __nodebug__, __target__("sse2"), \
48 __min_vector_width__(128)))
49#endif
50
51#if defined(__cplusplus) && (__cplusplus >= 201103L)
52#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
53#define __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR __DEFAULT_FN_ATTRS_SSE2 constexpr
54#else
55#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
56#define __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR __DEFAULT_FN_ATTRS_SSE2
57#endif
58
59#define __trunc64(x) \
60 (__m64) __builtin_shufflevector((__v2di)(x), __extension__(__v2di){}, 0)
61#define __zext128(x) \
62 (__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0, \
63 1, 2, 3)
64#define __anyext128(x) \
65 (__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0, \
66 1, -1, -1)
67#define __zeroupper64(x) \
68 (__m128i) __builtin_shufflevector((__v4si)(x), __extension__(__v4si){}, 0, \
69 1, 4, 5)
4170
4271/// Adds the 32-bit float values in the low-order bits of the operands.
4372///
......@@ -54,9 +83,8 @@ typedef unsigned int __v4su __attribute__((__vector_size__(16)));
5483/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the sum
5584/// of the lower 32 bits of both operands. The upper 96 bits are copied from
5685/// the upper 96 bits of the first source operand.
57static __inline__ __m128 __DEFAULT_FN_ATTRS
58_mm_add_ss(__m128 __a, __m128 __b)
59{
86static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
87_mm_add_ss(__m128 __a, __m128 __b) {
6088 __a[0] += __b[0];
6189 return __a;
6290}
......@@ -74,9 +102,8 @@ _mm_add_ss(__m128 __a, __m128 __b)
74102/// A 128-bit vector of [4 x float] containing one of the source operands.
75103/// \returns A 128-bit vector of [4 x float] containing the sums of both
76104/// operands.
77static __inline__ __m128 __DEFAULT_FN_ATTRS
78_mm_add_ps(__m128 __a, __m128 __b)
79{
105static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
106_mm_add_ps(__m128 __a, __m128 __b) {
80107 return (__m128)((__v4sf)__a + (__v4sf)__b);
81108}
82109
......@@ -96,9 +123,8 @@ _mm_add_ps(__m128 __a, __m128 __b)
96123/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the
97124/// difference of the lower 32 bits of both operands. The upper 96 bits are
98125/// copied from the upper 96 bits of the first source operand.
99static __inline__ __m128 __DEFAULT_FN_ATTRS
100_mm_sub_ss(__m128 __a, __m128 __b)
101{
126static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
127_mm_sub_ss(__m128 __a, __m128 __b) {
102128 __a[0] -= __b[0];
103129 return __a;
104130}
......@@ -117,9 +143,8 @@ _mm_sub_ss(__m128 __a, __m128 __b)
117143/// A 128-bit vector of [4 x float] containing the subtrahend.
118144/// \returns A 128-bit vector of [4 x float] containing the differences between
119145/// both operands.
120static __inline__ __m128 __DEFAULT_FN_ATTRS
121_mm_sub_ps(__m128 __a, __m128 __b)
122{
146static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
147_mm_sub_ps(__m128 __a, __m128 __b) {
123148 return (__m128)((__v4sf)__a - (__v4sf)__b);
124149}
125150
......@@ -139,9 +164,8 @@ _mm_sub_ps(__m128 __a, __m128 __b)
139164/// \returns A 128-bit vector of [4 x float] containing the product of the lower
140165/// 32 bits of both operands. The upper 96 bits are copied from the upper 96
141166/// bits of the first source operand.
142static __inline__ __m128 __DEFAULT_FN_ATTRS
143_mm_mul_ss(__m128 __a, __m128 __b)
144{
167static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
168_mm_mul_ss(__m128 __a, __m128 __b) {
145169 __a[0] *= __b[0];
146170 return __a;
147171}
......@@ -159,9 +183,8 @@ _mm_mul_ss(__m128 __a, __m128 __b)
159183/// A 128-bit vector of [4 x float] containing one of the source operands.
160184/// \returns A 128-bit vector of [4 x float] containing the products of both
161185/// operands.
162static __inline__ __m128 __DEFAULT_FN_ATTRS
163_mm_mul_ps(__m128 __a, __m128 __b)
164{
186static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
187_mm_mul_ps(__m128 __a, __m128 __b) {
165188 return (__m128)((__v4sf)__a * (__v4sf)__b);
166189}
167190
......@@ -181,9 +204,8 @@ _mm_mul_ps(__m128 __a, __m128 __b)
181204/// \returns A 128-bit vector of [4 x float] containing the quotients of the
182205/// lower 32 bits of both operands. The upper 96 bits are copied from the
183206/// upper 96 bits of the first source operand.
184static __inline__ __m128 __DEFAULT_FN_ATTRS
185_mm_div_ss(__m128 __a, __m128 __b)
186{
207static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
208_mm_div_ss(__m128 __a, __m128 __b) {
187209 __a[0] /= __b[0];
188210 return __a;
189211}
......@@ -200,9 +222,8 @@ _mm_div_ss(__m128 __a, __m128 __b)
200222/// A 128-bit vector of [4 x float] containing the divisor.
201223/// \returns A 128-bit vector of [4 x float] containing the quotients of both
202224/// operands.
203static __inline__ __m128 __DEFAULT_FN_ATTRS
204_mm_div_ps(__m128 __a, __m128 __b)
205{
225static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
226_mm_div_ps(__m128 __a, __m128 __b) {
206227 return (__m128)((__v4sf)__a / (__v4sf)__b);
207228}
208229
......@@ -416,9 +437,8 @@ _mm_max_ps(__m128 __a, __m128 __b)
416437/// A 128-bit vector containing one of the source operands.
417438/// \returns A 128-bit vector of [4 x float] containing the bitwise AND of the
418439/// values between both operands.
419static __inline__ __m128 __DEFAULT_FN_ATTRS
420_mm_and_ps(__m128 __a, __m128 __b)
421{
440static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
441_mm_and_ps(__m128 __a, __m128 __b) {
422442 return (__m128)((__v4su)__a & (__v4su)__b);
423443}
424444
......@@ -438,9 +458,8 @@ _mm_and_ps(__m128 __a, __m128 __b)
438458/// \returns A 128-bit vector of [4 x float] containing the bitwise AND of the
439459/// one's complement of the first operand and the values in the second
440460/// operand.
441static __inline__ __m128 __DEFAULT_FN_ATTRS
442_mm_andnot_ps(__m128 __a, __m128 __b)
443{
461static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
462_mm_andnot_ps(__m128 __a, __m128 __b) {
444463 return (__m128)(~(__v4su)__a & (__v4su)__b);
445464}
446465
......@@ -456,9 +475,8 @@ _mm_andnot_ps(__m128 __a, __m128 __b)
456475/// A 128-bit vector of [4 x float] containing one of the source operands.
457476/// \returns A 128-bit vector of [4 x float] containing the bitwise OR of the
458477/// values between both operands.
459static __inline__ __m128 __DEFAULT_FN_ATTRS
460_mm_or_ps(__m128 __a, __m128 __b)
461{
478static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
479_mm_or_ps(__m128 __a, __m128 __b) {
462480 return (__m128)((__v4su)__a | (__v4su)__b);
463481}
464482
......@@ -475,9 +493,8 @@ _mm_or_ps(__m128 __a, __m128 __b)
475493/// A 128-bit vector of [4 x float] containing one of the source operands.
476494/// \returns A 128-bit vector of [4 x float] containing the bitwise exclusive OR
477495/// of the values between both operands.
478static __inline__ __m128 __DEFAULT_FN_ATTRS
479_mm_xor_ps(__m128 __a, __m128 __b)
480{
496static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
497_mm_xor_ps(__m128 __a, __m128 __b) {
481498 return (__m128)((__v4su)__a ^ (__v4su)__b);
482499}
483500
......@@ -1448,10 +1465,10 @@ _mm_cvtss_si64(__m128 __a)
14481465/// \param __a
14491466/// A 128-bit vector of [4 x float].
14501467/// \returns A 64-bit integer vector containing the converted values.
1451static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
1468static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
14521469_mm_cvtps_pi32(__m128 __a)
14531470{
1454 return (__m64)__builtin_ia32_cvtps2pi((__v4sf)__a);
1471 return __trunc64(__builtin_ia32_cvtps2dq((__v4sf)__zeroupper64(__a)));
14551472}
14561473
14571474/// Converts two low-order float values in a 128-bit vector of
......@@ -1468,7 +1485,7 @@ _mm_cvtps_pi32(__m128 __a)
14681485/// \param __a
14691486/// A 128-bit vector of [4 x float].
14701487/// \returns A 64-bit integer vector containing the converted values.
1471static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
1488static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
14721489_mm_cvt_ps2pi(__m128 __a)
14731490{
14741491 return _mm_cvtps_pi32(__a);
......@@ -1558,10 +1575,10 @@ _mm_cvttss_si64(__m128 __a)
15581575/// \param __a
15591576/// A 128-bit vector of [4 x float].
15601577/// \returns A 64-bit integer vector containing the converted values.
1561static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
1578static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
15621579_mm_cvttps_pi32(__m128 __a)
15631580{
1564 return (__m64)__builtin_ia32_cvttps2pi((__v4sf)__a);
1581 return __trunc64(__builtin_ia32_cvttps2dq((__v4sf)__zeroupper64(__a)));
15651582}
15661583
15671584/// Converts the lower (first) two elements of a 128-bit vector of [4 x float]
......@@ -1579,7 +1596,7 @@ _mm_cvttps_pi32(__m128 __a)
15791596/// \param __a
15801597/// A 128-bit vector of [4 x float].
15811598/// \returns A 64-bit integer vector containing the converted values.
1582static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
1599static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
15831600_mm_cvtt_ps2pi(__m128 __a)
15841601{
15851602 return _mm_cvttps_pi32(__a);
......@@ -1601,9 +1618,8 @@ _mm_cvtt_ps2pi(__m128 __a)
16011618/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the
16021619/// converted value of the second operand. The upper 96 bits are copied from
16031620/// the upper 96 bits of the first operand.
1604static __inline__ __m128 __DEFAULT_FN_ATTRS
1605_mm_cvtsi32_ss(__m128 __a, int __b)
1606{
1621static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_cvtsi32_ss(__m128 __a,
1622 int __b) {
16071623 __a[0] = __b;
16081624 return __a;
16091625}
......@@ -1624,9 +1640,8 @@ _mm_cvtsi32_ss(__m128 __a, int __b)
16241640/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the
16251641/// converted value of the second operand. The upper 96 bits are copied from
16261642/// the upper 96 bits of the first operand.
1627static __inline__ __m128 __DEFAULT_FN_ATTRS
1628_mm_cvt_si2ss(__m128 __a, int __b)
1629{
1643static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_cvt_si2ss(__m128 __a,
1644 int __b) {
16301645 return _mm_cvtsi32_ss(__a, __b);
16311646}
16321647
......@@ -1648,9 +1663,8 @@ _mm_cvt_si2ss(__m128 __a, int __b)
16481663/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the
16491664/// converted value of the second operand. The upper 96 bits are copied from
16501665/// the upper 96 bits of the first operand.
1651static __inline__ __m128 __DEFAULT_FN_ATTRS
1652_mm_cvtsi64_ss(__m128 __a, long long __b)
1653{
1666static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
1667_mm_cvtsi64_ss(__m128 __a, long long __b) {
16541668 __a[0] = __b;
16551669 return __a;
16561670}
......@@ -1674,10 +1688,13 @@ _mm_cvtsi64_ss(__m128 __a, long long __b)
16741688/// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the
16751689/// converted value of the second operand. The upper 64 bits are copied from
16761690/// the upper 64 bits of the first operand.
1677static __inline__ __m128 __DEFAULT_FN_ATTRS_MMX
1691static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
16781692_mm_cvtpi32_ps(__m128 __a, __m64 __b)
16791693{
1680 return __builtin_ia32_cvtpi2ps((__v4sf)__a, (__v2si)__b);
1694 return (__m128)__builtin_shufflevector(
1695 (__v4sf)__a,
1696 __builtin_convertvector((__v4si)__zext128(__b), __v4sf),
1697 4, 5, 2, 3);
16811698}
16821699
16831700/// Converts two elements of a 64-bit vector of [2 x i32] into two
......@@ -1697,7 +1714,7 @@ _mm_cvtpi32_ps(__m128 __a, __m64 __b)
16971714/// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the
16981715/// converted value from the second operand. The upper 64 bits are copied
16991716/// from the upper 64 bits of the first operand.
1700static __inline__ __m128 __DEFAULT_FN_ATTRS_MMX
1717static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
17011718_mm_cvt_pi2ps(__m128 __a, __m64 __b)
17021719{
17031720 return _mm_cvtpi32_ps(__a, __b);
......@@ -1714,9 +1731,8 @@ _mm_cvt_pi2ps(__m128 __a, __m64 __b)
17141731/// A 128-bit vector of [4 x float]. The lower 32 bits of this operand are
17151732/// used in the extraction.
17161733/// \returns A 32-bit float containing the extracted value.
1717static __inline__ float __DEFAULT_FN_ATTRS
1718_mm_cvtss_f32(__m128 __a)
1719{
1734static __inline__ float __DEFAULT_FN_ATTRS_CONSTEXPR
1735_mm_cvtss_f32(__m128 __a) {
17201736 return __a[0];
17211737}
17221738
......@@ -1907,9 +1923,8 @@ _mm_undefined_ps(void)
19071923/// \returns An initialized 128-bit floating-point vector of [4 x float]. The
19081924/// lower 32 bits contain the value provided in the source operand. The
19091925/// upper 96 bits are set to zero.
1910static __inline__ __m128 __DEFAULT_FN_ATTRS
1911_mm_set_ss(float __w)
1912{
1926static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
1927_mm_set_ss(float __w) {
19131928 return __extension__ (__m128){ __w, 0.0f, 0.0f, 0.0f };
19141929}
19151930
......@@ -1925,9 +1940,8 @@ _mm_set_ss(float __w)
19251940/// A single-precision floating-point value used to initialize each vector
19261941/// element of the result.
19271942/// \returns An initialized 128-bit floating-point vector of [4 x float].
1928static __inline__ __m128 __DEFAULT_FN_ATTRS
1929_mm_set1_ps(float __w)
1930{
1943static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
1944_mm_set1_ps(float __w) {
19311945 return __extension__ (__m128){ __w, __w, __w, __w };
19321946}
19331947
......@@ -1944,9 +1958,8 @@ _mm_set1_ps(float __w)
19441958/// A single-precision floating-point value used to initialize each vector
19451959/// element of the result.
19461960/// \returns An initialized 128-bit floating-point vector of [4 x float].
1947static __inline__ __m128 __DEFAULT_FN_ATTRS
1948_mm_set_ps1(float __w)
1949{
1961static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
1962_mm_set_ps1(float __w) {
19501963 return _mm_set1_ps(__w);
19511964}
19521965
......@@ -1971,9 +1984,8 @@ _mm_set_ps1(float __w)
19711984/// A single-precision floating-point value used to initialize bits [31:0]
19721985/// of the result.
19731986/// \returns An initialized 128-bit floating-point vector of [4 x float].
1974static __inline__ __m128 __DEFAULT_FN_ATTRS
1975_mm_set_ps(float __z, float __y, float __x, float __w)
1976{
1987static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
1988_mm_set_ps(float __z, float __y, float __x, float __w) {
19771989 return __extension__ (__m128){ __w, __x, __y, __z };
19781990}
19791991
......@@ -1999,9 +2011,8 @@ _mm_set_ps(float __z, float __y, float __x, float __w)
19992011/// A single-precision floating-point value used to initialize bits [127:96]
20002012/// of the result.
20012013/// \returns An initialized 128-bit floating-point vector of [4 x float].
2002static __inline__ __m128 __DEFAULT_FN_ATTRS
2003_mm_setr_ps(float __z, float __y, float __x, float __w)
2004{
2014static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
2015_mm_setr_ps(float __z, float __y, float __x, float __w) {
20052016 return __extension__ (__m128){ __z, __y, __x, __w };
20062017}
20072018
......@@ -2014,9 +2025,8 @@ _mm_setr_ps(float __z, float __y, float __x, float __w)
20142025///
20152026/// \returns An initialized 128-bit floating-point vector of [4 x float] with
20162027/// all elements set to zero.
2017static __inline__ __m128 __DEFAULT_FN_ATTRS
2018_mm_setzero_ps(void)
2019{
2028static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
2029_mm_setzero_ps(void) {
20202030 return __extension__ (__m128){ 0.0f, 0.0f, 0.0f, 0.0f };
20212031}
20222032
......@@ -2231,10 +2241,10 @@ _mm_storer_ps(float *__p, __m128 __a)
22312241/// A pointer to an aligned memory location used to store the register value.
22322242/// \param __a
22332243/// A 64-bit integer containing the value to be stored.
2234static __inline__ void __DEFAULT_FN_ATTRS_MMX
2244static __inline__ void __DEFAULT_FN_ATTRS
22352245_mm_stream_pi(void *__p, __m64 __a)
22362246{
2237 __builtin_ia32_movntq((__m64 *)__p, __a);
2247 __builtin_nontemporal_store(__a, (__m64 *)__p);
22382248}
22392249
22402250/// Moves packed float values from a 128-bit vector of [4 x float] to a
......@@ -2296,7 +2306,7 @@ void _mm_sfence(void);
22962306/// 3: Bits [63:48] are copied to the destination.
22972307/// \returns A 16-bit integer containing the extracted 16 bits of packed data.
22982308#define _mm_extract_pi16(a, n) \
2299 ((int)__builtin_ia32_vec_ext_v4hi((__v4hi)a, (int)n))
2309 ((int)(unsigned short)__builtin_ia32_vec_ext_v4hi((__v4hi)a, (int)n))
23002310
23012311/// Copies data from the 64-bit vector of [4 x i16] to the destination,
23022312/// and inserts the lower 16-bits of an integer operand at the 16-bit offset
......@@ -2342,10 +2352,10 @@ void _mm_sfence(void);
23422352/// \param __b
23432353/// A 64-bit integer vector containing one of the source operands.
23442354/// \returns A 64-bit integer vector containing the comparison results.
2345static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
2355static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
23462356_mm_max_pi16(__m64 __a, __m64 __b)
23472357{
2348 return (__m64)__builtin_ia32_pmaxsw((__v4hi)__a, (__v4hi)__b);
2358 return (__m64)__builtin_elementwise_max((__v4hi)__a, (__v4hi)__b);
23492359}
23502360
23512361/// Compares each of the corresponding packed 8-bit unsigned integer
......@@ -2361,10 +2371,10 @@ _mm_max_pi16(__m64 __a, __m64 __b)
23612371/// \param __b
23622372/// A 64-bit integer vector containing one of the source operands.
23632373/// \returns A 64-bit integer vector containing the comparison results.
2364static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
2374static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
23652375_mm_max_pu8(__m64 __a, __m64 __b)
23662376{
2367 return (__m64)__builtin_ia32_pmaxub((__v8qi)__a, (__v8qi)__b);
2377 return (__m64)__builtin_elementwise_max((__v8qu)__a, (__v8qu)__b);
23682378}
23692379
23702380/// Compares each of the corresponding packed 16-bit integer values of
......@@ -2380,10 +2390,10 @@ _mm_max_pu8(__m64 __a, __m64 __b)
23802390/// \param __b
23812391/// A 64-bit integer vector containing one of the source operands.
23822392/// \returns A 64-bit integer vector containing the comparison results.
2383static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
2393static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
23842394_mm_min_pi16(__m64 __a, __m64 __b)
23852395{
2386 return (__m64)__builtin_ia32_pminsw((__v4hi)__a, (__v4hi)__b);
2396 return (__m64)__builtin_elementwise_min((__v4hi)__a, (__v4hi)__b);
23872397}
23882398
23892399/// Compares each of the corresponding packed 8-bit unsigned integer
......@@ -2399,10 +2409,10 @@ _mm_min_pi16(__m64 __a, __m64 __b)
23992409/// \param __b
24002410/// A 64-bit integer vector containing one of the source operands.
24012411/// \returns A 64-bit integer vector containing the comparison results.
2402static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
2412static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
24032413_mm_min_pu8(__m64 __a, __m64 __b)
24042414{
2405 return (__m64)__builtin_ia32_pminub((__v8qi)__a, (__v8qi)__b);
2415 return (__m64)__builtin_elementwise_min((__v8qu)__a, (__v8qu)__b);
24062416}
24072417
24082418/// Takes the most significant bit from each 8-bit element in a 64-bit
......@@ -2417,10 +2427,10 @@ _mm_min_pu8(__m64 __a, __m64 __b)
24172427/// A 64-bit integer vector containing the values with bits to be extracted.
24182428/// \returns The most significant bit from each 8-bit element in \a __a,
24192429/// written to bits [7:0].
2420static __inline__ int __DEFAULT_FN_ATTRS_MMX
2430static __inline__ int __DEFAULT_FN_ATTRS_SSE2
24212431_mm_movemask_pi8(__m64 __a)
24222432{
2423 return __builtin_ia32_pmovmskb((__v8qi)__a);
2433 return __builtin_ia32_pmovmskb128((__v16qi)__zext128(__a));
24242434}
24252435
24262436/// Multiplies packed 16-bit unsigned integer values and writes the
......@@ -2436,10 +2446,11 @@ _mm_movemask_pi8(__m64 __a)
24362446/// \param __b
24372447/// A 64-bit integer vector containing one of the source operands.
24382448/// \returns A 64-bit integer vector containing the products of both operands.
2439static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
2449static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
24402450_mm_mulhi_pu16(__m64 __a, __m64 __b)
24412451{
2442 return (__m64)__builtin_ia32_pmulhuw((__v4hi)__a, (__v4hi)__b);
2452 return __trunc64(__builtin_ia32_pmulhuw128((__v8hi)__anyext128(__a),
2453 (__v8hi)__anyext128(__b)));
24432454}
24442455
24452456/// Shuffles the 4 16-bit integers from a 64-bit integer vector to the
......@@ -2476,8 +2487,10 @@ _mm_mulhi_pu16(__m64 __a, __m64 __b)
24762487/// <c>_MM_SHUFFLE(b6, b4, b2, b0)</c> can create an 8-bit mask of the form
24772488/// <c>[b6, b4, b2, b0]</c>.
24782489/// \returns A 64-bit integer vector containing the shuffled values.
2479#define _mm_shuffle_pi16(a, n) \
2480 ((__m64)__builtin_ia32_pshufw((__v4hi)(__m64)(a), (n)))
2490#define _mm_shuffle_pi16(a, n) \
2491 ((__m64)__builtin_shufflevector((__v4hi)(__m64)(a), __extension__(__v4hi){}, \
2492 (n) & 0x3, ((n) >> 2) & 0x3, \
2493 ((n) >> 4) & 0x3, ((n) >> 6) & 0x3))
24812494
24822495/// Conditionally copies the values from each 8-bit element in the first
24832496/// 64-bit integer vector operand to the specified memory location, as
......@@ -2502,10 +2515,25 @@ _mm_mulhi_pu16(__m64 __a, __m64 __b)
25022515/// A pointer to a 64-bit memory location that will receive the conditionally
25032516/// copied integer values. The address of the memory location does not have
25042517/// to be aligned.
2505static __inline__ void __DEFAULT_FN_ATTRS_MMX
2518static __inline__ void __DEFAULT_FN_ATTRS_SSE2
25062519_mm_maskmove_si64(__m64 __d, __m64 __n, char *__p)
25072520{
2508 __builtin_ia32_maskmovq((__v8qi)__d, (__v8qi)__n, __p);
2521 // This is complex, because we need to support the case where __p is pointing
2522 // within the last 15 to 8 bytes of a page. In that case, using a 128-bit
2523 // write might cause a trap where a 64-bit maskmovq would not. (Memory
2524 // locations not selected by the mask bits might still cause traps.)
2525 __m128i __d128 = __anyext128(__d);
2526 __m128i __n128 = __zext128(__n);
2527 if (((__SIZE_TYPE__)__p & 0xfff) >= 4096-15 &&
2528 ((__SIZE_TYPE__)__p & 0xfff) <= 4096-8) {
2529 // If there's a risk of spurious trap due to a 128-bit write, back up the
2530 // pointer by 8 bytes and shift values in registers to match.
2531 __p -= 8;
2532 __d128 = __builtin_ia32_pslldqi128_byteshift((__v2di)__d128, 8);
2533 __n128 = __builtin_ia32_pslldqi128_byteshift((__v2di)__n128, 8);
2534 }
2535
2536 __builtin_ia32_maskmovdqu((__v16qi)__d128, (__v16qi)__n128, __p);
25092537}
25102538
25112539/// Computes the rounded averages of the packed unsigned 8-bit integer
......@@ -2521,10 +2549,11 @@ _mm_maskmove_si64(__m64 __d, __m64 __n, char *__p)
25212549/// \param __b
25222550/// A 64-bit integer vector containing one of the source operands.
25232551/// \returns A 64-bit integer vector containing the averages of both operands.
2524static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
2552static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
25252553_mm_avg_pu8(__m64 __a, __m64 __b)
25262554{
2527 return (__m64)__builtin_ia32_pavgb((__v8qi)__a, (__v8qi)__b);
2555 return __trunc64(__builtin_ia32_pavgb128((__v16qi)__anyext128(__a),
2556 (__v16qi)__anyext128(__b)));
25282557}
25292558
25302559/// Computes the rounded averages of the packed unsigned 16-bit integer
......@@ -2540,10 +2569,11 @@ _mm_avg_pu8(__m64 __a, __m64 __b)
25402569/// \param __b
25412570/// A 64-bit integer vector containing one of the source operands.
25422571/// \returns A 64-bit integer vector containing the averages of both operands.
2543static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
2572static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
25442573_mm_avg_pu16(__m64 __a, __m64 __b)
25452574{
2546 return (__m64)__builtin_ia32_pavgw((__v4hi)__a, (__v4hi)__b);
2575 return __trunc64(__builtin_ia32_pavgw128((__v8hi)__anyext128(__a),
2576 (__v8hi)__anyext128(__b)));
25472577}
25482578
25492579/// Subtracts the corresponding 8-bit unsigned integer values of the two
......@@ -2562,10 +2592,11 @@ _mm_avg_pu16(__m64 __a, __m64 __b)
25622592/// \returns A 64-bit integer vector whose lower 16 bits contain the sums of the
25632593/// sets of absolute differences between both operands. The upper bits are
25642594/// cleared.
2565static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
2595static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
25662596_mm_sad_pu8(__m64 __a, __m64 __b)
25672597{
2568 return (__m64)__builtin_ia32_psadbw((__v8qi)__a, (__v8qi)__b);
2598 return __trunc64(__builtin_ia32_psadbw128((__v16qi)__zext128(__a),
2599 (__v16qi)__zext128(__b)));
25692600}
25702601
25712602#if defined(__cplusplus)
......@@ -2741,9 +2772,8 @@ void _mm_setcsr(unsigned int __i);
27412772/// Bits [95:64] are written to bits [63:32] of the destination. \n
27422773/// Bits [127:96] are written to bits [127:96] of the destination.
27432774/// \returns A 128-bit vector of [4 x float] containing the interleaved values.
2744static __inline__ __m128 __DEFAULT_FN_ATTRS
2745_mm_unpackhi_ps(__m128 __a, __m128 __b)
2746{
2775static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
2776_mm_unpackhi_ps(__m128 __a, __m128 __b) {
27472777 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__b, 2, 6, 3, 7);
27482778}
27492779
......@@ -2763,9 +2793,8 @@ _mm_unpackhi_ps(__m128 __a, __m128 __b)
27632793/// Bits [31:0] are written to bits [63:32] of the destination. \n
27642794/// Bits [63:32] are written to bits [127:96] of the destination.
27652795/// \returns A 128-bit vector of [4 x float] containing the interleaved values.
2766static __inline__ __m128 __DEFAULT_FN_ATTRS
2767_mm_unpacklo_ps(__m128 __a, __m128 __b)
2768{
2796static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
2797_mm_unpacklo_ps(__m128 __a, __m128 __b) {
27692798 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__b, 0, 4, 1, 5);
27702799}
27712800
......@@ -2785,9 +2814,8 @@ _mm_unpacklo_ps(__m128 __a, __m128 __b)
27852814/// A 128-bit floating-point vector of [4 x float]. The lower 32 bits are
27862815/// written to the lower 32 bits of the result.
27872816/// \returns A 128-bit floating-point vector of [4 x float].
2788static __inline__ __m128 __DEFAULT_FN_ATTRS
2789_mm_move_ss(__m128 __a, __m128 __b)
2790{
2817static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
2818_mm_move_ss(__m128 __a, __m128 __b) {
27912819 __a[0] = __b[0];
27922820 return __a;
27932821}
......@@ -2807,9 +2835,8 @@ _mm_move_ss(__m128 __a, __m128 __b)
28072835/// A 128-bit floating-point vector of [4 x float]. The upper 64 bits are
28082836/// written to the lower 64 bits of the result.
28092837/// \returns A 128-bit floating-point vector of [4 x float].
2810static __inline__ __m128 __DEFAULT_FN_ATTRS
2811_mm_movehl_ps(__m128 __a, __m128 __b)
2812{
2838static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
2839_mm_movehl_ps(__m128 __a, __m128 __b) {
28132840 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__b, 6, 7, 2, 3);
28142841}
28152842
......@@ -2828,9 +2855,8 @@ _mm_movehl_ps(__m128 __a, __m128 __b)
28282855/// A 128-bit floating-point vector of [4 x float]. The lower 64 bits are
28292856/// written to the upper 64 bits of the result.
28302857/// \returns A 128-bit floating-point vector of [4 x float].
2831static __inline__ __m128 __DEFAULT_FN_ATTRS
2832_mm_movelh_ps(__m128 __a, __m128 __b)
2833{
2858static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
2859_mm_movelh_ps(__m128 __a, __m128 __b) {
28342860 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__b, 0, 1, 4, 5);
28352861}
28362862
......@@ -2846,22 +2872,10 @@ _mm_movelh_ps(__m128 __a, __m128 __b)
28462872/// from the corresponding elements in this operand.
28472873/// \returns A 128-bit vector of [4 x float] containing the copied and converted
28482874/// values from the operand.
2849static __inline__ __m128 __DEFAULT_FN_ATTRS_MMX
2875static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
28502876_mm_cvtpi16_ps(__m64 __a)
28512877{
2852 __m64 __b, __c;
2853 __m128 __r;
2854
2855 __b = _mm_setzero_si64();
2856 __b = _mm_cmpgt_pi16(__b, __a);
2857 __c = _mm_unpackhi_pi16(__a, __b);
2858 __r = _mm_setzero_ps();
2859 __r = _mm_cvtpi32_ps(__r, __c);
2860 __r = _mm_movelh_ps(__r, __r);
2861 __c = _mm_unpacklo_pi16(__a, __b);
2862 __r = _mm_cvtpi32_ps(__r, __c);
2863
2864 return __r;
2878 return __builtin_convertvector((__v4hi)__a, __v4sf);
28652879}
28662880
28672881/// Converts a 64-bit vector of 16-bit unsigned integer values into a
......@@ -2876,21 +2890,10 @@ _mm_cvtpi16_ps(__m64 __a)
28762890/// destination are copied from the corresponding elements in this operand.
28772891/// \returns A 128-bit vector of [4 x float] containing the copied and converted
28782892/// values from the operand.
2879static __inline__ __m128 __DEFAULT_FN_ATTRS_MMX
2893static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
28802894_mm_cvtpu16_ps(__m64 __a)
28812895{
2882 __m64 __b, __c;
2883 __m128 __r;
2884
2885 __b = _mm_setzero_si64();
2886 __c = _mm_unpackhi_pi16(__a, __b);
2887 __r = _mm_setzero_ps();
2888 __r = _mm_cvtpi32_ps(__r, __c);
2889 __r = _mm_movelh_ps(__r, __r);
2890 __c = _mm_unpacklo_pi16(__a, __b);
2891 __r = _mm_cvtpi32_ps(__r, __c);
2892
2893 return __r;
2896 return __builtin_convertvector((__v4hu)__a, __v4sf);
28942897}
28952898
28962899/// Converts the lower four 8-bit values from a 64-bit vector of [8 x i8]
......@@ -2905,16 +2908,12 @@ _mm_cvtpu16_ps(__m64 __a)
29052908/// from the corresponding lower 4 elements in this operand.
29062909/// \returns A 128-bit vector of [4 x float] containing the copied and converted
29072910/// values from the operand.
2908static __inline__ __m128 __DEFAULT_FN_ATTRS_MMX
2911static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
29092912_mm_cvtpi8_ps(__m64 __a)
29102913{
2911 __m64 __b;
2912
2913 __b = _mm_setzero_si64();
2914 __b = _mm_cmpgt_pi8(__b, __a);
2915 __b = _mm_unpacklo_pi8(__a, __b);
2916
2917 return _mm_cvtpi16_ps(__b);
2914 return __builtin_convertvector(
2915 __builtin_shufflevector((__v8qs)__a, __extension__ (__v8qs){},
2916 0, 1, 2, 3), __v4sf);
29182917}
29192918
29202919/// Converts the lower four unsigned 8-bit integer values from a 64-bit
......@@ -2930,15 +2929,12 @@ _mm_cvtpi8_ps(__m64 __a)
29302929/// operand.
29312930/// \returns A 128-bit vector of [4 x float] containing the copied and converted
29322931/// values from the source operand.
2933static __inline__ __m128 __DEFAULT_FN_ATTRS_MMX
2932static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
29342933_mm_cvtpu8_ps(__m64 __a)
29352934{
2936 __m64 __b;
2937
2938 __b = _mm_setzero_si64();
2939 __b = _mm_unpacklo_pi8(__a, __b);
2940
2941 return _mm_cvtpi16_ps(__b);
2935 return __builtin_convertvector(
2936 __builtin_shufflevector((__v8qu)__a, __extension__ (__v8qu){},
2937 0, 1, 2, 3), __v4sf);
29422938}
29432939
29442940/// Converts the two 32-bit signed integer values from each 64-bit vector
......@@ -2957,16 +2953,12 @@ _mm_cvtpu8_ps(__m64 __a)
29572953/// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the
29582954/// copied and converted values from the first operand. The upper 64 bits
29592955/// contain the copied and converted values from the second operand.
2960static __inline__ __m128 __DEFAULT_FN_ATTRS_MMX
2956static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
29612957_mm_cvtpi32x2_ps(__m64 __a, __m64 __b)
29622958{
2963 __m128 __c;
2964
2965 __c = _mm_setzero_ps();
2966 __c = _mm_cvtpi32_ps(__c, __b);
2967 __c = _mm_movelh_ps(__c, __c);
2968
2969 return _mm_cvtpi32_ps(__c, __a);
2959 return __builtin_convertvector(
2960 __builtin_shufflevector((__v2si)__a, (__v2si)__b,
2961 0, 1, 2, 3), __v4sf);
29702962}
29712963
29722964/// Converts each single-precision floating-point element of a 128-bit
......@@ -2986,16 +2978,11 @@ _mm_cvtpi32x2_ps(__m64 __a, __m64 __b)
29862978/// A 128-bit floating-point vector of [4 x float].
29872979/// \returns A 64-bit integer vector of [4 x i16] containing the converted
29882980/// values.
2989static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
2981static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
29902982_mm_cvtps_pi16(__m128 __a)
29912983{
2992 __m64 __b, __c;
2993
2994 __b = _mm_cvtps_pi32(__a);
2995 __a = _mm_movehl_ps(__a, __a);
2996 __c = _mm_cvtps_pi32(__a);
2997
2998 return _mm_packs_pi32(__b, __c);
2984 return __trunc64(__builtin_ia32_packssdw128(
2985 (__v4si)__builtin_ia32_cvtps2dq((__v4sf)__a), (__v4si)_mm_setzero_ps()));
29992986}
30002987
30012988/// Converts each single-precision floating-point element of a 128-bit
......@@ -3016,7 +3003,7 @@ _mm_cvtps_pi16(__m128 __a)
30163003/// 128-bit floating-point vector of [4 x float].
30173004/// \returns A 64-bit integer vector of [8 x i8]. The lower 32 bits contain the
30183005/// converted values and the uppper 32 bits are set to zero.
3019static __inline__ __m64 __DEFAULT_FN_ATTRS_MMX
3006static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
30203007_mm_cvtps_pi8(__m128 __a)
30213008{
30223009 __m64 __b, __c;
......@@ -3196,8 +3183,14 @@ do { \
31963183#define _m_psadbw _mm_sad_pu8
31973184#define _m_ _mm_
31983185
3186#undef __trunc64
3187#undef __zext128
3188#undef __anyext128
3189#undef __zeroupper64
31993190#undef __DEFAULT_FN_ATTRS
3200#undef __DEFAULT_FN_ATTRS_MMX
3191#undef __DEFAULT_FN_ATTRS_CONSTEXPR
3192#undef __DEFAULT_FN_ATTRS_SSE2
3193#undef __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
32013194
32023195/* Ugly hack for backwards-compatibility (compatible with gcc) */
32033196#if defined(__SSE2__) && !__building_module(_Builtin_intrinsics)